Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(sdks): surface success:false scrape errors in Go, Java, .NET, Ruby and PHP (#4971)
Gaurav Chadha88569f422248
diff --git a/apps/dot-net-sdk/Firecrawl.Tests/ScrapeErrorTests.cs b/apps/dot-net-sdk/Firecrawl.Tests/ScrapeErrorTests.csnew file mode 100644index 000000000..bec60ea93--- /dev/null+++ b/apps/dot-net-sdk/Firecrawl.Tests/ScrapeErrorTests.cs@@ -0,0 +1,48 @@+using System.Net;+using System.Text;+using Firecrawl.Exceptions;+using Xunit;++namespace Firecrawl.Tests;++public class ScrapeErrorTests+{+ private sealed class StubHandler : HttpMessageHandler+ {+ private readonly string _body;++ public StubHandler(string body) => _body = body;++ protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)+ => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)+ {+ Content = new StringContent(_body, Encoding.UTF8, "application/json"),+ });+ }++ private static FirecrawlClient ClientReturning(string body)+ => new(apiKey: "fc-test-key", apiUrl: "http://localhost", maxRetries: 0, httpClient: new HttpClient(new StubHandler(body)));++ [Fact]+ public async Task ScrapeAsync_ThrowsWithApiErrorOnSuccessFalse()+ {+ var client = ClientReturning(+ "{\"success\":false,\"code\":\"SCRAPE_DNS_RESOLUTION_ERROR\",\"error\":\"DNS resolution failed for hostname \\\"nonexistent.example\\\".\"}");++ var ex = await Assert.ThrowsAsync<FirecrawlException>(() => client.ScrapeAsync("https://nonexistent.example"));++ Assert.Equal(200, ex.StatusCode);+ Assert.Equal("SCRAPE_DNS_RESOLUTION_ERROR", ex.ErrorCode);+ Assert.Equal("DNS resolution failed for hostname \"nonexistent.example\".", ex.Message);+ }++ [Fact]+ public async Task ScrapeAsync_ReturnsDocumentOnSuccess()+ {+ var client = ClientReturning("{\"success\":true,\"data\":{\"markdown\":\"# Hello\"}}");++ var doc = await client.ScrapeAsync("https://example.com");++ Assert.Equal("# Hello", doc.Markdown);+ }+}diff --git a/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj b/apps/dot-net-sdk/Firecrawl/Firecrawl.csprojindex c21001936..a33cb5118 100644--- a/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj+++ b/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj@@ -8,7 +8,7 @@<!-- NuGet package metadata --><PackageId>firecrawl-sdk</PackageId>- <Version>1.14.1</Version>+ <Version>1.14.2</Version><Authors>Firecrawl</Authors><Company>Firecrawl</Company><Description>.NET SDK for the Firecrawl API - web scraping, crawling, and data extraction</Description>diff --git a/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs b/apps/dot-net-sdk/Firecrawl/FirecrawlClient.csindex 9f1bf1e90..d1cec4df0 100644--- a/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs+++ b/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs@@ -82,6 +82,10 @@ public class FirecrawlClientvar response = await _http.PostAsync<ApiResponse<Document>>("/v2/scrape", body, cancellationToken: cancellationToken);+ // Some scrape failures (e.g. SCRAPE_DNS_RESOLUTION_ERROR) arrive as HTTP 200 with success: false.+ if (!response.Success && response.Error is not null)+ throw new FirecrawlException(response.Error, 200, response.Code, response.Details);+return response.Data ?? throw new FirecrawlException("Scrape response contained no data");}@@ -755,7 +759,7 @@ public class FirecrawlClient// INTERNAL UTILITIES// ================================================================- private const string SdkOrigin = "[email protected]";+ private const string SdkOrigin = "[email protected]";private static Dictionary<string, object> BuildBody(object? options){diff --git a/apps/dot-net-sdk/Firecrawl/Models/ApiResponse.cs b/apps/dot-net-sdk/Firecrawl/Models/ApiResponse.csindex 0b5b6a055..e37c5fd2f 100644--- a/apps/dot-net-sdk/Firecrawl/Models/ApiResponse.cs+++ b/apps/dot-net-sdk/Firecrawl/Models/ApiResponse.cs@@ -12,4 +12,13 @@ internal class ApiResponse<T>[JsonPropertyName("data")]public T? Data { get; set; }++ [JsonPropertyName("error")]+ public string? Error { get; set; }++ [JsonPropertyName("code")]+ public string? Code { get; set; }++ [JsonPropertyName("details")]+ public object? Details { get; set; }}diff --git a/apps/go-sdk/firecrawl.go b/apps/go-sdk/firecrawl.goindex 902ebcfcd..8c9e50044 100644--- a/apps/go-sdk/firecrawl.go+++ b/apps/go-sdk/firecrawl.go@@ -100,6 +100,15 @@ func (c *Client) Scrape(ctx context.Context, url string, opts *ScrapeOptions) (*return nil, err}+ // Some scrape failures (e.g. SCRAPE_DNS_RESOLUTION_ERROR) arrive as HTTP 200 with success: false.+ var envelope struct {+ Success *bool `json:"success"`+ }+ if json.Unmarshal(raw, &envelope) == nil && envelope.Success != nil && !*envelope.Success {+ msg, code, requiresAction := extractError(raw, 200)+ return nil, &FirecrawlError{StatusCode: 200, ErrorCode: code, Message: msg, RequiresAction: requiresAction}+ }+doc, err := extractDataAs[Document](raw)if err != nil {return nil, errdiff --git a/apps/go-sdk/scrape_errors_test.go b/apps/go-sdk/scrape_errors_test.gonew file mode 100644index 000000000..282fc40a7--- /dev/null+++ b/apps/go-sdk/scrape_errors_test.go@@ -0,0 +1,65 @@+package firecrawl++import (+ "context"+ "errors"+ "net/http"+ "net/http/httptest"+ "testing"++ "github.com/firecrawl/firecrawl/apps/go-sdk/option"+)++func TestScrapeReturnsErrorOnSuccessFalse(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ w.WriteHeader(http.StatusOK)+ _, _ = w.Write([]byte(`{"success":false,"code":"SCRAPE_DNS_RESOLUTION_ERROR","error":"DNS resolution failed for hostname \"nonexistent.example\"."}`))+ }))+ defer server.Close()++ client, err := NewClient(option.WithAPIKey("fc-test"), option.WithAPIURL(server.URL))+ if err != nil {+ t.Fatalf("NewClient: %v", err)+ }++ doc, err := client.Scrape(context.Background(), "https://nonexistent.example", nil)+ if doc != nil {+ t.Errorf("expected nil document, got %+v", doc)+ }+ var fe *FirecrawlError+ if !errors.As(err, &fe) {+ t.Fatalf("expected *FirecrawlError, got %T: %v", err, err)+ }+ if fe.ErrorCode != "SCRAPE_DNS_RESOLUTION_ERROR" {+ t.Errorf("ErrorCode = %q, want SCRAPE_DNS_RESOLUTION_ERROR", fe.ErrorCode)+ }+ if fe.StatusCode != 200 {+ t.Errorf("StatusCode = %d, want 200", fe.StatusCode)+ }+ if fe.Message != `DNS resolution failed for hostname "nonexistent.example".` {+ t.Errorf("Message = %q", fe.Message)+ }+}++func TestScrapeReturnsDocumentOnSuccess(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ w.WriteHeader(http.StatusOK)+ _, _ = w.Write([]byte(`{"success":true,"data":{"markdown":"# Hello"}}`))+ }))+ defer server.Close()++ client, err := NewClient(option.WithAPIKey("fc-test"), option.WithAPIURL(server.URL))+ if err != nil {+ t.Fatalf("NewClient: %v", err)+ }++ doc, err := client.Scrape(context.Background(), "https://example.com", nil)+ if err != nil {+ t.Fatalf("Scrape: %v", err)+ }+ if doc.Markdown != "# Hello" {+ t.Errorf("Markdown = %q, want # Hello", doc.Markdown)+ }+}diff --git a/apps/java-sdk/build.gradle.kts b/apps/java-sdk/build.gradle.ktsindex f67fb6ce1..8b91e7062 100644--- a/apps/java-sdk/build.gradle.kts+++ b/apps/java-sdk/build.gradle.kts@@ -4,7 +4,7 @@ plugins {}group = "com.firecrawl"-version = "1.18.3"+version = "1.18.4"java {sourceCompatibility = JavaVersion.VERSION_11diff --git a/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java b/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.javaindex e4ad4fea7..c6646097d 100644--- a/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java+++ b/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java@@ -37,7 +37,7 @@ import java.util.concurrent.ForkJoinPool;public class FirecrawlClient {private static final String DEFAULT_API_URL = "https://api.firecrawl.dev";- private static final String SDK_ORIGIN = "[email protected]";+ private static final String SDK_ORIGIN = "[email protected]";private static final long DEFAULT_TIMEOUT_MS = 300_000; // 5 minutesprivate static final int DEFAULT_MAX_RETRIES = 3;private static final double DEFAULT_BACKOFF_FACTOR = 0.5;@@ -103,7 +103,18 @@ public class FirecrawlClient {mergeOptions(body, options);}body.putIfAbsent("origin", SDK_ORIGIN);- return extractData(http.post("/v2/scrape", body, Map.class), Document.class);+ Map raw = http.post("/v2/scrape", body, Map.class);+ // Some scrape failures (e.g. SCRAPE_DNS_RESOLUTION_ERROR) arrive as HTTP 200 with success: false.+ if (Boolean.FALSE.equals(raw.get("success"))) {+ Object error = raw.get("error");+ Object code = raw.get("code");+ throw new FirecrawlException(+ error != null ? String.valueOf(error) : "Scrape failed",+ 200,+ code != null ? String.valueOf(code) : null,+ raw.get("details"));+ }+ return extractData(raw, Document.class);}/**diff --git a/apps/java-sdk/src/test/java/com/firecrawl/ScrapeErrorMockTest.java b/apps/java-sdk/src/test/java/com/firecrawl/ScrapeErrorMockTest.javanew file mode 100644index 000000000..3f4e02b6f--- /dev/null+++ b/apps/java-sdk/src/test/java/com/firecrawl/ScrapeErrorMockTest.java@@ -0,0 +1,81 @@+package com.firecrawl;++import com.firecrawl.client.FirecrawlClient;+import com.firecrawl.errors.FirecrawlException;+import com.firecrawl.models.Document;+import com.sun.net.httpserver.HttpServer;+import org.junit.jupiter.api.AfterEach;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;++import java.io.IOException;+import java.io.OutputStream;+import java.net.InetSocketAddress;+import java.nio.charset.StandardCharsets;+import java.util.concurrent.atomic.AtomicReference;++import static org.junit.jupiter.api.Assertions.*;++/**+ * POST /v2/scrape error handling against a local mock HTTP server.+ */+class ScrapeErrorMockTest {++ private HttpServer server;+ private FirecrawlClient client;+ private final AtomicReference<String> responseBody = new AtomicReference<>();+ private final AtomicReference<String> lastMethod = new AtomicReference<>();+ private final AtomicReference<String> lastPath = new AtomicReference<>();++ @BeforeEach+ void setup() throws IOException {+ server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);+ server.createContext("/", exchange -> {+ lastMethod.set(exchange.getRequestMethod());+ lastPath.set(exchange.getRequestURI().getPath());+ byte[] bytes = responseBody.get().getBytes(StandardCharsets.UTF_8);+ exchange.getResponseHeaders().set("Content-Type", "application/json");+ exchange.sendResponseHeaders(200, bytes.length);+ try (OutputStream os = exchange.getResponseBody()) {+ os.write(bytes);+ }+ });+ server.start();+ client = FirecrawlClient.builder()+ .apiKey("fc-test-key")+ .apiUrl("http://127.0.0.1:" + server.getAddress().getPort())+ .maxRetries(0)+ .build();+ }++ @AfterEach+ void teardown() {+ server.stop(0);+ }++ @Test+ void testScrapeThrowsOnSuccessFalse() {+ responseBody.set("{\"success\":false,\"code\":\"SCRAPE_DNS_RESOLUTION_ERROR\","+ + "\"error\":\"DNS resolution failed for hostname \\\"nonexistent.example\\\".\"}");++ FirecrawlException e = assertThrows(FirecrawlException.class,+ () -> client.scrape("https://nonexistent.example"));++ assertEquals("POST", lastMethod.get());+ assertEquals("/v2/scrape", lastPath.get());+ assertEquals(200, e.getStatusCode());+ assertEquals("SCRAPE_DNS_RESOLUTION_ERROR", e.getErrorCode());+ assertEquals("DNS resolution failed for hostname \"nonexistent.example\".", e.getMessage());+ }++ @Test+ void testScrapeReturnsDocumentOnSuccess() {+ responseBody.set("{\"success\":true,\"data\":{\"markdown\":\"# Hello\"}}");++ Document doc = client.scrape("https://example.com");++ assertEquals("POST", lastMethod.get());+ assertEquals("/v2/scrape", lastPath.get());+ assertEquals("# Hello", doc.getMarkdown());+ }+}diff --git a/apps/php-sdk/CHANGELOG.md b/apps/php-sdk/CHANGELOG.mdindex 19f133b6b..62a581b1e 100644--- a/apps/php-sdk/CHANGELOG.md+++ b/apps/php-sdk/CHANGELOG.md@@ -5,6 +5,13 @@ All notable changes to the Firecrawl PHP SDK will be documented in this file.The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).+## [1.16.3] - 2026-10-06++### Fixed+- `FirecrawlException` raised for HTTP 200 responses with `success: false`+ (e.g. `SCRAPE_DNS_RESOLUTION_ERROR`) now carries the API error code and a+ 200 status code.+## [1.16.2] - 2026-10-06### Addeddiff --git a/apps/php-sdk/src/Client/FirecrawlClient.php b/apps/php-sdk/src/Client/FirecrawlClient.phpindex c057b9249..833501897 100644--- a/apps/php-sdk/src/Client/FirecrawlClient.php+++ b/apps/php-sdk/src/Client/FirecrawlClient.php@@ -824,9 +824,14 @@ final class FirecrawlClientif (($response['success'] ?? null) === false) {$error = $response['error'] ?? null;- throw new FirecrawlException(is_string($error) && $error !== ''- ? $error- : 'The API reported the request as unsuccessful.');+ $code = $response['code'] ?? null;++ throw new FirecrawlException(+ is_string($error) && $error !== '' ? $error : 'The API reported the request as unsuccessful.',+ 200,+ is_string($code) ? $code : null,+ $response['details'] ?? null,+ );}return $response;diff --git a/apps/php-sdk/src/Version.php b/apps/php-sdk/src/Version.phpindex 8ff31edbf..2daf427cd 100644--- a/apps/php-sdk/src/Version.php+++ b/apps/php-sdk/src/Version.php@@ -6,5 +6,5 @@ namespace Firecrawl;final class Version{- public const SDK_VERSION = '1.16.2';+ public const SDK_VERSION = '1.16.3';}diff --git a/apps/php-sdk/tests/Unit/ScrapeErrorTest.php b/apps/php-sdk/tests/Unit/ScrapeErrorTest.phpnew file mode 100644index 000000000..7723de12c--- /dev/null+++ b/apps/php-sdk/tests/Unit/ScrapeErrorTest.php@@ -0,0 +1,25 @@+<?php++declare(strict_types=1);++use Firecrawl\Exceptions\FirecrawlException;+use GuzzleHttp\Psr7\Response;++it('throws with the API error code when scrape returns success false', function (): void {+ $client = fakeFirecrawlClient([+ new Response(200, [], json_encode([+ 'success' => false,+ 'code' => 'SCRAPE_DNS_RESOLUTION_ERROR',+ 'error' => 'DNS resolution failed for hostname "nonexistent.example".',+ ])),+ ]);++ try {+ $client->scrape('https://nonexistent.example');+ $this->fail('Expected FirecrawlException');+ } catch (FirecrawlException $e) {+ expect($e->getStatusCode())->toBe(200);+ expect($e->getErrorCode())->toBe('SCRAPE_DNS_RESOLUTION_ERROR');+ expect($e->getMessage())->toBe('DNS resolution failed for hostname "nonexistent.example".');+ }+});diff --git a/apps/ruby-sdk/lib/firecrawl/client.rb b/apps/ruby-sdk/lib/firecrawl/client.rbindex 1c3808f56..55a7da16e 100644--- a/apps/ruby-sdk/lib/firecrawl/client.rb+++ b/apps/ruby-sdk/lib/firecrawl/client.rb@@ -81,6 +81,10 @@ module Firecrawlbody.merge!(options.to_h) if optionsbody["origin"] ||= "ruby-sdk@#{Firecrawl::VERSION}"raw = @http.post("/v2/scrape", body)+ # Some scrape failures (e.g. SCRAPE_DNS_RESOLUTION_ERROR) arrive as HTTP 200 with success: false.+ if raw["success"] == false+ raise FirecrawlError.new(raw["error"] || "Scrape failed", status_code: 200, error_code: raw["code"], details: raw["details"])+ enddata = raw["data"] || rawModels::Document.new(data)enddiff --git a/apps/ruby-sdk/lib/firecrawl/version.rb b/apps/ruby-sdk/lib/firecrawl/version.rbindex 9e8041c26..c4008c8e6 100644--- a/apps/ruby-sdk/lib/firecrawl/version.rb+++ b/apps/ruby-sdk/lib/firecrawl/version.rb@@ -1,5 +1,5 @@# frozen_string_literal: truemodule Firecrawl- VERSION = "1.17.3"+ VERSION = "1.17.4"enddiff --git a/apps/ruby-sdk/test/firecrawl/client_test.rb b/apps/ruby-sdk/test/firecrawl/client_test.rbindex c7b1f7754..d1ce330b5 100644--- a/apps/ruby-sdk/test/firecrawl/client_test.rb+++ b/apps/ruby-sdk/test/firecrawl/client_test.rb@@ -186,6 +186,20 @@ class ClientTest < Minitest::Testassert_raises(ArgumentError) { @client.scrape(nil) }end+ def test_scrape_raises_on_success_false+ stub_request(:post, "#{BASE_URL}/v2/scrape")+ .to_return(+ status: 200,+ body: JSON.generate(success: false, code: "SCRAPE_DNS_RESOLUTION_ERROR", error: "DNS resolution failed for hostname \"nonexistent.example\"."),+ headers: { "Content-Type" => "application/json" }+ )++ error = assert_raises(Firecrawl::FirecrawlError) { @client.scrape("https://nonexistent.example") }+ assert_equal 200, error.status_code+ assert_equal "SCRAPE_DNS_RESOLUTION_ERROR", error.error_code+ assert_equal "DNS resolution failed for hostname \"nonexistent.example\".", error.message+ end+def test_scrape_with_product_formatstub_request(:post, "#{BASE_URL}/v2/scrape").to_return(