Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(php-sdk): address review findings on budgets, deadlines, and coverage
Kris Oei71e03b083fa4
diff --git a/apps/php-sdk/CHANGELOG.md b/apps/php-sdk/CHANGELOG.mdindex c4cd4e974..056885888 100644--- a/apps/php-sdk/CHANGELOG.md+++ b/apps/php-sdk/CHANGELOG.md@@ -15,6 +15,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0Requires `laravel/ai ^0.9` (PHP 8.3+, Laravel 12+) in the consuming app.- `CrawlOptions::with(idempotencyKey:)`: `startCrawl()` now sends the`x-idempotency-key` header, matching the existing batch scrape support.+- `startCrawl()` and `getCrawlStatus()` accept an optional per-request+ timeout in seconds.### Fixed- `scrape()`, `search()`, and `map()` now throw `FirecrawlException` when the APIdiff --git a/apps/php-sdk/README.md b/apps/php-sdk/README.mdindex 38d5caae6..a9b8badcc 100644--- a/apps/php-sdk/README.md+++ b/apps/php-sdk/README.md@@ -396,6 +396,9 @@ The SDK ships native tool classes for the [Laravel AI SDK](https://laravel.com/dcomposer require laravel/ai```+Note for contributors: `laravel/ai` is also a dev dependency of this package,+so running the SDK's own test suite requires PHP 8.3+.+Add Firecrawl capabilities to any agent, no MCP server or manual HTTP calls needed.The tools resolve the `FirecrawlClient` from the container, so your existing`config/firecrawl.php` / `FIRECRAWL_API_KEY` setup is reused as-is:diff --git a/apps/php-sdk/src/Client/FirecrawlClient.php b/apps/php-sdk/src/Client/FirecrawlClient.phpindex 3d53aa066..a0da68063 100644--- a/apps/php-sdk/src/Client/FirecrawlClient.php+++ b/apps/php-sdk/src/Client/FirecrawlClient.php@@ -259,8 +259,11 @@ final class FirecrawlClient/*** Start an async crawl job and return immediately.*/- public function startCrawl(string $url, ?CrawlOptions $options = null): CrawlResponse- {+ public function startCrawl(+ string $url,+ ?CrawlOptions $options = null,+ ?float $requestTimeoutSeconds = null,+ ): CrawlResponse {$body = ['url' => $url];$extraHeaders = [];@@ -273,15 +276,17 @@ final class FirecrawlClient$body = array_merge($body, $options->toArray());}- return CrawlResponse::fromArray($this->http->post('/v2/crawl', $body, $extraHeaders));+ return CrawlResponse::fromArray(+ $this->http->post('/v2/crawl', $body, $extraHeaders, $requestTimeoutSeconds),+ );}/*** Get the status and results of a crawl job.*/- public function getCrawlStatus(string $jobId): CrawlJob+ public function getCrawlStatus(string $jobId, ?float $requestTimeoutSeconds = null): CrawlJob{- return CrawlJob::fromArray($this->http->get("/v2/crawl/{$jobId}"));+ return CrawlJob::fromArray($this->http->get("/v2/crawl/{$jobId}", $requestTimeoutSeconds));}/**diff --git a/apps/php-sdk/src/Client/FirecrawlHttpClient.php b/apps/php-sdk/src/Client/FirecrawlHttpClient.phpindex 0b2284d7d..8f8536aeb 100644--- a/apps/php-sdk/src/Client/FirecrawlHttpClient.php+++ b/apps/php-sdk/src/Client/FirecrawlHttpClient.php@@ -44,15 +44,15 @@ final class FirecrawlHttpClient* @param array<string, string> $extraHeaders* @return array<string, mixed>*/- public function post(string $path, array $body, array $extraHeaders = []): array+ public function post(string $path, array $body, array $extraHeaders = [], ?float $timeoutSeconds = null): array{- return $this->request('POST', $this->baseUrl . $path, $body, $extraHeaders);+ return $this->request('POST', $this->baseUrl . $path, $body, $extraHeaders, timeoutSeconds: $timeoutSeconds);}/** @return array<string, mixed> */- public function get(string $path): array+ public function get(string $path, ?float $timeoutSeconds = null): array{- return $this->request('GET', $this->baseUrl . $path);+ return $this->request('GET', $this->baseUrl . $path, timeoutSeconds: $timeoutSeconds);}/** @return array<string, mixed> */@@ -134,6 +134,7 @@ final class FirecrawlHttpClientarray $body = [],array $extraHeaders = [],?array $multipart = null,+ ?float $timeoutSeconds = null,): array {$defaultHeaders = ['Accept' => 'application/json',@@ -156,6 +157,10 @@ final class FirecrawlHttpClientRequestOptions::HTTP_ERRORS => false,];+ if ($timeoutSeconds !== null) {+ $options[RequestOptions::TIMEOUT] = $timeoutSeconds;+ }+if ($multipart !== null) {$options[RequestOptions::MULTIPART] = $multipart;} elseif (($method === 'POST' || $method === 'PATCH') && $body !== []) {diff --git a/apps/php-sdk/src/Laravel/Tools/FirecrawlCrawl.php b/apps/php-sdk/src/Laravel/Tools/FirecrawlCrawl.phpindex 36fbf821a..205ad692f 100644--- a/apps/php-sdk/src/Laravel/Tools/FirecrawlCrawl.php+++ b/apps/php-sdk/src/Laravel/Tools/FirecrawlCrawl.php@@ -51,6 +51,7 @@ class FirecrawlCrawl extends FirecrawlToolintegration: self::INTEGRATION,idempotencyKey: $this->newIdempotencyKey(),),+ requestTimeoutSeconds: $this->remainingSeconds($deadline),);$jobId = $start->getId();@@ -58,7 +59,7 @@ class FirecrawlCrawl extends FirecrawlToolthrow new FirecrawlException('Crawl start did not return a job ID.');}- $job = $this->client()->getCrawlStatus($jobId);+ $job = $this->client()->getCrawlStatus($jobId, $this->remainingSeconds($deadline));while (!$job->isDone()) {if (time() >= $deadline) {@@ -68,13 +69,18 @@ class FirecrawlCrawl extends FirecrawlTool}sleep($this->pollIntervalSeconds);- $job = $this->client()->getCrawlStatus($jobId);+ $job = $this->client()->getCrawlStatus($jobId, $this->remainingSeconds($deadline));}return $this->toJson($this->formatJob($job));});}+ private function remainingSeconds(int $deadline): float+ {+ return (float) max($deadline - time(), 1);+ }+/*** The API only accepts UUID-formatted idempotency keys.*/@@ -100,20 +106,21 @@ class FirecrawlCrawl extends FirecrawlTool$omitted = 0;foreach ($job->getData() as $document) {- $markdown = $this->truncate($this->documentContent($document), $this->pageCharacterLimit);+ $page = [+ 'url' => $document->getMetadata()['sourceURL']+ ?? $document->getMetadata()['url']+ ?? null,+ 'markdown' => $this->truncate($this->documentContent($document), $this->pageCharacterLimit),+ ];+ $length = mb_strlen($this->toJson($page));- if ($pages !== [] && $used + mb_strlen($markdown) > $this->outputCharacterBudget) {+ if ($pages !== [] && $used + $length > $this->outputCharacterBudget) {$omitted++;continue;}- $used += mb_strlen($markdown);- $pages[] = [- 'url' => $document->getMetadata()['sourceURL']- ?? $document->getMetadata()['url']- ?? null,- 'markdown' => $markdown,- ];+ $used += $length;+ $pages[] = $page;}$result = [@@ -128,7 +135,8 @@ class FirecrawlCrawl extends FirecrawlTool}if ($job->getNext() !== null && $job->getNext() !== '') {- $result['note'] = 'More pages exist on the server than fit in this response.';+ $result['note'] = 'More pages exist on the server than fit in this response. '+ . 'Use a smaller limit or scrape specific pages with firecrawl_scrape.';}return $result;diff --git a/apps/php-sdk/src/Laravel/Tools/FirecrawlTool.php b/apps/php-sdk/src/Laravel/Tools/FirecrawlTool.phpindex 5f31e492e..e4882ab93 100644--- a/apps/php-sdk/src/Laravel/Tools/FirecrawlTool.php+++ b/apps/php-sdk/src/Laravel/Tools/FirecrawlTool.php@@ -67,6 +67,10 @@ abstract class FirecrawlTool implements Tool$json = $this->toJson([...$items, ['omitted' => $total - count($items)]]);}+ if (mb_strlen($json) > $this->outputCharacterBudget) {+ $json = $this->toJson([['omitted' => $total]]);+ }+return $json;}@@ -97,6 +101,7 @@ abstract class FirecrawlTool implements Tool/*** Trim tool output so large pages do not overflow the model context.+ * The truncation notice fits inside the cap.*/protected function truncate(string $content, int $maxCharacters): string{@@ -105,8 +110,12 @@ abstract class FirecrawlTool implements Tool}$total = mb_strlen($content);+ $suffix = "\n\n[Truncated: {$total} characters total.]";++ if ($maxCharacters <= mb_strlen($suffix)) {+ return mb_substr($content, 0, $maxCharacters);+ }- return mb_substr($content, 0, $maxCharacters)- . "\n\n[Truncated: showing the first {$maxCharacters} of {$total} characters.]";+ return mb_substr($content, 0, $maxCharacters - mb_strlen($suffix)) . $suffix;}}diff --git a/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlCrawlTest.php b/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlCrawlTest.phpindex a11d7f198..c9b1890f4 100644--- a/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlCrawlTest.php+++ b/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlCrawlTest.php@@ -173,3 +173,20 @@ it('exposes name and a schema with required url and optional limit', function ()expect($jsonSchema['required'])->toContain('url');expect($jsonSchema['required'])->not->toContain('limit');});++it('bounds each HTTP request by the remaining deadline', function (): void {+ $history = new ArrayObject();+ $client = fakeFirecrawlClient([+ new Response(200, [], json_encode(['success' => true, 'id' => 'job-1'])),+ new Response(200, [], json_encode([+ 'success' => true, 'status' => 'completed', 'total' => 0, 'completed' => 0, 'data' => [],+ ])),+ ], $history);++ (new FirecrawlCrawl($client))->handle(new Request(['url' => 'https://example.com']));++ foreach ([0, 1] as $i) {+ $timeout = $history[$i]['options'][GuzzleHttp\RequestOptions::TIMEOUT] ?? null;+ expect($timeout)->toBeFloat()->toBeGreaterThan(0)->toBeLessThanOrEqual(55.0);+ }+});diff --git a/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlMapTest.php b/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlMapTest.phpindex 24bac94f7..e303fc73f 100644--- a/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlMapTest.php+++ b/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlMapTest.php@@ -116,3 +116,19 @@ it('drops tail links and reports the cut when output exceeds the budget', functiexpect($decoded)->not->toHaveCount(4);expect(array_key_exists('omitted', end($decoded)))->toBeTrue();});++it('reduces a single oversized item to an omitted marker', function (): void {+ $client = fakeFirecrawlClient([+ new Response(200, [], json_encode(['success' => true, 'data' => ['links' => [+ ['url' => 'https://example.com/' . str_repeat('x', 400)],+ ]]])),+ ]);++ $tool = new class ($client) extends FirecrawlMap {+ protected int $outputCharacterBudget = 100;+ };++ $result = $tool->handle(new Request(['url' => 'https://example.com']));++ expect(json_decode($result, true))->toBe([['omitted' => 1]]);+});diff --git a/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlScrapeTest.php b/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlScrapeTest.phpindex 6cf07a106..b8907f6ad 100644--- a/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlScrapeTest.php+++ b/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlScrapeTest.php@@ -89,8 +89,9 @@ it('truncates markdown longer than 80000 characters', function (): void {$result = (new FirecrawlScrape($client))->handle(new Request(['url' => 'https://example.com']));- expect($result)->toStartWith($prefix);- expect($result)->toEndWith('[Truncated: showing the first 80000 of 80100 characters.]');+ expect(mb_strlen($result))->toBeLessThanOrEqual(80000);+ expect($result)->toStartWith(str_repeat('a', 1000));+ expect($result)->toEndWith('[Truncated: 80100 characters total.]');});it('resolves the client from the container when none is injected', function (): void {diff --git a/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlToolsTest.php b/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlToolsTest.phpindex f9e8e104f..a437b4aae 100644--- a/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlToolsTest.php+++ b/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlToolsTest.php@@ -23,9 +23,23 @@ it('returns one instance of each core tool', function (): void {it('passes an explicit client through to every tool', function (): void {$client = fakeFirecrawlClient([new Response(200, [], json_encode(['success' => true, 'data' => ['markdown' => '# Hi']])),+ new Response(200, [], json_encode(['success' => true, 'data' => [+ 'web' => [['title' => 'T', 'url' => 'https://a.com', 'description' => 'D']],+ ]])),+ new Response(200, [], json_encode(['success' => true, 'data' => [+ 'links' => [['url' => 'https://a.com']],+ ]])),+ new Response(200, [], json_encode(['success' => true, 'id' => 'job-1'])),+ new Response(200, [], json_encode([+ 'success' => true, 'status' => 'completed', 'total' => 0, 'completed' => 0, 'data' => [],+ ])),]);- [$scrape] = FirecrawlTools::all($client);+ [$scrape, $search, $map, $crawl] = FirecrawlTools::all($client);expect($scrape->handle(new Request(['url' => 'https://example.com'])))->toBe('# Hi');+ expect($search->handle(new Request(['query' => 'hi'])))->toContain('https://a.com');+ expect($map->handle(new Request(['url' => 'https://example.com'])))->toContain('https://a.com');+ expect(json_decode($crawl->handle(new Request(['url' => 'https://example.com'])), true)['status'])+ ->toBe('completed');});