Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(php-sdk): bound crawl tool lifecycle and tool output budgets
Kris Oei686bdc4586b3
diff --git a/apps/php-sdk/CHANGELOG.md b/apps/php-sdk/CHANGELOG.mdindex 415a3cdb5..c4cd4e974 100644--- a/apps/php-sdk/CHANGELOG.md+++ b/apps/php-sdk/CHANGELOG.md@@ -13,6 +13,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0`FirecrawlTools::all()` helper. Drop them into any `laravel/ai` agent's `tools()`array; API key and config are reused from the existing Laravel integration.Requires `laravel/ai ^0.9` (PHP 8.3+, Laravel 12+) in the consuming app.+- `CrawlOptions::with(idempotencyKey:)`: `startCrawl()` now sends the+ `x-idempotency-key` header, matching the existing batch scrape support.### Fixed- `scrape()`, `search()`, and `map()` now throw `FirecrawlException` when the APIdiff --git a/apps/php-sdk/README.md b/apps/php-sdk/README.mdindex 0b346f541..38d5caae6 100644--- a/apps/php-sdk/README.md+++ b/apps/php-sdk/README.md@@ -461,11 +461,13 @@ new FirecrawlScrape(FirecrawlClient::create(apiKey: 'fc-other-key'));Tool failures (rate limits, timeouts, invalid URLs) are returned to the modelas readable error strings rather than thrown, so agent runs degrade gracefully.-`firecrawl_crawl` waits up to 55 seconds for the crawl to finish. If your-agent runs inside a queued job, keep the crawl limit small or raise the-worker's job timeout; on timeout the model receives a readable message and-the crawl may still complete on the server. Extend `FirecrawlCrawl` and-override `$timeoutSeconds` to change how long it waits.+`firecrawl_crawl` waits up to 55 seconds for the crawl to finish and returns+a JSON object with the crawl status and pages, so failed or partial crawls+are visible to the model. If your agent runs inside a queued job, keep the+crawl limit small or raise the worker's job timeout; on timeout the model+receives a readable message and the crawl may still complete on the server.+Extend `FirecrawlCrawl` and override `$timeoutSeconds` to change how long it+waits.## Licensediff --git a/apps/php-sdk/src/Client/FirecrawlClient.php b/apps/php-sdk/src/Client/FirecrawlClient.phpindex c25c924c7..3d53aa066 100644--- a/apps/php-sdk/src/Client/FirecrawlClient.php+++ b/apps/php-sdk/src/Client/FirecrawlClient.php@@ -262,11 +262,18 @@ final class FirecrawlClientpublic function startCrawl(string $url, ?CrawlOptions $options = null): CrawlResponse{$body = ['url' => $url];+ $extraHeaders = [];+if ($options !== null) {+ $idempotencyKey = $options->getIdempotencyKey();+ if ($idempotencyKey !== null && $idempotencyKey !== '') {+ $extraHeaders['x-idempotency-key'] = $idempotencyKey;+ }+$body = array_merge($body, $options->toArray());}- return CrawlResponse::fromArray($this->http->post('/v2/crawl', $body));+ return CrawlResponse::fromArray($this->http->post('/v2/crawl', $body, $extraHeaders));}/**diff --git a/apps/php-sdk/src/Laravel/Tools/FirecrawlCrawl.php b/apps/php-sdk/src/Laravel/Tools/FirecrawlCrawl.phpindex d0bd7368e..57de61109 100644--- a/apps/php-sdk/src/Laravel/Tools/FirecrawlCrawl.php+++ b/apps/php-sdk/src/Laravel/Tools/FirecrawlCrawl.php@@ -4,9 +4,9 @@ declare(strict_types=1);namespace Firecrawl\Laravel\Tools;-use Firecrawl\Exceptions\JobTimeoutException;+use Firecrawl\Exceptions\FirecrawlException;+use Firecrawl\Models\CrawlJob;use Firecrawl\Models\CrawlOptions;-use Firecrawl\Models\Document;use Firecrawl\Models\ScrapeOptions;use Illuminate\Contracts\JsonSchema\JsonSchema;use Laravel\Ai\Tools\Request;@@ -14,11 +14,17 @@ use Laravel\Ai\Tools\Request;class FirecrawlCrawl extends FirecrawlTool{/**+ * One wall-clock deadline covering the start request and all polling.* Kept below typical queue worker timeouts (Laravel defaults to 60* seconds); override to wait longer.*/protected int $timeoutSeconds = 55;+ protected int $pollIntervalSeconds = 2;++ /** Per-page output ceiling; override to change. */+ protected int $pageCharacterLimit = 15000;+public function name(): string{return 'firecrawl_crawl';@@ -26,46 +32,96 @@ class FirecrawlCrawl extends FirecrawlToolpublic function description(): string{- return 'Crawl a website with Firecrawl starting from a URL, following its links and returning '- . 'each crawled page as a {url, markdown} object in a JSON array. This is a slower, '- . 'multi-page operation. Prefer firecrawl_scrape when you only need one known page, and '- . 'keep the page limit small. Waits up to about a minute for the crawl to finish.';+ return 'Crawl a website with Firecrawl starting from a URL, following its links. Returns a '+ . 'JSON object with the crawl status and a pages array of {url, markdown} objects. This '+ . 'is a slower, multi-page operation. Prefer firecrawl_scrape when you only need one '+ . 'known page, and keep the page limit small. Waits up to about a minute.';}public function handle(Request $request): string{return $this->guard(function () use ($request): string {$limit = min(max($request->integer('limit') ?: 5, 1), 25);+ $deadline = time() + $this->timeoutSeconds;++ $start = $this->client()->startCrawl(+ (string) $request->string('url'),+ CrawlOptions::with(+ limit: $limit,+ scrapeOptions: ScrapeOptions::with(formats: ['markdown']),+ integration: self::INTEGRATION,+ idempotencyKey: bin2hex(random_bytes(16)),+ ),+ );++ $jobId = $start->getId();+ if ($jobId === null || $jobId === '') {+ throw new FirecrawlException('Crawl start did not return a job ID.');+ }++ $job = $this->client()->getCrawlStatus($jobId);++ while (!$job->isDone()) {+ if (time() >= $deadline) {+ return "The crawl did not finish within {$this->timeoutSeconds} seconds. It may still "+ . 'complete on the server. Use a smaller limit, or scrape key pages individually '+ . 'with firecrawl_scrape.';+ }++ sleep($this->pollIntervalSeconds);+ $job = $this->client()->getCrawlStatus($jobId);+ }++ return $this->toJson($this->formatJob($job));+ });+ }++ /**+ * The status field keeps failed or cancelled crawls visible even when+ * partial pages came back. Pagination cursors are reported, not+ * followed, so the result stays one bounded response.+ *+ * @return array<string, mixed>+ */+ private function formatJob(CrawlJob $job): array+ {+ $pages = [];+ $used = 0;+ $omitted = 0;++ foreach ($job->getData() as $document) {+ $markdown = $this->truncate($this->documentContent($document), $this->pageCharacterLimit);- try {- $job = $this->client()->crawl(- (string) $request->string('url'),- CrawlOptions::with(- limit: $limit,- scrapeOptions: ScrapeOptions::with(formats: ['markdown']),- integration: self::INTEGRATION,- ),- timeoutSec: $this->timeoutSeconds,- );- } catch (JobTimeoutException) {- return "The crawl did not finish within {$this->timeoutSeconds} seconds. It may still "- . 'complete on the server. Use a smaller limit, or scrape key pages individually '- . 'with firecrawl_scrape.';+ if ($pages !== [] && $used + mb_strlen($markdown) > $this->outputCharacterBudget) {+ $omitted++;+ continue;}- $pages = array_map(fn (Document $document): array => [+ $used += mb_strlen($markdown);+ $pages[] = ['url' => $document->getMetadata()['sourceURL']?? $document->getMetadata()['url']?? null,- 'markdown' => $this->truncate($this->documentContent($document), 15000),- ], $job->getData());+ 'markdown' => $markdown,+ ];+ }- if ($pages === []) {- return "Crawl finished with status [{$job->getStatus()}] but returned no pages.";- }+ $result = [+ 'status' => $job->getStatus(),+ 'completed' => $job->getCompleted(),+ 'total' => $job->getTotal(),+ 'pages' => $pages,+ ];- return $this->toJson($pages);- });+ if ($omitted > 0) {+ $result['omittedPages'] = $omitted;+ }++ if ($job->getNext() !== null && $job->getNext() !== '') {+ $result['note'] = 'More pages exist on the server than fit in this response.';+ }++ return $result;}/** @return array<string, \Illuminate\JsonSchema\Types\Type> */diff --git a/apps/php-sdk/src/Laravel/Tools/FirecrawlMap.php b/apps/php-sdk/src/Laravel/Tools/FirecrawlMap.phpindex b40aeb18f..e01be7f8d 100644--- a/apps/php-sdk/src/Laravel/Tools/FirecrawlMap.php+++ b/apps/php-sdk/src/Laravel/Tools/FirecrawlMap.php@@ -48,7 +48,7 @@ class FirecrawlMap extends FirecrawlToolreturn 'No URLs discovered.';}- return $this->toJson($links);+ return $this->toBudgetedJson($links);});}diff --git a/apps/php-sdk/src/Laravel/Tools/FirecrawlSearch.php b/apps/php-sdk/src/Laravel/Tools/FirecrawlSearch.phpindex 64fffa777..479570592 100644--- a/apps/php-sdk/src/Laravel/Tools/FirecrawlSearch.php+++ b/apps/php-sdk/src/Laravel/Tools/FirecrawlSearch.php@@ -42,7 +42,7 @@ class FirecrawlSearch extends FirecrawlToolreturn 'No results found.';}- return $this->toJson($web);+ return $this->toBudgetedJson($web);});}diff --git a/apps/php-sdk/src/Laravel/Tools/FirecrawlTool.php b/apps/php-sdk/src/Laravel/Tools/FirecrawlTool.phpindex c79d8abd6..778c496df 100644--- a/apps/php-sdk/src/Laravel/Tools/FirecrawlTool.php+++ b/apps/php-sdk/src/Laravel/Tools/FirecrawlTool.php@@ -16,6 +16,9 @@ abstract class FirecrawlTool implements Tool// deployed version, including older self-hosted instances.protected const INTEGRATION = '_laravel-ai';+ /** Whole-result output ceiling; override to change. */+ protected int $outputCharacterBudget = 100000;+public function __construct(private ?FirecrawlClient $client = null,) {}@@ -48,6 +51,25 @@ abstract class FirecrawlTool implements Tool}}+ /**+ * Encode list items as JSON, dropping tail items when the result would+ * exceed the output budget; a final {"omitted": N} element reports the cut.+ *+ * @param list<array<string, mixed>> $items+ */+ protected function toBudgetedJson(array $items): string+ {+ $json = $this->toJson($items);+ $total = count($items);++ while (mb_strlen($json) > $this->outputCharacterBudget && count($items) > 1) {+ $items = array_slice($items, 0, (int) ceil(count($items) / 2));+ $json = $this->toJson([...$items, ['omitted' => $total - count($items)]]);+ }++ return $json;+ }+/** @param array<int|string, mixed> $data */protected function toJson(array $data): string{diff --git a/apps/php-sdk/src/Models/CrawlOptions.php b/apps/php-sdk/src/Models/CrawlOptions.phpindex 988029f33..e05e34d8d 100644--- a/apps/php-sdk/src/Models/CrawlOptions.php+++ b/apps/php-sdk/src/Models/CrawlOptions.php@@ -29,6 +29,7 @@ final class CrawlOptionsprivate readonly ?bool $regexOnFullURL = null,private readonly ?bool $zeroDataRetention = null,private readonly ?string $integration = null,+ private readonly ?string $idempotencyKey = null,) {}/**@@ -54,15 +55,22 @@ final class CrawlOptions?bool $regexOnFullURL = null,?bool $zeroDataRetention = null,?string $integration = null,+ ?string $idempotencyKey = null,): self {return new self($prompt, $excludePaths, $includePaths, $maxDiscoveryDepth, $sitemap,$ignoreQueryParameters, $deduplicateSimilarURLs, $limit, $crawlEntireDomain,$allowExternalLinks, $allowSubdomains, $delay, $maxConcurrency, $webhook,$scrapeOptions, $regexOnFullURL, $zeroDataRetention, $integration,+ $idempotencyKey,);}+ public function getIdempotencyKey(): ?string+ {+ return $this->idempotencyKey;+ }+/** @return array<string, mixed> */public function toArray(): array{diff --git a/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlCrawlTest.php b/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlCrawlTest.phpindex 3b909060e..aacadacf6 100644--- a/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlCrawlTest.php+++ b/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlCrawlTest.php@@ -8,7 +8,7 @@ use Illuminate\JsonSchema\JsonSchemaTypeFactory;use Laravel\Ai\ObjectSchema;use Laravel\Ai\Tools\Request;-it('crawls a site and returns pages as JSON', function (): void {+it('crawls a site and returns status and pages as JSON', function (): void {$history = new ArrayObject();$client = fakeFirecrawlClient([new Response(200, [], json_encode(['success' => true, 'id' => 'job-1'])),@@ -26,7 +26,12 @@ it('crawls a site and returns pages as JSON', function (): void {$result = (new FirecrawlCrawl($client))->handle(new Request(['url' => 'https://example.com']));expect(json_decode($result, true))->toBe([- ['url' => 'https://example.com/p1', 'markdown' => '# Page 1'],+ 'status' => 'completed',+ 'completed' => 1,+ 'total' => 1,+ 'pages' => [+ ['url' => 'https://example.com/p1', 'markdown' => '# Page 1'],+ ],]);$body = json_decode((string) $history[0]['request']->getBody(), true);@@ -34,6 +39,7 @@ it('crawls a site and returns pages as JSON', function (): void {expect($body['limit'])->toBe(5);expect($body['scrapeOptions']['formats'])->toBe(['markdown']);expect($body['integration'])->toBe('_laravel-ai');+ expect($history[0]['request']->getHeaderLine('x-idempotency-key'))->toMatch('/^[0-9a-f]{32}$/');});it('clamps the page limit between 1 and 25', function (): void {@@ -51,20 +57,78 @@ it('clamps the page limit between 1 and 25', function (): void {expect($body['limit'])->toBe(25);});-it('reports crawls that finish without pages', function (): void {+it('reuses the same idempotency key across HTTP retries of the start request', function (): void {+ $history = new ArrayObject();$client = fakeFirecrawlClient([+ new Response(502, [], json_encode(['error' => 'bad gateway'])),new Response(200, [], json_encode(['success' => true, 'id' => 'job-1'])),new Response(200, [], json_encode([- 'success' => true, 'status' => 'failed', 'total' => 0, 'completed' => 0, 'data' => [],+ 'success' => true, 'status' => 'completed', 'total' => 0, 'completed' => 0, 'data' => [],])),- ]);+ ], $history);++ (new FirecrawlCrawl($client))->handle(new Request(['url' => 'https://example.com']));++ $firstKey = $history[0]['request']->getHeaderLine('x-idempotency-key');+ $retryKey = $history[1]['request']->getHeaderLine('x-idempotency-key');+ expect($history[0]['request']->getMethod())->toBe('POST');+ expect($history[1]['request']->getMethod())->toBe('POST');+ expect($firstKey)->toMatch('/^[0-9a-f]{32}$/');+ expect($retryKey)->toBe($firstKey);+});++it('surfaces failed crawls with partial pages and does not follow pagination', function (): void {+ $history = new ArrayObject();+ $client = fakeFirecrawlClient([+ new Response(200, [], json_encode(['success' => true, 'id' => 'job-1'])),+ new Response(200, [], json_encode([+ 'success' => true,+ 'status' => 'failed',+ 'total' => 5,+ 'completed' => 2,+ 'next' => 'https://api.firecrawl.dev/v2/crawl/job-1?skip=1',+ 'data' => [+ ['markdown' => '# Partial page', 'metadata' => ['sourceURL' => 'https://example.com/p1']],+ ],+ ])),+ ], $history);$result = (new FirecrawlCrawl($client))->handle(new Request(['url' => 'https://example.com']));+ $decoded = json_decode($result, true);++ expect($decoded['status'])->toBe('failed');+ expect($decoded['pages'])->toHaveCount(1);+ expect($decoded['note'])->toContain('More pages exist');+ expect($history)->toHaveCount(2);+});++it('applies the whole-result budget across pages', function (): void {+ $client = fakeFirecrawlClient([+ new Response(200, [], json_encode(['success' => true, 'id' => 'job-1'])),+ new Response(200, [], json_encode([+ 'success' => true,+ 'status' => 'completed',+ 'total' => 3,+ 'completed' => 3,+ 'data' => [+ ['markdown' => str_repeat('a', 40), 'metadata' => ['sourceURL' => 'https://example.com/1']],+ ['markdown' => str_repeat('b', 40), 'metadata' => ['sourceURL' => 'https://example.com/2']],+ ['markdown' => str_repeat('c', 40), 'metadata' => ['sourceURL' => 'https://example.com/3']],+ ],+ ])),+ ]);++ $tool = new class ($client) extends FirecrawlCrawl {+ protected int $outputCharacterBudget = 60;+ };++ $decoded = json_decode($tool->handle(new Request(['url' => 'https://example.com'])), true);- expect($result)->toBe('Crawl finished with status [failed] but returned no pages.');+ expect($decoded['pages'])->toHaveCount(1);+ expect($decoded['omittedPages'])->toBe(2);});-it('reports a friendly message when the crawl exceeds its own timeout', function (): void {+it('returns a timeout message when the crawl runs long', function (): void {$client = fakeFirecrawlClient([new Response(200, [], json_encode(['success' => true, 'id' => 'job-1'])),new Response(200, [], json_encode([@@ -77,14 +141,13 @@ it('reports a friendly message when the crawl exceeds its own timeout', function$tool = new class ($client) extends FirecrawlCrawl {protected int $timeoutSeconds = 1;+ protected int $pollIntervalSeconds = 1;};$result = $tool->handle(new Request(['url' => 'https://example.com']));- expect($result)->toBe(- 'The crawl did not finish within 1 seconds. It may still complete on the server. '- . 'Use a smaller limit, or scrape key pages individually with firecrawl_scrape.',- );+ expect($result)->toContain('did not finish within 1 seconds');+ expect($result)->toContain('may still complete on the server');});it('returns API failures as readable strings instead of throwing', function (): void {diff --git a/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlMapTest.php b/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlMapTest.phpindex f17076f2a..24bac94f7 100644--- a/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlMapTest.php+++ b/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlMapTest.php@@ -94,3 +94,25 @@ it('exposes name and a schema with required url', function (): void {$jsonSchema = (new ObjectSchema($types))->toSchema();expect($jsonSchema['required'])->toContain('url');});++it('drops tail links and reports the cut when output exceeds the budget', function (): void {+ $links = [];+ for ($i = 1; $i <= 4; $i++) {+ $links[] = ['url' => "https://example.com/page-{$i}"];+ }++ $client = fakeFirecrawlClient([+ new Response(200, [], json_encode(['success' => true, 'data' => ['links' => $links]])),+ ]);++ $tool = new class ($client) extends FirecrawlMap {+ protected int $outputCharacterBudget = 100;+ };++ $result = $tool->handle(new Request(['url' => 'https://example.com']));+ $decoded = json_decode($result, true);++ expect(mb_strlen($result))->toBeLessThanOrEqual(100);+ expect($decoded)->not->toHaveCount(4);+ expect(array_key_exists('omitted', end($decoded)))->toBeTrue();+});