Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(php-sdk): bound crawl tool lifecycle and tool output budgets

Kris Oei686bdc4586b3
diff --git a/apps/php-sdk/CHANGELOG.md b/apps/php-sdk/CHANGELOG.md
index 415a3cdb5..c4cd4e974 100644
--- a/apps/php-sdk/CHANGELOG.md
+++ b/apps/php-sdk/CHANGELOG.md
@@ -13,6 +13,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
`FirecrawlTools::all()` helper. Drop them into any `laravel/ai` agent's `tools()`
array; API key and config are reused from the existing Laravel integration.
Requires `laravel/ai ^0.9` (PHP 8.3+, Laravel 12+) in the consuming app.
+- `CrawlOptions::with(idempotencyKey:)`: `startCrawl()` now sends the
+ `x-idempotency-key` header, matching the existing batch scrape support.
### Fixed
- `scrape()`, `search()`, and `map()` now throw `FirecrawlException` when the API
diff --git a/apps/php-sdk/README.md b/apps/php-sdk/README.md
index 0b346f541..38d5caae6 100644
--- a/apps/php-sdk/README.md
+++ b/apps/php-sdk/README.md
@@ -461,11 +461,13 @@ new FirecrawlScrape(FirecrawlClient::create(apiKey: 'fc-other-key'));
Tool failures (rate limits, timeouts, invalid URLs) are returned to the model
as readable error strings rather than thrown, so agent runs degrade gracefully.
-`firecrawl_crawl` waits up to 55 seconds for the crawl to finish. If your
-agent runs inside a queued job, keep the crawl limit small or raise the
-worker's job timeout; on timeout the model receives a readable message and
-the crawl may still complete on the server. Extend `FirecrawlCrawl` and
-override `$timeoutSeconds` to change how long it waits.
+`firecrawl_crawl` waits up to 55 seconds for the crawl to finish and returns
+a JSON object with the crawl status and pages, so failed or partial crawls
+are visible to the model. If your agent runs inside a queued job, keep the
+crawl limit small or raise the worker's job timeout; on timeout the model
+receives a readable message and the crawl may still complete on the server.
+Extend `FirecrawlCrawl` and override `$timeoutSeconds` to change how long it
+waits.
## License
diff --git a/apps/php-sdk/src/Client/FirecrawlClient.php b/apps/php-sdk/src/Client/FirecrawlClient.php
index c25c924c7..3d53aa066 100644
--- a/apps/php-sdk/src/Client/FirecrawlClient.php
+++ b/apps/php-sdk/src/Client/FirecrawlClient.php
@@ -262,11 +262,18 @@ final class FirecrawlClient
public function startCrawl(string $url, ?CrawlOptions $options = null): CrawlResponse
{
$body = ['url' => $url];
+ $extraHeaders = [];
+
if ($options !== null) {
+ $idempotencyKey = $options->getIdempotencyKey();
+ if ($idempotencyKey !== null && $idempotencyKey !== '') {
+ $extraHeaders['x-idempotency-key'] = $idempotencyKey;
+ }
+
$body = array_merge($body, $options->toArray());
}
- return CrawlResponse::fromArray($this->http->post('/v2/crawl', $body));
+ return CrawlResponse::fromArray($this->http->post('/v2/crawl', $body, $extraHeaders));
}
/**
diff --git a/apps/php-sdk/src/Laravel/Tools/FirecrawlCrawl.php b/apps/php-sdk/src/Laravel/Tools/FirecrawlCrawl.php
index d0bd7368e..57de61109 100644
--- a/apps/php-sdk/src/Laravel/Tools/FirecrawlCrawl.php
+++ b/apps/php-sdk/src/Laravel/Tools/FirecrawlCrawl.php
@@ -4,9 +4,9 @@ declare(strict_types=1);
namespace Firecrawl\Laravel\Tools;
-use Firecrawl\Exceptions\JobTimeoutException;
+use Firecrawl\Exceptions\FirecrawlException;
+use Firecrawl\Models\CrawlJob;
use Firecrawl\Models\CrawlOptions;
-use Firecrawl\Models\Document;
use Firecrawl\Models\ScrapeOptions;
use Illuminate\Contracts\JsonSchema\JsonSchema;
use Laravel\Ai\Tools\Request;
@@ -14,11 +14,17 @@ use Laravel\Ai\Tools\Request;
class FirecrawlCrawl extends FirecrawlTool
{
/**
+ * One wall-clock deadline covering the start request and all polling.
* Kept below typical queue worker timeouts (Laravel defaults to 60
* seconds); override to wait longer.
*/
protected int $timeoutSeconds = 55;
+ protected int $pollIntervalSeconds = 2;
+
+ /** Per-page output ceiling; override to change. */
+ protected int $pageCharacterLimit = 15000;
+
public function name(): string
{
return 'firecrawl_crawl';
@@ -26,46 +32,96 @@ class FirecrawlCrawl extends FirecrawlTool
public function description(): string
{
- return 'Crawl a website with Firecrawl starting from a URL, following its links and returning '
- . 'each crawled page as a {url, markdown} object in a JSON array. This is a slower, '
- . 'multi-page operation. Prefer firecrawl_scrape when you only need one known page, and '
- . 'keep the page limit small. Waits up to about a minute for the crawl to finish.';
+ return 'Crawl a website with Firecrawl starting from a URL, following its links. Returns a '
+ . 'JSON object with the crawl status and a pages array of {url, markdown} objects. This '
+ . 'is a slower, multi-page operation. Prefer firecrawl_scrape when you only need one '
+ . 'known page, and keep the page limit small. Waits up to about a minute.';
}
public function handle(Request $request): string
{
return $this->guard(function () use ($request): string {
$limit = min(max($request->integer('limit') ?: 5, 1), 25);
+ $deadline = time() + $this->timeoutSeconds;
+
+ $start = $this->client()->startCrawl(
+ (string) $request->string('url'),
+ CrawlOptions::with(
+ limit: $limit,
+ scrapeOptions: ScrapeOptions::with(formats: ['markdown']),
+ integration: self::INTEGRATION,
+ idempotencyKey: bin2hex(random_bytes(16)),
+ ),
+ );
+
+ $jobId = $start->getId();
+ if ($jobId === null || $jobId === '') {
+ throw new FirecrawlException('Crawl start did not return a job ID.');
+ }
+
+ $job = $this->client()->getCrawlStatus($jobId);
+
+ while (!$job->isDone()) {
+ if (time() >= $deadline) {
+ return "The crawl did not finish within {$this->timeoutSeconds} seconds. It may still "
+ . 'complete on the server. Use a smaller limit, or scrape key pages individually '
+ . 'with firecrawl_scrape.';
+ }
+
+ sleep($this->pollIntervalSeconds);
+ $job = $this->client()->getCrawlStatus($jobId);
+ }
+
+ return $this->toJson($this->formatJob($job));
+ });
+ }
+
+ /**
+ * The status field keeps failed or cancelled crawls visible even when
+ * partial pages came back. Pagination cursors are reported, not
+ * followed, so the result stays one bounded response.
+ *
+ * @return array<string, mixed>
+ */
+ private function formatJob(CrawlJob $job): array
+ {
+ $pages = [];
+ $used = 0;
+ $omitted = 0;
+
+ foreach ($job->getData() as $document) {
+ $markdown = $this->truncate($this->documentContent($document), $this->pageCharacterLimit);
- try {
- $job = $this->client()->crawl(
- (string) $request->string('url'),
- CrawlOptions::with(
- limit: $limit,
- scrapeOptions: ScrapeOptions::with(formats: ['markdown']),
- integration: self::INTEGRATION,
- ),
- timeoutSec: $this->timeoutSeconds,
- );
- } catch (JobTimeoutException) {
- return "The crawl did not finish within {$this->timeoutSeconds} seconds. It may still "
- . 'complete on the server. Use a smaller limit, or scrape key pages individually '
- . 'with firecrawl_scrape.';
+ if ($pages !== [] && $used + mb_strlen($markdown) > $this->outputCharacterBudget) {
+ $omitted++;
+ continue;
}
- $pages = array_map(fn (Document $document): array => [
+ $used += mb_strlen($markdown);
+ $pages[] = [
'url' => $document->getMetadata()['sourceURL']
?? $document->getMetadata()['url']
?? null,
- 'markdown' => $this->truncate($this->documentContent($document), 15000),
- ], $job->getData());
+ 'markdown' => $markdown,
+ ];
+ }
- if ($pages === []) {
- return "Crawl finished with status [{$job->getStatus()}] but returned no pages.";
- }
+ $result = [
+ 'status' => $job->getStatus(),
+ 'completed' => $job->getCompleted(),
+ 'total' => $job->getTotal(),
+ 'pages' => $pages,
+ ];
- return $this->toJson($pages);
- });
+ if ($omitted > 0) {
+ $result['omittedPages'] = $omitted;
+ }
+
+ if ($job->getNext() !== null && $job->getNext() !== '') {
+ $result['note'] = 'More pages exist on the server than fit in this response.';
+ }
+
+ return $result;
}
/** @return array<string, \Illuminate\JsonSchema\Types\Type> */
diff --git a/apps/php-sdk/src/Laravel/Tools/FirecrawlMap.php b/apps/php-sdk/src/Laravel/Tools/FirecrawlMap.php
index b40aeb18f..e01be7f8d 100644
--- a/apps/php-sdk/src/Laravel/Tools/FirecrawlMap.php
+++ b/apps/php-sdk/src/Laravel/Tools/FirecrawlMap.php
@@ -48,7 +48,7 @@ class FirecrawlMap extends FirecrawlTool
return 'No URLs discovered.';
}
- return $this->toJson($links);
+ return $this->toBudgetedJson($links);
});
}
diff --git a/apps/php-sdk/src/Laravel/Tools/FirecrawlSearch.php b/apps/php-sdk/src/Laravel/Tools/FirecrawlSearch.php
index 64fffa777..479570592 100644
--- a/apps/php-sdk/src/Laravel/Tools/FirecrawlSearch.php
+++ b/apps/php-sdk/src/Laravel/Tools/FirecrawlSearch.php
@@ -42,7 +42,7 @@ class FirecrawlSearch extends FirecrawlTool
return 'No results found.';
}
- return $this->toJson($web);
+ return $this->toBudgetedJson($web);
});
}
diff --git a/apps/php-sdk/src/Laravel/Tools/FirecrawlTool.php b/apps/php-sdk/src/Laravel/Tools/FirecrawlTool.php
index c79d8abd6..778c496df 100644
--- a/apps/php-sdk/src/Laravel/Tools/FirecrawlTool.php
+++ b/apps/php-sdk/src/Laravel/Tools/FirecrawlTool.php
@@ -16,6 +16,9 @@ abstract class FirecrawlTool implements Tool
// deployed version, including older self-hosted instances.
protected const INTEGRATION = '_laravel-ai';
+ /** Whole-result output ceiling; override to change. */
+ protected int $outputCharacterBudget = 100000;
+
public function __construct(
private ?FirecrawlClient $client = null,
) {}
@@ -48,6 +51,25 @@ abstract class FirecrawlTool implements Tool
}
}
+ /**
+ * Encode list items as JSON, dropping tail items when the result would
+ * exceed the output budget; a final {"omitted": N} element reports the cut.
+ *
+ * @param list<array<string, mixed>> $items
+ */
+ protected function toBudgetedJson(array $items): string
+ {
+ $json = $this->toJson($items);
+ $total = count($items);
+
+ while (mb_strlen($json) > $this->outputCharacterBudget && count($items) > 1) {
+ $items = array_slice($items, 0, (int) ceil(count($items) / 2));
+ $json = $this->toJson([...$items, ['omitted' => $total - count($items)]]);
+ }
+
+ return $json;
+ }
+
/** @param array<int|string, mixed> $data */
protected function toJson(array $data): string
{
diff --git a/apps/php-sdk/src/Models/CrawlOptions.php b/apps/php-sdk/src/Models/CrawlOptions.php
index 988029f33..e05e34d8d 100644
--- a/apps/php-sdk/src/Models/CrawlOptions.php
+++ b/apps/php-sdk/src/Models/CrawlOptions.php
@@ -29,6 +29,7 @@ final class CrawlOptions
private readonly ?bool $regexOnFullURL = null,
private readonly ?bool $zeroDataRetention = null,
private readonly ?string $integration = null,
+ private readonly ?string $idempotencyKey = null,
) {}
/**
@@ -54,15 +55,22 @@ final class CrawlOptions
?bool $regexOnFullURL = null,
?bool $zeroDataRetention = null,
?string $integration = null,
+ ?string $idempotencyKey = null,
): self {
return new self(
$prompt, $excludePaths, $includePaths, $maxDiscoveryDepth, $sitemap,
$ignoreQueryParameters, $deduplicateSimilarURLs, $limit, $crawlEntireDomain,
$allowExternalLinks, $allowSubdomains, $delay, $maxConcurrency, $webhook,
$scrapeOptions, $regexOnFullURL, $zeroDataRetention, $integration,
+ $idempotencyKey,
);
}
+ public function getIdempotencyKey(): ?string
+ {
+ return $this->idempotencyKey;
+ }
+
/** @return array<string, mixed> */
public function toArray(): array
{
diff --git a/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlCrawlTest.php b/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlCrawlTest.php
index 3b909060e..aacadacf6 100644
--- a/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlCrawlTest.php
+++ b/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlCrawlTest.php
@@ -8,7 +8,7 @@ use Illuminate\JsonSchema\JsonSchemaTypeFactory;
use Laravel\Ai\ObjectSchema;
use Laravel\Ai\Tools\Request;
-it('crawls a site and returns pages as JSON', function (): void {
+it('crawls a site and returns status and pages as JSON', function (): void {
$history = new ArrayObject();
$client = fakeFirecrawlClient([
new Response(200, [], json_encode(['success' => true, 'id' => 'job-1'])),
@@ -26,7 +26,12 @@ it('crawls a site and returns pages as JSON', function (): void {
$result = (new FirecrawlCrawl($client))->handle(new Request(['url' => 'https://example.com']));
expect(json_decode($result, true))->toBe([
- ['url' => 'https://example.com/p1', 'markdown' => '# Page 1'],
+ 'status' => 'completed',
+ 'completed' => 1,
+ 'total' => 1,
+ 'pages' => [
+ ['url' => 'https://example.com/p1', 'markdown' => '# Page 1'],
+ ],
]);
$body = json_decode((string) $history[0]['request']->getBody(), true);
@@ -34,6 +39,7 @@ it('crawls a site and returns pages as JSON', function (): void {
expect($body['limit'])->toBe(5);
expect($body['scrapeOptions']['formats'])->toBe(['markdown']);
expect($body['integration'])->toBe('_laravel-ai');
+ expect($history[0]['request']->getHeaderLine('x-idempotency-key'))->toMatch('/^[0-9a-f]{32}$/');
});
it('clamps the page limit between 1 and 25', function (): void {
@@ -51,20 +57,78 @@ it('clamps the page limit between 1 and 25', function (): void {
expect($body['limit'])->toBe(25);
});
-it('reports crawls that finish without pages', function (): void {
+it('reuses the same idempotency key across HTTP retries of the start request', function (): void {
+ $history = new ArrayObject();
$client = fakeFirecrawlClient([
+ new Response(502, [], json_encode(['error' => 'bad gateway'])),
new Response(200, [], json_encode(['success' => true, 'id' => 'job-1'])),
new Response(200, [], json_encode([
- 'success' => true, 'status' => 'failed', 'total' => 0, 'completed' => 0, 'data' => [],
+ 'success' => true, 'status' => 'completed', 'total' => 0, 'completed' => 0, 'data' => [],
])),
- ]);
+ ], $history);
+
+ (new FirecrawlCrawl($client))->handle(new Request(['url' => 'https://example.com']));
+
+ $firstKey = $history[0]['request']->getHeaderLine('x-idempotency-key');
+ $retryKey = $history[1]['request']->getHeaderLine('x-idempotency-key');
+ expect($history[0]['request']->getMethod())->toBe('POST');
+ expect($history[1]['request']->getMethod())->toBe('POST');
+ expect($firstKey)->toMatch('/^[0-9a-f]{32}$/');
+ expect($retryKey)->toBe($firstKey);
+});
+
+it('surfaces failed crawls with partial pages and does not follow pagination', function (): void {
+ $history = new ArrayObject();
+ $client = fakeFirecrawlClient([
+ new Response(200, [], json_encode(['success' => true, 'id' => 'job-1'])),
+ new Response(200, [], json_encode([
+ 'success' => true,
+ 'status' => 'failed',
+ 'total' => 5,
+ 'completed' => 2,
+ 'next' => 'https://api.firecrawl.dev/v2/crawl/job-1?skip=1',
+ 'data' => [
+ ['markdown' => '# Partial page', 'metadata' => ['sourceURL' => 'https://example.com/p1']],
+ ],
+ ])),
+ ], $history);
$result = (new FirecrawlCrawl($client))->handle(new Request(['url' => 'https://example.com']));
+ $decoded = json_decode($result, true);
+
+ expect($decoded['status'])->toBe('failed');
+ expect($decoded['pages'])->toHaveCount(1);
+ expect($decoded['note'])->toContain('More pages exist');
+ expect($history)->toHaveCount(2);
+});
+
+it('applies the whole-result budget across pages', function (): void {
+ $client = fakeFirecrawlClient([
+ new Response(200, [], json_encode(['success' => true, 'id' => 'job-1'])),
+ new Response(200, [], json_encode([
+ 'success' => true,
+ 'status' => 'completed',
+ 'total' => 3,
+ 'completed' => 3,
+ 'data' => [
+ ['markdown' => str_repeat('a', 40), 'metadata' => ['sourceURL' => 'https://example.com/1']],
+ ['markdown' => str_repeat('b', 40), 'metadata' => ['sourceURL' => 'https://example.com/2']],
+ ['markdown' => str_repeat('c', 40), 'metadata' => ['sourceURL' => 'https://example.com/3']],
+ ],
+ ])),
+ ]);
+
+ $tool = new class ($client) extends FirecrawlCrawl {
+ protected int $outputCharacterBudget = 60;
+ };
+
+ $decoded = json_decode($tool->handle(new Request(['url' => 'https://example.com'])), true);
- expect($result)->toBe('Crawl finished with status [failed] but returned no pages.');
+ expect($decoded['pages'])->toHaveCount(1);
+ expect($decoded['omittedPages'])->toBe(2);
});
-it('reports a friendly message when the crawl exceeds its own timeout', function (): void {
+it('returns a timeout message when the crawl runs long', function (): void {
$client = fakeFirecrawlClient([
new Response(200, [], json_encode(['success' => true, 'id' => 'job-1'])),
new Response(200, [], json_encode([
@@ -77,14 +141,13 @@ it('reports a friendly message when the crawl exceeds its own timeout', function
$tool = new class ($client) extends FirecrawlCrawl {
protected int $timeoutSeconds = 1;
+ protected int $pollIntervalSeconds = 1;
};
$result = $tool->handle(new Request(['url' => 'https://example.com']));
- expect($result)->toBe(
- 'The crawl did not finish within 1 seconds. It may still complete on the server. '
- . 'Use a smaller limit, or scrape key pages individually with firecrawl_scrape.',
- );
+ expect($result)->toContain('did not finish within 1 seconds');
+ expect($result)->toContain('may still complete on the server');
});
it('returns API failures as readable strings instead of throwing', function (): void {
diff --git a/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlMapTest.php b/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlMapTest.php
index f17076f2a..24bac94f7 100644
--- a/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlMapTest.php
+++ b/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlMapTest.php
@@ -94,3 +94,25 @@ it('exposes name and a schema with required url', function (): void {
$jsonSchema = (new ObjectSchema($types))->toSchema();
expect($jsonSchema['required'])->toContain('url');
});
+
+it('drops tail links and reports the cut when output exceeds the budget', function (): void {
+ $links = [];
+ for ($i = 1; $i <= 4; $i++) {
+ $links[] = ['url' => "https://example.com/page-{$i}"];
+ }
+
+ $client = fakeFirecrawlClient([
+ new Response(200, [], json_encode(['success' => true, 'data' => ['links' => $links]])),
+ ]);
+
+ $tool = new class ($client) extends FirecrawlMap {
+ protected int $outputCharacterBudget = 100;
+ };
+
+ $result = $tool->handle(new Request(['url' => 'https://example.com']));
+ $decoded = json_decode($result, true);
+
+ expect(mb_strlen($result))->toBeLessThanOrEqual(100);
+ expect($decoded)->not->toHaveCount(4);
+ expect(array_key_exists('omitted', end($decoded)))->toBeTrue();
+});