Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(php-sdk): address review findings on budgets, deadlines, and coverage

Kris Oei71e03b083fa4
diff --git a/apps/php-sdk/CHANGELOG.md b/apps/php-sdk/CHANGELOG.md
index c4cd4e974..056885888 100644
--- a/apps/php-sdk/CHANGELOG.md
+++ b/apps/php-sdk/CHANGELOG.md
@@ -15,6 +15,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
Requires `laravel/ai ^0.9` (PHP 8.3+, Laravel 12+) in the consuming app.
- `CrawlOptions::with(idempotencyKey:)`: `startCrawl()` now sends the
`x-idempotency-key` header, matching the existing batch scrape support.
+- `startCrawl()` and `getCrawlStatus()` accept an optional per-request
+ timeout in seconds.
### Fixed
- `scrape()`, `search()`, and `map()` now throw `FirecrawlException` when the API
diff --git a/apps/php-sdk/README.md b/apps/php-sdk/README.md
index 38d5caae6..a9b8badcc 100644
--- a/apps/php-sdk/README.md
+++ b/apps/php-sdk/README.md
@@ -396,6 +396,9 @@ The SDK ships native tool classes for the [Laravel AI SDK](https://laravel.com/d
composer require laravel/ai
```
+Note for contributors: `laravel/ai` is also a dev dependency of this package,
+so running the SDK's own test suite requires PHP 8.3+.
+
Add Firecrawl capabilities to any agent, no MCP server or manual HTTP calls needed.
The tools resolve the `FirecrawlClient` from the container, so your existing
`config/firecrawl.php` / `FIRECRAWL_API_KEY` setup is reused as-is:
diff --git a/apps/php-sdk/src/Client/FirecrawlClient.php b/apps/php-sdk/src/Client/FirecrawlClient.php
index 3d53aa066..a0da68063 100644
--- a/apps/php-sdk/src/Client/FirecrawlClient.php
+++ b/apps/php-sdk/src/Client/FirecrawlClient.php
@@ -259,8 +259,11 @@ final class FirecrawlClient
/**
* Start an async crawl job and return immediately.
*/
- public function startCrawl(string $url, ?CrawlOptions $options = null): CrawlResponse
- {
+ public function startCrawl(
+ string $url,
+ ?CrawlOptions $options = null,
+ ?float $requestTimeoutSeconds = null,
+ ): CrawlResponse {
$body = ['url' => $url];
$extraHeaders = [];
@@ -273,15 +276,17 @@ final class FirecrawlClient
$body = array_merge($body, $options->toArray());
}
- return CrawlResponse::fromArray($this->http->post('/v2/crawl', $body, $extraHeaders));
+ return CrawlResponse::fromArray(
+ $this->http->post('/v2/crawl', $body, $extraHeaders, $requestTimeoutSeconds),
+ );
}
/**
* Get the status and results of a crawl job.
*/
- public function getCrawlStatus(string $jobId): CrawlJob
+ public function getCrawlStatus(string $jobId, ?float $requestTimeoutSeconds = null): CrawlJob
{
- return CrawlJob::fromArray($this->http->get("/v2/crawl/{$jobId}"));
+ return CrawlJob::fromArray($this->http->get("/v2/crawl/{$jobId}", $requestTimeoutSeconds));
}
/**
diff --git a/apps/php-sdk/src/Client/FirecrawlHttpClient.php b/apps/php-sdk/src/Client/FirecrawlHttpClient.php
index 0b2284d7d..8f8536aeb 100644
--- a/apps/php-sdk/src/Client/FirecrawlHttpClient.php
+++ b/apps/php-sdk/src/Client/FirecrawlHttpClient.php
@@ -44,15 +44,15 @@ final class FirecrawlHttpClient
* @param array<string, string> $extraHeaders
* @return array<string, mixed>
*/
- public function post(string $path, array $body, array $extraHeaders = []): array
+ public function post(string $path, array $body, array $extraHeaders = [], ?float $timeoutSeconds = null): array
{
- return $this->request('POST', $this->baseUrl . $path, $body, $extraHeaders);
+ return $this->request('POST', $this->baseUrl . $path, $body, $extraHeaders, timeoutSeconds: $timeoutSeconds);
}
/** @return array<string, mixed> */
- public function get(string $path): array
+ public function get(string $path, ?float $timeoutSeconds = null): array
{
- return $this->request('GET', $this->baseUrl . $path);
+ return $this->request('GET', $this->baseUrl . $path, timeoutSeconds: $timeoutSeconds);
}
/** @return array<string, mixed> */
@@ -134,6 +134,7 @@ final class FirecrawlHttpClient
array $body = [],
array $extraHeaders = [],
?array $multipart = null,
+ ?float $timeoutSeconds = null,
): array {
$defaultHeaders = [
'Accept' => 'application/json',
@@ -156,6 +157,10 @@ final class FirecrawlHttpClient
RequestOptions::HTTP_ERRORS => false,
];
+ if ($timeoutSeconds !== null) {
+ $options[RequestOptions::TIMEOUT] = $timeoutSeconds;
+ }
+
if ($multipart !== null) {
$options[RequestOptions::MULTIPART] = $multipart;
} elseif (($method === 'POST' || $method === 'PATCH') && $body !== []) {
diff --git a/apps/php-sdk/src/Laravel/Tools/FirecrawlCrawl.php b/apps/php-sdk/src/Laravel/Tools/FirecrawlCrawl.php
index 36fbf821a..205ad692f 100644
--- a/apps/php-sdk/src/Laravel/Tools/FirecrawlCrawl.php
+++ b/apps/php-sdk/src/Laravel/Tools/FirecrawlCrawl.php
@@ -51,6 +51,7 @@ class FirecrawlCrawl extends FirecrawlTool
integration: self::INTEGRATION,
idempotencyKey: $this->newIdempotencyKey(),
),
+ requestTimeoutSeconds: $this->remainingSeconds($deadline),
);
$jobId = $start->getId();
@@ -58,7 +59,7 @@ class FirecrawlCrawl extends FirecrawlTool
throw new FirecrawlException('Crawl start did not return a job ID.');
}
- $job = $this->client()->getCrawlStatus($jobId);
+ $job = $this->client()->getCrawlStatus($jobId, $this->remainingSeconds($deadline));
while (!$job->isDone()) {
if (time() >= $deadline) {
@@ -68,13 +69,18 @@ class FirecrawlCrawl extends FirecrawlTool
}
sleep($this->pollIntervalSeconds);
- $job = $this->client()->getCrawlStatus($jobId);
+ $job = $this->client()->getCrawlStatus($jobId, $this->remainingSeconds($deadline));
}
return $this->toJson($this->formatJob($job));
});
}
+ private function remainingSeconds(int $deadline): float
+ {
+ return (float) max($deadline - time(), 1);
+ }
+
/**
* The API only accepts UUID-formatted idempotency keys.
*/
@@ -100,20 +106,21 @@ class FirecrawlCrawl extends FirecrawlTool
$omitted = 0;
foreach ($job->getData() as $document) {
- $markdown = $this->truncate($this->documentContent($document), $this->pageCharacterLimit);
+ $page = [
+ 'url' => $document->getMetadata()['sourceURL']
+ ?? $document->getMetadata()['url']
+ ?? null,
+ 'markdown' => $this->truncate($this->documentContent($document), $this->pageCharacterLimit),
+ ];
+ $length = mb_strlen($this->toJson($page));
- if ($pages !== [] && $used + mb_strlen($markdown) > $this->outputCharacterBudget) {
+ if ($pages !== [] && $used + $length > $this->outputCharacterBudget) {
$omitted++;
continue;
}
- $used += mb_strlen($markdown);
- $pages[] = [
- 'url' => $document->getMetadata()['sourceURL']
- ?? $document->getMetadata()['url']
- ?? null,
- 'markdown' => $markdown,
- ];
+ $used += $length;
+ $pages[] = $page;
}
$result = [
@@ -128,7 +135,8 @@ class FirecrawlCrawl extends FirecrawlTool
}
if ($job->getNext() !== null && $job->getNext() !== '') {
- $result['note'] = 'More pages exist on the server than fit in this response.';
+ $result['note'] = 'More pages exist on the server than fit in this response. '
+ . 'Use a smaller limit or scrape specific pages with firecrawl_scrape.';
}
return $result;
diff --git a/apps/php-sdk/src/Laravel/Tools/FirecrawlTool.php b/apps/php-sdk/src/Laravel/Tools/FirecrawlTool.php
index 5f31e492e..e4882ab93 100644
--- a/apps/php-sdk/src/Laravel/Tools/FirecrawlTool.php
+++ b/apps/php-sdk/src/Laravel/Tools/FirecrawlTool.php
@@ -67,6 +67,10 @@ abstract class FirecrawlTool implements Tool
$json = $this->toJson([...$items, ['omitted' => $total - count($items)]]);
}
+ if (mb_strlen($json) > $this->outputCharacterBudget) {
+ $json = $this->toJson([['omitted' => $total]]);
+ }
+
return $json;
}
@@ -97,6 +101,7 @@ abstract class FirecrawlTool implements Tool
/**
* Trim tool output so large pages do not overflow the model context.
+ * The truncation notice fits inside the cap.
*/
protected function truncate(string $content, int $maxCharacters): string
{
@@ -105,8 +110,12 @@ abstract class FirecrawlTool implements Tool
}
$total = mb_strlen($content);
+ $suffix = "\n\n[Truncated: {$total} characters total.]";
+
+ if ($maxCharacters <= mb_strlen($suffix)) {
+ return mb_substr($content, 0, $maxCharacters);
+ }
- return mb_substr($content, 0, $maxCharacters)
- . "\n\n[Truncated: showing the first {$maxCharacters} of {$total} characters.]";
+ return mb_substr($content, 0, $maxCharacters - mb_strlen($suffix)) . $suffix;
}
}
diff --git a/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlCrawlTest.php b/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlCrawlTest.php
index a11d7f198..c9b1890f4 100644
--- a/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlCrawlTest.php
+++ b/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlCrawlTest.php
@@ -173,3 +173,20 @@ it('exposes name and a schema with required url and optional limit', function ()
expect($jsonSchema['required'])->toContain('url');
expect($jsonSchema['required'])->not->toContain('limit');
});
+
+it('bounds each HTTP request by the remaining deadline', function (): void {
+ $history = new ArrayObject();
+ $client = fakeFirecrawlClient([
+ new Response(200, [], json_encode(['success' => true, 'id' => 'job-1'])),
+ new Response(200, [], json_encode([
+ 'success' => true, 'status' => 'completed', 'total' => 0, 'completed' => 0, 'data' => [],
+ ])),
+ ], $history);
+
+ (new FirecrawlCrawl($client))->handle(new Request(['url' => 'https://example.com']));
+
+ foreach ([0, 1] as $i) {
+ $timeout = $history[$i]['options'][GuzzleHttp\RequestOptions::TIMEOUT] ?? null;
+ expect($timeout)->toBeFloat()->toBeGreaterThan(0)->toBeLessThanOrEqual(55.0);
+ }
+});
diff --git a/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlMapTest.php b/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlMapTest.php
index 24bac94f7..e303fc73f 100644
--- a/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlMapTest.php
+++ b/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlMapTest.php
@@ -116,3 +116,19 @@ it('drops tail links and reports the cut when output exceeds the budget', functi
expect($decoded)->not->toHaveCount(4);
expect(array_key_exists('omitted', end($decoded)))->toBeTrue();
});
+
+it('reduces a single oversized item to an omitted marker', function (): void {
+ $client = fakeFirecrawlClient([
+ new Response(200, [], json_encode(['success' => true, 'data' => ['links' => [
+ ['url' => 'https://example.com/' . str_repeat('x', 400)],
+ ]]])),
+ ]);
+
+ $tool = new class ($client) extends FirecrawlMap {
+ protected int $outputCharacterBudget = 100;
+ };
+
+ $result = $tool->handle(new Request(['url' => 'https://example.com']));
+
+ expect(json_decode($result, true))->toBe([['omitted' => 1]]);
+});
diff --git a/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlScrapeTest.php b/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlScrapeTest.php
index 6cf07a106..b8907f6ad 100644
--- a/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlScrapeTest.php
+++ b/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlScrapeTest.php
@@ -89,8 +89,9 @@ it('truncates markdown longer than 80000 characters', function (): void {
$result = (new FirecrawlScrape($client))->handle(new Request(['url' => 'https://example.com']));
- expect($result)->toStartWith($prefix);
- expect($result)->toEndWith('[Truncated: showing the first 80000 of 80100 characters.]');
+ expect(mb_strlen($result))->toBeLessThanOrEqual(80000);
+ expect($result)->toStartWith(str_repeat('a', 1000));
+ expect($result)->toEndWith('[Truncated: 80100 characters total.]');
});
it('resolves the client from the container when none is injected', function (): void {
diff --git a/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlToolsTest.php b/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlToolsTest.php
index f9e8e104f..a437b4aae 100644
--- a/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlToolsTest.php
+++ b/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlToolsTest.php
@@ -23,9 +23,23 @@ it('returns one instance of each core tool', function (): void {
it('passes an explicit client through to every tool', function (): void {
$client = fakeFirecrawlClient([
new Response(200, [], json_encode(['success' => true, 'data' => ['markdown' => '# Hi']])),
+ new Response(200, [], json_encode(['success' => true, 'data' => [
+ 'web' => [['title' => 'T', 'url' => 'https://a.com', 'description' => 'D']],
+ ]])),
+ new Response(200, [], json_encode(['success' => true, 'data' => [
+ 'links' => [['url' => 'https://a.com']],
+ ]])),
+ new Response(200, [], json_encode(['success' => true, 'id' => 'job-1'])),
+ new Response(200, [], json_encode([
+ 'success' => true, 'status' => 'completed', 'total' => 0, 'completed' => 0, 'data' => [],
+ ])),
]);
- [$scrape] = FirecrawlTools::all($client);
+ [$scrape, $search, $map, $crawl] = FirecrawlTools::all($client);
expect($scrape->handle(new Request(['url' => 'https://example.com'])))->toBe('# Hi');
+ expect($search->handle(new Request(['query' => 'hi'])))->toContain('https://a.com');
+ expect($map->handle(new Request(['url' => 'https://example.com'])))->toContain('https://a.com');
+ expect(json_decode($crawl->handle(new Request(['url' => 'https://example.com'])), true)['status'])
+ ->toBe('completed');
});