Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(php-sdk): bound crawl wait time for queue-friendly execution

Kris Oeif27395db9f65
diff --git a/apps/php-sdk/README.md b/apps/php-sdk/README.md
index 491b5abcb..31c80cef3 100644
--- a/apps/php-sdk/README.md
+++ b/apps/php-sdk/README.md
@@ -461,6 +461,12 @@ new FirecrawlScrape(FirecrawlClient::create(apiKey: 'fc-other-key'));
Tool failures (rate limits, timeouts, invalid URLs) are returned to the model
as readable error strings rather than thrown, so agent runs degrade gracefully.
+`firecrawl_crawl` waits up to 55 seconds for the crawl to finish. If your
+agent runs inside a queued job, keep the crawl limit small or raise the
+worker's job timeout, since a killed job orphans a crawl that is still
+running (and billed) on the server. Extend `FirecrawlCrawl` and override
+`$timeoutSeconds` to change how long it waits.
+
## License
MIT
diff --git a/apps/php-sdk/src/Laravel/Tools/FirecrawlCrawl.php b/apps/php-sdk/src/Laravel/Tools/FirecrawlCrawl.php
index ba701dab2..9bcd8bab0 100644
--- a/apps/php-sdk/src/Laravel/Tools/FirecrawlCrawl.php
+++ b/apps/php-sdk/src/Laravel/Tools/FirecrawlCrawl.php
@@ -4,6 +4,7 @@ declare(strict_types=1);
namespace Firecrawl\Laravel\Tools;
+use Firecrawl\Exceptions\JobTimeoutException;
use Firecrawl\Models\CrawlOptions;
use Firecrawl\Models\Document;
use Firecrawl\Models\ScrapeOptions;
@@ -12,6 +13,13 @@ use Laravel\Ai\Tools\Request;
class FirecrawlCrawl extends FirecrawlTool
{
+ /**
+ * Extenders can override this to wait longer, but the default keeps the
+ * crawl inside typical queue worker timeouts (e.g. Laravel's default of
+ * 60 seconds) so a billed crawl is not orphaned when the job is killed.
+ */
+ protected int $timeoutSeconds = 55;
+
public function name(): string
{
return 'firecrawl_crawl';
@@ -21,8 +29,8 @@ class FirecrawlCrawl extends FirecrawlTool
{
return 'Crawl a website with Firecrawl starting from a URL, following its links and returning '
. 'each crawled page as a {url, markdown} object in a JSON array. This is a slower, '
- . 'multi-page operation that waits for the crawl to finish. Prefer firecrawl_scrape when '
- . 'you only need one known page, and keep the page limit small.';
+ . 'multi-page operation. Prefer firecrawl_scrape when you only need one known page, and '
+ . 'keep the page limit small. Waits up to about a minute for the crawl to finish.';
}
public function handle(Request $request): string
@@ -30,14 +38,21 @@ class FirecrawlCrawl extends FirecrawlTool
return $this->guard(function () use ($request): string {
$limit = min(max($request->integer('limit') ?: 5, 1), 25);
- $job = $this->client()->crawl(
- (string) $request->string('url'),
- CrawlOptions::with(
- limit: $limit,
- scrapeOptions: ScrapeOptions::with(formats: ['markdown']),
- integration: self::INTEGRATION,
- ),
- );
+ try {
+ $job = $this->client()->crawl(
+ (string) $request->string('url'),
+ CrawlOptions::with(
+ limit: $limit,
+ scrapeOptions: ScrapeOptions::with(formats: ['markdown']),
+ integration: self::INTEGRATION,
+ ),
+ timeoutSec: $this->timeoutSeconds,
+ );
+ } catch (JobTimeoutException) {
+ return "The crawl did not finish within {$this->timeoutSeconds} seconds. It may still "
+ . 'complete on the server. Use a smaller limit, or scrape key pages individually '
+ . 'with firecrawl_scrape.';
+ }
$pages = array_map(fn (Document $document): array => [
'url' => $document->getMetadata()['sourceURL']
diff --git a/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlCrawlTest.php b/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlCrawlTest.php
index 2f8ebf63c..0388446f6 100644
--- a/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlCrawlTest.php
+++ b/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlCrawlTest.php
@@ -66,6 +66,31 @@ it('reports crawls that finish without pages', function (): void {
expect($result)->toBe('Crawl finished with status [failed] but returned no pages.');
});
+it('reports a friendly message when the crawl exceeds its own timeout', function (): void {
+ $client = fakeFirecrawlClient([
+ new Response(200, [], json_encode(['success' => true, 'id' => 'job-1'])),
+ new Response(200, [], json_encode([
+ 'success' => true, 'status' => 'scraping', 'total' => 0, 'completed' => 0, 'data' => [],
+ ])),
+ new Response(200, [], json_encode([
+ 'success' => true, 'status' => 'scraping', 'total' => 0, 'completed' => 0, 'data' => [],
+ ])),
+ ]);
+
+ // Anonymous subclass overrides the wait so the test does not sleep for
+ // the full production timeout.
+ $tool = new class ($client) extends FirecrawlCrawl {
+ protected int $timeoutSeconds = 1;
+ };
+
+ $result = $tool->handle(new Request(['url' => 'https://example.com']));
+
+ expect($result)->toBe(
+ 'The crawl did not finish within 1 seconds. It may still complete on the server. '
+ . 'Use a smaller limit, or scrape key pages individually with firecrawl_scrape.',
+ );
+});
+
it('returns API failures as readable strings instead of throwing', function (): void {
$client = fakeFirecrawlClient([
new Response(400, [], json_encode(['error' => 'Invalid URL'])),