Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(php-sdk): bound crawl wait time for queue-friendly execution
Kris Oeif27395db9f65
diff --git a/apps/php-sdk/README.md b/apps/php-sdk/README.mdindex 491b5abcb..31c80cef3 100644--- a/apps/php-sdk/README.md+++ b/apps/php-sdk/README.md@@ -461,6 +461,12 @@ new FirecrawlScrape(FirecrawlClient::create(apiKey: 'fc-other-key'));Tool failures (rate limits, timeouts, invalid URLs) are returned to the modelas readable error strings rather than thrown, so agent runs degrade gracefully.+`firecrawl_crawl` waits up to 55 seconds for the crawl to finish. If your+agent runs inside a queued job, keep the crawl limit small or raise the+worker's job timeout, since a killed job orphans a crawl that is still+running (and billed) on the server. Extend `FirecrawlCrawl` and override+`$timeoutSeconds` to change how long it waits.+## LicenseMITdiff --git a/apps/php-sdk/src/Laravel/Tools/FirecrawlCrawl.php b/apps/php-sdk/src/Laravel/Tools/FirecrawlCrawl.phpindex ba701dab2..9bcd8bab0 100644--- a/apps/php-sdk/src/Laravel/Tools/FirecrawlCrawl.php+++ b/apps/php-sdk/src/Laravel/Tools/FirecrawlCrawl.php@@ -4,6 +4,7 @@ declare(strict_types=1);namespace Firecrawl\Laravel\Tools;+use Firecrawl\Exceptions\JobTimeoutException;use Firecrawl\Models\CrawlOptions;use Firecrawl\Models\Document;use Firecrawl\Models\ScrapeOptions;@@ -12,6 +13,13 @@ use Laravel\Ai\Tools\Request;class FirecrawlCrawl extends FirecrawlTool{+ /**+ * Extenders can override this to wait longer, but the default keeps the+ * crawl inside typical queue worker timeouts (e.g. Laravel's default of+ * 60 seconds) so a billed crawl is not orphaned when the job is killed.+ */+ protected int $timeoutSeconds = 55;+public function name(): string{return 'firecrawl_crawl';@@ -21,8 +29,8 @@ class FirecrawlCrawl extends FirecrawlTool{return 'Crawl a website with Firecrawl starting from a URL, following its links and returning '. 'each crawled page as a {url, markdown} object in a JSON array. This is a slower, '- . 'multi-page operation that waits for the crawl to finish. Prefer firecrawl_scrape when '- . 'you only need one known page, and keep the page limit small.';+ . 'multi-page operation. Prefer firecrawl_scrape when you only need one known page, and '+ . 'keep the page limit small. Waits up to about a minute for the crawl to finish.';}public function handle(Request $request): string@@ -30,14 +38,21 @@ class FirecrawlCrawl extends FirecrawlToolreturn $this->guard(function () use ($request): string {$limit = min(max($request->integer('limit') ?: 5, 1), 25);- $job = $this->client()->crawl(- (string) $request->string('url'),- CrawlOptions::with(- limit: $limit,- scrapeOptions: ScrapeOptions::with(formats: ['markdown']),- integration: self::INTEGRATION,- ),- );+ try {+ $job = $this->client()->crawl(+ (string) $request->string('url'),+ CrawlOptions::with(+ limit: $limit,+ scrapeOptions: ScrapeOptions::with(formats: ['markdown']),+ integration: self::INTEGRATION,+ ),+ timeoutSec: $this->timeoutSeconds,+ );+ } catch (JobTimeoutException) {+ return "The crawl did not finish within {$this->timeoutSeconds} seconds. It may still "+ . 'complete on the server. Use a smaller limit, or scrape key pages individually '+ . 'with firecrawl_scrape.';+ }$pages = array_map(fn (Document $document): array => ['url' => $document->getMetadata()['sourceURL']diff --git a/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlCrawlTest.php b/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlCrawlTest.phpindex 2f8ebf63c..0388446f6 100644--- a/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlCrawlTest.php+++ b/apps/php-sdk/tests/Unit/LaravelTools/FirecrawlCrawlTest.php@@ -66,6 +66,31 @@ it('reports crawls that finish without pages', function (): void {expect($result)->toBe('Crawl finished with status [failed] but returned no pages.');});+it('reports a friendly message when the crawl exceeds its own timeout', function (): void {+ $client = fakeFirecrawlClient([+ new Response(200, [], json_encode(['success' => true, 'id' => 'job-1'])),+ new Response(200, [], json_encode([+ 'success' => true, 'status' => 'scraping', 'total' => 0, 'completed' => 0, 'data' => [],+ ])),+ new Response(200, [], json_encode([+ 'success' => true, 'status' => 'scraping', 'total' => 0, 'completed' => 0, 'data' => [],+ ])),+ ]);++ // Anonymous subclass overrides the wait so the test does not sleep for+ // the full production timeout.+ $tool = new class ($client) extends FirecrawlCrawl {+ protected int $timeoutSeconds = 1;+ };++ $result = $tool->handle(new Request(['url' => 'https://example.com']));++ expect($result)->toBe(+ 'The crawl did not finish within 1 seconds. It may still complete on the server. '+ . 'Use a smaller limit, or scrape key pages individually with firecrawl_scrape.',+ );+});+it('returns API failures as readable strings instead of throwing', function (): void {$client = fakeFirecrawlClient([new Response(400, [], json_encode(['error' => 'Invalid URL'])),