Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(php-sdk): address all cubic-dev code review issues and simplify CI
Devin AI715186d7921c
diff --git a/.github/scripts/check_version_has_incremented.py b/.github/scripts/check_version_has_incremented.pyindex 295e46594..cbab64e0d 100644--- a/.github/scripts/check_version_has_incremented.py+++ b/.github/scripts/check_version_has_incremented.py@@ -108,10 +108,10 @@ def get_packagist_version(package_name: str) -> str:# Filter out dev versions and find highest stable versionstable_versions = []for v in versions:- if "dev" not in v and v.startswith("v"):- stable_versions.append(v.lstrip("v"))- elif "dev" not in v and re.match(r"^\d", v):- stable_versions.append(v)+ normalized = v.lstrip("v")+ parsed = parse_version(normalized)+ if "dev" not in v and not parsed.is_prerelease and re.match(r"^\d", normalized):+ stable_versions.append(normalized)if not stable_versions:return "0.0.0"stable_versions.sort(key=lambda x: parse_version(x), reverse=True)diff --git a/.github/workflows/publish-php-sdk.yml b/.github/workflows/publish-php-sdk.ymlindex df15eb6a8..2d1b9aa03 100644--- a/.github/workflows/publish-php-sdk.yml+++ b/.github/workflows/publish-php-sdk.yml@@ -50,5 +50,6 @@ jobs:- name: Notify Packagistif: ${{ env.VERSION_INCREMENTED == 'true' }}run: |- curl -s -X POST "https://packagist.org/api/update-package?username=${{ secrets.PACKAGIST_USERNAME }}&apiToken=${{ secrets.PACKAGIST_TOKEN }}" \+ curl -sfS -X POST "https://packagist.org/api/update-package?username=${{ secrets.PACKAGIST_USERNAME }}&apiToken=${{ secrets.PACKAGIST_TOKEN }}" \+ -H "Content-Type: application/json" \-d '{"repository":{"url":"https://github.com/firecrawl/firecrawl"}}'diff --git a/.github/workflows/test-php-sdk.yml b/.github/workflows/test-php-sdk.ymlindex 40d1ff7d5..3a1ffc63c 100644--- a/.github/workflows/test-php-sdk.yml+++ b/.github/workflows/test-php-sdk.yml@@ -15,9 +15,13 @@ on:workflow_dispatch:jobs:- lint:- name: Static Analysis- runs-on: blacksmith-2vcpu-ubuntu-2404+ build-and-test:+ name: Build and Test+ runs-on: blacksmith-4vcpu-ubuntu-2404+ if: >-+ github.event_name == 'workflow_dispatch' ||+ github.event_name == 'pull_request' ||+ (github.event_name == 'push' && github.ref == 'refs/heads/main')steps:- uses: actions/checkout@v5@@ -26,6 +30,7 @@ jobs:with:php-version: '8.3'tools: composer:v2+ coverage: none- name: Get Composer cache directoryid: composer-cache@@ -52,65 +57,6 @@ jobs:working-directory: ./apps/php-sdkrun: vendor/bin/phpstan analyse --no-progress- test:- name: Test (PHP ${{ matrix.php }}, Laravel ${{ matrix.laravel }})- runs-on: blacksmith-2vcpu-ubuntu-2404- if: >-- github.event_name == 'workflow_dispatch' ||- github.event_name == 'pull_request' ||- (github.event_name == 'push' && github.ref == 'refs/heads/main')- strategy:- fail-fast: false- matrix:- include:- - php: '8.1'- laravel: '10.*'- testbench: '8.*'- - php: '8.2'- laravel: '10.*'- testbench: '8.*'- - php: '8.2'- laravel: '11.*'- testbench: '9.*'- - php: '8.3'- laravel: '11.*'- testbench: '9.*'- - php: '8.3'- laravel: '12.*'- testbench: '10.*'- - php: '8.4'- laravel: '12.*'- testbench: '10.*'- steps:- - uses: actions/checkout@v5-- - name: Set up PHP ${{ matrix.php }}- uses: shivammathur/setup-php@v2- with:- php-version: ${{ matrix.php }}- tools: composer:v2- coverage: none-- - name: Get Composer cache directory- id: composer-cache- working-directory: ./apps/php-sdk- run: echo "dir=$(composer config cache-files-dir)" >> $GITHUB_OUTPUT-- - name: Cache Composer dependencies- uses: actions/cache@v4- with:- path: ${{ steps.composer-cache.outputs.dir }}- key: ${{ runner.os }}-php${{ matrix.php }}-laravel${{ matrix.laravel }}-composer-${{ hashFiles('apps/php-sdk/composer.json') }}- restore-keys: |- ${{ runner.os }}-php${{ matrix.php }}-laravel${{ matrix.laravel }}-composer-- ${{ runner.os }}-php${{ matrix.php }}-composer--- - name: Install dependencies with Laravel ${{ matrix.laravel }}- working-directory: ./apps/php-sdk- run: |- composer require "illuminate/contracts:${{ matrix.laravel }}" "illuminate/support:${{ matrix.laravel }}" "orchestra/testbench:${{ matrix.testbench }}" --no-interaction --no-update- composer update --prefer-dist --no-interaction-- name: Run unit testsworking-directory: ./apps/php-sdkrun: vendor/bin/pest --cidiff --git a/apps/php-sdk/src/Client/FirecrawlClient.php b/apps/php-sdk/src/Client/FirecrawlClient.phpindex 592908c5d..46b11b721 100644--- a/apps/php-sdk/src/Client/FirecrawlClient.php+++ b/apps/php-sdk/src/Client/FirecrawlClient.php@@ -58,7 +58,7 @@ final class FirecrawlClientfloat $backoffFactor = self::DEFAULT_BACKOFF_FACTOR,?ClientInterface $httpClient = null,): self {- $resolvedKey = $apiKey ?: (getenv('FIRECRAWL_API_KEY') ?: '');+ $resolvedKey = trim($apiKey ?: (getenv('FIRECRAWL_API_KEY') ?: ''));if ($resolvedKey === '') {throw new FirecrawlException('API key is required. Pass it directly or set the FIRECRAWL_API_KEY environment variable.',@@ -67,6 +67,12 @@ final class FirecrawlClient$resolvedUrl = $apiUrl ?: (getenv('FIRECRAWL_API_URL') ?: self::DEFAULT_API_URL);+ if (!preg_match('#^https?://#i', $resolvedUrl)) {+ throw new FirecrawlException(+ 'API URL must be a fully qualified URL including scheme (e.g. https://api.firecrawl.dev).',+ );+ }+$http = new FirecrawlHttpClient($resolvedKey,$resolvedUrl,@@ -339,6 +345,8 @@ final class FirecrawlClientthrow new FirecrawlException('Agent start did not return a job ID');}+ $this->ensureValidPollInterval($pollIntervalSec);+$deadline = time() + $timeoutSec;while (time() < $deadline) {$status = $this->getAgentStatus($start->getId());@@ -456,6 +464,13 @@ final class FirecrawlClient// INTERNAL POLLING HELPERS// ================================================================+ private function ensureValidPollInterval(int $pollIntervalSec): void+ {+ if ($pollIntervalSec < 1) {+ throw new FirecrawlException('Poll interval must be at least 1 second, got ' . $pollIntervalSec);+ }+ }+private function pollCrawl(?string $jobId,int $pollIntervalSec,@@ -465,6 +480,8 @@ final class FirecrawlClientthrow new FirecrawlException('Crawl start did not return a job ID');}+ $this->ensureValidPollInterval($pollIntervalSec);+$deadline = time() + $timeoutSec;while (time() < $deadline) {$job = $this->getCrawlStatus($jobId);@@ -486,6 +503,8 @@ final class FirecrawlClientthrow new FirecrawlException('Batch scrape start did not return a job ID');}+ $this->ensureValidPollInterval($pollIntervalSec);+$deadline = time() + $timeoutSec;while (time() < $deadline) {$job = $this->getBatchScrapeStatus($jobId);@@ -498,10 +517,23 @@ final class FirecrawlClientthrow new JobTimeoutException($jobId, $timeoutSec, 'Batch scrape');}+ private function assertSameOrigin(string $url): void+ {+ $baseHost = parse_url($this->http->getBaseUrl(), PHP_URL_HOST);+ $nextHost = parse_url($url, PHP_URL_HOST);++ if ($baseHost === null || $nextHost === null || strcasecmp($baseHost, $nextHost) !== 0) {+ throw new FirecrawlException(+ 'Pagination URL origin does not match the API base URL. Refusing to follow: ' . $url,+ );+ }+ }+private function paginateCrawl(CrawlJob $job): CrawlJob{$current = $job;while ($current->getNext() !== null && $current->getNext() !== '') {+ $this->assertSameOrigin($current->getNext());$nextRaw = $this->http->getAbsolute($current->getNext());$nextPage = CrawlJob::fromArray($nextRaw);@@ -519,6 +551,7 @@ final class FirecrawlClient{$current = $job;while ($current->getNext() !== null && $current->getNext() !== '') {+ $this->assertSameOrigin($current->getNext());$nextRaw = $this->http->getAbsolute($current->getNext());$nextPage = BatchScrapeJob::fromArray($nextRaw);diff --git a/apps/php-sdk/src/Client/FirecrawlHttpClient.php b/apps/php-sdk/src/Client/FirecrawlHttpClient.phpindex 47ebf560d..d038ff13f 100644--- a/apps/php-sdk/src/Client/FirecrawlHttpClient.php+++ b/apps/php-sdk/src/Client/FirecrawlHttpClient.php@@ -67,6 +67,11 @@ final class FirecrawlHttpClientreturn $this->request('DELETE', $this->baseUrl . $path);}+ public function getBaseUrl(): string+ {+ return $this->baseUrl;+ }+/*** @param array<string, mixed> $body* @param array<string, string> $extraHeaders@@ -103,6 +108,9 @@ final class FirecrawlHttpClient$responseBody = (string) $response->getBody();if ($statusCode >= 200 && $statusCode < 300) {+ if ($responseBody === '' || $responseBody === '{}') {+ return [];+ }/** @var array<string, mixed> */return json_decode($responseBody, true, 512, JSON_THROW_ON_ERROR);}diff --git a/apps/php-sdk/src/Laravel/Facades/Firecrawl.php b/apps/php-sdk/src/Laravel/Facades/Firecrawl.phpindex e6208cd5e..dd5963d6e 100644--- a/apps/php-sdk/src/Laravel/Facades/Firecrawl.php+++ b/apps/php-sdk/src/Laravel/Facades/Firecrawl.php@@ -31,13 +31,13 @@ use Illuminate\Support\Facades\Facade;* @method static BrowserDeleteResponse stopInteractiveBrowser(string $jobId)* @method static CrawlJob crawl(string $url, ?CrawlOptions $options = null, int $pollIntervalSec = 2, int $timeoutSec = 300)* @method static CrawlJob getCrawlStatus(string $jobId)- * @method static array cancelCrawl(string $jobId)- * @method static BatchScrapeJob batchScrape(array $urls, ?BatchScrapeOptions $options = null, int $pollIntervalSec = 2, int $timeoutSec = 300)- * @method static array cancelBatchScrape(string $jobId)+ * @method static array<string, mixed> cancelCrawl(string $jobId)+ * @method static BatchScrapeJob batchScrape(list<string> $urls, ?BatchScrapeOptions $options = null, int $pollIntervalSec = 2, int $timeoutSec = 300)+ * @method static array<string, mixed> cancelBatchScrape(string $jobId)* @method static MapData map(string $url, ?MapOptions $options = null)* @method static SearchData search(string $query, ?SearchOptions $options = null)* @method static AgentStatusResponse agent(AgentOptions $options, int $pollIntervalSec = 2, int $timeoutSec = 300)- * @method static array cancelAgent(string $jobId)+ * @method static array<string, mixed> cancelAgent(string $jobId)* @method static BrowserCreateResponse browser(?int $ttl = null, ?int $activityTtl = null, ?bool $streamWebView = null)* @method static BrowserExecuteResponse browserExecute(string $sessionId, string $code, string $language = 'bash', ?int $timeout = null)* @method static BrowserDeleteResponse deleteBrowser(string $sessionId)diff --git a/apps/php-sdk/src/Laravel/FirecrawlServiceProvider.php b/apps/php-sdk/src/Laravel/FirecrawlServiceProvider.phpindex 7373805ff..e61a3f4c6 100644--- a/apps/php-sdk/src/Laravel/FirecrawlServiceProvider.php+++ b/apps/php-sdk/src/Laravel/FirecrawlServiceProvider.php@@ -5,22 +5,29 @@ declare(strict_types=1);namespace Firecrawl\Laravel;use Firecrawl\Client\FirecrawlClient;-use Illuminate\Contracts\Support\DeferrableProvider;use Illuminate\Support\ServiceProvider;-class FirecrawlServiceProvider extends ServiceProvider implements DeferrableProvider+class FirecrawlServiceProvider extends ServiceProvider{public function register(): void{$this->mergeConfigFrom(__DIR__ . '/../../config/firecrawl.php', 'firecrawl');$this->app->singleton(FirecrawlClient::class, function ($app): FirecrawlClient {- /** @var array{api_key: string|null, api_url: string|null, timeout: float, max_retries: int, backoff_factor: float} $config */+ /** @var array<string, mixed> $config */$config = $app['config']->get('firecrawl', []);+ $apiKey = isset($config['api_key']) && is_string($config['api_key'])+ ? trim($config['api_key'])+ : null;++ if ($apiKey === '') {+ $apiKey = null;+ }+return FirecrawlClient::create(- apiKey: $config['api_key'] ?? null,- apiUrl: $config['api_url'] ?? null,+ apiKey: $apiKey,+ apiUrl: isset($config['api_url']) && is_string($config['api_url']) ? $config['api_url'] : null,timeoutSeconds: (float) ($config['timeout'] ?? 300),maxRetries: (int) ($config['max_retries'] ?? 3),backoffFactor: (float) ($config['backoff_factor'] ?? 0.5),diff --git a/apps/php-sdk/src/Models/AgentOptions.php b/apps/php-sdk/src/Models/AgentOptions.phpindex bd24edf53..49061561a 100644--- a/apps/php-sdk/src/Models/AgentOptions.php+++ b/apps/php-sdk/src/Models/AgentOptions.php@@ -21,6 +21,10 @@ final class AgentOptionsprivate readonly ?WebhookConfig $webhook = null,) {}+ /**+ * @param list<string>|null $urls+ * @param array<string, mixed>|null $schema+ */public static function with(?array $urls = null,?string $prompt = null,diff --git a/apps/php-sdk/src/Models/BatchScrapeJob.php b/apps/php-sdk/src/Models/BatchScrapeJob.phpindex a10ee89ed..703cb4aa2 100644--- a/apps/php-sdk/src/Models/BatchScrapeJob.php+++ b/apps/php-sdk/src/Models/BatchScrapeJob.php@@ -38,7 +38,7 @@ final class BatchScrapeJobstatus: $raw['status'] ?? null,completed: (int) ($raw['completed'] ?? 0),total: (int) ($raw['total'] ?? 0),- creditsUsed: $raw['creditsUsed'] ?? null,+ creditsUsed: isset($raw['creditsUsed']) ? (int) $raw['creditsUsed'] : null,expiresAt: $raw['expiresAt'] ?? null,next: $raw['next'] ?? null,data: $docs,diff --git a/apps/php-sdk/src/Models/CrawlJob.php b/apps/php-sdk/src/Models/CrawlJob.phpindex 99ee8b63a..5610a4966 100644--- a/apps/php-sdk/src/Models/CrawlJob.php+++ b/apps/php-sdk/src/Models/CrawlJob.php@@ -38,7 +38,7 @@ final class CrawlJobstatus: $raw['status'] ?? null,total: (int) ($raw['total'] ?? 0),completed: (int) ($raw['completed'] ?? 0),- creditsUsed: $raw['creditsUsed'] ?? null,+ creditsUsed: isset($raw['creditsUsed']) ? (int) $raw['creditsUsed'] : null,expiresAt: $raw['expiresAt'] ?? null,next: $raw['next'] ?? null,data: $docs,diff --git a/apps/php-sdk/src/Models/CrawlOptions.php b/apps/php-sdk/src/Models/CrawlOptions.phpindex fc535e26f..988029f33 100644--- a/apps/php-sdk/src/Models/CrawlOptions.php+++ b/apps/php-sdk/src/Models/CrawlOptions.php@@ -31,6 +31,10 @@ final class CrawlOptionsprivate readonly ?string $integration = null,) {}+ /**+ * @param list<string>|null $excludePaths+ * @param list<string>|null $includePaths+ */public static function with(?string $prompt = null,?array $excludePaths = null,diff --git a/apps/php-sdk/src/Models/CreditUsage.php b/apps/php-sdk/src/Models/CreditUsage.phpindex 181319633..ba981f67f 100644--- a/apps/php-sdk/src/Models/CreditUsage.php+++ b/apps/php-sdk/src/Models/CreditUsage.php@@ -16,11 +16,13 @@ final class CreditUsage/** @param array<string, mixed> $data */public static function fromArray(array $data): self{+ $inner = $data['data'] ?? $data;+return new self(- remainingCredits: (int) ($data['remainingCredits'] ?? 0),- planCredits: isset($data['planCredits']) ? (int) $data['planCredits'] : null,- billingPeriodStart: $data['billingPeriodStart'] ?? null,- billingPeriodEnd: $data['billingPeriodEnd'] ?? null,+ remainingCredits: (int) ($inner['remainingCredits'] ?? 0),+ planCredits: isset($inner['planCredits']) ? (int) $inner['planCredits'] : null,+ billingPeriodStart: $inner['billingPeriodStart'] ?? null,+ billingPeriodEnd: $inner['billingPeriodEnd'] ?? null,);}diff --git a/apps/php-sdk/src/Models/JsonFormat.php b/apps/php-sdk/src/Models/JsonFormat.phpindex 226574c61..31362cb97 100644--- a/apps/php-sdk/src/Models/JsonFormat.php+++ b/apps/php-sdk/src/Models/JsonFormat.php@@ -12,6 +12,9 @@ final class JsonFormatprivate readonly ?array $schema = null,) {}+ /**+ * @param array<string, mixed>|null $schema+ */public static function with(?string $prompt = null,?array $schema = null,diff --git a/apps/php-sdk/src/Models/LocationConfig.php b/apps/php-sdk/src/Models/LocationConfig.phpindex 5b8c189b1..51049c5f3 100644--- a/apps/php-sdk/src/Models/LocationConfig.php+++ b/apps/php-sdk/src/Models/LocationConfig.php@@ -12,6 +12,9 @@ final class LocationConfigprivate readonly ?array $languages = null,) {}+ /**+ * @param list<string>|null $languages+ */public static function with(?string $country = null,?array $languages = null,diff --git a/apps/php-sdk/src/Models/MapData.php b/apps/php-sdk/src/Models/MapData.phpindex 2c8627628..8d9d900cd 100644--- a/apps/php-sdk/src/Models/MapData.php+++ b/apps/php-sdk/src/Models/MapData.php@@ -17,6 +17,9 @@ final class MapDatapublic static function fromArray(array $data): self{$rawLinks = $data['links'] ?? [];+ if (!is_array($rawLinks)) {+ return new self(links: []);+ }$normalized = [];foreach ($rawLinks as $link) {diff --git a/apps/php-sdk/src/Models/ScrapeOptions.php b/apps/php-sdk/src/Models/ScrapeOptions.phpindex f48267dee..b91126530 100644--- a/apps/php-sdk/src/Models/ScrapeOptions.php+++ b/apps/php-sdk/src/Models/ScrapeOptions.php@@ -35,6 +35,14 @@ final class ScrapeOptionsprivate readonly ?string $integration = null,) {}+ /**+ * @param list<string|JsonFormat>|null $formats+ * @param array<string, string>|null $headers+ * @param list<string>|null $includeTags+ * @param list<string>|null $excludeTags+ * @param list<mixed>|null $parsers+ * @param list<array<string, mixed>>|null $actions+ */public static function with(?array $formats = null,?array $headers = null,diff --git a/apps/php-sdk/src/Models/SearchOptions.php b/apps/php-sdk/src/Models/SearchOptions.phpindex d5c630806..077a7b50b 100644--- a/apps/php-sdk/src/Models/SearchOptions.php+++ b/apps/php-sdk/src/Models/SearchOptions.php@@ -22,6 +22,10 @@ final class SearchOptionsprivate readonly ?string $integration = null,) {}+ /**+ * @param list<mixed>|null $sources+ * @param list<mixed>|null $categories+ */public static function with(?array $sources = null,?array $categories = null,diff --git a/apps/php-sdk/src/Models/WebhookConfig.php b/apps/php-sdk/src/Models/WebhookConfig.phpindex 2aaad89ae..e14be2e33 100644--- a/apps/php-sdk/src/Models/WebhookConfig.php+++ b/apps/php-sdk/src/Models/WebhookConfig.php@@ -16,6 +16,11 @@ final class WebhookConfigprivate readonly ?array $events = null,) {}+ /**+ * @param array<string, string>|null $headers+ * @param array<string, string>|null $metadata+ * @param list<string>|null $events+ */public static function with(string $url,?array $headers = null,