Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Scrape-evals dispatch workflow and .kulala req file (#3217)
Rafael Miller59e0d3aa853d
diff --git a/.github/workflows/scrape-evals.yml b/.github/workflows/scrape-evals.ymlnew file mode 100644index 000000000..8d99fc4cd--- /dev/null+++ b/.github/workflows/scrape-evals.yml@@ -0,0 +1,131 @@+name: Scrape Evals+on:+ issue_comment:+ types: [created]+ pull_request:+ types: [opened]++jobs:+ quality-eval:+ if: |+ (github.event_name == 'issue_comment' &&+ github.event.issue.pull_request != null &&+ (contains(github.event.comment.body, '#scrape-quality-eval') ||+ contains(github.event.issue.title, '#scrape-quality-eval'))) ||+ (github.event_name == 'pull_request' &&+ (contains(github.event.pull_request.body, '#scrape-quality-eval') ||+ contains(github.event.pull_request.title, '#scrape-quality-eval')))+ runs-on: ubuntu-latest+ steps:+ - name: Check org membership+ id: check-member+ uses: actions/github-script@v7+ with:+ github-token: ${{ secrets.SCRAPE_EVALS_PAT }}+ script: |+ try {+ await github.rest.orgs.checkMembershipForUser({+ org: 'firecrawl',+ username: context.actor,+ });+ return true;+ } catch {+ return false;+ }++ - name: Dispatch quality eval+ if: steps.check-member.outputs.result == 'true'+ uses: actions/github-script@v7+ with:+ github-token: ${{ secrets.SCRAPE_EVALS_PAT }}+ script: |+ let prNumber, prBranch, commitSha, headRepo;+ if (context.eventName === 'pull_request') {+ prNumber = context.payload.pull_request.number;+ prBranch = context.payload.pull_request.head.ref;+ commitSha = context.payload.pull_request.head.sha;+ headRepo = context.payload.pull_request.head.repo.full_name;+ } else {+ const pr = await github.rest.pulls.get({+ owner: context.repo.owner,+ repo: context.repo.repo,+ pull_number: context.issue.number,+ });+ prNumber = pr.data.number;+ prBranch = pr.data.head.ref;+ commitSha = pr.data.head.sha;+ headRepo = pr.data.head.repo.full_name;+ }+ await github.rest.repos.createDispatchEvent({+ owner: 'firecrawl',+ repo: 'scrape-evals',+ event_type: 'firecrawl-pr-eval',+ client_payload: {+ pr_number: prNumber,+ pr_branch: prBranch,+ commit_sha: commitSha,+ repo: headRepo,+ },+ });++ ocr-eval:+ if: |+ (github.event_name == 'issue_comment' &&+ github.event.issue.pull_request != null &&+ (contains(github.event.comment.body, '#scrape-ocr-eval') ||+ contains(github.event.issue.title, '#scrape-ocr-eval'))) ||+ (github.event_name == 'pull_request' &&+ (contains(github.event.pull_request.body, '#scrape-ocr-eval') ||+ contains(github.event.pull_request.title, '#scrape-ocr-eval')))+ runs-on: ubuntu-latest+ steps:+ - name: Check org membership+ id: check-member+ uses: actions/github-script@v7+ with:+ github-token: ${{ secrets.SCRAPE_EVALS_PAT }}+ script: |+ try {+ await github.rest.orgs.checkMembershipForUser({+ org: 'firecrawl',+ username: context.actor,+ });+ return true;+ } catch {+ return false;+ }++ - name: Dispatch OCR eval+ if: steps.check-member.outputs.result == 'true'+ uses: actions/github-script@v7+ with:+ github-token: ${{ secrets.SCRAPE_EVALS_PAT }}+ script: |+ let prNumber, prBranch, commitSha, headRepo;+ if (context.eventName === 'pull_request') {+ prNumber = context.payload.pull_request.number;+ prBranch = context.payload.pull_request.head.ref;+ commitSha = context.payload.pull_request.head.sha;+ headRepo = context.payload.pull_request.head.repo.full_name;+ } else {+ const pr = await github.rest.pulls.get({+ owner: context.repo.owner,+ repo: context.repo.repo,+ pull_number: context.issue.number,+ });+ prNumber = pr.data.number;+ prBranch = pr.data.head.ref;+ commitSha = pr.data.head.sha;+ headRepo = pr.data.head.repo.full_name;+ }+ await github.rest.repos.createDispatchEvent({+ owner: 'firecrawl',+ repo: 'scrape-evals',+ event_type: 'firecrawl-ocr-eval',+ client_payload: {+ pr_number: prNumber,+ pr_branch: prBranch,+ commit_sha: commitSha,+ repo: headRepo,+ },+ });diff --git a/apps/api/requests.kulala.http b/apps/api/requests.kulala.httpnew file mode 100644index 000000000..d901e42b9--- /dev/null+++ b/apps/api/requests.kulala.http@@ -0,0 +1,147 @@+# Pick your baseUrl here:+# @baseUrl = http://localhost:3002+@baseUrl = https://api.firecrawl.dev++### Scrape Website+# @name scrape+POST {{baseUrl}}/v2/scrape HTTP/1.1+Authorization: Bearer {{TEST_API_KEY}}+content-type: application/json++{+ "url": "https://www.a-pdf.com/ocr/demo.pdf",+ "parsers": [{ "type": "pdf" }],+ "maxAge": 0+}++### Crawl Website+# @name crawl+POST {{baseUrl}}/v2/crawl HTTP/1.1+Authorization: Bearer {{$dotenv TEST_API_KEY}}+content-type: application/json++{+ "url":"https://firecrawl.dev"+}++### Check Crawl Status+@crawlId = {{crawl.response.body.$.id}}+# @name crawlStatus+GET {{baseUrl}}/v2/crawl/{{crawlId}} HTTP/1.1+Authorization: Bearer {{$dotenv TEST_API_KEY}}++### Cancel Crawl+@crawlId = {{crawl.response.body.$.id}}+# @name cancelCrawl+DELETE {{baseUrl}}/v2/crawl/{{crawlId}} HTTP/1.1+Authorization: Bearer {{$dotenv TEST_API_KEY}}++### Extract website+# @name extract+POST {{baseUrl}}/v2/extract HTTP/1.1+Authorization: Bearer {{$dotenv TEST_API_KEY}}+content-type: application/json++{+ "urls": ["https://firecrawl.dev"],+ "schema": {+ "type": "object",+ "properties": {+ "companyName": {+ "type": "string"+ },+ "companyDescription": {+ "type": "string"+ }+ }+ },+ "agent": {+ "model": "fire-1"+ },+ "origin": "api-sdk"+}++### Check Extract Status+@extractId = {{extract.response.body.$.id}}+# @name extractStatus+GET {{baseUrl}}/v2/extract/{{extractId}} HTTP/1.1+Authorization: Bearer {{$dotenv TEST_API_KEY}}++### Batch Scrape Websites+# @name batchScrape+POST {{baseUrl}}/v2/batch/scrape HTTP/1.1+Authorization: Bearer {{$dotenv TEST_API_KEY}}+content-type: application/json++{+ "urls": [+ "firecrawl.dev",+ "mendable.ai"+ ]+}++### Check Batch Scrape Status+@batchScrapeId = {{batchScrape.response.body.$.id}}+# @name batchScrapeStatus+GET {{baseUrl}}/v2/batch/scrape/{{batchScrapeId}} HTTP/1.1+Authorization: Bearer {{$dotenv TEST_API_KEY}}++### Map Website+# @name map+POST {{baseUrl}}/v2/map HTTP/1.1+Authorization: Bearer {{$dotenv TEST_API_KEY}}+content-type: application/json++{+ "url": "firecrawl.dev",+ "sitemapOnly": true+}++### Search+# @name search+POST {{baseUrl}}/v2/search HTTP/1.1+Authorization: Bearer {{$dotenv TEST_API_KEY}}+content-type: application/json++{+ "query": "firecrawl",+ "limit": 50+}++### Get Active Crawls+# @name activeCrawls+GET {{baseUrl}}/v2/crawl/active HTTP/1.1+Authorization: Bearer {{$dotenv TEST_API_KEY}}++### Generate LLMs TXT+# @name generateLlmsTxt+POST {{baseUrl}}/v1/llmstxt HTTP/1.1+Authorization: Bearer {{$dotenv TEST_API_KEY}}+content-type: application/json++{+ "url": "https://firecrawl.dev",+ "maxUrls": 1,+ "showFullText": false,+ "cache": true+}+++### Check Generate LLMs TXT Status+@generateLlmsTxtId = {{generateLlmsTxt.response.body.$.id}}+# @name generateLlmsTxtStatus+GET {{baseUrl}}/v1/llmstxt/{{generateLlmsTxtId}} HTTP/1.1+Authorization: Bearer {{$dotenv TEST_API_KEY}}++### Search with Wallet+@paymentHeaderSearch = 'EXAMPLE-PAYMENT-HEADER'+# @name searchWithWallet+POST {{baseUrl}}/v1/x402/search HTTP/1.1+Authorization: Bearer {{$dotenv TEST_API_KEY}}+content-type: application/json+X-Payment: {{paymentHeaderSearch}}++{+ "query": "Premier League fixtures for the 2025/26 season",+ "limit": 2+}