Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
ci: remove disabled prod test job and archived workflows (#5374)
David Tippette1dc7a43954b
diff --git a/.github/archive/js-sdk.yml b/.github/archive/js-sdk.ymldeleted file mode 100644index e873d8ee6..000000000--- a/.github/archive/js-sdk.yml+++ /dev/null@@ -1,53 +0,0 @@-name: Run JavaScript SDK E2E Tests--on: []--env:- ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}- BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }}- FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}- HOST: ${{ secrets.HOST }}- LLAMAPARSE_API_KEY: ${{ secrets.LLAMAPARSE_API_KEY }}- NUM_WORKERS_PER_QUEUE: ${{ secrets.NUM_WORKERS_PER_QUEUE }}- OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}- PLAYWRIGHT_MICROSERVICE_URL: ${{ secrets.PLAYWRIGHT_MICROSERVICE_URL }}- PORT: ${{ secrets.PORT }}- REDIS_URL: ${{ secrets.REDIS_URL }}- SUPABASE_ANON_TOKEN: ${{ secrets.SUPABASE_ANON_TOKEN }}- SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }}- SUPABASE_URL: ${{ secrets.SUPABASE_URL }}- TEST_API_KEY: ${{ secrets.TEST_API_KEY }}- HDX_NODE_BETA_MODE: 1--jobs:- build:- runs-on: blacksmith-4vcpu-ubuntu-2404- services:- redis:- image: redis- ports:- - 6379:6379-- steps:- - uses: actions/checkout@v3- - name: Set up Node.js- uses: actions/setup-node@v3- with:- node-version: "20"- - name: Install pnpm- run: npm install -g pnpm- - name: Install dependencies for API- run: pnpm install- working-directory: ./apps/api- - name: Start the application- run: npm start &- working-directory: ./apps/api- - name: Start workers- run: npm run workers &- working-directory: ./apps/api- - name: Install dependencies for JavaScript SDK- run: pnpm install- working-directory: ./apps/js-sdk/firecrawl- - name: Run E2E tests for JavaScript SDK- run: npm run test- working-directory: ./apps/js-sdk/firecrawl\ No newline at end of filediff --git a/.github/archive/publish-rust-sdk.yml b/.github/archive/publish-rust-sdk.ymldeleted file mode 100644index a3ed6c1b8..000000000--- a/.github/archive/publish-rust-sdk.yml+++ /dev/null@@ -1,42 +0,0 @@-name: Publish Rust SDK--on: []--env:- CRATES_IO_TOKEN: ${{ secrets.CRATES_IO_TOKEN }}--jobs:- build-and-publish:- runs-on: blacksmith-4vcpu-ubuntu-2404-- steps:- - name: Checkout repository- uses: actions/checkout@v3-- - name: Set up Rust- uses: actions-rs/toolchain@v1- with:- toolchain: stable- default: true- profile: minimal-- - name: Install dependencies- run: cargo build --release-- - name: Run version check script- id: version_check_script- run: |- VERSION_INCREMENTED=$(cargo search --limit 1 my_crate_name | grep my_crate_name)- echo "VERSION_INCREMENTED=$VERSION_INCREMENTED" >> $GITHUB_ENV-- - name: Build the package- if: ${{ env.VERSION_INCREMENTED == 'true' }}- run: cargo package- working-directory: ./apps/rust-sdk-- - name: Publish to crates.io- if: ${{ env.VERSION_INCREMENTED == 'true' }}- env:- CARGO_REG_TOKEN: ${{ secrets.CRATES_IO_TOKEN }}- run: cargo publish- working-directory: ./apps/rust-sdk\ No newline at end of filediff --git a/.github/archive/python-sdk.yml b/.github/archive/python-sdk.ymldeleted file mode 100644index d1f716de3..000000000--- a/.github/archive/python-sdk.yml+++ /dev/null@@ -1,65 +0,0 @@-name: Run Python SDK E2E Tests--on: []--env:- ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}- BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }}- FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}- HOST: ${{ secrets.HOST }}- LLAMAPARSE_API_KEY: ${{ secrets.LLAMAPARSE_API_KEY }}- NUM_WORKERS_PER_QUEUE: ${{ secrets.NUM_WORKERS_PER_QUEUE }}- OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}- PLAYWRIGHT_MICROSERVICE_URL: ${{ secrets.PLAYWRIGHT_MICROSERVICE_URL }}- PORT: ${{ secrets.PORT }}- REDIS_URL: ${{ secrets.REDIS_URL }}- SUPABASE_ANON_TOKEN: ${{ secrets.SUPABASE_ANON_TOKEN }}- SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }}- SUPABASE_URL: ${{ secrets.SUPABASE_URL }}- TEST_API_KEY: ${{ secrets.TEST_API_KEY }}- HDX_NODE_BETA_MODE: 1--jobs:- build:- runs-on: blacksmith-4vcpu-ubuntu-2404- strategy:- matrix:- python-version: ["3.10"]- services:- redis:- image: redis- ports:- - 6379:6379-- steps:- - uses: actions/checkout@v3- - name: Set up Node.js- uses: actions/setup-node@v3- with:- node-version: "20"- - name: Install pnpm- run: npm install -g pnpm- - name: Install dependencies for API- run: pnpm install- working-directory: ./apps/api- - name: Start the application- run: npm start &- working-directory: ./apps/api- id: start_app- - name: Start workers- run: npm run workers &- working-directory: ./apps/api- id: start_workers- - name: Set up Python ${{ matrix.python-version }}- uses: actions/setup-python@v4- with:- python-version: ${{ matrix.python-version }}- - name: Install Python dependencies- run: |- python -m pip install --upgrade pip- pip install -r requirements.txt- working-directory: ./apps/python-sdk- - name: Run E2E tests for Python SDK- run: |- python -m pytest firecrawl/__tests__/e2e/v2- working-directory: ./apps/python-sdkdiff --git a/.github/archive/rust-sdk.yml b/.github/archive/rust-sdk.ymldeleted file mode 100644index a4f9b2d9b..000000000--- a/.github/archive/rust-sdk.yml+++ /dev/null@@ -1,56 +0,0 @@-name: Run Rust SDK E2E Tests--on: []--env:- ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}- BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }}- FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}- HOST: ${{ secrets.HOST }}- LLAMAPARSE_API_KEY: ${{ secrets.LLAMAPARSE_API_KEY }}- NUM_WORKERS_PER_QUEUE: ${{ secrets.NUM_WORKERS_PER_QUEUE }}- OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}- PLAYWRIGHT_MICROSERVICE_URL: ${{ secrets.PLAYWRIGHT_MICROSERVICE_URL }}- PORT: ${{ secrets.PORT }}- REDIS_URL: ${{ secrets.REDIS_URL }}- SUPABASE_ANON_TOKEN: ${{ secrets.SUPABASE_ANON_TOKEN }}- SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }}- SUPABASE_URL: ${{ secrets.SUPABASE_URL }}- TEST_API_KEY: ${{ secrets.TEST_API_KEY }}- HDX_NODE_BETA_MODE: 1---jobs:- build:- runs-on: blacksmith-4vcpu-ubuntu-2404- services:- redis:- image: redis- ports:- - 6379:6379-- steps:- - name: Checkout repository- uses: actions/checkout@v3- - name: Install pnpm- run: npm install -g pnpm- - name: Install dependencies for API- run: pnpm install- working-directory: ./apps/api- - name: Start the application- run: npm start &- working-directory: ./apps/api- id: start_app- - name: Start workers- run: npm run workers &- working-directory: ./apps/api- id: start_workers- - name: Set up Rust- uses: actions/setup-rust@v1- with:- rust-version: stable- - name: Try the lib build- working-directory: ./apps/rust-sdk- run: cargo build- - name: Run E2E tests for Rust SDK- run: cargo test --test e2e_with_authdiff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.ymlindex c23244dd7..6f3c3cb97 100644--- a/.github/workflows/test-server.yml+++ b/.github/workflows/test-server.yml@@ -535,173 +535,6 @@ jobs:name: Logs (kubernetes, ${{ matrix.ai }}, ${{ matrix.search }}, ${{ matrix.engine }}, ${{ matrix.proxy }}, ${{ matrix.nuq }})path: logs/logs.zip- # temp disabled- # prod-test:- # name: Production environment tests- # runs-on: big-runner- # services:- # redis:- # image: redis- # ports:- # - 6379:6379- # rabbitmq:- # image: rabbitmq- # ports:- # - 5672:5672- # env:- # BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }}- # OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}- # REDIS_URL: ${{ secrets.REDIS_URL }}- # SUPABASE_ANON_TOKEN: ${{ secrets.SUPABASE_ANON_TOKEN }}- # SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }}- # SUPABASE_URL: ${{ secrets.SUPABASE_URL }}- # SUPABASE_REPLICA_URL: ${{ secrets.SUPABASE_REPLICA_URL }}- # INDEX_SUPABASE_SERVICE_TOKEN: ${{ secrets.INDEX_SUPABASE_SERVICE_TOKEN }}- # INDEX_SUPABASE_ANON_TOKEN: ${{ secrets.INDEX_SUPABASE_ANON_TOKEN }}- # INDEX_SUPABASE_URL: ${{ secrets.INDEX_SUPABASE_URL }}- # TEST_API_KEY: ${{ secrets.TEST_API_KEY }}- # TEST_TEAM_ID: ${{ secrets.TEST_TEAM_ID }}- # TEST_API_KEY_CONCURRENCY: ${{ secrets.TEST_API_KEY_CONCURRENCY }}- # TEST_TEAM_ID_CONCURRENCY: ${{ secrets.TEST_TEAM_ID_CONCURRENCY }}- # TEST_API_KEY_ZDR: ${{ secrets.TEST_API_KEY_ZDR }}- # TEST_TEAM_ID_ZDR: ${{ secrets.TEST_TEAM_ID_ZDR }}- # FIRE_ENGINE_BETA_URL: ${{ secrets.FIRE_ENGINE_BETA_URL }}- # FIRE_ENGINE_STAGING_URL: ${{ secrets.FIRE_ENGINE_STAGING_URL }}- # USE_DB_AUTHENTICATION: true- # ENV: ${{ secrets.ENV }}- # RUNPOD_MU_POD_ID: ${{ secrets.RUNPOD_MU_POD_ID }}- # RUNPOD_MUV2_POD_ID: ${{ secrets.RUNPOD_MUV2_POD_ID }}- # RUNPOD_MU_API_KEY: ${{ secrets.RUNPOD_MU_API_KEY }}- # GCS_CREDENTIALS: ${{ secrets.GCS_CREDENTIALS }}- # GCS_BUCKET_NAME: ${{ secrets.GCS_BUCKET_NAME }}- # GCS_INDEX_BUCKET_NAME: ${{ secrets.GCS_INDEX_BUCKET_NAME }}- # GCS_MEDIA_BUCKET_NAME: ${{ secrets.GCS_MEDIA_BUCKET_NAME }}- # GOOGLE_GENERATIVE_AI_API_KEY: ${{ secrets.GOOGLE_GENERATIVE_AI_API_KEY }}- # GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}- # ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}- # VERTEX_CREDENTIALS: ${{ secrets.VERTEX_CREDENTIALS }}- # USE_GO_MARKDOWN_PARSER: true- # SENTRY_ENVIRONMENT: dev- # IDMUX_URL: ${{ secrets.IDMUX_URL }}- # LOG_ENCRYPTION_KEY: ${{ secrets.LOG_ENCRYPTION_KEY }}- # TEST_SUITE_WEBSITE: ${{ secrets.TEST_SUITE_WEBSITE }}- # NUQ_DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgres- # NUQ_RABBITMQ_URL: amqp://localhost:5672- # HOST: 0.0.0.0- # steps:- # - uses: actions/checkout@v5-- # - name: Tailscale- # uses: tailscale/github-action@v4- # with:- # oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }}- # oauth-secret: ${{ secrets.TS_OAUTH_SECRET }}- # tags: tag:ci- # use-cache: 'true'-- # - uses: pnpm/action-setup@v4- # with:- # version: 11.4.0-- # - run: pnpm config set store-dir ~/.pnpm-store- # - run: mkdir -p ~/.pnpm-store-- # - uses: actions/setup-node@v6- # with:- # node-version: 22- # cache: pnpm- # cache-dependency-path: |- # apps/api/pnpm-lock.yaml-- # - run: pnpm fetch- # working-directory: apps/api-- # - name: Restore native lib- # id: napi_restore- # uses: actions/cache/restore@v4- # with:- # path: |- # apps/api/native/*.node- # apps/api/native/index.js- # apps/api/native/index.d.ts- # # note: this key is not ideal, need to find a better solution- # key: ${{ runner.os }}-napi-${{ hashFiles('apps/api/native/Cargo.toml', 'apps/api/native/package.json', 'apps/api/native/src/**') }}-- # - name: Build native lib- # if: steps.napi_restore.outputs.cache-hit != 'true'- # run: pnpm install- # working-directory: apps/api/native-- # - name: Cache native lib- # if: steps.napi_restore.outputs.cache-hit != 'true'- # uses: actions/cache/save@v4- # with:- # path: |- # apps/api/native/*.node- # apps/api/native/index.js- # apps/api/native/index.d.ts- # key: ${{ steps.napi_restore.outputs.cache-primary-key }}-- # - uses: actions/setup-go@v6- # with:- # go-version: 1.24- # cache-dependency-path: apps/api/sharedLibs/go-html-to-md/go.sum-- # - name: Restore Go lib- # id: golib_restore- # uses: actions/cache/restore@v4- # with:- # path: apps/api/sharedLibs/go-html-to-md/libhtml-to-markdown.so- # key: ${{ runner.os }}-golib-${{ hashFiles('apps/api/sharedLibs/go-html-to-md/go.sum', 'apps/api/sharedLibs/go-html-to-md/*.go') }}-- # - name: Build go-html-to-md- # if: steps.golib_restore.outputs.cache-hit != 'true'- # run: |- # cd apps/api/sharedLibs/go-html-to-md- # go mod tidy- # go build -o libhtml-to-markdown.so -buildmode=c-shared html-to-markdown.go-- # - name: Cache Go lib- # if: steps.golib_restore.outputs.cache-hit != 'true'- # uses: actions/cache/save@v4- # with:- # path: apps/api/sharedLibs/go-html-to-md/libhtml-to-markdown.so- # key: ${{ steps.golib_restore.outputs.cache-primary-key }}-- # - name: Install API dependencies- # run: pnpm install --frozen-lockfile --ignore-scripts- # working-directory: apps/api- # env:- # npm_config_ignore_scripts: "true"-- # - name: Run Docker Postgres- # run: |- # docker build -t firecrawl/nuq-postgres:latest ./apps/nuq-postgres- # docker run -d -p 5432:5432 -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=postgres --name postgres firecrawl/nuq-postgres:latest-- # - name: Run tests- # run: pnpm harness pnpm test:snips- # working-directory: apps/api- # env:- # npm_config_ignore_scripts: "true" # required currently to prevent re-building cached native lib-- # - name: Create logs directory- # if: always()- # run: |- # mkdir -p logs- # cp ./apps/api/firecrawl.log logs/firecrawl.log- # cd logs- # zip -r logs.zip ./*- # echo "${{ secrets.LOG_ENCRYPTION_KEY }}" | gpg --batch --yes --passphrase-fd 0 -c logs.zip- # rm logs.zip-- # - uses: actions/upload-artifact@v4- # if: always()- # with:- # name: Encrypted Logs- # path: logs/logs.zip.gpg- # retention-days: 5-nuq-fdb-core:name: NuQ FoundationDB core testsruns-on: blacksmith-4vcpu-ubuntu-2404