Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

ci: remove disabled prod test job and archived workflows (#5374)

David Tippette1dc7a43954b
diff --git a/.github/archive/js-sdk.yml b/.github/archive/js-sdk.yml
deleted file mode 100644
index e873d8ee6..000000000
--- a/.github/archive/js-sdk.yml
+++ /dev/null
@@ -1,53 +0,0 @@
-name: Run JavaScript SDK E2E Tests
-
-on: []
-
-env:
- ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
- BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }}
- FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
- HOST: ${{ secrets.HOST }}
- LLAMAPARSE_API_KEY: ${{ secrets.LLAMAPARSE_API_KEY }}
- NUM_WORKERS_PER_QUEUE: ${{ secrets.NUM_WORKERS_PER_QUEUE }}
- OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
- PLAYWRIGHT_MICROSERVICE_URL: ${{ secrets.PLAYWRIGHT_MICROSERVICE_URL }}
- PORT: ${{ secrets.PORT }}
- REDIS_URL: ${{ secrets.REDIS_URL }}
- SUPABASE_ANON_TOKEN: ${{ secrets.SUPABASE_ANON_TOKEN }}
- SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }}
- SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
- TEST_API_KEY: ${{ secrets.TEST_API_KEY }}
- HDX_NODE_BETA_MODE: 1
-
-jobs:
- build:
- runs-on: blacksmith-4vcpu-ubuntu-2404
- services:
- redis:
- image: redis
- ports:
- - 6379:6379
-
- steps:
- - uses: actions/checkout@v3
- - name: Set up Node.js
- uses: actions/setup-node@v3
- with:
- node-version: "20"
- - name: Install pnpm
- run: npm install -g pnpm
- - name: Install dependencies for API
- run: pnpm install
- working-directory: ./apps/api
- - name: Start the application
- run: npm start &
- working-directory: ./apps/api
- - name: Start workers
- run: npm run workers &
- working-directory: ./apps/api
- - name: Install dependencies for JavaScript SDK
- run: pnpm install
- working-directory: ./apps/js-sdk/firecrawl
- - name: Run E2E tests for JavaScript SDK
- run: npm run test
- working-directory: ./apps/js-sdk/firecrawl
\ No newline at end of file
diff --git a/.github/archive/publish-rust-sdk.yml b/.github/archive/publish-rust-sdk.yml
deleted file mode 100644
index a3ed6c1b8..000000000
--- a/.github/archive/publish-rust-sdk.yml
+++ /dev/null
@@ -1,42 +0,0 @@
-name: Publish Rust SDK
-
-on: []
-
-env:
- CRATES_IO_TOKEN: ${{ secrets.CRATES_IO_TOKEN }}
-
-jobs:
- build-and-publish:
- runs-on: blacksmith-4vcpu-ubuntu-2404
-
- steps:
- - name: Checkout repository
- uses: actions/checkout@v3
-
- - name: Set up Rust
- uses: actions-rs/toolchain@v1
- with:
- toolchain: stable
- default: true
- profile: minimal
-
- - name: Install dependencies
- run: cargo build --release
-
- - name: Run version check script
- id: version_check_script
- run: |
- VERSION_INCREMENTED=$(cargo search --limit 1 my_crate_name | grep my_crate_name)
- echo "VERSION_INCREMENTED=$VERSION_INCREMENTED" >> $GITHUB_ENV
-
- - name: Build the package
- if: ${{ env.VERSION_INCREMENTED == 'true' }}
- run: cargo package
- working-directory: ./apps/rust-sdk
-
- - name: Publish to crates.io
- if: ${{ env.VERSION_INCREMENTED == 'true' }}
- env:
- CARGO_REG_TOKEN: ${{ secrets.CRATES_IO_TOKEN }}
- run: cargo publish
- working-directory: ./apps/rust-sdk
\ No newline at end of file
diff --git a/.github/archive/python-sdk.yml b/.github/archive/python-sdk.yml
deleted file mode 100644
index d1f716de3..000000000
--- a/.github/archive/python-sdk.yml
+++ /dev/null
@@ -1,65 +0,0 @@
-name: Run Python SDK E2E Tests
-
-on: []
-
-env:
- ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
- BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }}
- FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
- HOST: ${{ secrets.HOST }}
- LLAMAPARSE_API_KEY: ${{ secrets.LLAMAPARSE_API_KEY }}
- NUM_WORKERS_PER_QUEUE: ${{ secrets.NUM_WORKERS_PER_QUEUE }}
- OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
- PLAYWRIGHT_MICROSERVICE_URL: ${{ secrets.PLAYWRIGHT_MICROSERVICE_URL }}
- PORT: ${{ secrets.PORT }}
- REDIS_URL: ${{ secrets.REDIS_URL }}
- SUPABASE_ANON_TOKEN: ${{ secrets.SUPABASE_ANON_TOKEN }}
- SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }}
- SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
- TEST_API_KEY: ${{ secrets.TEST_API_KEY }}
- HDX_NODE_BETA_MODE: 1
-
-jobs:
- build:
- runs-on: blacksmith-4vcpu-ubuntu-2404
- strategy:
- matrix:
- python-version: ["3.10"]
- services:
- redis:
- image: redis
- ports:
- - 6379:6379
-
- steps:
- - uses: actions/checkout@v3
- - name: Set up Node.js
- uses: actions/setup-node@v3
- with:
- node-version: "20"
- - name: Install pnpm
- run: npm install -g pnpm
- - name: Install dependencies for API
- run: pnpm install
- working-directory: ./apps/api
- - name: Start the application
- run: npm start &
- working-directory: ./apps/api
- id: start_app
- - name: Start workers
- run: npm run workers &
- working-directory: ./apps/api
- id: start_workers
- - name: Set up Python ${{ matrix.python-version }}
- uses: actions/setup-python@v4
- with:
- python-version: ${{ matrix.python-version }}
- - name: Install Python dependencies
- run: |
- python -m pip install --upgrade pip
- pip install -r requirements.txt
- working-directory: ./apps/python-sdk
- - name: Run E2E tests for Python SDK
- run: |
- python -m pytest firecrawl/__tests__/e2e/v2
- working-directory: ./apps/python-sdk
diff --git a/.github/archive/rust-sdk.yml b/.github/archive/rust-sdk.yml
deleted file mode 100644
index a4f9b2d9b..000000000
--- a/.github/archive/rust-sdk.yml
+++ /dev/null
@@ -1,56 +0,0 @@
-name: Run Rust SDK E2E Tests
-
-on: []
-
-env:
- ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
- BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }}
- FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
- HOST: ${{ secrets.HOST }}
- LLAMAPARSE_API_KEY: ${{ secrets.LLAMAPARSE_API_KEY }}
- NUM_WORKERS_PER_QUEUE: ${{ secrets.NUM_WORKERS_PER_QUEUE }}
- OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
- PLAYWRIGHT_MICROSERVICE_URL: ${{ secrets.PLAYWRIGHT_MICROSERVICE_URL }}
- PORT: ${{ secrets.PORT }}
- REDIS_URL: ${{ secrets.REDIS_URL }}
- SUPABASE_ANON_TOKEN: ${{ secrets.SUPABASE_ANON_TOKEN }}
- SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }}
- SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
- TEST_API_KEY: ${{ secrets.TEST_API_KEY }}
- HDX_NODE_BETA_MODE: 1
-
-
-jobs:
- build:
- runs-on: blacksmith-4vcpu-ubuntu-2404
- services:
- redis:
- image: redis
- ports:
- - 6379:6379
-
- steps:
- - name: Checkout repository
- uses: actions/checkout@v3
- - name: Install pnpm
- run: npm install -g pnpm
- - name: Install dependencies for API
- run: pnpm install
- working-directory: ./apps/api
- - name: Start the application
- run: npm start &
- working-directory: ./apps/api
- id: start_app
- - name: Start workers
- run: npm run workers &
- working-directory: ./apps/api
- id: start_workers
- - name: Set up Rust
- uses: actions/setup-rust@v1
- with:
- rust-version: stable
- - name: Try the lib build
- working-directory: ./apps/rust-sdk
- run: cargo build
- - name: Run E2E tests for Rust SDK
- run: cargo test --test e2e_with_auth
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.yml
index c23244dd7..6f3c3cb97 100644
--- a/.github/workflows/test-server.yml
+++ b/.github/workflows/test-server.yml
@@ -535,173 +535,6 @@ jobs:
name: Logs (kubernetes, ${{ matrix.ai }}, ${{ matrix.search }}, ${{ matrix.engine }}, ${{ matrix.proxy }}, ${{ matrix.nuq }})
path: logs/logs.zip
- # temp disabled
- # prod-test:
- # name: Production environment tests
- # runs-on: big-runner
- # services:
- # redis:
- # image: redis
- # ports:
- # - 6379:6379
- # rabbitmq:
- # image: rabbitmq
- # ports:
- # - 5672:5672
- # env:
- # BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }}
- # OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
- # REDIS_URL: ${{ secrets.REDIS_URL }}
- # SUPABASE_ANON_TOKEN: ${{ secrets.SUPABASE_ANON_TOKEN }}
- # SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }}
- # SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
- # SUPABASE_REPLICA_URL: ${{ secrets.SUPABASE_REPLICA_URL }}
- # INDEX_SUPABASE_SERVICE_TOKEN: ${{ secrets.INDEX_SUPABASE_SERVICE_TOKEN }}
- # INDEX_SUPABASE_ANON_TOKEN: ${{ secrets.INDEX_SUPABASE_ANON_TOKEN }}
- # INDEX_SUPABASE_URL: ${{ secrets.INDEX_SUPABASE_URL }}
- # TEST_API_KEY: ${{ secrets.TEST_API_KEY }}
- # TEST_TEAM_ID: ${{ secrets.TEST_TEAM_ID }}
- # TEST_API_KEY_CONCURRENCY: ${{ secrets.TEST_API_KEY_CONCURRENCY }}
- # TEST_TEAM_ID_CONCURRENCY: ${{ secrets.TEST_TEAM_ID_CONCURRENCY }}
- # TEST_API_KEY_ZDR: ${{ secrets.TEST_API_KEY_ZDR }}
- # TEST_TEAM_ID_ZDR: ${{ secrets.TEST_TEAM_ID_ZDR }}
- # FIRE_ENGINE_BETA_URL: ${{ secrets.FIRE_ENGINE_BETA_URL }}
- # FIRE_ENGINE_STAGING_URL: ${{ secrets.FIRE_ENGINE_STAGING_URL }}
- # USE_DB_AUTHENTICATION: true
- # ENV: ${{ secrets.ENV }}
- # RUNPOD_MU_POD_ID: ${{ secrets.RUNPOD_MU_POD_ID }}
- # RUNPOD_MUV2_POD_ID: ${{ secrets.RUNPOD_MUV2_POD_ID }}
- # RUNPOD_MU_API_KEY: ${{ secrets.RUNPOD_MU_API_KEY }}
- # GCS_CREDENTIALS: ${{ secrets.GCS_CREDENTIALS }}
- # GCS_BUCKET_NAME: ${{ secrets.GCS_BUCKET_NAME }}
- # GCS_INDEX_BUCKET_NAME: ${{ secrets.GCS_INDEX_BUCKET_NAME }}
- # GCS_MEDIA_BUCKET_NAME: ${{ secrets.GCS_MEDIA_BUCKET_NAME }}
- # GOOGLE_GENERATIVE_AI_API_KEY: ${{ secrets.GOOGLE_GENERATIVE_AI_API_KEY }}
- # GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
- # ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
- # VERTEX_CREDENTIALS: ${{ secrets.VERTEX_CREDENTIALS }}
- # USE_GO_MARKDOWN_PARSER: true
- # SENTRY_ENVIRONMENT: dev
- # IDMUX_URL: ${{ secrets.IDMUX_URL }}
- # LOG_ENCRYPTION_KEY: ${{ secrets.LOG_ENCRYPTION_KEY }}
- # TEST_SUITE_WEBSITE: ${{ secrets.TEST_SUITE_WEBSITE }}
- # NUQ_DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgres
- # NUQ_RABBITMQ_URL: amqp://localhost:5672
- # HOST: 0.0.0.0
- # steps:
- # - uses: actions/checkout@v5
-
- # - name: Tailscale
- # uses: tailscale/github-action@v4
- # with:
- # oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }}
- # oauth-secret: ${{ secrets.TS_OAUTH_SECRET }}
- # tags: tag:ci
- # use-cache: 'true'
-
- # - uses: pnpm/action-setup@v4
- # with:
- # version: 11.4.0
-
- # - run: pnpm config set store-dir ~/.pnpm-store
- # - run: mkdir -p ~/.pnpm-store
-
- # - uses: actions/setup-node@v6
- # with:
- # node-version: 22
- # cache: pnpm
- # cache-dependency-path: |
- # apps/api/pnpm-lock.yaml
-
- # - run: pnpm fetch
- # working-directory: apps/api
-
- # - name: Restore native lib
- # id: napi_restore
- # uses: actions/cache/restore@v4
- # with:
- # path: |
- # apps/api/native/*.node
- # apps/api/native/index.js
- # apps/api/native/index.d.ts
- # # note: this key is not ideal, need to find a better solution
- # key: ${{ runner.os }}-napi-${{ hashFiles('apps/api/native/Cargo.toml', 'apps/api/native/package.json', 'apps/api/native/src/**') }}
-
- # - name: Build native lib
- # if: steps.napi_restore.outputs.cache-hit != 'true'
- # run: pnpm install
- # working-directory: apps/api/native
-
- # - name: Cache native lib
- # if: steps.napi_restore.outputs.cache-hit != 'true'
- # uses: actions/cache/save@v4
- # with:
- # path: |
- # apps/api/native/*.node
- # apps/api/native/index.js
- # apps/api/native/index.d.ts
- # key: ${{ steps.napi_restore.outputs.cache-primary-key }}
-
- # - uses: actions/setup-go@v6
- # with:
- # go-version: 1.24
- # cache-dependency-path: apps/api/sharedLibs/go-html-to-md/go.sum
-
- # - name: Restore Go lib
- # id: golib_restore
- # uses: actions/cache/restore@v4
- # with:
- # path: apps/api/sharedLibs/go-html-to-md/libhtml-to-markdown.so
- # key: ${{ runner.os }}-golib-${{ hashFiles('apps/api/sharedLibs/go-html-to-md/go.sum', 'apps/api/sharedLibs/go-html-to-md/*.go') }}
-
- # - name: Build go-html-to-md
- # if: steps.golib_restore.outputs.cache-hit != 'true'
- # run: |
- # cd apps/api/sharedLibs/go-html-to-md
- # go mod tidy
- # go build -o libhtml-to-markdown.so -buildmode=c-shared html-to-markdown.go
-
- # - name: Cache Go lib
- # if: steps.golib_restore.outputs.cache-hit != 'true'
- # uses: actions/cache/save@v4
- # with:
- # path: apps/api/sharedLibs/go-html-to-md/libhtml-to-markdown.so
- # key: ${{ steps.golib_restore.outputs.cache-primary-key }}
-
- # - name: Install API dependencies
- # run: pnpm install --frozen-lockfile --ignore-scripts
- # working-directory: apps/api
- # env:
- # npm_config_ignore_scripts: "true"
-
- # - name: Run Docker Postgres
- # run: |
- # docker build -t firecrawl/nuq-postgres:latest ./apps/nuq-postgres
- # docker run -d -p 5432:5432 -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=postgres --name postgres firecrawl/nuq-postgres:latest
-
- # - name: Run tests
- # run: pnpm harness pnpm test:snips
- # working-directory: apps/api
- # env:
- # npm_config_ignore_scripts: "true" # required currently to prevent re-building cached native lib
-
- # - name: Create logs directory
- # if: always()
- # run: |
- # mkdir -p logs
- # cp ./apps/api/firecrawl.log logs/firecrawl.log
- # cd logs
- # zip -r logs.zip ./*
- # echo "${{ secrets.LOG_ENCRYPTION_KEY }}" | gpg --batch --yes --passphrase-fd 0 -c logs.zip
- # rm logs.zip
-
- # - uses: actions/upload-artifact@v4
- # if: always()
- # with:
- # name: Encrypted Logs
- # path: logs/logs.zip.gpg
- # retention-days: 5
-
nuq-fdb-core:
name: NuQ FoundationDB core tests
runs-on: blacksmith-4vcpu-ubuntu-2404