Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(ci): playwright cache + prod tests

delong33ceaf6fdb61a
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.yml
index 4944514fe..dbc828d2e 100644
--- a/.github/workflows/test-server.yml
+++ b/.github/workflows/test-server.yml
@@ -53,13 +53,16 @@ jobs:
name: Self-hosted environment tests
strategy:
matrix:
- engine: ["playwright", "fetch"]
- # engine: ["fetch"]
- proxy: ["proxy", "no-proxy"]
- search: ["searxng", "google"]
- ai: ["openai", "no-ai"]
+ # engine: ["playwright", "fetch"]
+ # proxy: ["proxy", "no-proxy"]
+ proxy: ["no-proxy"] # temp disabled proxy tests
+ # search: ["searxng", "google"]
+ # ai: ["openai", "no-ai"]
+ engine: ["playwright"]
+ search: ["google"]
+ ai: ["no-ai"]
fail-fast: false
- runs-on: big-runner # ubuntu-latest
+ runs-on: big-runner
needs: audit
services:
redis:
@@ -69,7 +72,7 @@ jobs:
env:
HOST: 0.0.0.0
TEST_SUITE_SELF_HOSTED: true
- TEST_SUITE_WEBSITE: ${{ matrix.proxy == 'proxy' && secrets.TEST_SUITE_WEBSITE || 'http://127.0.0.1:4321' }}
+ TEST_SUITE_WEBSITE: ${{ matrix.proxy == 'no-proxy' && 'http://127.0.0.1:4321' || '' }}
OPENAI_API_KEY: ${{ matrix.ai == 'openai' && secrets.OPENAI_API_KEY || '' }}
SEARXNG_ENDPOINT: ${{ matrix.search == 'searxng' && 'http://localhost:3434' || '' }}
PLAYWRIGHT_MICROSERVICE_URL: ${{ matrix.engine == 'playwright' && 'http://localhost:3003/scrape' || '' }}
@@ -77,7 +80,6 @@ jobs:
PROXY_USERNAME: ${{ matrix.proxy == 'proxy' && secrets.PROXY_USERNAME || '' }}
PROXY_PASSWORD: ${{ matrix.proxy == 'proxy' && secrets.PROXY_PASSWORD || '' }}
NUQ_DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgres
- NUQ_WORKER_COUNT: 2
USE_GO_MARKDOWN_PARSER: true
ALLOW_LOCAL_WEBHOOKS: true
steps:
@@ -156,14 +158,31 @@ jobs:
path: apps/api/sharedLibs/go-html-to-md/libhtml-to-markdown.so
key: ${{ steps.golib_restore.outputs.cache-primary-key }}
+ - name: Restore Playwright cache
+ if: matrix.engine == 'playwright'
+ id: pw_cache
+ uses: actions/cache/restore@v4
+ with:
+ path: ~/.cache/ms-playwright
+ key: ${{ runner.os }}-${{ runner.arch }}-pw-chromium-${{ hashFiles('apps/playwright-service-ts/pnpm-lock.yaml', 'apps/playwright-service-ts/package.json') }}
+ restore-keys: |
+ ${{ runner.os }}-${{ runner.arch }}-pw-chromium-
+
- name: Install Playwright dependencies
if: matrix.engine == 'playwright'
run: |
pnpm install
pnpm exec playwright install-deps
- pnpm exec playwright install
+ pnpm exec playwright install chromium
working-directory: ./apps/playwright-service-ts
-
+
+ - name: Cache Playwright
+ if: steps.pw_cache.outputs.cache-hit != 'true'
+ uses: actions/cache/save@v4
+ with:
+ path: ~/.cache/ms-playwright
+ key: ${{ steps.pw_cache.outputs.cache-primary-key }}
+
- name: Set up SearXNG
if: matrix.search == 'searxng'
run: |
@@ -242,111 +261,170 @@ jobs:
name: Logs (kubernetes, ${{ matrix.ai }}, ${{ matrix.search }}, ${{ matrix.engine }}, ${{ matrix.proxy }})
path: logs/logs.zip
- # prod-test:
- # name: Production environment tests
- # runs-on: big-runner
- # needs: audit
- # services:
- # redis:
- # image: redis
- # ports:
- # - 6379:6379
- # rabbitmq:
- # image: rabbitmq
- # ports:
- # - 5672:5672
- # env:
- # BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }}
- # OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
- # REDIS_URL: ${{ secrets.REDIS_URL }}
- # SUPABASE_ANON_TOKEN: ${{ secrets.SUPABASE_ANON_TOKEN }}
- # SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }}
- # SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
- # SUPABASE_REPLICA_URL: ${{ secrets.SUPABASE_REPLICA_URL }}
- # INDEX_SUPABASE_SERVICE_TOKEN: ${{ secrets.INDEX_SUPABASE_SERVICE_TOKEN }}
- # INDEX_SUPABASE_ANON_TOKEN: ${{ secrets.INDEX_SUPABASE_ANON_TOKEN }}
- # INDEX_SUPABASE_URL: ${{ secrets.INDEX_SUPABASE_URL }}
- # TEST_API_KEY: ${{ secrets.TEST_API_KEY }}
- # TEST_TEAM_ID: ${{ secrets.TEST_TEAM_ID }}
- # TEST_API_KEY_CONCURRENCY: ${{ secrets.TEST_API_KEY_CONCURRENCY }}
- # TEST_TEAM_ID_CONCURRENCY: ${{ secrets.TEST_TEAM_ID_CONCURRENCY }}
- # TEST_API_KEY_ZDR: ${{ secrets.TEST_API_KEY_ZDR }}
- # TEST_TEAM_ID_ZDR: ${{ secrets.TEST_TEAM_ID_ZDR }}
- # FIRE_ENGINE_BETA_URL: ${{ secrets.FIRE_ENGINE_BETA_URL }}
- # FIRE_ENGINE_STAGING_URL: ${{ secrets.FIRE_ENGINE_STAGING_URL }}
- # USE_DB_AUTHENTICATION: true
- # SERPER_API_KEY: ${{ secrets.SERPER_API_KEY }}
- # ENV: ${{ secrets.ENV }}
- # RUNPOD_MU_POD_ID: ${{ secrets.RUNPOD_MU_POD_ID }}
- # RUNPOD_MUV2_POD_ID: ${{ secrets.RUNPOD_MUV2_POD_ID }}
- # RUNPOD_MU_API_KEY: ${{ secrets.RUNPOD_MU_API_KEY }}
- # GCS_CREDENTIALS: ${{ secrets.GCS_CREDENTIALS }}
- # GCS_BUCKET_NAME: ${{ secrets.GCS_BUCKET_NAME }}
- # GCS_INDEX_BUCKET_NAME: ${{ secrets.GCS_INDEX_BUCKET_NAME }}
- # GCS_MEDIA_BUCKET_NAME: ${{ secrets.GCS_MEDIA_BUCKET_NAME }}
- # GOOGLE_GENERATIVE_AI_API_KEY: ${{ secrets.GOOGLE_GENERATIVE_AI_API_KEY }}
- # GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
- # ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
- # VERTEX_CREDENTIALS: ${{ secrets.VERTEX_CREDENTIALS }}
- # USE_GO_MARKDOWN_PARSER: true
- # SENTRY_ENVIRONMENT: dev
- # IDMUX_URL: ${{ secrets.IDMUX_URL }}
- # LOG_ENCRYPTION_KEY: ${{ secrets.LOG_ENCRYPTION_KEY }}
- # NUQ_DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgres
- # NUQ_RABBITMQ_URL: amqp://localhost:5672
- # HOST: 0.0.0.0
- # steps:
- # - uses: actions/checkout@v5
- # - name: Run Docker Postgres
- # run: |
- # docker build -t firecrawl/nuq-postgres:latest ./apps/nuq-postgres
- # docker run -d -p 5432:5432 -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=postgres --name postgres firecrawl/nuq-postgres:latest
- # - name: Tailscale
- # uses: tailscale/github-action@v4
- # with:
- # oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }}
- # oauth-secret: ${{ secrets.TS_OAUTH_SECRET }}
- # tags: tag:ci
- # use-cache: 'true'
- # - name: Install pnpm
- # uses: pnpm/action-setup@v4
- # with:
- # version: 10
- # - name: Set up Node.js
- # uses: actions/setup-node@v6
- # with:
- # node-version: '22'
- # cache: 'pnpm'
- # cache-dependency-path: './apps/api/pnpm-lock.yaml'
- # - name: Install dependencies
- # run: pnpm install
- # working-directory: ./apps/api
- # - name: Set up Go
- # uses: actions/setup-go@v6
- # with:
- # go-version: '1.24'
- # cache-dependency-path: ./apps/api/sharedLibs/go-html-to-md/go.sum
- # - name: Build go-html-to-md
- # run: |
- # go mod tidy
- # go build -o libhtml-to-markdown.so -buildmode=c-shared html-to-markdown.go
- # chmod +x libhtml-to-markdown.so
- # working-directory: ./apps/api/sharedLibs/go-html-to-md
- # - name: Build the application and run snippet tests
- # run: pnpm harness pnpm test:snips
- # working-directory: ./apps/api
- # - name: Create logs directory
- # if: always()
- # run: |
- # mkdir -p logs
- # cp ./apps/api/firecrawl.log logs/firecrawl.log
- # cd logs
- # zip -r logs.zip ./*
- # echo "${{ secrets.LOG_ENCRYPTION_KEY }}" | gpg --batch --yes --passphrase-fd 0 -c logs.zip
- # rm logs.zip
- # - uses: actions/upload-artifact@v4
- # if: always()
- # with:
- # name: Encrypted Logs
- # path: logs/logs.zip.gpg
- # retention-days: 5
+ prod-test:
+ name: Production environment tests
+ runs-on: big-runner
+ needs: audit
+ services:
+ redis:
+ image: redis
+ ports:
+ - 6379:6379
+ rabbitmq:
+ image: rabbitmq
+ ports:
+ - 5672:5672
+ env:
+ BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }}
+ OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+ REDIS_URL: ${{ secrets.REDIS_URL }}
+ SUPABASE_ANON_TOKEN: ${{ secrets.SUPABASE_ANON_TOKEN }}
+ SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }}
+ SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
+ SUPABASE_REPLICA_URL: ${{ secrets.SUPABASE_REPLICA_URL }}
+ INDEX_SUPABASE_SERVICE_TOKEN: ${{ secrets.INDEX_SUPABASE_SERVICE_TOKEN }}
+ INDEX_SUPABASE_ANON_TOKEN: ${{ secrets.INDEX_SUPABASE_ANON_TOKEN }}
+ INDEX_SUPABASE_URL: ${{ secrets.INDEX_SUPABASE_URL }}
+ TEST_API_KEY: ${{ secrets.TEST_API_KEY }}
+ TEST_TEAM_ID: ${{ secrets.TEST_TEAM_ID }}
+ TEST_API_KEY_CONCURRENCY: ${{ secrets.TEST_API_KEY_CONCURRENCY }}
+ TEST_TEAM_ID_CONCURRENCY: ${{ secrets.TEST_TEAM_ID_CONCURRENCY }}
+ TEST_API_KEY_ZDR: ${{ secrets.TEST_API_KEY_ZDR }}
+ TEST_TEAM_ID_ZDR: ${{ secrets.TEST_TEAM_ID_ZDR }}
+ FIRE_ENGINE_BETA_URL: ${{ secrets.FIRE_ENGINE_BETA_URL }}
+ FIRE_ENGINE_STAGING_URL: ${{ secrets.FIRE_ENGINE_STAGING_URL }}
+ USE_DB_AUTHENTICATION: true
+ SERPER_API_KEY: ${{ secrets.SERPER_API_KEY }}
+ ENV: ${{ secrets.ENV }}
+ RUNPOD_MU_POD_ID: ${{ secrets.RUNPOD_MU_POD_ID }}
+ RUNPOD_MUV2_POD_ID: ${{ secrets.RUNPOD_MUV2_POD_ID }}
+ RUNPOD_MU_API_KEY: ${{ secrets.RUNPOD_MU_API_KEY }}
+ GCS_CREDENTIALS: ${{ secrets.GCS_CREDENTIALS }}
+ GCS_BUCKET_NAME: ${{ secrets.GCS_BUCKET_NAME }}
+ GCS_INDEX_BUCKET_NAME: ${{ secrets.GCS_INDEX_BUCKET_NAME }}
+ GCS_MEDIA_BUCKET_NAME: ${{ secrets.GCS_MEDIA_BUCKET_NAME }}
+ GOOGLE_GENERATIVE_AI_API_KEY: ${{ secrets.GOOGLE_GENERATIVE_AI_API_KEY }}
+ GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
+ ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+ VERTEX_CREDENTIALS: ${{ secrets.VERTEX_CREDENTIALS }}
+ USE_GO_MARKDOWN_PARSER: true
+ SENTRY_ENVIRONMENT: dev
+ IDMUX_URL: ${{ secrets.IDMUX_URL }}
+ LOG_ENCRYPTION_KEY: ${{ secrets.LOG_ENCRYPTION_KEY }}
+ TEST_SUITE_WEBSITE: ${{ secrets.TEST_SUITE_WEBSITE }}
+ NUQ_DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgres
+ NUQ_RABBITMQ_URL: amqp://localhost:5672
+ HOST: 0.0.0.0
+ steps:
+ - uses: actions/checkout@v5
+
+ - name: Tailscale
+ uses: tailscale/github-action@v4
+ with:
+ oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }}
+ oauth-secret: ${{ secrets.TS_OAUTH_SECRET }}
+ tags: tag:ci
+ use-cache: 'true'
+
+ - uses: pnpm/action-setup@v4
+ with:
+ version: 10
+
+ - run: pnpm config set store-dir ~/.pnpm-store
+ - run: mkdir -p ~/.pnpm-store
+
+ - uses: actions/setup-node@v6
+ with:
+ node-version: 22
+ cache: pnpm
+ cache-dependency-path: |
+ apps/api/pnpm-lock.yaml
+
+ - run: pnpm fetch
+ working-directory: apps/api
+
+ - name: Restore native lib
+ id: napi_restore
+ uses: actions/cache/restore@v4
+ with:
+ path: |
+ apps/api/native/*.node
+ apps/api/native/index.js
+ apps/api/native/index.d.ts
+ # note: this key is not ideal, need to find a better solution
+ key: ${{ runner.os }}-napi-${{ hashFiles('apps/api/native/Cargo.toml', 'apps/api/native/package.json', 'apps/api/native/src/**') }}
+
+ - name: Build native lib
+ if: steps.napi_restore.outputs.cache-hit != 'true'
+ run: pnpm install
+ working-directory: apps/api/native
+
+ - name: Cache native lib
+ if: steps.napi_restore.outputs.cache-hit != 'true'
+ uses: actions/cache/save@v4
+ with:
+ path: |
+ apps/api/native/*.node
+ apps/api/native/index.js
+ apps/api/native/index.d.ts
+ key: ${{ steps.napi_restore.outputs.cache-primary-key }}
+
+ - uses: actions/setup-go@v6
+ with:
+ go-version: 1.24
+ cache-dependency-path: apps/api/sharedLibs/go-html-to-md/go.sum
+
+ - name: Restore Go lib
+ id: golib_restore
+ uses: actions/cache/restore@v4
+ with:
+ path: apps/api/sharedLibs/go-html-to-md/libhtml-to-markdown.so
+ key: ${{ runner.os }}-golib-${{ hashFiles('apps/api/sharedLibs/go-html-to-md/go.sum', 'apps/api/sharedLibs/go-html-to-md/*.go') }}
+
+ - name: Build go-html-to-md
+ if: steps.golib_restore.outputs.cache-hit != 'true'
+ run: |
+ cd apps/api/sharedLibs/go-html-to-md
+ go mod tidy
+ go build -o libhtml-to-markdown.so -buildmode=c-shared html-to-markdown.go
+
+ - name: Cache Go lib
+ if: steps.golib_restore.outputs.cache-hit != 'true'
+ uses: actions/cache/save@v4
+ with:
+ path: apps/api/sharedLibs/go-html-to-md/libhtml-to-markdown.so
+ key: ${{ steps.golib_restore.outputs.cache-primary-key }}
+
+ - name: Install API dependencies
+ run: pnpm install --frozen-lockfile --ignore-scripts
+ working-directory: apps/api
+ env:
+ npm_config_ignore_scripts: "true"
+
+ - name: Run Docker Postgres
+ run: |
+ docker build -t firecrawl/nuq-postgres:latest ./apps/nuq-postgres
+ docker run -d -p 5432:5432 -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=postgres --name postgres firecrawl/nuq-postgres:latest
+
+ - name: Run tests
+ run: pnpm harness pnpm test:snips
+ working-directory: apps/api
+ env:
+ npm_config_ignore_scripts: "true" # required currently to prevent re-building cached native lib
+
+ - name: Create logs directory
+ if: always()
+ run: |
+ mkdir -p logs
+ cp ./apps/api/firecrawl.log logs/firecrawl.log
+ cd logs
+ zip -r logs.zip ./*
+ echo "${{ secrets.LOG_ENCRYPTION_KEY }}" | gpg --batch --yes --passphrase-fd 0 -c logs.zip
+ rm logs.zip
+
+ - uses: actions/upload-artifact@v4
+ if: always()
+ with:
+ name: Encrypted Logs
+ path: logs/logs.zip.gpg
+ retention-days: 5