Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(ci): playwright cache + prod tests
delong33ceaf6fdb61a
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.ymlindex 4944514fe..dbc828d2e 100644--- a/.github/workflows/test-server.yml+++ b/.github/workflows/test-server.yml@@ -53,13 +53,16 @@ jobs:name: Self-hosted environment testsstrategy:matrix:- engine: ["playwright", "fetch"]- # engine: ["fetch"]- proxy: ["proxy", "no-proxy"]- search: ["searxng", "google"]- ai: ["openai", "no-ai"]+ # engine: ["playwright", "fetch"]+ # proxy: ["proxy", "no-proxy"]+ proxy: ["no-proxy"] # temp disabled proxy tests+ # search: ["searxng", "google"]+ # ai: ["openai", "no-ai"]+ engine: ["playwright"]+ search: ["google"]+ ai: ["no-ai"]fail-fast: false- runs-on: big-runner # ubuntu-latest+ runs-on: big-runnerneeds: auditservices:redis:@@ -69,7 +72,7 @@ jobs:env:HOST: 0.0.0.0TEST_SUITE_SELF_HOSTED: true- TEST_SUITE_WEBSITE: ${{ matrix.proxy == 'proxy' && secrets.TEST_SUITE_WEBSITE || 'http://127.0.0.1:4321' }}+ TEST_SUITE_WEBSITE: ${{ matrix.proxy == 'no-proxy' && 'http://127.0.0.1:4321' || '' }}OPENAI_API_KEY: ${{ matrix.ai == 'openai' && secrets.OPENAI_API_KEY || '' }}SEARXNG_ENDPOINT: ${{ matrix.search == 'searxng' && 'http://localhost:3434' || '' }}PLAYWRIGHT_MICROSERVICE_URL: ${{ matrix.engine == 'playwright' && 'http://localhost:3003/scrape' || '' }}@@ -77,7 +80,6 @@ jobs:PROXY_USERNAME: ${{ matrix.proxy == 'proxy' && secrets.PROXY_USERNAME || '' }}PROXY_PASSWORD: ${{ matrix.proxy == 'proxy' && secrets.PROXY_PASSWORD || '' }}NUQ_DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgres- NUQ_WORKER_COUNT: 2USE_GO_MARKDOWN_PARSER: trueALLOW_LOCAL_WEBHOOKS: truesteps:@@ -156,14 +158,31 @@ jobs:path: apps/api/sharedLibs/go-html-to-md/libhtml-to-markdown.sokey: ${{ steps.golib_restore.outputs.cache-primary-key }}+ - name: Restore Playwright cache+ if: matrix.engine == 'playwright'+ id: pw_cache+ uses: actions/cache/restore@v4+ with:+ path: ~/.cache/ms-playwright+ key: ${{ runner.os }}-${{ runner.arch }}-pw-chromium-${{ hashFiles('apps/playwright-service-ts/pnpm-lock.yaml', 'apps/playwright-service-ts/package.json') }}+ restore-keys: |+ ${{ runner.os }}-${{ runner.arch }}-pw-chromium-+- name: Install Playwright dependenciesif: matrix.engine == 'playwright'run: |pnpm installpnpm exec playwright install-deps- pnpm exec playwright install+ pnpm exec playwright install chromiumworking-directory: ./apps/playwright-service-ts-++ - name: Cache Playwright+ if: steps.pw_cache.outputs.cache-hit != 'true'+ uses: actions/cache/save@v4+ with:+ path: ~/.cache/ms-playwright+ key: ${{ steps.pw_cache.outputs.cache-primary-key }}+- name: Set up SearXNGif: matrix.search == 'searxng'run: |@@ -242,111 +261,170 @@ jobs:name: Logs (kubernetes, ${{ matrix.ai }}, ${{ matrix.search }}, ${{ matrix.engine }}, ${{ matrix.proxy }})path: logs/logs.zip- # prod-test:- # name: Production environment tests- # runs-on: big-runner- # needs: audit- # services:- # redis:- # image: redis- # ports:- # - 6379:6379- # rabbitmq:- # image: rabbitmq- # ports:- # - 5672:5672- # env:- # BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }}- # OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}- # REDIS_URL: ${{ secrets.REDIS_URL }}- # SUPABASE_ANON_TOKEN: ${{ secrets.SUPABASE_ANON_TOKEN }}- # SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }}- # SUPABASE_URL: ${{ secrets.SUPABASE_URL }}- # SUPABASE_REPLICA_URL: ${{ secrets.SUPABASE_REPLICA_URL }}- # INDEX_SUPABASE_SERVICE_TOKEN: ${{ secrets.INDEX_SUPABASE_SERVICE_TOKEN }}- # INDEX_SUPABASE_ANON_TOKEN: ${{ secrets.INDEX_SUPABASE_ANON_TOKEN }}- # INDEX_SUPABASE_URL: ${{ secrets.INDEX_SUPABASE_URL }}- # TEST_API_KEY: ${{ secrets.TEST_API_KEY }}- # TEST_TEAM_ID: ${{ secrets.TEST_TEAM_ID }}- # TEST_API_KEY_CONCURRENCY: ${{ secrets.TEST_API_KEY_CONCURRENCY }}- # TEST_TEAM_ID_CONCURRENCY: ${{ secrets.TEST_TEAM_ID_CONCURRENCY }}- # TEST_API_KEY_ZDR: ${{ secrets.TEST_API_KEY_ZDR }}- # TEST_TEAM_ID_ZDR: ${{ secrets.TEST_TEAM_ID_ZDR }}- # FIRE_ENGINE_BETA_URL: ${{ secrets.FIRE_ENGINE_BETA_URL }}- # FIRE_ENGINE_STAGING_URL: ${{ secrets.FIRE_ENGINE_STAGING_URL }}- # USE_DB_AUTHENTICATION: true- # SERPER_API_KEY: ${{ secrets.SERPER_API_KEY }}- # ENV: ${{ secrets.ENV }}- # RUNPOD_MU_POD_ID: ${{ secrets.RUNPOD_MU_POD_ID }}- # RUNPOD_MUV2_POD_ID: ${{ secrets.RUNPOD_MUV2_POD_ID }}- # RUNPOD_MU_API_KEY: ${{ secrets.RUNPOD_MU_API_KEY }}- # GCS_CREDENTIALS: ${{ secrets.GCS_CREDENTIALS }}- # GCS_BUCKET_NAME: ${{ secrets.GCS_BUCKET_NAME }}- # GCS_INDEX_BUCKET_NAME: ${{ secrets.GCS_INDEX_BUCKET_NAME }}- # GCS_MEDIA_BUCKET_NAME: ${{ secrets.GCS_MEDIA_BUCKET_NAME }}- # GOOGLE_GENERATIVE_AI_API_KEY: ${{ secrets.GOOGLE_GENERATIVE_AI_API_KEY }}- # GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}- # ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}- # VERTEX_CREDENTIALS: ${{ secrets.VERTEX_CREDENTIALS }}- # USE_GO_MARKDOWN_PARSER: true- # SENTRY_ENVIRONMENT: dev- # IDMUX_URL: ${{ secrets.IDMUX_URL }}- # LOG_ENCRYPTION_KEY: ${{ secrets.LOG_ENCRYPTION_KEY }}- # NUQ_DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgres- # NUQ_RABBITMQ_URL: amqp://localhost:5672- # HOST: 0.0.0.0- # steps:- # - uses: actions/checkout@v5- # - name: Run Docker Postgres- # run: |- # docker build -t firecrawl/nuq-postgres:latest ./apps/nuq-postgres- # docker run -d -p 5432:5432 -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=postgres --name postgres firecrawl/nuq-postgres:latest- # - name: Tailscale- # uses: tailscale/github-action@v4- # with:- # oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }}- # oauth-secret: ${{ secrets.TS_OAUTH_SECRET }}- # tags: tag:ci- # use-cache: 'true'- # - name: Install pnpm- # uses: pnpm/action-setup@v4- # with:- # version: 10- # - name: Set up Node.js- # uses: actions/setup-node@v6- # with:- # node-version: '22'- # cache: 'pnpm'- # cache-dependency-path: './apps/api/pnpm-lock.yaml'- # - name: Install dependencies- # run: pnpm install- # working-directory: ./apps/api- # - name: Set up Go- # uses: actions/setup-go@v6- # with:- # go-version: '1.24'- # cache-dependency-path: ./apps/api/sharedLibs/go-html-to-md/go.sum- # - name: Build go-html-to-md- # run: |- # go mod tidy- # go build -o libhtml-to-markdown.so -buildmode=c-shared html-to-markdown.go- # chmod +x libhtml-to-markdown.so- # working-directory: ./apps/api/sharedLibs/go-html-to-md- # - name: Build the application and run snippet tests- # run: pnpm harness pnpm test:snips- # working-directory: ./apps/api- # - name: Create logs directory- # if: always()- # run: |- # mkdir -p logs- # cp ./apps/api/firecrawl.log logs/firecrawl.log- # cd logs- # zip -r logs.zip ./*- # echo "${{ secrets.LOG_ENCRYPTION_KEY }}" | gpg --batch --yes --passphrase-fd 0 -c logs.zip- # rm logs.zip- # - uses: actions/upload-artifact@v4- # if: always()- # with:- # name: Encrypted Logs- # path: logs/logs.zip.gpg- # retention-days: 5+ prod-test:+ name: Production environment tests+ runs-on: big-runner+ needs: audit+ services:+ redis:+ image: redis+ ports:+ - 6379:6379+ rabbitmq:+ image: rabbitmq+ ports:+ - 5672:5672+ env:+ BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }}+ OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}+ REDIS_URL: ${{ secrets.REDIS_URL }}+ SUPABASE_ANON_TOKEN: ${{ secrets.SUPABASE_ANON_TOKEN }}+ SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }}+ SUPABASE_URL: ${{ secrets.SUPABASE_URL }}+ SUPABASE_REPLICA_URL: ${{ secrets.SUPABASE_REPLICA_URL }}+ INDEX_SUPABASE_SERVICE_TOKEN: ${{ secrets.INDEX_SUPABASE_SERVICE_TOKEN }}+ INDEX_SUPABASE_ANON_TOKEN: ${{ secrets.INDEX_SUPABASE_ANON_TOKEN }}+ INDEX_SUPABASE_URL: ${{ secrets.INDEX_SUPABASE_URL }}+ TEST_API_KEY: ${{ secrets.TEST_API_KEY }}+ TEST_TEAM_ID: ${{ secrets.TEST_TEAM_ID }}+ TEST_API_KEY_CONCURRENCY: ${{ secrets.TEST_API_KEY_CONCURRENCY }}+ TEST_TEAM_ID_CONCURRENCY: ${{ secrets.TEST_TEAM_ID_CONCURRENCY }}+ TEST_API_KEY_ZDR: ${{ secrets.TEST_API_KEY_ZDR }}+ TEST_TEAM_ID_ZDR: ${{ secrets.TEST_TEAM_ID_ZDR }}+ FIRE_ENGINE_BETA_URL: ${{ secrets.FIRE_ENGINE_BETA_URL }}+ FIRE_ENGINE_STAGING_URL: ${{ secrets.FIRE_ENGINE_STAGING_URL }}+ USE_DB_AUTHENTICATION: true+ SERPER_API_KEY: ${{ secrets.SERPER_API_KEY }}+ ENV: ${{ secrets.ENV }}+ RUNPOD_MU_POD_ID: ${{ secrets.RUNPOD_MU_POD_ID }}+ RUNPOD_MUV2_POD_ID: ${{ secrets.RUNPOD_MUV2_POD_ID }}+ RUNPOD_MU_API_KEY: ${{ secrets.RUNPOD_MU_API_KEY }}+ GCS_CREDENTIALS: ${{ secrets.GCS_CREDENTIALS }}+ GCS_BUCKET_NAME: ${{ secrets.GCS_BUCKET_NAME }}+ GCS_INDEX_BUCKET_NAME: ${{ secrets.GCS_INDEX_BUCKET_NAME }}+ GCS_MEDIA_BUCKET_NAME: ${{ secrets.GCS_MEDIA_BUCKET_NAME }}+ GOOGLE_GENERATIVE_AI_API_KEY: ${{ secrets.GOOGLE_GENERATIVE_AI_API_KEY }}+ GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}+ ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}+ VERTEX_CREDENTIALS: ${{ secrets.VERTEX_CREDENTIALS }}+ USE_GO_MARKDOWN_PARSER: true+ SENTRY_ENVIRONMENT: dev+ IDMUX_URL: ${{ secrets.IDMUX_URL }}+ LOG_ENCRYPTION_KEY: ${{ secrets.LOG_ENCRYPTION_KEY }}+ TEST_SUITE_WEBSITE: ${{ secrets.TEST_SUITE_WEBSITE }}+ NUQ_DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgres+ NUQ_RABBITMQ_URL: amqp://localhost:5672+ HOST: 0.0.0.0+ steps:+ - uses: actions/checkout@v5++ - name: Tailscale+ uses: tailscale/github-action@v4+ with:+ oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }}+ oauth-secret: ${{ secrets.TS_OAUTH_SECRET }}+ tags: tag:ci+ use-cache: 'true'++ - uses: pnpm/action-setup@v4+ with:+ version: 10++ - run: pnpm config set store-dir ~/.pnpm-store+ - run: mkdir -p ~/.pnpm-store++ - uses: actions/setup-node@v6+ with:+ node-version: 22+ cache: pnpm+ cache-dependency-path: |+ apps/api/pnpm-lock.yaml++ - run: pnpm fetch+ working-directory: apps/api++ - name: Restore native lib+ id: napi_restore+ uses: actions/cache/restore@v4+ with:+ path: |+ apps/api/native/*.node+ apps/api/native/index.js+ apps/api/native/index.d.ts+ # note: this key is not ideal, need to find a better solution+ key: ${{ runner.os }}-napi-${{ hashFiles('apps/api/native/Cargo.toml', 'apps/api/native/package.json', 'apps/api/native/src/**') }}++ - name: Build native lib+ if: steps.napi_restore.outputs.cache-hit != 'true'+ run: pnpm install+ working-directory: apps/api/native++ - name: Cache native lib+ if: steps.napi_restore.outputs.cache-hit != 'true'+ uses: actions/cache/save@v4+ with:+ path: |+ apps/api/native/*.node+ apps/api/native/index.js+ apps/api/native/index.d.ts+ key: ${{ steps.napi_restore.outputs.cache-primary-key }}++ - uses: actions/setup-go@v6+ with:+ go-version: 1.24+ cache-dependency-path: apps/api/sharedLibs/go-html-to-md/go.sum++ - name: Restore Go lib+ id: golib_restore+ uses: actions/cache/restore@v4+ with:+ path: apps/api/sharedLibs/go-html-to-md/libhtml-to-markdown.so+ key: ${{ runner.os }}-golib-${{ hashFiles('apps/api/sharedLibs/go-html-to-md/go.sum', 'apps/api/sharedLibs/go-html-to-md/*.go') }}++ - name: Build go-html-to-md+ if: steps.golib_restore.outputs.cache-hit != 'true'+ run: |+ cd apps/api/sharedLibs/go-html-to-md+ go mod tidy+ go build -o libhtml-to-markdown.so -buildmode=c-shared html-to-markdown.go++ - name: Cache Go lib+ if: steps.golib_restore.outputs.cache-hit != 'true'+ uses: actions/cache/save@v4+ with:+ path: apps/api/sharedLibs/go-html-to-md/libhtml-to-markdown.so+ key: ${{ steps.golib_restore.outputs.cache-primary-key }}++ - name: Install API dependencies+ run: pnpm install --frozen-lockfile --ignore-scripts+ working-directory: apps/api+ env:+ npm_config_ignore_scripts: "true"++ - name: Run Docker Postgres+ run: |+ docker build -t firecrawl/nuq-postgres:latest ./apps/nuq-postgres+ docker run -d -p 5432:5432 -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=postgres --name postgres firecrawl/nuq-postgres:latest++ - name: Run tests+ run: pnpm harness pnpm test:snips+ working-directory: apps/api+ env:+ npm_config_ignore_scripts: "true" # required currently to prevent re-building cached native lib++ - name: Create logs directory+ if: always()+ run: |+ mkdir -p logs+ cp ./apps/api/firecrawl.log logs/firecrawl.log+ cd logs+ zip -r logs.zip ./*+ echo "${{ secrets.LOG_ENCRYPTION_KEY }}" | gpg --batch --yes --passphrase-fd 0 -c logs.zip+ rm logs.zip++ - uses: actions/upload-artifact@v4+ if: always()+ with:+ name: Encrypted Logs+ path: logs/logs.zip.gpg+ retention-days: 5