Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(ci): revamp (#2124)
Gergő Móriczfc5f38f23383
diff --git a/.github/k8s/playwright.yml b/.github/k8s/playwright.ymldeleted file mode 100644index cd33ec6b5..000000000--- a/.github/k8s/playwright.yml+++ /dev/null@@ -1,62 +0,0 @@-apiVersion: apps/v1-kind: Deployment-metadata:- name: firecrawl-playwright-service-spec:- replicas: 1- selector:- matchLabels:- app: firecrawl-playwright-service- template:- metadata:- labels:- app: firecrawl-playwright-service- spec:- containers:- - name: playwright-service- image: firecrawl/playwright-service:latest- imagePullPolicy: Never- envFrom:- - configMapRef:- name: firecrawl-config- - configMapRef:- name: firecrawl-config-dynamic- - secretRef:- name: firecrawl-secret-dynamic- env:- - name: PORT- value: "3003"- ports:- - containerPort: 3003- resources:- requests:- memory: "1G"- cpu: "500m"- limits:- memory: "3G"- cpu: "1000m"- livenessProbe:- httpGet:- path: /health- port: 3003- initialDelaySeconds: 30- periodSeconds: 10- readinessProbe:- httpGet:- path: /health- port: 3003- initialDelaySeconds: 30- periodSeconds: 10-----apiVersion: v1-kind: Service-metadata:- name: firecrawl-playwright-service-spec:- selector:- app: firecrawl-playwright-service- ports:- - name: http- protocol: TCP- port: 3003- targetPort: 3003\ No newline at end of filediff --git a/.github/k8s/searxng.yml b/.github/k8s/searxng.ymldeleted file mode 100644index 03c9f88e6..000000000--- a/.github/k8s/searxng.yml+++ /dev/null@@ -1,59 +0,0 @@-apiVersion: v1-kind: ConfigMap-metadata:- name: searxng-settings-data:- settings.yml: |- use_default_settings: true- search:- formats: [html, json, csv]- server:- secret_key: 'fcsecret'-----apiVersion: apps/v1-kind: Deployment-metadata:- name: firecrawl-searxng-spec:- replicas: 1- selector:- matchLabels:- app: firecrawl-searxng- template:- metadata:- labels:- app: firecrawl-searxng- spec:- containers:- - name: searxng- image: searxng/searxng:latest- imagePullPolicy: IfNotPresent- ports:- - containerPort: 8080- volumeMounts:- - name: settings- mountPath: /etc/searxng- resources:- requests:- memory: "256Mi"- cpu: "100m"- limits:- memory: "512Mi"- cpu: "200m"- volumes:- - name: settings- configMap:- name: searxng-settings-----apiVersion: v1-kind: Service-metadata:- name: firecrawl-searxng-spec:- selector:- app: firecrawl-searxng- ports:- - name: http- protocol: TCP- port: 8080- targetPort: 8080diff --git a/.github/k8s/server-nsh.yml b/.github/k8s/server-nsh.ymldeleted file mode 100644index 7778ed32a..000000000--- a/.github/k8s/server-nsh.yml+++ /dev/null@@ -1,114 +0,0 @@-apiVersion: apps/v1-kind: Deployment-metadata:- name: tailscale-proxy-spec:- replicas: 1- selector:- matchLabels:- app: tailscale-proxy- template:- metadata:- labels:- app: tailscale-proxy- spec:- containers:- - name: proxy- image: firecrawl/tailscale-proxy:latest- imagePullPolicy: Never- ports:- - containerPort: 8080- securityContext:- capabilities:- add: ["NET_ADMIN", "NET_RAW"]- allowPrivilegeEscalation: false- volumeMounts:- - name: dev-net-tun- mountPath: /dev/net/tun- readinessProbe:- tcpSocket:- port: 8080- initialDelaySeconds: 3- periodSeconds: 5- timeoutSeconds: 3- successThreshold: 1- failureThreshold: 3- env:- - name: TS_AUTHKEY- valueFrom:- secretKeyRef:- name: tailscale- key: TS_AUTHKEY- - name: TS_ADVERTISE_TAGS- valueFrom:- secretKeyRef:- name: tailscale- key: TS_ADVERTISE_TAGS- - name: FIRE_ENGINE_BETA_URL- valueFrom:- secretKeyRef:- name: tailscale- key: FIRE_ENGINE_BETA_URL- resources:- requests:- memory: "128Mi"- cpu: "100m"- limits:- memory: "256Mi"- cpu: "200m"- volumes:- - name: dev-net-tun- hostPath:- path: /dev/net/tun- type: CharDevice-----apiVersion: v1-kind: Service-metadata:- name: tailscale-proxy-spec:- selector:- app: tailscale-proxy- ports:- - name: http- protocol: TCP- port: 8080- targetPort: 8080-----apiVersion: apps/v1-kind: Deployment-metadata:- name: firecrawl-index-worker-spec:- replicas: 1- selector:- matchLabels:- app: firecrawl-index-worker- template:- metadata:- labels:- app: firecrawl-index-worker- spec:- containers:- - name: index-worker- image: firecrawl/firecrawl:latest- imagePullPolicy: Never- command: [ "node" ]- args: [ "--max-old-space-size=2048", "dist/src/services/indexing/index-worker.js" ]- envFrom:- - configMapRef:- name: firecrawl-config- - configMapRef:- name: firecrawl-config-dynamic- - secretRef:- name: firecrawl-secret-dynamic- env:- - name: FLY_PROCESS_GROUP- value: "index-worker"- resources:- requests:- memory: "1G"- cpu: "500m"- limits:- memory: "3G"- cpu: "1000m"\ No newline at end of filediff --git a/.github/k8s/server.yml b/.github/k8s/server.ymldeleted file mode 100644index 5d55bb080..000000000--- a/.github/k8s/server.yml+++ /dev/null@@ -1,305 +0,0 @@-apiVersion: v1-kind: ConfigMap-metadata:- name: firecrawl-config-data:- HOST: "0.0.0.0"- REDIS_URL: "redis://firecrawl-dragonfly:6379"- REDIS_RATE_LIMIT_URL: "redis://firecrawl-dragonfly:6379"- REDIS_EVICT_URL: "redis://firecrawl-dragonfly:6379"- SENTRY_ENVIRONMENT: "dev"- ENV: "production"- LOGGING_LEVEL: "DEBUG"- IS_KUBERNETES: "true"- USE_GO_MARKDOWN_PARSER: "true"- NUQ_DATABASE_URL: "postgres://postgres:postgres@nuq-postgres:5432/postgres"- NUQ_DATABASE_URL_LISTEN: "postgres://postgres:postgres@nuq-postgres:5432/postgres"-----apiVersion: apps/v1-kind: Deployment-metadata:- name: nuq-postgres-spec:- replicas: 1- selector:- matchLabels:- app: nuq-postgres- template:- metadata:- labels:- app: nuq-postgres- spec:- containers:- - name: postgres- image: firecrawl/nuq-postgres:latest- imagePullPolicy: Never- ports:- - containerPort: 5432- env:- - name: POSTGRES_USER- value: "postgres"- - name: POSTGRES_PASSWORD- value: "postgres"- - name: POSTGRES_DB- value: "postgres"- resources:- requests:- memory: "512Mi"- cpu: "250m"- limits:- memory: "1Gi"- cpu: "500m"------apiVersion: v1-kind: Service-metadata:- name: nuq-postgres-spec:- selector:- app: nuq-postgres- ports:- - name: postgres- protocol: TCP- port: 5432- targetPort: 5432-----apiVersion: apps/v1-kind: Deployment-metadata:- name: firecrawl-dragonfly-spec:- replicas: 1- selector:- matchLabels:- app: firecrawl-dragonfly- template:- metadata:- labels:- app: firecrawl-dragonfly- spec:- containers:- - name: dragonfly- image: ghcr.io/dragonflydb/dragonfly:v1.31.2- args:- - "--logtostderr"- - "--cluster_mode=emulated"- - "--lock_on_hashtags"- - "--bind=::"- - "--maxmemory=4gb"- - "--dir=/data"- ports:- - containerPort: 6379- readinessProbe:- tcpSocket:- port: 6379- initialDelaySeconds: 3- periodSeconds: 5- timeoutSeconds: 3- successThreshold: 1- failureThreshold: 3- resources:- requests:- memory: "1Gi"- cpu: "500m"- limits:- memory: "4Gi"- cpu: "1000m"-----apiVersion: v1-kind: Service-metadata:- name: firecrawl-dragonfly-spec:- selector:- app: firecrawl-dragonfly- ports:- - name: redis- protocol: TCP- port: 6379- targetPort: 6379-----apiVersion: apps/v1-kind: Deployment-metadata:- name: firecrawl-app-spec:- replicas: 1- selector:- matchLabels:- app: firecrawl-app- template:- metadata:- labels:- app: firecrawl-app- spec:- containers:- - name: firecrawl-app- image: firecrawl/firecrawl:latest- imagePullPolicy: Never- command: ["node"]- args: ["dist/src/index.js"]- ports:- - containerPort: 3002- envFrom:- - configMapRef:- name: firecrawl-config- - configMapRef:- name: firecrawl-config-dynamic- - secretRef:- name: firecrawl-secret-dynamic- env:- - name: FLY_PROCESS_GROUP- value: "app"- - name: PORT- value: "3002"- - name: NODE_OPTIONS- value: "--no-node-snapshot --max-old-space-size=6144"- resources:- requests:- memory: "1G"- cpu: "500m"- limits:- memory: "6G"- cpu: "1000m"- livenessProbe:- httpGet:- path: /v0/health/liveness- port: 3002- initialDelaySeconds: 5- periodSeconds: 5- timeoutSeconds: 5- successThreshold: 1- failureThreshold: 3- readinessProbe:- httpGet:- path: /v0/health/readiness- port: 3002- initialDelaySeconds: 5- periodSeconds: 5- timeoutSeconds: 5- successThreshold: 1- failureThreshold: 3-----apiVersion: v1-kind: Service-metadata:- name: firecrawl-app- namespace: default- labels:- app: firecrawl-app-spec:- selector:- app: firecrawl-app- ports:- - name: http- protocol: TCP- port: 3002- targetPort: 3002- type: NodePort-----apiVersion: apps/v1-kind: Deployment-metadata:- name: firecrawl-worker-spec:- replicas: 1- selector:- matchLabels:- app: firecrawl-worker- template:- metadata:- labels:- app: firecrawl-worker- spec:- containers:- - name: worker- image: firecrawl/firecrawl:latest- imagePullPolicy: Never- command: [ "node" ]- args: [ "dist/src/services/queue-worker.js", "--max-old-space-size=2048" ]- envFrom:- - configMapRef:- name: firecrawl-config- - configMapRef:- name: firecrawl-config-dynamic- - secretRef:- name: firecrawl-secret-dynamic- env:- - name: FLY_PROCESS_GROUP- value: "worker"- - name: PORT- value: "3005"- - name: NODE_OPTIONS- value: "--no-node-snapshot"- - name: GOMEMLIMIT- value: "1024MiB"- - name: SCRAPE_WORKER_MAX_OLD_SPACE_SIZE- value: "768"- resources:- requests:- memory: "2G"- cpu: "500m"- limits:- memory: "4G"- cpu: "1000m"- livenessProbe:- httpGet:- path: /liveness- port: 3005- initialDelaySeconds: 5- periodSeconds: 5- timeoutSeconds: 5- successThreshold: 1- failureThreshold: 3-----apiVersion: apps/v1-kind: Deployment-metadata:- name: firecrawl-nuq-worker-spec:- replicas: 5- selector:- matchLabels:- app: firecrawl-nuq-worker- template:- metadata:- labels:- app: firecrawl-nuq-worker- spec:- containers:- - name: nuq-worker- image: firecrawl/firecrawl:latest- imagePullPolicy: Never- command: [ "node" ]- args: [ "dist/src/services/worker/nuq-worker.js", "--max-old-space-size=2048" ]- envFrom:- - configMapRef:- name: firecrawl-config- - configMapRef:- name: firecrawl-config-dynamic- - secretRef:- name: firecrawl-secret-dynamic- env:- - name: PORT- value: "3005"- - name: NODE_OPTIONS- value: "--no-node-snapshot"- - name: GOMEMLIMIT- value: "1024MiB"- resources:- requests:- memory: "2G"- cpu: "500m"- limits:- memory: "4G"- cpu: "1000m"- livenessProbe:- httpGet:- path: /health- port: 3005- initialDelaySeconds: 10- periodSeconds: 10- timeoutSeconds: 10- successThreshold: 1- failureThreshold: 3\ No newline at end of filediff --git a/.github/k8s/tailscale-proxy/Dockerfile b/.github/k8s/tailscale-proxy/Dockerfiledeleted file mode 100644index 77bd48e43..000000000--- a/.github/k8s/tailscale-proxy/Dockerfile+++ /dev/null@@ -1,28 +0,0 @@-FROM debian:bookworm-slim--# Install prerequisites, iptables, nginx, and Tailscale-RUN apt-get update \- && apt-get install -y --no-install-recommends \- ca-certificates \- curl \- gnupg \- iptables \- iproute2 \- nginx \- && curl -fsSL https://pkgs.tailscale.com/stable/debian/bookworm.noarmor.gpg \- -o /usr/share/keyrings/tailscale-archive-keyring.gpg \- && curl -fsSL https://pkgs.tailscale.com/stable/debian/bookworm.tailscale-keyring.list \- -o /etc/apt/sources.list.d/tailscale.list \- && apt-get update \- && apt-get install -y --no-install-recommends tailscale \- && rm -rf /var/lib/apt/lists/*--# Configure nginx as a reverse proxy-COPY nginx.conf /etc/nginx/nginx.conf.template--# Start script that sets up tailscale and nginx-COPY entrypoint.sh /entrypoint.sh-RUN chmod +x /entrypoint.sh--EXPOSE 8080-ENTRYPOINT ["/entrypoint.sh"]\ No newline at end of filediff --git a/.github/k8s/tailscale-proxy/entrypoint.sh b/.github/k8s/tailscale-proxy/entrypoint.shdeleted file mode 100644index a1a81639f..000000000--- a/.github/k8s/tailscale-proxy/entrypoint.sh+++ /dev/null@@ -1,48 +0,0 @@-#!/bin/sh-set -eu--# Require auth key-if [ -z "${TS_AUTHKEY:-}" ]; then- echo "[tailscale-proxy] TS_AUTHKEY is not set; refusing to start." >&2- exit 1-fi--# Compute hostname: allow override via TS_HOSTNAME, else default to pod name-TS_HOSTNAME=${TS_HOSTNAME:-}-if [ -z "${TS_HOSTNAME}" ]; then- # Kubernetes sets HOSTNAME to the pod name- TS_HOSTNAME="firecrawl-proxy-${HOSTNAME:-container}"-fi--# Start tailscaled-tailscaled --state=mem: &--# Wait for tailscaled control socket to be ready (up to ~15s)-i=0-until tailscale version >/dev/null 2>&1 || [ $i -ge 30 ]; do- i=$((i+1))- sleep 0.5-done--# Prepare optional flags-ADVERTISE_FLAGS=""-if [ -n "${TS_ADVERTISE_TAGS:-}" ]; then- ADVERTISE_FLAGS="--advertise-tags=${TS_ADVERTISE_TAGS}"-fi--set -x-tailscale up \- --authkey="${TS_AUTHKEY}" \- --hostname="${TS_HOSTNAME}" \- --accept-routes=true \- --accept-dns=true \- --ssh=false \- ${ADVERTISE_FLAGS} \- --reset-set +x--# Replace FIRE_ENGINE_BETA_URL in nginx config-sed "s|\$FIRE_ENGINE_BETA_URL|${FIRE_ENGINE_BETA_URL}|g" /etc/nginx/nginx.conf.template > /etc/nginx/nginx.conf--# Start nginx-exec nginx -g "daemon off;"\ No newline at end of filediff --git a/.github/k8s/tailscale-proxy/nginx.conf b/.github/k8s/tailscale-proxy/nginx.confdeleted file mode 100644index ca48512ac..000000000--- a/.github/k8s/tailscale-proxy/nginx.conf+++ /dev/null@@ -1,24 +0,0 @@-user www-data;-worker_processes auto;-error_log /dev/stderr notice;-pid /run/nginx.pid;--events {- worker_connections 1024;-}--http {- access_log /dev/stdout combined;-- server {- listen 8080;-- location / {- proxy_pass $FIRE_ENGINE_BETA_URL;- proxy_set_header Host $host;- proxy_set_header X-Real-IP $remote_addr;- proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;- proxy_set_header X-Forwarded-Proto $scheme;- }- }-}\ No newline at end of filediff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.ymlindex 973591180..bdb4ab8ac 100644--- a/.github/workflows/test-server.yml+++ b/.github/workflows/test-server.yml@@ -4,12 +4,24 @@ on:pull_request:branches:- main- - nsc/v2jobs:+ audit:+ runs-on: ubuntu-latest+ steps:+ - uses: actions/checkout@v5+ - name: Install pnpm+ uses: pnpm/action-setup@v4+ with:+ version: 10+ - name: Audit NPM Packages+ run: |+ pnpm dlx audit-ci@^7 --directory apps/api --config apps/api/audit-ci.jsonc+ pnpm dlx audit-ci@^7 --directory apps/playwright-service-ts --config apps/playwright-service-ts/audit-ci.jsoncbuild-images:name: Build imagesruns-on: big-runner+ needs: auditsteps:- uses: actions/checkout@v5- name: Set up Docker Buildx@@ -22,22 +34,6 @@ jobs:--cache-to type=gha,mode=max \--load \./apps/api- - name: Build nuq-postgres image- run: |- docker buildx build \- -t firecrawl/nuq-postgres:latest \- --cache-from type=gha \- --cache-to type=gha,mode=max \- --load \- ./apps/nuq-postgres- - name: Build tailscale-proxy image- run: |- docker buildx build \- -t firecrawl/tailscale-proxy:latest \- --cache-from type=gha \- --cache-to type=gha,mode=max \- --load \- .github/k8s/tailscale-proxy- name: Build Playwright imagerun: |docker buildx build \@@ -46,25 +42,8 @@ jobs:--cache-to type=gha,mode=max \--load \./apps/playwright-service-ts- - name: Export images as tarballs- run: |- docker save -o firecrawl-firecrawl.tar firecrawl/firecrawl:latest- docker save -o firecrawl-nuq-postgres.tar firecrawl/nuq-postgres:latest- docker save -o firecrawl-tailscale-proxy.tar firecrawl/tailscale-proxy:latest- docker save -o firecrawl-playwright-service.tar firecrawl/playwright-service:latest- - name: Upload images artifact- uses: actions/upload-artifact@v4- with:- name: built-images- path: |- firecrawl-firecrawl.tar- firecrawl-nuq-postgres.tar- firecrawl-tailscale-proxy.tar- firecrawl-playwright-service.tar-test:- name: Self-hosted (Kubernetes) environment tests- needs: build-images+ name: Self-hosted environment testsstrategy:matrix:ai: ["openai", "no-ai"]@@ -74,50 +53,28 @@ jobs:proxy: ["proxy"]fail-fast: falseruns-on: big-runner+ needs: auditservices:redis:image: redisports:- 6379:6379+ env:+ OPENAI_API_KEY: ${{ matrix.ai == 'openai' && secrets.OPENAI_API_KEY || '' }}+ SEARXNG_ENDPOINT: ${{ matrix.search == 'searxng' && 'http://localhost:3434' || '' }}+ PLAYWRIGHT_MICROSERVICE_URL: ${{ matrix.engine == 'playwright' && 'http://localhost:3003/scrape' || '' }}+ PROXY_SERVER: ${{ matrix.proxy == 'proxy' && secrets.PROXY_SERVER || '' }}+ PROXY_USERNAME: ${{ matrix.proxy == 'proxy' && secrets.PROXY_USERNAME || '' }}+ PROXY_PASSWORD: ${{ matrix.proxy == 'proxy' && secrets.PROXY_PASSWORD || '' }}+ NUQ_DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgres+ NUQ_REDUCE_NOISE: true+ HOST: 0.0.0.0steps:- uses: actions/checkout@v5- - name: Start minikube- uses: medyagh/[email protected]- - name: Download images artifact- uses: actions/download-artifact@v4- with:- name: built-images- path: ./images- - name: Load images into minikube- run: |- eval $(minikube docker-env)- docker load -i ./images/firecrawl-firecrawl.tar- docker load -i ./images/firecrawl-nuq-postgres.tar- - name: Load Playwright image into minikube- if: matrix.engine == 'playwright'+ - name: Run Docker Postgresrun: |- eval $(minikube docker-env)- docker load -i ./images/firecrawl-playwright-service.tar- - name: Create configmap- run: |- kubectl create configmap firecrawl-config-dynamic -n default \- --from-literal=SEARXNG_ENDPOINT=${{ matrix.search == 'searxng' && 'http://firecrawl-searxng:8080/search' || '' }} \- --from-literal=PLAYWRIGHT_MICROSERVICE_URL=${{ matrix.engine == 'playwright' && 'http://firecrawl-playwright-service:3003/scrape' || '' }} \- --from-literal=PROXY_SERVER=${{ matrix.proxy == 'proxy' && secrets.PROXY_SERVER || '' }} \- --from-literal=PROXY_USERNAME=${{ matrix.proxy == 'proxy' && secrets.PROXY_USERNAME || '' }}- - name: Create secret- run: |- kubectl create secret generic firecrawl-secret-dynamic \- --from-literal=OPENAI_API_KEY=${{ matrix.ai == 'openai' && secrets.OPENAI_API_KEY || '' }} \- --from-literal=PROXY_PASSWORD=${{ matrix.proxy == 'proxy' && secrets.PROXY_PASSWORD || '' }}- - name: Deploy server- run: kubectl apply -f .github/k8s/server.yml- - name: Deploy playwright- if: matrix.engine == 'playwright'- run: kubectl apply -f .github/k8s/playwright.yml- - name: Deploy searxng- if: matrix.search == 'searxng'- run: kubectl apply -f .github/k8s/searxng.yml+ docker build -t firecrawl/nuq-postgres:latest ./apps/nuq-postgres+ docker run -d -p 5432:5432 -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=postgres --name postgres firecrawl/nuq-postgres:latest- name: Install pnpmuses: pnpm/action-setup@v4with:@@ -125,78 +82,68 @@ jobs:- name: Set up Node.jsuses: actions/setup-node@v4with:- node-version: '20'+ node-version: '22'cache: 'pnpm'cache-dependency-path: './apps/api/pnpm-lock.yaml'- name: Install dependenciesrun: pnpm installworking-directory: ./apps/api- - name: Wait for server- run: kubectl wait --for=condition=ready pod -l app=firecrawl-app --timeout=60s -o json- - name: Debug wait failure (server)- if: failure()- run: |- echo "Pods with label app=firecrawl-app:" || true- kubectl get pods -l app=firecrawl-app -o wide || true- echo "\nDescribe pods:" || true- kubectl describe pods -l app=firecrawl-app || true- echo "\nRecent events:" || true- kubectl get events --sort-by=.lastTimestamp | tail -n 100 || true- echo "\nLogs from firecrawl-app deployment (if any):" || true- kubectl logs deployment/firecrawl-app --all-containers=true --tail=200 || true- - name: Wait for playwright+ - uses: actions-rust-lang/setup-rust-toolchain@v1+ - name: Install Playwright dependenciesif: matrix.engine == 'playwright'- run: kubectl wait --for=condition=ready pod -l app=firecrawl-playwright-service --timeout=60s -o json- - name: Debug wait failure (playwright)- if: failure() && matrix.engine == 'playwright'run: |- echo "Pods with label app=firecrawl-playwright-service:" || true- kubectl get pods -l app=firecrawl-playwright-service -o wide || true- echo "\nDescribe pods:" || true- kubectl describe pods -l app=firecrawl-playwright-service || true- echo "\nRecent events:" || true- kubectl get events --sort-by=.lastTimestamp | tail -n 100 || true- echo "\nLogs from playwright deployment (if any):" || true- kubectl logs deployment/firecrawl-playwright-service --all-containers=true --tail=200 || true- - name: Wait for searxng+ pnpm install+ pnpm exec playwright install-deps+ pnpm exec playwright install+ working-directory: ./apps/playwright-service-ts+ - name: Set up Go+ uses: actions/setup-go@v5+ with:+ go-version: '1.24'+ cache-dependency-path: ./apps/api/sharedLibs/go-html-to-md/go.sum+ - name: Build go-html-to-md+ run: |+ go mod tidy+ go build -o libhtml-to-markdown.so -buildmode=c-shared html-to-markdown.go+ chmod +x libhtml-to-markdown.so+ working-directory: ./apps/api/sharedLibs/go-html-to-md+ - name: Set up SearXNGif: matrix.search == 'searxng'- run: kubectl wait --for=condition=ready pod -l app=firecrawl-searxng --timeout=60s -o json- - name: Debug wait failure (searxng)- if: failure() && matrix.search == 'searxng'run: |- echo "Pods with label app=firecrawl-searxng:" || true- kubectl get pods -l app=firecrawl-searxng -o wide || true- echo "\nDescribe pods:" || true- kubectl describe pods -l app=firecrawl-searxng || true- echo "\nRecent events:" || true- kubectl get events --sort-by=.lastTimestamp | tail -n 100 || true- echo "\nLogs from searxng deployment (if any):" || true- kubectl logs deployment/firecrawl-searxng --all-containers=true --tail=200 || true- - name: Run snippet tests+ mkdir searxng++ echo "use_default_settings: true+ search:+ formats: [html, json, csv]+ server:+ secret_key: 'fcsecret'" > searxng/settings.yml++ docker run -d -p 3434:8080 -v "${PWD}/searxng:/etc/searxng" --name searxng searxng/searxng+ pnpx wait-on tcp:3434 -t 30s+ working-directory: ./+ - name: Start playwright+ if: matrix.engine == 'playwright'+ run: pnpm run dev > playwright.log 2>&1 &+ working-directory: ./apps/playwright-service-tsenv:- TEST_SUITE_SELF_HOSTED: true- OPENAI_API_KEY: ${{ matrix.ai == 'openai' && '<IS SET>' || '' }}- PROXY_SERVER: ${{ matrix.proxy == 'proxy' && secrets.PROXY_SERVER || '' }}- PLAYWRIGHT_MICROSERVICE_URL: ${{ matrix.engine == 'playwright' && '<IS SET>' || '' }}- run: TEST_API_URL=$(minikube service firecrawl-app --url) pnpm run test:snips+ PORT: 3003+ - name: Wait for playwright+ if: matrix.engine == 'playwright'+ run: pnpx wait-on tcp:3003 -t 15s+ - name: Start the application and run snippet tests+ run: pnpm harness pnpm test:snipsworking-directory: ./apps/api- - name: Create logs directory- if: always()- run: mkdir -p logs- name: Copy log filesif: always()run: |- kubectl logs deployment/firecrawl-app > logs/api.log || true- kubectl logs deployment/firecrawl-worker > logs/worker.log || true- kubectl logs deployment/firecrawl-nuq-worker > logs/nuq-worker.log || true- kubectl logs deployment/firecrawl-dragonfly > logs/dragonfly.log || true- kubectl logs deployment/nuq-postgres > logs/postgres.log || true+ mkdir -p logs+ cp ./apps/api/firecrawl.log logs/firecrawl.log- name: Copy SearXNG logsif: always() && matrix.search == 'searxng'- run: kubectl logs deployment/firecrawl-searxng > logs/searxng.log || true+ run: docker logs searxng > logs/searxng.log && docker kill searxng- name: Copy Playwright logsif: always() && matrix.engine == 'playwright'- run: kubectl logs deployment/firecrawl-playwright-service > logs/playwright.log || true+ run: cp ./apps/playwright-service-ts/playwright.log logs/playwright.log- name: Zip logsif: always()run: |@@ -208,129 +155,68 @@ jobs:name: Logs (kubernetes, ${{ matrix.ai }}, ${{ matrix.search }}, ${{ matrix.engine }}, ${{ matrix.proxy }})path: logs/logs.zip- docker-test:- name: Self-hosted (Docker) environment tests- runs-on: ubuntu-latest- steps:- - uses: actions/checkout@v5- - name: Compose- env:- PROXY_SERVER: ${{ secrets.PROXY_SERVER }}- PROXY_USERNAME: ${{ secrets.PROXY_USERNAME }}- PROXY_PASSWORD: ${{ secrets.PROXY_PASSWORD }}- run: |- docker compose up -d- - name: Install pnpm- uses: pnpm/action-setup@v4- with:- version: 10- - name: Set up Node.js- uses: actions/setup-node@v4- with:- node-version: "20"- cache: "pnpm"- cache-dependency-path: './apps/api/pnpm-lock.yaml'- - name: Install dependencies- run: pnpm install- working-directory: ./apps/api- - name: Wait for server- run: pnpx wait-on http://localhost:3002 -t 30s- - name: Run snippet tests- env:- PROXY_SERVER: ${{ secrets.PROXY_SERVER }}- PLAYWRIGHT_MICROSERVICE_URL: ${{ secrets.PLAYWRIGHT_MICROSERVICE_URL }}- run: pnpm run test:snips- working-directory: ./apps/api- - name: Create logs directory- if: always()- run: mkdir -p logs- - name: Copy log files- if: always()- run: |- docker compose logs api > logs/api.log || true- docker compose logs playwright-service > logs/playwright.log || true- docker compose logs nuq-postgres > logs/postgres.log || true- - name: Zip logs- if: always()- run: |- cd logs- zip -r logs.zip ./*- - uses: actions/upload-artifact@v4- if: always()- with:- name: Logs (docker, no-ai, google, playwright, proxy)- path: logs/logs.zip-prod-test:name: Production environment tests- needs: build-imagesruns-on: big-runner+ needs: audit+ services:+ redis:+ image: redis+ ports:+ - 6379:6379+ env:+ BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }}+ OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}+ REDIS_URL: ${{ secrets.REDIS_URL }}+ SUPABASE_ANON_TOKEN: ${{ secrets.SUPABASE_ANON_TOKEN }}+ SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }}+ SUPABASE_URL: ${{ secrets.SUPABASE_URL }}+ SUPABASE_REPLICA_URL: ${{ secrets.SUPABASE_REPLICA_URL }}+ INDEX_SUPABASE_SERVICE_TOKEN: ${{ secrets.INDEX_SUPABASE_SERVICE_TOKEN }}+ INDEX_SUPABASE_ANON_TOKEN: ${{ secrets.INDEX_SUPABASE_ANON_TOKEN }}+ INDEX_SUPABASE_URL: ${{ secrets.INDEX_SUPABASE_URL }}+ TEST_API_KEY: ${{ secrets.TEST_API_KEY }}+ TEST_TEAM_ID: ${{ secrets.TEST_TEAM_ID }}+ TEST_API_KEY_CONCURRENCY: ${{ secrets.TEST_API_KEY_CONCURRENCY }}+ TEST_TEAM_ID_CONCURRENCY: ${{ secrets.TEST_TEAM_ID_CONCURRENCY }}+ TEST_API_KEY_ZDR: ${{ secrets.TEST_API_KEY_ZDR }}+ TEST_TEAM_ID_ZDR: ${{ secrets.TEST_TEAM_ID_ZDR }}+ FIRE_ENGINE_BETA_URL: ${{ secrets.FIRE_ENGINE_BETA_URL }}+ FIRE_ENGINE_STAGING_URL: ${{ secrets.FIRE_ENGINE_STAGING_URL }}+ USE_DB_AUTHENTICATION: true+ SERPER_API_KEY: ${{ secrets.SERPER_API_KEY }}+ ENV: ${{ secrets.ENV }}+ RUNPOD_MU_POD_ID: ${{ secrets.RUNPOD_MU_POD_ID }}+ RUNPOD_MUV2_POD_ID: ${{ secrets.RUNPOD_MUV2_POD_ID }}+ RUNPOD_MU_API_KEY: ${{ secrets.RUNPOD_MU_API_KEY }}+ GCS_CREDENTIALS: ${{ secrets.GCS_CREDENTIALS }}+ GCS_BUCKET_NAME: ${{ secrets.GCS_BUCKET_NAME }}+ GCS_INDEX_BUCKET_NAME: ${{ secrets.GCS_INDEX_BUCKET_NAME }}+ GCS_MEDIA_BUCKET_NAME: ${{ secrets.GCS_MEDIA_BUCKET_NAME }}+ GOOGLE_GENERATIVE_AI_API_KEY: ${{ secrets.GOOGLE_GENERATIVE_AI_API_KEY }}+ GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}+ ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}+ VERTEX_CREDENTIALS: ${{ secrets.VERTEX_CREDENTIALS }}+ USE_GO_MARKDOWN_PARSER: true+ SENTRY_ENVIRONMENT: dev+ IDMUX_URL: ${{ secrets.IDMUX_URL }}+ LOG_ENCRYPTION_KEY: ${{ secrets.LOG_ENCRYPTION_KEY }}+ NUQ_DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgres+ NUQ_REDUCE_NOISE: true+ HOST: 0.0.0.0steps:- uses: actions/checkout@v5- - name: Start minikube- uses: medyagh/[email protected]- - name: Download images artifact- uses: actions/download-artifact@v4- with:- name: built-images- path: ./images+ - name: Run Docker Postgres+ run: |+ docker build -t firecrawl/nuq-postgres:latest ./apps/nuq-postgres+ docker run -d -p 5432:5432 -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=postgres --name postgres firecrawl/nuq-postgres:latest- name: Tailscaleuses: tailscale/github-action@v3with:oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }}oauth-secret: ${{ secrets.TS_OAUTH_SECRET }}tags: tag:ci- - name: Load prebuilt images into minikube- run: |- eval $(minikube docker-env)- docker load -i ./images/firecrawl-firecrawl.tar- docker load -i ./images/firecrawl-nuq-postgres.tar- docker load -i ./images/firecrawl-tailscale-proxy.tar- - name: Create Tailscale secret- run: |- if [ -z "${{ secrets.TS_AUTHKEY }}" ]; then- echo "secrets.TS_AUTHKEY is empty; set a reusable/preauthorized or ephemeral key" >&2- exit 1- fi- kubectl create secret generic tailscale \- --from-literal=TS_AUTHKEY=${{ secrets.TS_AUTHKEY }} \- --from-literal=TS_ADVERTISE_TAGS=${{ secrets.TS_ADVERTISE_TAGS }} \- --from-literal=FIRE_ENGINE_BETA_URL=${{ secrets.FIRE_ENGINE_BETA_URL }} \- --dry-run=client -o yaml | kubectl apply -f -- - name: Create configmap- run: |- kubectl create configmap firecrawl-config-dynamic -n default \- --from-literal=USE_DB_AUTHENTICATION=true \- --from-literal=SUPABASE_ANON_TOKEN=${{ secrets.SUPABASE_ANON_TOKEN }} \- --from-literal=SUPABASE_URL=${{ secrets.SUPABASE_URL }} \- --from-literal=SUPABASE_REPLICA_URL=${{ secrets.SUPABASE_REPLICA_URL }} \- --from-literal=INDEX_SUPABASE_ANON_TOKEN=${{ secrets.INDEX_SUPABASE_ANON_TOKEN }} \- --from-literal=INDEX_SUPABASE_URL=${{ secrets.INDEX_SUPABASE_URL }} \- --from-literal=FIRE_ENGINE_BETA_URL=http://tailscale-proxy:8080 \- --from-literal=RUNPOD_MU_POD_ID=${{ secrets.RUNPOD_MU_POD_ID }} \- --from-literal=RUNPOD_MUV2_POD_ID=${{ secrets.RUNPOD_MUV2_POD_ID }} \- --from-literal=GCS_BUCKET_NAME=${{ secrets.GCS_BUCKET_NAME }} \- --from-literal=GCS_MEDIA_BUCKET_NAME=${{ secrets.GCS_MEDIA_BUCKET_NAME }} \- --from-literal=GCS_INDEX_BUCKET_NAME=${{ secrets.GCS_INDEX_BUCKET_NAME }}- - name: Create secret- run: |- kubectl create secret generic firecrawl-secret-dynamic \- --from-literal=BULL_AUTH_KEY=${{ secrets.BULL_AUTH_KEY }} \- --from-literal=OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }} \- --from-literal=SUPABASE_SERVICE_TOKEN=${{ secrets.SUPABASE_SERVICE_TOKEN }} \- --from-literal=INDEX_SUPABASE_SERVICE_TOKEN=${{ secrets.INDEX_SUPABASE_SERVICE_TOKEN }} \- --from-literal=SERPER_API_KEY=${{ secrets.SERPER_API_KEY }} \- --from-literal=RUNPOD_MU_API_KEY=${{ secrets.RUNPOD_MU_API_KEY }} \- --from-literal=GCS_CREDENTIALS=${{ secrets.GCS_CREDENTIALS }} \- --from-literal=GOOGLE_GENERATIVE_AI_API_KEY=${{ secrets.GOOGLE_GENERATIVE_AI_API_KEY }} \- --from-literal=GROQ_API_KEY=${{ secrets.GROQ_API_KEY }} \- --from-literal=ANTHROPIC_API_KEY=${{ secrets.ANTHROPIC_API_KEY }} \- --from-literal=VERTEX_CREDENTIALS=${{ secrets.VERTEX_CREDENTIALS }} \- --from-literal=LOG_ENCRYPTION_KEY=${{ secrets.LOG_ENCRYPTION_KEY }}- - name: Deploy server- run: |- kubectl apply -f .github/k8s/server.yml- kubectl apply -f .github/k8s/server-nsh.yml+ use-cache: 'true'- name: Install pnpmuses: pnpm/action-setup@v4with:@@ -338,90 +224,31 @@ jobs:- name: Set up Node.jsuses: actions/setup-node@v4with:- node-version: "20"- cache: "pnpm"+ node-version: '22'+ cache: 'pnpm'cache-dependency-path: './apps/api/pnpm-lock.yaml'- name: Install dependenciesrun: pnpm installworking-directory: ./apps/api- - name: Wait for API- run: kubectl wait --for=condition=ready pod -l app=firecrawl-app --timeout=60s -o json- - name: Debug wait failure (API)- if: failure()- run: |- echo "Pods with label app=firecrawl-app:" || true- kubectl get pods -l app=firecrawl-app -o wide || true- echo "\nDescribe pods:" || true- kubectl describe pods -l app=firecrawl-app || true- echo "\nRecent events:" || true- kubectl get events --sort-by=.lastTimestamp | tail -n 100 || true- - name: Wait for nuq-postgres- run: kubectl wait --for=condition=ready pod -l app=nuq-postgres --timeout=60s -o json- - name: Debug wait failure (nuq-postgres)- if: failure()- run: |- echo "Pods with label app=nuq-postgres:" || true- kubectl get pods -l app=nuq-postgres -o wide || true- echo "\nDescribe pods:" || true- kubectl describe pods -l app=nuq-postgres || true- echo "\nRecent events:" || true- kubectl get events --sort-by=.lastTimestamp | tail -n 100 || true- echo "\nLogs from nuq-postgres deployment (if any):" || true- kubectl logs deployment/nuq-postgres --all-containers=true --tail=200 || true- - name: Wait for tailscale-proxy- run: kubectl wait --for=condition=ready pod -l app=tailscale-proxy --timeout=60s -o json- - name: Debug wait failure (tailscale-proxy)- if: failure()- run: |- echo "Pods with label app=tailscale-proxy:" || true- kubectl get pods -l app=tailscale-proxy -o wide || true- echo "\nDescribe pods:" || true- kubectl describe pods -l app=tailscale-proxy || true- echo "\nRecent events:" || true- kubectl get events --sort-by=.lastTimestamp | tail -n 100 || true- - name: Wait for dragonfly (redis)- run: kubectl wait --for=condition=ready pod -l app=firecrawl-dragonfly --timeout=60s -o json- - name: Debug wait failure (dragonfly)- if: failure()- run: |- echo "Pods with label app=firecrawl-dragonfly:" || true- kubectl get pods -l app=firecrawl-dragonfly -o wide || true- echo "\nDescribe pods:" || true- kubectl describe pods -l app=firecrawl-dragonfly || true- echo "\nRecent events:" || true- kubectl get events --sort-by=.lastTimestamp | tail -n 100 || true- - name: Run snippet tests- env:- IDMUX_URL: ${{ secrets.IDMUX_URL }}- USE_DB_AUTHENTICATION: true- SUPABASE_ANON_TOKEN: ${{ secrets.SUPABASE_ANON_TOKEN }}- SUPABASE_URL: ${{ secrets.SUPABASE_URL }}- SUPABASE_REPLICA_URL: ${{ secrets.SUPABASE_REPLICA_URL }}- SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }}- GCS_BUCKET_NAME: ${{ secrets.GCS_BUCKET_NAME }}- GCS_CREDENTIALS: ${{ secrets.GCS_CREDENTIALS }}+ - name: Set up Go+ uses: actions/setup-go@v5+ with:+ go-version: '1.24'+ cache-dependency-path: ./apps/api/sharedLibs/go-html-to-md/go.sum+ - name: Build go-html-to-mdrun: |- TEST_API_URL=$(minikube service firecrawl-app --url) pnpm run test:snips+ go mod tidy+ go build -o libhtml-to-markdown.so -buildmode=c-shared html-to-markdown.go+ chmod +x libhtml-to-markdown.so+ working-directory: ./apps/api/sharedLibs/go-html-to-md+ - name: Build the application and run snippet tests+ run: pnpm harness pnpm test:snipsworking-directory: ./apps/api- - name: Kill instances- if: always()- run: pkill -9 node- name: Create logs directory- if: always()- run: mkdir -p logs- - name: Copy log files- if: always()- run: |- kubectl logs deployment/firecrawl-app > logs/api.log || true- kubectl logs deployment/firecrawl-worker > logs/worker.log || true- kubectl logs deployment/firecrawl-index-worker > logs/index-worker.log || true- kubectl logs deployment/firecrawl-nuq-worker > logs/nuq-worker.log || true- kubectl logs deployment/firecrawl-dragonfly > logs/dragonfly.log || true- kubectl logs deployment/nuq-postgres > logs/postgres.log || true- kubectl logs deployment/tailscale-proxy > logs/tailscale-proxy.log || true- - name: Zip and encrypt logsif: always()run: |+ mkdir -p logs+ cp ./apps/api/firecrawl.log logs/firecrawl.logcd logszip -r logs.zip ./*echo "${{ secrets.LOG_ENCRYPTION_KEY }}" | gpg --batch --yes --passphrase-fd 0 -c logs.zipdiff --git a/apps/api/.gitignore b/apps/api/.gitignoreindex 52345155e..e31e58112 100644--- a/apps/api/.gitignore+++ b/apps/api/.gitignore@@ -10,4 +10,6 @@ dump.rdb.rdb.sentryclirc-.env.*\ No newline at end of file+.env.*++firecrawl.log\ No newline at end of filediff --git a/apps/api/jest.config.js b/apps/api/jest.config.jsdeleted file mode 100644index 285445232..000000000--- a/apps/api/jest.config.js+++ /dev/null@@ -1,8 +0,0 @@-module.exports = {- preset: "ts-jest",- testEnvironment: "node",- setupFiles: ["./jest.setup.js"],- // ignore dist folder root dir- modulePathIgnorePatterns: ["<rootDir>/dist/"],--};diff --git a/apps/api/jest.config.ts b/apps/api/jest.config.tsnew file mode 100644index 000000000..c25eb711f--- /dev/null+++ b/apps/api/jest.config.ts@@ -0,0 +1,14 @@+import type { Config } from "jest";++const config: Config = {+ verbose: true,+ preset: "ts-jest",+ testEnvironment: "node",+ testPathIgnorePatterns: ["<rootDir>/dist/"],+ forceExit: true,+ detectOpenHandles: true,+ openHandlesTimeout: 120000,+ watchAll: false,+};++export default config;diff --git a/apps/api/jest.setup.js b/apps/api/jest.setup.jsdeleted file mode 100644index 0b3b09b7c..000000000--- a/apps/api/jest.setup.js+++ /dev/null@@ -1 +0,0 @@-// global.fetch = require('jest-fetch-mock');diff --git a/apps/api/package.json b/apps/api/package.jsonindex 1716cd45a..5f9b70dd2 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -14,11 +14,11 @@"start:dev": "tsc-watch --onSuccess \"node dist/src/index.js\"","build": "tsc","build:nosentry": "tsc",- "test": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false --testPathIgnorePatterns='src/__tests__/e2e_noAuth/*'",- "test:local-no-auth": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false --testPathIgnorePatterns='src/__tests__/e2e_withAuth/*'",- "test:full": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false --testPathIgnorePatterns='(src/__tests__/e2e_noAuth|src/__tests__/e2e_withAuth)'",- "test:prod": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false --testPathIgnorePatterns='(src/__tests__/e2e_noAuth|src/__tests__/e2e_full_withAuth|src/scraper/scrapeURL)'",- "test:snips": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false src/__tests__/snips/**/*.test.ts",+ "test": "jest --testPathIgnorePatterns=\"src/__tests__/e2e_noAuth/*\"",+ "test:local-no-auth": "jest --testPathIgnorePatterns=\"src/__tests__/e2e_withAuth/*\"",+ "test:full": "jest --testPathIgnorePatterns=\"(src/__tests__/e2e_noAuth|src/__tests__/e2e_withAuth)\"",+ "test:prod": "jest --testPathIgnorePatterns=\"(src/__tests__/e2e_noAuth|src/__tests__/e2e_full_withAuth|src/scraper/scrapeURL)\"",+ "test:snips": "jest \"src/__tests__/snips/**/*.test.ts\"","harness": "tsx src/harness.ts","workers": "tsc-watch --onSuccess \"node dist/src/services/queue-worker.js\"","worker:production": "node dist/src/services/queue-worker.js",diff --git a/apps/api/src/__tests__/snips/v1/zdr.test.ts b/apps/api/src/__tests__/snips/v1/zdr.test.tsindex 27d7b81db..41da3a5cb 100644--- a/apps/api/src/__tests__/snips/v1/zdr.test.ts+++ b/apps/api/src/__tests__/snips/v1/zdr.test.ts@@ -1,4 +1,3 @@-import { exec, spawn } from "node:child_process";import { getJobFromGCS } from "../../../lib/gcs-jobs";import { supabase_service } from "../../../services/supabase";import {@@ -31,66 +30,15 @@ if (process.env.TEST_SUITE_SELF_HOSTED) {expect(true).toBe(true);});} else {- function getOutput(command: string): Promise<string> {- return new Promise((resolve, reject) => {- try {- const cmd = spawn(command, { shell: true });- let out = "";- cmd.stdout?.on("data", data => {- out += data;- });- cmd.stderr?.on("data", data => {- out += data;- });- cmd.on("error", e => {- reject(e);- });- cmd.on("close", code => {- if (code !== 0) {- reject(new Error(`Command ${command} failed with code ${code}`));- } else {- resolve(out);- }- });- } catch (e) {- reject(e);- }- });- }-- async function getServerLogs() {+ async function getLogs() {let logs: string;- if (!process.env.GITHUB_ACTIONS) {- try {- await stat("api.log");- } catch (e) {- console.warn("No api.log file found");- return [];- }- logs = await readFile("api.log", "utf8");- } else {- logs = await getOutput("kubectl logs deployment/firecrawl-app");- }- return logs- .split("\n")- .filter(- x => x.trim().length > 0 && !logIgnoreList.some(y => x.includes(y)),- );- }-- async function getWorkerLogs() {- let logs: string;- if (!process.env.GITHUB_ACTIONS) {- try {- await stat("worker.log");- } catch (e) {- console.warn("No worker.log file found");- return [];- }- logs = await readFile("worker.log", "utf8");- } else {- logs = await getOutput("kubectl logs deployment/firecrawl-nuq-worker");+ try {+ await stat("firecrawl.log");+ } catch (e) {+ console.warn("No firecrawl.log file found");+ return [];}+ logs = await readFile("firecrawl.log", "utf8");return logs.split("\n").filter(@@ -154,8 +102,7 @@ if (process.env.TEST_SUITE_SELF_HOSTED) {}, 60000);it("should clean up a crawl", async () => {- const preServerLogs = await getServerLogs();- const preWorkerLogs = await getWorkerLogs();+ const preLogs = await getLogs();let identity = await idmux({name: `zdr/${scope}/crawl`,@@ -181,23 +128,13 @@ if (process.env.TEST_SUITE_SELF_HOSTED) {await new Promise(resolve => setTimeout(resolve, 2500));- const postServerLogs = (await getServerLogs()).slice(- preServerLogs.length,- );- const postWorkerLogs = (await getWorkerLogs()).slice(- preWorkerLogs.length,- );+ const postLogs = (await getLogs()).slice(preLogs.length);- if (postWorkerLogs.length > 0 || postServerLogs.length > 0) {- console.warn(- "Logs changed during crawl",- postServerLogs,- postWorkerLogs,- );+ if (postLogs.length > 0) {+ console.warn("Logs changed during crawl", postLogs);}- expect(postServerLogs).toHaveLength(0);- expect(postWorkerLogs).toHaveLength(0);+ expect(postLogs).toHaveLength(0);const { data, error } = await supabase_service.from("firecrawl_jobs")@@ -251,8 +188,7 @@ if (process.env.TEST_SUITE_SELF_HOSTED) {}, 600000);it("should clean up a batch scrape", async () => {- const preServerLogs = await getServerLogs();- const preWorkerLogs = await getWorkerLogs();+ const preLogs = await getLogs();let identity = await idmux({name: `zdr/${scope}/batch-scrape`,@@ -277,23 +213,13 @@ if (process.env.TEST_SUITE_SELF_HOSTED) {await new Promise(resolve => setTimeout(resolve, 2500));- const postServerLogs = (await getServerLogs()).slice(- preServerLogs.length,- );- const postWorkerLogs = (await getWorkerLogs()).slice(- preWorkerLogs.length,- );+ const postLogs = (await getLogs()).slice(preLogs.length);- if (postWorkerLogs.length > 0 || postServerLogs.length > 0) {- console.warn(- "Logs changed during batch scrape",- postServerLogs,- postWorkerLogs,- );+ if (postLogs.length > 0) {+ console.warn("Logs changed during batch scrape", postLogs);}- expect(postServerLogs).toHaveLength(0);- expect(postWorkerLogs).toHaveLength(0);+ expect(postLogs).toHaveLength(0);const { data, error } = await supabase_service.from("firecrawl_jobs")diff --git a/apps/api/src/__tests__/snips/v2/zdr.test.ts b/apps/api/src/__tests__/snips/v2/zdr.test.tsindex 61937aa7c..4cae9a903 100644--- a/apps/api/src/__tests__/snips/v2/zdr.test.ts+++ b/apps/api/src/__tests__/snips/v2/zdr.test.ts@@ -9,7 +9,6 @@ import {idmux,} from "./lib";import { readFile, stat } from "node:fs/promises";-import { spawn } from "node:child_process";const logIgnoreList = ["Billing queue created",@@ -31,66 +30,15 @@ if (process.env.TEST_SUITE_SELF_HOSTED) {expect(true).toBe(true);});} else {- function getOutput(command: string): Promise<string> {- return new Promise((resolve, reject) => {- try {- const cmd = spawn(command, { shell: true });- let out = "";- cmd.stdout?.on("data", data => {- out += data;- });- cmd.stderr?.on("data", data => {- out += data;- });- cmd.on("error", e => {- reject(e);- });- cmd.on("close", code => {- if (code !== 0) {- reject(new Error(`Command ${command} failed with code ${code}`));- } else {- resolve(out);- }- });- } catch (e) {- reject(e);- }- });- }-- async function getServerLogs() {+ async function getLogs() {let logs: string;- if (!process.env.GITHUB_ACTIONS) {- try {- await stat("api.log");- } catch (e) {- console.warn("No api.log file found");- return [];- }- logs = await readFile("api.log", "utf8");- } else {- logs = await getOutput("kubectl logs deployment/firecrawl-app");- }- return logs- .split("\n")- .filter(- x => x.trim().length > 0 && !logIgnoreList.some(y => x.includes(y)),- );- }-- async function getWorkerLogs() {- let logs: string;- if (!process.env.GITHUB_ACTIONS) {- try {- await stat("worker.log");- } catch (e) {- console.warn("No worker.log file found");- return [];- }- logs = await readFile("worker.log", "utf8");- } else {- logs = await getOutput("kubectl logs deployment/firecrawl-nuq-worker");+ try {+ await stat("firecrawl.log");+ } catch (e) {+ console.warn("No firecrawl.log file found");+ return [];}+ logs = await readFile("firecrawl.log", "utf8");return logs.split("\n").filter(@@ -154,8 +102,7 @@ if (process.env.TEST_SUITE_SELF_HOSTED) {}, 60000);it("should clean up a crawl", async () => {- const preServerLogs = await getServerLogs();- const preWorkerLogs = await getWorkerLogs();+ const preLogs = await getLogs();let identity = await idmux({name: `zdr/${scope}/crawl`,@@ -179,23 +126,13 @@ if (process.env.TEST_SUITE_SELF_HOSTED) {identity,);- const postServerLogs = (await getServerLogs()).slice(- preServerLogs.length,- );- const postWorkerLogs = (await getWorkerLogs()).slice(- preWorkerLogs.length,- );+ const postLogs = (await getLogs()).slice(preLogs.length);- if (postWorkerLogs.length > 0 || postServerLogs.length > 0) {- console.warn(- "Logs changed during crawl",- postServerLogs,- postWorkerLogs,- );+ if (postLogs.length > 0) {+ console.warn("Logs changed during crawl", postLogs);}- expect(postServerLogs).toHaveLength(0);- expect(postWorkerLogs).toHaveLength(0);+ expect(postLogs).toHaveLength(0);const { data, error } = await supabase_service.from("firecrawl_jobs")@@ -249,8 +186,7 @@ if (process.env.TEST_SUITE_SELF_HOSTED) {}, 600000);it("should clean up a batch scrape", async () => {- const preServerLogs = await getServerLogs();- const preWorkerLogs = await getWorkerLogs();+ const preLogs = await getLogs();let identity = await idmux({name: `zdr/${scope}/batch-scrape`,@@ -273,23 +209,13 @@ if (process.env.TEST_SUITE_SELF_HOSTED) {identity,);- const postServerLogs = (await getServerLogs()).slice(- preServerLogs.length,- );- const postWorkerLogs = (await getWorkerLogs()).slice(- preWorkerLogs.length,- );+ const postLogs = (await getLogs()).slice(preLogs.length);- if (postWorkerLogs.length > 0 || postServerLogs.length > 0) {- console.warn(- "Logs changed during batch scrape",- postServerLogs,- postWorkerLogs,- );+ if (postLogs.length > 0) {+ console.warn("Logs changed during batch scrape", postLogs);}- expect(postServerLogs).toHaveLength(0);- expect(postWorkerLogs).toHaveLength(0);+ expect(postLogs).toHaveLength(0);const { data, error } = await supabase_service.from("firecrawl_jobs")diff --git a/apps/api/src/harness.ts b/apps/api/src/harness.tsindex b9c9a4f76..dea00afb4 100644--- a/apps/api/src/harness.ts+++ b/apps/api/src/harness.ts@@ -3,6 +3,7 @@ import { type ChildProcess, spawn } from "child_process";import * as net from "net";import { basename } from "path";import { HTML_TO_MARKDOWN_PATH } from "./natives";+import { createWriteStream } from "fs";const childProcesses = new Set<ChildProcess>();@@ -64,6 +65,8 @@ function formatDuration(nanoseconds: bigint): string {return `${minutes}m ${remainingSeconds.toFixed(0)}s`;}+const stream = createWriteStream("firecrawl.log");+const logger = {section(message: string) {console.log(@@ -110,6 +113,7 @@ const logger = {const color = getProcessColor(name);const label = `${color}${name.padEnd(12)}${colors.reset}`;console.log(`${label} ${line}`);+ stream.write(`${name.padEnd(12)} ${line}\n`);},};diff --git a/apps/nuq-postgres/Dockerfile b/apps/nuq-postgres/Dockerfileindex 942b2fbe1..cbbe41cd9 100644--- a/apps/nuq-postgres/Dockerfile+++ b/apps/nuq-postgres/Dockerfile@@ -17,8 +17,5 @@ RUN set -eux; \sed -ri "s/^#?shared_preload_libraries\s*=.*/shared_preload_libraries = 'pg_cron'/" "$conf_sample"; \printf "\n# Added for pg_cron\ncron.database_name = 'postgres'\n" >> "$conf_sample"-# Create required extensions before executing our SQL-RUN printf 'CREATE EXTENSION IF NOT EXISTS pgcrypto;\nCREATE EXTENSION IF NOT EXISTS pg_cron;\n' > /docker-entrypoint-initdb.d/010-extensions.sql-# Copy nuq.sql so it is executed as part of the initdb sequence-COPY nuq.sql /docker-entrypoint-initdb.d/020-nuq.sql\ No newline at end of file+COPY nuq.sql /docker-entrypoint-initdb.d/010-nuq.sql\ No newline at end of filediff --git a/apps/nuq-postgres/nuq.sql b/apps/nuq-postgres/nuq.sqlindex 5d3ad041c..96977fe3f 100644--- a/apps/nuq-postgres/nuq.sql+++ b/apps/nuq-postgres/nuq.sql@@ -1,3 +1,6 @@+CREATE EXTENSION IF NOT EXISTS pgcrypto;+CREATE EXTENSION IF NOT EXISTS pg_cron;+CREATE SCHEMA IF NOT EXISTS nuq;DO $$ BEGIN