Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(ci): revamp (#2124)

Gergő Móriczfc5f38f23383
diff --git a/.github/k8s/playwright.yml b/.github/k8s/playwright.yml
deleted file mode 100644
index cd33ec6b5..000000000
--- a/.github/k8s/playwright.yml
+++ /dev/null
@@ -1,62 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: firecrawl-playwright-service
-spec:
- replicas: 1
- selector:
- matchLabels:
- app: firecrawl-playwright-service
- template:
- metadata:
- labels:
- app: firecrawl-playwright-service
- spec:
- containers:
- - name: playwright-service
- image: firecrawl/playwright-service:latest
- imagePullPolicy: Never
- envFrom:
- - configMapRef:
- name: firecrawl-config
- - configMapRef:
- name: firecrawl-config-dynamic
- - secretRef:
- name: firecrawl-secret-dynamic
- env:
- - name: PORT
- value: "3003"
- ports:
- - containerPort: 3003
- resources:
- requests:
- memory: "1G"
- cpu: "500m"
- limits:
- memory: "3G"
- cpu: "1000m"
- livenessProbe:
- httpGet:
- path: /health
- port: 3003
- initialDelaySeconds: 30
- periodSeconds: 10
- readinessProbe:
- httpGet:
- path: /health
- port: 3003
- initialDelaySeconds: 30
- periodSeconds: 10
----
-apiVersion: v1
-kind: Service
-metadata:
- name: firecrawl-playwright-service
-spec:
- selector:
- app: firecrawl-playwright-service
- ports:
- - name: http
- protocol: TCP
- port: 3003
- targetPort: 3003
\ No newline at end of file
diff --git a/.github/k8s/searxng.yml b/.github/k8s/searxng.yml
deleted file mode 100644
index 03c9f88e6..000000000
--- a/.github/k8s/searxng.yml
+++ /dev/null
@@ -1,59 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: searxng-settings
-data:
- settings.yml: |
- use_default_settings: true
- search:
- formats: [html, json, csv]
- server:
- secret_key: 'fcsecret'
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: firecrawl-searxng
-spec:
- replicas: 1
- selector:
- matchLabels:
- app: firecrawl-searxng
- template:
- metadata:
- labels:
- app: firecrawl-searxng
- spec:
- containers:
- - name: searxng
- image: searxng/searxng:latest
- imagePullPolicy: IfNotPresent
- ports:
- - containerPort: 8080
- volumeMounts:
- - name: settings
- mountPath: /etc/searxng
- resources:
- requests:
- memory: "256Mi"
- cpu: "100m"
- limits:
- memory: "512Mi"
- cpu: "200m"
- volumes:
- - name: settings
- configMap:
- name: searxng-settings
----
-apiVersion: v1
-kind: Service
-metadata:
- name: firecrawl-searxng
-spec:
- selector:
- app: firecrawl-searxng
- ports:
- - name: http
- protocol: TCP
- port: 8080
- targetPort: 8080
diff --git a/.github/k8s/server-nsh.yml b/.github/k8s/server-nsh.yml
deleted file mode 100644
index 7778ed32a..000000000
--- a/.github/k8s/server-nsh.yml
+++ /dev/null
@@ -1,114 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: tailscale-proxy
-spec:
- replicas: 1
- selector:
- matchLabels:
- app: tailscale-proxy
- template:
- metadata:
- labels:
- app: tailscale-proxy
- spec:
- containers:
- - name: proxy
- image: firecrawl/tailscale-proxy:latest
- imagePullPolicy: Never
- ports:
- - containerPort: 8080
- securityContext:
- capabilities:
- add: ["NET_ADMIN", "NET_RAW"]
- allowPrivilegeEscalation: false
- volumeMounts:
- - name: dev-net-tun
- mountPath: /dev/net/tun
- readinessProbe:
- tcpSocket:
- port: 8080
- initialDelaySeconds: 3
- periodSeconds: 5
- timeoutSeconds: 3
- successThreshold: 1
- failureThreshold: 3
- env:
- - name: TS_AUTHKEY
- valueFrom:
- secretKeyRef:
- name: tailscale
- key: TS_AUTHKEY
- - name: TS_ADVERTISE_TAGS
- valueFrom:
- secretKeyRef:
- name: tailscale
- key: TS_ADVERTISE_TAGS
- - name: FIRE_ENGINE_BETA_URL
- valueFrom:
- secretKeyRef:
- name: tailscale
- key: FIRE_ENGINE_BETA_URL
- resources:
- requests:
- memory: "128Mi"
- cpu: "100m"
- limits:
- memory: "256Mi"
- cpu: "200m"
- volumes:
- - name: dev-net-tun
- hostPath:
- path: /dev/net/tun
- type: CharDevice
----
-apiVersion: v1
-kind: Service
-metadata:
- name: tailscale-proxy
-spec:
- selector:
- app: tailscale-proxy
- ports:
- - name: http
- protocol: TCP
- port: 8080
- targetPort: 8080
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: firecrawl-index-worker
-spec:
- replicas: 1
- selector:
- matchLabels:
- app: firecrawl-index-worker
- template:
- metadata:
- labels:
- app: firecrawl-index-worker
- spec:
- containers:
- - name: index-worker
- image: firecrawl/firecrawl:latest
- imagePullPolicy: Never
- command: [ "node" ]
- args: [ "--max-old-space-size=2048", "dist/src/services/indexing/index-worker.js" ]
- envFrom:
- - configMapRef:
- name: firecrawl-config
- - configMapRef:
- name: firecrawl-config-dynamic
- - secretRef:
- name: firecrawl-secret-dynamic
- env:
- - name: FLY_PROCESS_GROUP
- value: "index-worker"
- resources:
- requests:
- memory: "1G"
- cpu: "500m"
- limits:
- memory: "3G"
- cpu: "1000m"
\ No newline at end of file
diff --git a/.github/k8s/server.yml b/.github/k8s/server.yml
deleted file mode 100644
index 5d55bb080..000000000
--- a/.github/k8s/server.yml
+++ /dev/null
@@ -1,305 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: firecrawl-config
-data:
- HOST: "0.0.0.0"
- REDIS_URL: "redis://firecrawl-dragonfly:6379"
- REDIS_RATE_LIMIT_URL: "redis://firecrawl-dragonfly:6379"
- REDIS_EVICT_URL: "redis://firecrawl-dragonfly:6379"
- SENTRY_ENVIRONMENT: "dev"
- ENV: "production"
- LOGGING_LEVEL: "DEBUG"
- IS_KUBERNETES: "true"
- USE_GO_MARKDOWN_PARSER: "true"
- NUQ_DATABASE_URL: "postgres://postgres:postgres@nuq-postgres:5432/postgres"
- NUQ_DATABASE_URL_LISTEN: "postgres://postgres:postgres@nuq-postgres:5432/postgres"
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: nuq-postgres
-spec:
- replicas: 1
- selector:
- matchLabels:
- app: nuq-postgres
- template:
- metadata:
- labels:
- app: nuq-postgres
- spec:
- containers:
- - name: postgres
- image: firecrawl/nuq-postgres:latest
- imagePullPolicy: Never
- ports:
- - containerPort: 5432
- env:
- - name: POSTGRES_USER
- value: "postgres"
- - name: POSTGRES_PASSWORD
- value: "postgres"
- - name: POSTGRES_DB
- value: "postgres"
- resources:
- requests:
- memory: "512Mi"
- cpu: "250m"
- limits:
- memory: "1Gi"
- cpu: "500m"
-
----
-apiVersion: v1
-kind: Service
-metadata:
- name: nuq-postgres
-spec:
- selector:
- app: nuq-postgres
- ports:
- - name: postgres
- protocol: TCP
- port: 5432
- targetPort: 5432
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: firecrawl-dragonfly
-spec:
- replicas: 1
- selector:
- matchLabels:
- app: firecrawl-dragonfly
- template:
- metadata:
- labels:
- app: firecrawl-dragonfly
- spec:
- containers:
- - name: dragonfly
- image: ghcr.io/dragonflydb/dragonfly:v1.31.2
- args:
- - "--logtostderr"
- - "--cluster_mode=emulated"
- - "--lock_on_hashtags"
- - "--bind=::"
- - "--maxmemory=4gb"
- - "--dir=/data"
- ports:
- - containerPort: 6379
- readinessProbe:
- tcpSocket:
- port: 6379
- initialDelaySeconds: 3
- periodSeconds: 5
- timeoutSeconds: 3
- successThreshold: 1
- failureThreshold: 3
- resources:
- requests:
- memory: "1Gi"
- cpu: "500m"
- limits:
- memory: "4Gi"
- cpu: "1000m"
----
-apiVersion: v1
-kind: Service
-metadata:
- name: firecrawl-dragonfly
-spec:
- selector:
- app: firecrawl-dragonfly
- ports:
- - name: redis
- protocol: TCP
- port: 6379
- targetPort: 6379
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: firecrawl-app
-spec:
- replicas: 1
- selector:
- matchLabels:
- app: firecrawl-app
- template:
- metadata:
- labels:
- app: firecrawl-app
- spec:
- containers:
- - name: firecrawl-app
- image: firecrawl/firecrawl:latest
- imagePullPolicy: Never
- command: ["node"]
- args: ["dist/src/index.js"]
- ports:
- - containerPort: 3002
- envFrom:
- - configMapRef:
- name: firecrawl-config
- - configMapRef:
- name: firecrawl-config-dynamic
- - secretRef:
- name: firecrawl-secret-dynamic
- env:
- - name: FLY_PROCESS_GROUP
- value: "app"
- - name: PORT
- value: "3002"
- - name: NODE_OPTIONS
- value: "--no-node-snapshot --max-old-space-size=6144"
- resources:
- requests:
- memory: "1G"
- cpu: "500m"
- limits:
- memory: "6G"
- cpu: "1000m"
- livenessProbe:
- httpGet:
- path: /v0/health/liveness
- port: 3002
- initialDelaySeconds: 5
- periodSeconds: 5
- timeoutSeconds: 5
- successThreshold: 1
- failureThreshold: 3
- readinessProbe:
- httpGet:
- path: /v0/health/readiness
- port: 3002
- initialDelaySeconds: 5
- periodSeconds: 5
- timeoutSeconds: 5
- successThreshold: 1
- failureThreshold: 3
----
-apiVersion: v1
-kind: Service
-metadata:
- name: firecrawl-app
- namespace: default
- labels:
- app: firecrawl-app
-spec:
- selector:
- app: firecrawl-app
- ports:
- - name: http
- protocol: TCP
- port: 3002
- targetPort: 3002
- type: NodePort
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: firecrawl-worker
-spec:
- replicas: 1
- selector:
- matchLabels:
- app: firecrawl-worker
- template:
- metadata:
- labels:
- app: firecrawl-worker
- spec:
- containers:
- - name: worker
- image: firecrawl/firecrawl:latest
- imagePullPolicy: Never
- command: [ "node" ]
- args: [ "dist/src/services/queue-worker.js", "--max-old-space-size=2048" ]
- envFrom:
- - configMapRef:
- name: firecrawl-config
- - configMapRef:
- name: firecrawl-config-dynamic
- - secretRef:
- name: firecrawl-secret-dynamic
- env:
- - name: FLY_PROCESS_GROUP
- value: "worker"
- - name: PORT
- value: "3005"
- - name: NODE_OPTIONS
- value: "--no-node-snapshot"
- - name: GOMEMLIMIT
- value: "1024MiB"
- - name: SCRAPE_WORKER_MAX_OLD_SPACE_SIZE
- value: "768"
- resources:
- requests:
- memory: "2G"
- cpu: "500m"
- limits:
- memory: "4G"
- cpu: "1000m"
- livenessProbe:
- httpGet:
- path: /liveness
- port: 3005
- initialDelaySeconds: 5
- periodSeconds: 5
- timeoutSeconds: 5
- successThreshold: 1
- failureThreshold: 3
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: firecrawl-nuq-worker
-spec:
- replicas: 5
- selector:
- matchLabels:
- app: firecrawl-nuq-worker
- template:
- metadata:
- labels:
- app: firecrawl-nuq-worker
- spec:
- containers:
- - name: nuq-worker
- image: firecrawl/firecrawl:latest
- imagePullPolicy: Never
- command: [ "node" ]
- args: [ "dist/src/services/worker/nuq-worker.js", "--max-old-space-size=2048" ]
- envFrom:
- - configMapRef:
- name: firecrawl-config
- - configMapRef:
- name: firecrawl-config-dynamic
- - secretRef:
- name: firecrawl-secret-dynamic
- env:
- - name: PORT
- value: "3005"
- - name: NODE_OPTIONS
- value: "--no-node-snapshot"
- - name: GOMEMLIMIT
- value: "1024MiB"
- resources:
- requests:
- memory: "2G"
- cpu: "500m"
- limits:
- memory: "4G"
- cpu: "1000m"
- livenessProbe:
- httpGet:
- path: /health
- port: 3005
- initialDelaySeconds: 10
- periodSeconds: 10
- timeoutSeconds: 10
- successThreshold: 1
- failureThreshold: 3
\ No newline at end of file
diff --git a/.github/k8s/tailscale-proxy/Dockerfile b/.github/k8s/tailscale-proxy/Dockerfile
deleted file mode 100644
index 77bd48e43..000000000
--- a/.github/k8s/tailscale-proxy/Dockerfile
+++ /dev/null
@@ -1,28 +0,0 @@
-FROM debian:bookworm-slim
-
-# Install prerequisites, iptables, nginx, and Tailscale
-RUN apt-get update \
- && apt-get install -y --no-install-recommends \
- ca-certificates \
- curl \
- gnupg \
- iptables \
- iproute2 \
- nginx \
- && curl -fsSL https://pkgs.tailscale.com/stable/debian/bookworm.noarmor.gpg \
- -o /usr/share/keyrings/tailscale-archive-keyring.gpg \
- && curl -fsSL https://pkgs.tailscale.com/stable/debian/bookworm.tailscale-keyring.list \
- -o /etc/apt/sources.list.d/tailscale.list \
- && apt-get update \
- && apt-get install -y --no-install-recommends tailscale \
- && rm -rf /var/lib/apt/lists/*
-
-# Configure nginx as a reverse proxy
-COPY nginx.conf /etc/nginx/nginx.conf.template
-
-# Start script that sets up tailscale and nginx
-COPY entrypoint.sh /entrypoint.sh
-RUN chmod +x /entrypoint.sh
-
-EXPOSE 8080
-ENTRYPOINT ["/entrypoint.sh"]
\ No newline at end of file
diff --git a/.github/k8s/tailscale-proxy/entrypoint.sh b/.github/k8s/tailscale-proxy/entrypoint.sh
deleted file mode 100644
index a1a81639f..000000000
--- a/.github/k8s/tailscale-proxy/entrypoint.sh
+++ /dev/null
@@ -1,48 +0,0 @@
-#!/bin/sh
-set -eu
-
-# Require auth key
-if [ -z "${TS_AUTHKEY:-}" ]; then
- echo "[tailscale-proxy] TS_AUTHKEY is not set; refusing to start." >&2
- exit 1
-fi
-
-# Compute hostname: allow override via TS_HOSTNAME, else default to pod name
-TS_HOSTNAME=${TS_HOSTNAME:-}
-if [ -z "${TS_HOSTNAME}" ]; then
- # Kubernetes sets HOSTNAME to the pod name
- TS_HOSTNAME="firecrawl-proxy-${HOSTNAME:-container}"
-fi
-
-# Start tailscaled
-tailscaled --state=mem: &
-
-# Wait for tailscaled control socket to be ready (up to ~15s)
-i=0
-until tailscale version >/dev/null 2>&1 || [ $i -ge 30 ]; do
- i=$((i+1))
- sleep 0.5
-done
-
-# Prepare optional flags
-ADVERTISE_FLAGS=""
-if [ -n "${TS_ADVERTISE_TAGS:-}" ]; then
- ADVERTISE_FLAGS="--advertise-tags=${TS_ADVERTISE_TAGS}"
-fi
-
-set -x
-tailscale up \
- --authkey="${TS_AUTHKEY}" \
- --hostname="${TS_HOSTNAME}" \
- --accept-routes=true \
- --accept-dns=true \
- --ssh=false \
- ${ADVERTISE_FLAGS} \
- --reset
-set +x
-
-# Replace FIRE_ENGINE_BETA_URL in nginx config
-sed "s|\$FIRE_ENGINE_BETA_URL|${FIRE_ENGINE_BETA_URL}|g" /etc/nginx/nginx.conf.template > /etc/nginx/nginx.conf
-
-# Start nginx
-exec nginx -g "daemon off;"
\ No newline at end of file
diff --git a/.github/k8s/tailscale-proxy/nginx.conf b/.github/k8s/tailscale-proxy/nginx.conf
deleted file mode 100644
index ca48512ac..000000000
--- a/.github/k8s/tailscale-proxy/nginx.conf
+++ /dev/null
@@ -1,24 +0,0 @@
-user www-data;
-worker_processes auto;
-error_log /dev/stderr notice;
-pid /run/nginx.pid;
-
-events {
- worker_connections 1024;
-}
-
-http {
- access_log /dev/stdout combined;
-
- server {
- listen 8080;
-
- location / {
- proxy_pass $FIRE_ENGINE_BETA_URL;
- proxy_set_header Host $host;
- proxy_set_header X-Real-IP $remote_addr;
- proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
- proxy_set_header X-Forwarded-Proto $scheme;
- }
- }
-}
\ No newline at end of file
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.yml
index 973591180..bdb4ab8ac 100644
--- a/.github/workflows/test-server.yml
+++ b/.github/workflows/test-server.yml
@@ -4,12 +4,24 @@ on:
pull_request:
branches:
- main
- - nsc/v2
jobs:
+ audit:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v5
+ - name: Install pnpm
+ uses: pnpm/action-setup@v4
+ with:
+ version: 10
+ - name: Audit NPM Packages
+ run: |
+ pnpm dlx audit-ci@^7 --directory apps/api --config apps/api/audit-ci.jsonc
+ pnpm dlx audit-ci@^7 --directory apps/playwright-service-ts --config apps/playwright-service-ts/audit-ci.jsonc
build-images:
name: Build images
runs-on: big-runner
+ needs: audit
steps:
- uses: actions/checkout@v5
- name: Set up Docker Buildx
@@ -22,22 +34,6 @@ jobs:
--cache-to type=gha,mode=max \
--load \
./apps/api
- - name: Build nuq-postgres image
- run: |
- docker buildx build \
- -t firecrawl/nuq-postgres:latest \
- --cache-from type=gha \
- --cache-to type=gha,mode=max \
- --load \
- ./apps/nuq-postgres
- - name: Build tailscale-proxy image
- run: |
- docker buildx build \
- -t firecrawl/tailscale-proxy:latest \
- --cache-from type=gha \
- --cache-to type=gha,mode=max \
- --load \
- .github/k8s/tailscale-proxy
- name: Build Playwright image
run: |
docker buildx build \
@@ -46,25 +42,8 @@ jobs:
--cache-to type=gha,mode=max \
--load \
./apps/playwright-service-ts
- - name: Export images as tarballs
- run: |
- docker save -o firecrawl-firecrawl.tar firecrawl/firecrawl:latest
- docker save -o firecrawl-nuq-postgres.tar firecrawl/nuq-postgres:latest
- docker save -o firecrawl-tailscale-proxy.tar firecrawl/tailscale-proxy:latest
- docker save -o firecrawl-playwright-service.tar firecrawl/playwright-service:latest
- - name: Upload images artifact
- uses: actions/upload-artifact@v4
- with:
- name: built-images
- path: |
- firecrawl-firecrawl.tar
- firecrawl-nuq-postgres.tar
- firecrawl-tailscale-proxy.tar
- firecrawl-playwright-service.tar
-
test:
- name: Self-hosted (Kubernetes) environment tests
- needs: build-images
+ name: Self-hosted environment tests
strategy:
matrix:
ai: ["openai", "no-ai"]
@@ -74,50 +53,28 @@ jobs:
proxy: ["proxy"]
fail-fast: false
runs-on: big-runner
+ needs: audit
services:
redis:
image: redis
ports:
- 6379:6379
+ env:
+ OPENAI_API_KEY: ${{ matrix.ai == 'openai' && secrets.OPENAI_API_KEY || '' }}
+ SEARXNG_ENDPOINT: ${{ matrix.search == 'searxng' && 'http://localhost:3434' || '' }}
+ PLAYWRIGHT_MICROSERVICE_URL: ${{ matrix.engine == 'playwright' && 'http://localhost:3003/scrape' || '' }}
+ PROXY_SERVER: ${{ matrix.proxy == 'proxy' && secrets.PROXY_SERVER || '' }}
+ PROXY_USERNAME: ${{ matrix.proxy == 'proxy' && secrets.PROXY_USERNAME || '' }}
+ PROXY_PASSWORD: ${{ matrix.proxy == 'proxy' && secrets.PROXY_PASSWORD || '' }}
+ NUQ_DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgres
+ NUQ_REDUCE_NOISE: true
+ HOST: 0.0.0.0
steps:
- uses: actions/checkout@v5
- - name: Start minikube
- uses: medyagh/[email protected]
- - name: Download images artifact
- uses: actions/download-artifact@v4
- with:
- name: built-images
- path: ./images
- - name: Load images into minikube
- run: |
- eval $(minikube docker-env)
- docker load -i ./images/firecrawl-firecrawl.tar
- docker load -i ./images/firecrawl-nuq-postgres.tar
- - name: Load Playwright image into minikube
- if: matrix.engine == 'playwright'
+ - name: Run Docker Postgres
run: |
- eval $(minikube docker-env)
- docker load -i ./images/firecrawl-playwright-service.tar
- - name: Create configmap
- run: |
- kubectl create configmap firecrawl-config-dynamic -n default \
- --from-literal=SEARXNG_ENDPOINT=${{ matrix.search == 'searxng' && 'http://firecrawl-searxng:8080/search' || '' }} \
- --from-literal=PLAYWRIGHT_MICROSERVICE_URL=${{ matrix.engine == 'playwright' && 'http://firecrawl-playwright-service:3003/scrape' || '' }} \
- --from-literal=PROXY_SERVER=${{ matrix.proxy == 'proxy' && secrets.PROXY_SERVER || '' }} \
- --from-literal=PROXY_USERNAME=${{ matrix.proxy == 'proxy' && secrets.PROXY_USERNAME || '' }}
- - name: Create secret
- run: |
- kubectl create secret generic firecrawl-secret-dynamic \
- --from-literal=OPENAI_API_KEY=${{ matrix.ai == 'openai' && secrets.OPENAI_API_KEY || '' }} \
- --from-literal=PROXY_PASSWORD=${{ matrix.proxy == 'proxy' && secrets.PROXY_PASSWORD || '' }}
- - name: Deploy server
- run: kubectl apply -f .github/k8s/server.yml
- - name: Deploy playwright
- if: matrix.engine == 'playwright'
- run: kubectl apply -f .github/k8s/playwright.yml
- - name: Deploy searxng
- if: matrix.search == 'searxng'
- run: kubectl apply -f .github/k8s/searxng.yml
+ docker build -t firecrawl/nuq-postgres:latest ./apps/nuq-postgres
+ docker run -d -p 5432:5432 -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=postgres --name postgres firecrawl/nuq-postgres:latest
- name: Install pnpm
uses: pnpm/action-setup@v4
with:
@@ -125,78 +82,68 @@ jobs:
- name: Set up Node.js
uses: actions/setup-node@v4
with:
- node-version: '20'
+ node-version: '22'
cache: 'pnpm'
cache-dependency-path: './apps/api/pnpm-lock.yaml'
- name: Install dependencies
run: pnpm install
working-directory: ./apps/api
- - name: Wait for server
- run: kubectl wait --for=condition=ready pod -l app=firecrawl-app --timeout=60s -o json
- - name: Debug wait failure (server)
- if: failure()
- run: |
- echo "Pods with label app=firecrawl-app:" || true
- kubectl get pods -l app=firecrawl-app -o wide || true
- echo "\nDescribe pods:" || true
- kubectl describe pods -l app=firecrawl-app || true
- echo "\nRecent events:" || true
- kubectl get events --sort-by=.lastTimestamp | tail -n 100 || true
- echo "\nLogs from firecrawl-app deployment (if any):" || true
- kubectl logs deployment/firecrawl-app --all-containers=true --tail=200 || true
- - name: Wait for playwright
+ - uses: actions-rust-lang/setup-rust-toolchain@v1
+ - name: Install Playwright dependencies
if: matrix.engine == 'playwright'
- run: kubectl wait --for=condition=ready pod -l app=firecrawl-playwright-service --timeout=60s -o json
- - name: Debug wait failure (playwright)
- if: failure() && matrix.engine == 'playwright'
run: |
- echo "Pods with label app=firecrawl-playwright-service:" || true
- kubectl get pods -l app=firecrawl-playwright-service -o wide || true
- echo "\nDescribe pods:" || true
- kubectl describe pods -l app=firecrawl-playwright-service || true
- echo "\nRecent events:" || true
- kubectl get events --sort-by=.lastTimestamp | tail -n 100 || true
- echo "\nLogs from playwright deployment (if any):" || true
- kubectl logs deployment/firecrawl-playwright-service --all-containers=true --tail=200 || true
- - name: Wait for searxng
+ pnpm install
+ pnpm exec playwright install-deps
+ pnpm exec playwright install
+ working-directory: ./apps/playwright-service-ts
+ - name: Set up Go
+ uses: actions/setup-go@v5
+ with:
+ go-version: '1.24'
+ cache-dependency-path: ./apps/api/sharedLibs/go-html-to-md/go.sum
+ - name: Build go-html-to-md
+ run: |
+ go mod tidy
+ go build -o libhtml-to-markdown.so -buildmode=c-shared html-to-markdown.go
+ chmod +x libhtml-to-markdown.so
+ working-directory: ./apps/api/sharedLibs/go-html-to-md
+ - name: Set up SearXNG
if: matrix.search == 'searxng'
- run: kubectl wait --for=condition=ready pod -l app=firecrawl-searxng --timeout=60s -o json
- - name: Debug wait failure (searxng)
- if: failure() && matrix.search == 'searxng'
run: |
- echo "Pods with label app=firecrawl-searxng:" || true
- kubectl get pods -l app=firecrawl-searxng -o wide || true
- echo "\nDescribe pods:" || true
- kubectl describe pods -l app=firecrawl-searxng || true
- echo "\nRecent events:" || true
- kubectl get events --sort-by=.lastTimestamp | tail -n 100 || true
- echo "\nLogs from searxng deployment (if any):" || true
- kubectl logs deployment/firecrawl-searxng --all-containers=true --tail=200 || true
- - name: Run snippet tests
+ mkdir searxng
+
+ echo "use_default_settings: true
+ search:
+ formats: [html, json, csv]
+ server:
+ secret_key: 'fcsecret'" > searxng/settings.yml
+
+ docker run -d -p 3434:8080 -v "${PWD}/searxng:/etc/searxng" --name searxng searxng/searxng
+ pnpx wait-on tcp:3434 -t 30s
+ working-directory: ./
+ - name: Start playwright
+ if: matrix.engine == 'playwright'
+ run: pnpm run dev > playwright.log 2>&1 &
+ working-directory: ./apps/playwright-service-ts
env:
- TEST_SUITE_SELF_HOSTED: true
- OPENAI_API_KEY: ${{ matrix.ai == 'openai' && '<IS SET>' || '' }}
- PROXY_SERVER: ${{ matrix.proxy == 'proxy' && secrets.PROXY_SERVER || '' }}
- PLAYWRIGHT_MICROSERVICE_URL: ${{ matrix.engine == 'playwright' && '<IS SET>' || '' }}
- run: TEST_API_URL=$(minikube service firecrawl-app --url) pnpm run test:snips
+ PORT: 3003
+ - name: Wait for playwright
+ if: matrix.engine == 'playwright'
+ run: pnpx wait-on tcp:3003 -t 15s
+ - name: Start the application and run snippet tests
+ run: pnpm harness pnpm test:snips
working-directory: ./apps/api
- - name: Create logs directory
- if: always()
- run: mkdir -p logs
- name: Copy log files
if: always()
run: |
- kubectl logs deployment/firecrawl-app > logs/api.log || true
- kubectl logs deployment/firecrawl-worker > logs/worker.log || true
- kubectl logs deployment/firecrawl-nuq-worker > logs/nuq-worker.log || true
- kubectl logs deployment/firecrawl-dragonfly > logs/dragonfly.log || true
- kubectl logs deployment/nuq-postgres > logs/postgres.log || true
+ mkdir -p logs
+ cp ./apps/api/firecrawl.log logs/firecrawl.log
- name: Copy SearXNG logs
if: always() && matrix.search == 'searxng'
- run: kubectl logs deployment/firecrawl-searxng > logs/searxng.log || true
+ run: docker logs searxng > logs/searxng.log && docker kill searxng
- name: Copy Playwright logs
if: always() && matrix.engine == 'playwright'
- run: kubectl logs deployment/firecrawl-playwright-service > logs/playwright.log || true
+ run: cp ./apps/playwright-service-ts/playwright.log logs/playwright.log
- name: Zip logs
if: always()
run: |
@@ -208,129 +155,68 @@ jobs:
name: Logs (kubernetes, ${{ matrix.ai }}, ${{ matrix.search }}, ${{ matrix.engine }}, ${{ matrix.proxy }})
path: logs/logs.zip
- docker-test:
- name: Self-hosted (Docker) environment tests
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v5
- - name: Compose
- env:
- PROXY_SERVER: ${{ secrets.PROXY_SERVER }}
- PROXY_USERNAME: ${{ secrets.PROXY_USERNAME }}
- PROXY_PASSWORD: ${{ secrets.PROXY_PASSWORD }}
- run: |
- docker compose up -d
- - name: Install pnpm
- uses: pnpm/action-setup@v4
- with:
- version: 10
- - name: Set up Node.js
- uses: actions/setup-node@v4
- with:
- node-version: "20"
- cache: "pnpm"
- cache-dependency-path: './apps/api/pnpm-lock.yaml'
- - name: Install dependencies
- run: pnpm install
- working-directory: ./apps/api
- - name: Wait for server
- run: pnpx wait-on http://localhost:3002 -t 30s
- - name: Run snippet tests
- env:
- PROXY_SERVER: ${{ secrets.PROXY_SERVER }}
- PLAYWRIGHT_MICROSERVICE_URL: ${{ secrets.PLAYWRIGHT_MICROSERVICE_URL }}
- run: pnpm run test:snips
- working-directory: ./apps/api
- - name: Create logs directory
- if: always()
- run: mkdir -p logs
- - name: Copy log files
- if: always()
- run: |
- docker compose logs api > logs/api.log || true
- docker compose logs playwright-service > logs/playwright.log || true
- docker compose logs nuq-postgres > logs/postgres.log || true
- - name: Zip logs
- if: always()
- run: |
- cd logs
- zip -r logs.zip ./*
- - uses: actions/upload-artifact@v4
- if: always()
- with:
- name: Logs (docker, no-ai, google, playwright, proxy)
- path: logs/logs.zip
-
prod-test:
name: Production environment tests
- needs: build-images
runs-on: big-runner
+ needs: audit
+ services:
+ redis:
+ image: redis
+ ports:
+ - 6379:6379
+ env:
+ BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }}
+ OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+ REDIS_URL: ${{ secrets.REDIS_URL }}
+ SUPABASE_ANON_TOKEN: ${{ secrets.SUPABASE_ANON_TOKEN }}
+ SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }}
+ SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
+ SUPABASE_REPLICA_URL: ${{ secrets.SUPABASE_REPLICA_URL }}
+ INDEX_SUPABASE_SERVICE_TOKEN: ${{ secrets.INDEX_SUPABASE_SERVICE_TOKEN }}
+ INDEX_SUPABASE_ANON_TOKEN: ${{ secrets.INDEX_SUPABASE_ANON_TOKEN }}
+ INDEX_SUPABASE_URL: ${{ secrets.INDEX_SUPABASE_URL }}
+ TEST_API_KEY: ${{ secrets.TEST_API_KEY }}
+ TEST_TEAM_ID: ${{ secrets.TEST_TEAM_ID }}
+ TEST_API_KEY_CONCURRENCY: ${{ secrets.TEST_API_KEY_CONCURRENCY }}
+ TEST_TEAM_ID_CONCURRENCY: ${{ secrets.TEST_TEAM_ID_CONCURRENCY }}
+ TEST_API_KEY_ZDR: ${{ secrets.TEST_API_KEY_ZDR }}
+ TEST_TEAM_ID_ZDR: ${{ secrets.TEST_TEAM_ID_ZDR }}
+ FIRE_ENGINE_BETA_URL: ${{ secrets.FIRE_ENGINE_BETA_URL }}
+ FIRE_ENGINE_STAGING_URL: ${{ secrets.FIRE_ENGINE_STAGING_URL }}
+ USE_DB_AUTHENTICATION: true
+ SERPER_API_KEY: ${{ secrets.SERPER_API_KEY }}
+ ENV: ${{ secrets.ENV }}
+ RUNPOD_MU_POD_ID: ${{ secrets.RUNPOD_MU_POD_ID }}
+ RUNPOD_MUV2_POD_ID: ${{ secrets.RUNPOD_MUV2_POD_ID }}
+ RUNPOD_MU_API_KEY: ${{ secrets.RUNPOD_MU_API_KEY }}
+ GCS_CREDENTIALS: ${{ secrets.GCS_CREDENTIALS }}
+ GCS_BUCKET_NAME: ${{ secrets.GCS_BUCKET_NAME }}
+ GCS_INDEX_BUCKET_NAME: ${{ secrets.GCS_INDEX_BUCKET_NAME }}
+ GCS_MEDIA_BUCKET_NAME: ${{ secrets.GCS_MEDIA_BUCKET_NAME }}
+ GOOGLE_GENERATIVE_AI_API_KEY: ${{ secrets.GOOGLE_GENERATIVE_AI_API_KEY }}
+ GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
+ ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+ VERTEX_CREDENTIALS: ${{ secrets.VERTEX_CREDENTIALS }}
+ USE_GO_MARKDOWN_PARSER: true
+ SENTRY_ENVIRONMENT: dev
+ IDMUX_URL: ${{ secrets.IDMUX_URL }}
+ LOG_ENCRYPTION_KEY: ${{ secrets.LOG_ENCRYPTION_KEY }}
+ NUQ_DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgres
+ NUQ_REDUCE_NOISE: true
+ HOST: 0.0.0.0
steps:
- uses: actions/checkout@v5
- - name: Start minikube
- uses: medyagh/[email protected]
- - name: Download images artifact
- uses: actions/download-artifact@v4
- with:
- name: built-images
- path: ./images
+ - name: Run Docker Postgres
+ run: |
+ docker build -t firecrawl/nuq-postgres:latest ./apps/nuq-postgres
+ docker run -d -p 5432:5432 -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=postgres --name postgres firecrawl/nuq-postgres:latest
- name: Tailscale
uses: tailscale/github-action@v3
with:
oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }}
oauth-secret: ${{ secrets.TS_OAUTH_SECRET }}
tags: tag:ci
- - name: Load prebuilt images into minikube
- run: |
- eval $(minikube docker-env)
- docker load -i ./images/firecrawl-firecrawl.tar
- docker load -i ./images/firecrawl-nuq-postgres.tar
- docker load -i ./images/firecrawl-tailscale-proxy.tar
- - name: Create Tailscale secret
- run: |
- if [ -z "${{ secrets.TS_AUTHKEY }}" ]; then
- echo "secrets.TS_AUTHKEY is empty; set a reusable/preauthorized or ephemeral key" >&2
- exit 1
- fi
- kubectl create secret generic tailscale \
- --from-literal=TS_AUTHKEY=${{ secrets.TS_AUTHKEY }} \
- --from-literal=TS_ADVERTISE_TAGS=${{ secrets.TS_ADVERTISE_TAGS }} \
- --from-literal=FIRE_ENGINE_BETA_URL=${{ secrets.FIRE_ENGINE_BETA_URL }} \
- --dry-run=client -o yaml | kubectl apply -f -
- - name: Create configmap
- run: |
- kubectl create configmap firecrawl-config-dynamic -n default \
- --from-literal=USE_DB_AUTHENTICATION=true \
- --from-literal=SUPABASE_ANON_TOKEN=${{ secrets.SUPABASE_ANON_TOKEN }} \
- --from-literal=SUPABASE_URL=${{ secrets.SUPABASE_URL }} \
- --from-literal=SUPABASE_REPLICA_URL=${{ secrets.SUPABASE_REPLICA_URL }} \
- --from-literal=INDEX_SUPABASE_ANON_TOKEN=${{ secrets.INDEX_SUPABASE_ANON_TOKEN }} \
- --from-literal=INDEX_SUPABASE_URL=${{ secrets.INDEX_SUPABASE_URL }} \
- --from-literal=FIRE_ENGINE_BETA_URL=http://tailscale-proxy:8080 \
- --from-literal=RUNPOD_MU_POD_ID=${{ secrets.RUNPOD_MU_POD_ID }} \
- --from-literal=RUNPOD_MUV2_POD_ID=${{ secrets.RUNPOD_MUV2_POD_ID }} \
- --from-literal=GCS_BUCKET_NAME=${{ secrets.GCS_BUCKET_NAME }} \
- --from-literal=GCS_MEDIA_BUCKET_NAME=${{ secrets.GCS_MEDIA_BUCKET_NAME }} \
- --from-literal=GCS_INDEX_BUCKET_NAME=${{ secrets.GCS_INDEX_BUCKET_NAME }}
- - name: Create secret
- run: |
- kubectl create secret generic firecrawl-secret-dynamic \
- --from-literal=BULL_AUTH_KEY=${{ secrets.BULL_AUTH_KEY }} \
- --from-literal=OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }} \
- --from-literal=SUPABASE_SERVICE_TOKEN=${{ secrets.SUPABASE_SERVICE_TOKEN }} \
- --from-literal=INDEX_SUPABASE_SERVICE_TOKEN=${{ secrets.INDEX_SUPABASE_SERVICE_TOKEN }} \
- --from-literal=SERPER_API_KEY=${{ secrets.SERPER_API_KEY }} \
- --from-literal=RUNPOD_MU_API_KEY=${{ secrets.RUNPOD_MU_API_KEY }} \
- --from-literal=GCS_CREDENTIALS=${{ secrets.GCS_CREDENTIALS }} \
- --from-literal=GOOGLE_GENERATIVE_AI_API_KEY=${{ secrets.GOOGLE_GENERATIVE_AI_API_KEY }} \
- --from-literal=GROQ_API_KEY=${{ secrets.GROQ_API_KEY }} \
- --from-literal=ANTHROPIC_API_KEY=${{ secrets.ANTHROPIC_API_KEY }} \
- --from-literal=VERTEX_CREDENTIALS=${{ secrets.VERTEX_CREDENTIALS }} \
- --from-literal=LOG_ENCRYPTION_KEY=${{ secrets.LOG_ENCRYPTION_KEY }}
- - name: Deploy server
- run: |
- kubectl apply -f .github/k8s/server.yml
- kubectl apply -f .github/k8s/server-nsh.yml
+ use-cache: 'true'
- name: Install pnpm
uses: pnpm/action-setup@v4
with:
@@ -338,90 +224,31 @@ jobs:
- name: Set up Node.js
uses: actions/setup-node@v4
with:
- node-version: "20"
- cache: "pnpm"
+ node-version: '22'
+ cache: 'pnpm'
cache-dependency-path: './apps/api/pnpm-lock.yaml'
- name: Install dependencies
run: pnpm install
working-directory: ./apps/api
- - name: Wait for API
- run: kubectl wait --for=condition=ready pod -l app=firecrawl-app --timeout=60s -o json
- - name: Debug wait failure (API)
- if: failure()
- run: |
- echo "Pods with label app=firecrawl-app:" || true
- kubectl get pods -l app=firecrawl-app -o wide || true
- echo "\nDescribe pods:" || true
- kubectl describe pods -l app=firecrawl-app || true
- echo "\nRecent events:" || true
- kubectl get events --sort-by=.lastTimestamp | tail -n 100 || true
- - name: Wait for nuq-postgres
- run: kubectl wait --for=condition=ready pod -l app=nuq-postgres --timeout=60s -o json
- - name: Debug wait failure (nuq-postgres)
- if: failure()
- run: |
- echo "Pods with label app=nuq-postgres:" || true
- kubectl get pods -l app=nuq-postgres -o wide || true
- echo "\nDescribe pods:" || true
- kubectl describe pods -l app=nuq-postgres || true
- echo "\nRecent events:" || true
- kubectl get events --sort-by=.lastTimestamp | tail -n 100 || true
- echo "\nLogs from nuq-postgres deployment (if any):" || true
- kubectl logs deployment/nuq-postgres --all-containers=true --tail=200 || true
- - name: Wait for tailscale-proxy
- run: kubectl wait --for=condition=ready pod -l app=tailscale-proxy --timeout=60s -o json
- - name: Debug wait failure (tailscale-proxy)
- if: failure()
- run: |
- echo "Pods with label app=tailscale-proxy:" || true
- kubectl get pods -l app=tailscale-proxy -o wide || true
- echo "\nDescribe pods:" || true
- kubectl describe pods -l app=tailscale-proxy || true
- echo "\nRecent events:" || true
- kubectl get events --sort-by=.lastTimestamp | tail -n 100 || true
- - name: Wait for dragonfly (redis)
- run: kubectl wait --for=condition=ready pod -l app=firecrawl-dragonfly --timeout=60s -o json
- - name: Debug wait failure (dragonfly)
- if: failure()
- run: |
- echo "Pods with label app=firecrawl-dragonfly:" || true
- kubectl get pods -l app=firecrawl-dragonfly -o wide || true
- echo "\nDescribe pods:" || true
- kubectl describe pods -l app=firecrawl-dragonfly || true
- echo "\nRecent events:" || true
- kubectl get events --sort-by=.lastTimestamp | tail -n 100 || true
- - name: Run snippet tests
- env:
- IDMUX_URL: ${{ secrets.IDMUX_URL }}
- USE_DB_AUTHENTICATION: true
- SUPABASE_ANON_TOKEN: ${{ secrets.SUPABASE_ANON_TOKEN }}
- SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
- SUPABASE_REPLICA_URL: ${{ secrets.SUPABASE_REPLICA_URL }}
- SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }}
- GCS_BUCKET_NAME: ${{ secrets.GCS_BUCKET_NAME }}
- GCS_CREDENTIALS: ${{ secrets.GCS_CREDENTIALS }}
+ - name: Set up Go
+ uses: actions/setup-go@v5
+ with:
+ go-version: '1.24'
+ cache-dependency-path: ./apps/api/sharedLibs/go-html-to-md/go.sum
+ - name: Build go-html-to-md
run: |
- TEST_API_URL=$(minikube service firecrawl-app --url) pnpm run test:snips
+ go mod tidy
+ go build -o libhtml-to-markdown.so -buildmode=c-shared html-to-markdown.go
+ chmod +x libhtml-to-markdown.so
+ working-directory: ./apps/api/sharedLibs/go-html-to-md
+ - name: Build the application and run snippet tests
+ run: pnpm harness pnpm test:snips
working-directory: ./apps/api
- - name: Kill instances
- if: always()
- run: pkill -9 node
- name: Create logs directory
- if: always()
- run: mkdir -p logs
- - name: Copy log files
- if: always()
- run: |
- kubectl logs deployment/firecrawl-app > logs/api.log || true
- kubectl logs deployment/firecrawl-worker > logs/worker.log || true
- kubectl logs deployment/firecrawl-index-worker > logs/index-worker.log || true
- kubectl logs deployment/firecrawl-nuq-worker > logs/nuq-worker.log || true
- kubectl logs deployment/firecrawl-dragonfly > logs/dragonfly.log || true
- kubectl logs deployment/nuq-postgres > logs/postgres.log || true
- kubectl logs deployment/tailscale-proxy > logs/tailscale-proxy.log || true
- - name: Zip and encrypt logs
if: always()
run: |
+ mkdir -p logs
+ cp ./apps/api/firecrawl.log logs/firecrawl.log
cd logs
zip -r logs.zip ./*
echo "${{ secrets.LOG_ENCRYPTION_KEY }}" | gpg --batch --yes --passphrase-fd 0 -c logs.zip
diff --git a/apps/api/.gitignore b/apps/api/.gitignore
index 52345155e..e31e58112 100644
--- a/apps/api/.gitignore
+++ b/apps/api/.gitignore
@@ -10,4 +10,6 @@ dump.rdb
.rdb
.sentryclirc
-.env.*
\ No newline at end of file
+.env.*
+
+firecrawl.log
\ No newline at end of file
diff --git a/apps/api/jest.config.js b/apps/api/jest.config.js
deleted file mode 100644
index 285445232..000000000
--- a/apps/api/jest.config.js
+++ /dev/null
@@ -1,8 +0,0 @@
-module.exports = {
- preset: "ts-jest",
- testEnvironment: "node",
- setupFiles: ["./jest.setup.js"],
- // ignore dist folder root dir
- modulePathIgnorePatterns: ["<rootDir>/dist/"],
-
-};
diff --git a/apps/api/jest.config.ts b/apps/api/jest.config.ts
new file mode 100644
index 000000000..c25eb711f
--- /dev/null
+++ b/apps/api/jest.config.ts
@@ -0,0 +1,14 @@
+import type { Config } from "jest";
+
+const config: Config = {
+ verbose: true,
+ preset: "ts-jest",
+ testEnvironment: "node",
+ testPathIgnorePatterns: ["<rootDir>/dist/"],
+ forceExit: true,
+ detectOpenHandles: true,
+ openHandlesTimeout: 120000,
+ watchAll: false,
+};
+
+export default config;
diff --git a/apps/api/jest.setup.js b/apps/api/jest.setup.js
deleted file mode 100644
index 0b3b09b7c..000000000
--- a/apps/api/jest.setup.js
+++ /dev/null
@@ -1 +0,0 @@
-// global.fetch = require('jest-fetch-mock');
diff --git a/apps/api/package.json b/apps/api/package.json
index 1716cd45a..5f9b70dd2 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -14,11 +14,11 @@
"start:dev": "tsc-watch --onSuccess \"node dist/src/index.js\"",
"build": "tsc",
"build:nosentry": "tsc",
- "test": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false --testPathIgnorePatterns='src/__tests__/e2e_noAuth/*'",
- "test:local-no-auth": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false --testPathIgnorePatterns='src/__tests__/e2e_withAuth/*'",
- "test:full": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false --testPathIgnorePatterns='(src/__tests__/e2e_noAuth|src/__tests__/e2e_withAuth)'",
- "test:prod": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false --testPathIgnorePatterns='(src/__tests__/e2e_noAuth|src/__tests__/e2e_full_withAuth|src/scraper/scrapeURL)'",
- "test:snips": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false src/__tests__/snips/**/*.test.ts",
+ "test": "jest --testPathIgnorePatterns=\"src/__tests__/e2e_noAuth/*\"",
+ "test:local-no-auth": "jest --testPathIgnorePatterns=\"src/__tests__/e2e_withAuth/*\"",
+ "test:full": "jest --testPathIgnorePatterns=\"(src/__tests__/e2e_noAuth|src/__tests__/e2e_withAuth)\"",
+ "test:prod": "jest --testPathIgnorePatterns=\"(src/__tests__/e2e_noAuth|src/__tests__/e2e_full_withAuth|src/scraper/scrapeURL)\"",
+ "test:snips": "jest \"src/__tests__/snips/**/*.test.ts\"",
"harness": "tsx src/harness.ts",
"workers": "tsc-watch --onSuccess \"node dist/src/services/queue-worker.js\"",
"worker:production": "node dist/src/services/queue-worker.js",
diff --git a/apps/api/src/__tests__/snips/v1/zdr.test.ts b/apps/api/src/__tests__/snips/v1/zdr.test.ts
index 27d7b81db..41da3a5cb 100644
--- a/apps/api/src/__tests__/snips/v1/zdr.test.ts
+++ b/apps/api/src/__tests__/snips/v1/zdr.test.ts
@@ -1,4 +1,3 @@
-import { exec, spawn } from "node:child_process";
import { getJobFromGCS } from "../../../lib/gcs-jobs";
import { supabase_service } from "../../../services/supabase";
import {
@@ -31,66 +30,15 @@ if (process.env.TEST_SUITE_SELF_HOSTED) {
expect(true).toBe(true);
});
} else {
- function getOutput(command: string): Promise<string> {
- return new Promise((resolve, reject) => {
- try {
- const cmd = spawn(command, { shell: true });
- let out = "";
- cmd.stdout?.on("data", data => {
- out += data;
- });
- cmd.stderr?.on("data", data => {
- out += data;
- });
- cmd.on("error", e => {
- reject(e);
- });
- cmd.on("close", code => {
- if (code !== 0) {
- reject(new Error(`Command ${command} failed with code ${code}`));
- } else {
- resolve(out);
- }
- });
- } catch (e) {
- reject(e);
- }
- });
- }
-
- async function getServerLogs() {
+ async function getLogs() {
let logs: string;
- if (!process.env.GITHUB_ACTIONS) {
- try {
- await stat("api.log");
- } catch (e) {
- console.warn("No api.log file found");
- return [];
- }
- logs = await readFile("api.log", "utf8");
- } else {
- logs = await getOutput("kubectl logs deployment/firecrawl-app");
- }
- return logs
- .split("\n")
- .filter(
- x => x.trim().length > 0 && !logIgnoreList.some(y => x.includes(y)),
- );
- }
-
- async function getWorkerLogs() {
- let logs: string;
- if (!process.env.GITHUB_ACTIONS) {
- try {
- await stat("worker.log");
- } catch (e) {
- console.warn("No worker.log file found");
- return [];
- }
- logs = await readFile("worker.log", "utf8");
- } else {
- logs = await getOutput("kubectl logs deployment/firecrawl-nuq-worker");
+ try {
+ await stat("firecrawl.log");
+ } catch (e) {
+ console.warn("No firecrawl.log file found");
+ return [];
}
+ logs = await readFile("firecrawl.log", "utf8");
return logs
.split("\n")
.filter(
@@ -154,8 +102,7 @@ if (process.env.TEST_SUITE_SELF_HOSTED) {
}, 60000);
it("should clean up a crawl", async () => {
- const preServerLogs = await getServerLogs();
- const preWorkerLogs = await getWorkerLogs();
+ const preLogs = await getLogs();
let identity = await idmux({
name: `zdr/${scope}/crawl`,
@@ -181,23 +128,13 @@ if (process.env.TEST_SUITE_SELF_HOSTED) {
await new Promise(resolve => setTimeout(resolve, 2500));
- const postServerLogs = (await getServerLogs()).slice(
- preServerLogs.length,
- );
- const postWorkerLogs = (await getWorkerLogs()).slice(
- preWorkerLogs.length,
- );
+ const postLogs = (await getLogs()).slice(preLogs.length);
- if (postWorkerLogs.length > 0 || postServerLogs.length > 0) {
- console.warn(
- "Logs changed during crawl",
- postServerLogs,
- postWorkerLogs,
- );
+ if (postLogs.length > 0) {
+ console.warn("Logs changed during crawl", postLogs);
}
- expect(postServerLogs).toHaveLength(0);
- expect(postWorkerLogs).toHaveLength(0);
+ expect(postLogs).toHaveLength(0);
const { data, error } = await supabase_service
.from("firecrawl_jobs")
@@ -251,8 +188,7 @@ if (process.env.TEST_SUITE_SELF_HOSTED) {
}, 600000);
it("should clean up a batch scrape", async () => {
- const preServerLogs = await getServerLogs();
- const preWorkerLogs = await getWorkerLogs();
+ const preLogs = await getLogs();
let identity = await idmux({
name: `zdr/${scope}/batch-scrape`,
@@ -277,23 +213,13 @@ if (process.env.TEST_SUITE_SELF_HOSTED) {
await new Promise(resolve => setTimeout(resolve, 2500));
- const postServerLogs = (await getServerLogs()).slice(
- preServerLogs.length,
- );
- const postWorkerLogs = (await getWorkerLogs()).slice(
- preWorkerLogs.length,
- );
+ const postLogs = (await getLogs()).slice(preLogs.length);
- if (postWorkerLogs.length > 0 || postServerLogs.length > 0) {
- console.warn(
- "Logs changed during batch scrape",
- postServerLogs,
- postWorkerLogs,
- );
+ if (postLogs.length > 0) {
+ console.warn("Logs changed during batch scrape", postLogs);
}
- expect(postServerLogs).toHaveLength(0);
- expect(postWorkerLogs).toHaveLength(0);
+ expect(postLogs).toHaveLength(0);
const { data, error } = await supabase_service
.from("firecrawl_jobs")
diff --git a/apps/api/src/__tests__/snips/v2/zdr.test.ts b/apps/api/src/__tests__/snips/v2/zdr.test.ts
index 61937aa7c..4cae9a903 100644
--- a/apps/api/src/__tests__/snips/v2/zdr.test.ts
+++ b/apps/api/src/__tests__/snips/v2/zdr.test.ts
@@ -9,7 +9,6 @@ import {
idmux,
} from "./lib";
import { readFile, stat } from "node:fs/promises";
-import { spawn } from "node:child_process";
const logIgnoreList = [
"Billing queue created",
@@ -31,66 +30,15 @@ if (process.env.TEST_SUITE_SELF_HOSTED) {
expect(true).toBe(true);
});
} else {
- function getOutput(command: string): Promise<string> {
- return new Promise((resolve, reject) => {
- try {
- const cmd = spawn(command, { shell: true });
- let out = "";
- cmd.stdout?.on("data", data => {
- out += data;
- });
- cmd.stderr?.on("data", data => {
- out += data;
- });
- cmd.on("error", e => {
- reject(e);
- });
- cmd.on("close", code => {
- if (code !== 0) {
- reject(new Error(`Command ${command} failed with code ${code}`));
- } else {
- resolve(out);
- }
- });
- } catch (e) {
- reject(e);
- }
- });
- }
-
- async function getServerLogs() {
+ async function getLogs() {
let logs: string;
- if (!process.env.GITHUB_ACTIONS) {
- try {
- await stat("api.log");
- } catch (e) {
- console.warn("No api.log file found");
- return [];
- }
- logs = await readFile("api.log", "utf8");
- } else {
- logs = await getOutput("kubectl logs deployment/firecrawl-app");
- }
- return logs
- .split("\n")
- .filter(
- x => x.trim().length > 0 && !logIgnoreList.some(y => x.includes(y)),
- );
- }
-
- async function getWorkerLogs() {
- let logs: string;
- if (!process.env.GITHUB_ACTIONS) {
- try {
- await stat("worker.log");
- } catch (e) {
- console.warn("No worker.log file found");
- return [];
- }
- logs = await readFile("worker.log", "utf8");
- } else {
- logs = await getOutput("kubectl logs deployment/firecrawl-nuq-worker");
+ try {
+ await stat("firecrawl.log");
+ } catch (e) {
+ console.warn("No firecrawl.log file found");
+ return [];
}
+ logs = await readFile("firecrawl.log", "utf8");
return logs
.split("\n")
.filter(
@@ -154,8 +102,7 @@ if (process.env.TEST_SUITE_SELF_HOSTED) {
}, 60000);
it("should clean up a crawl", async () => {
- const preServerLogs = await getServerLogs();
- const preWorkerLogs = await getWorkerLogs();
+ const preLogs = await getLogs();
let identity = await idmux({
name: `zdr/${scope}/crawl`,
@@ -179,23 +126,13 @@ if (process.env.TEST_SUITE_SELF_HOSTED) {
identity,
);
- const postServerLogs = (await getServerLogs()).slice(
- preServerLogs.length,
- );
- const postWorkerLogs = (await getWorkerLogs()).slice(
- preWorkerLogs.length,
- );
+ const postLogs = (await getLogs()).slice(preLogs.length);
- if (postWorkerLogs.length > 0 || postServerLogs.length > 0) {
- console.warn(
- "Logs changed during crawl",
- postServerLogs,
- postWorkerLogs,
- );
+ if (postLogs.length > 0) {
+ console.warn("Logs changed during crawl", postLogs);
}
- expect(postServerLogs).toHaveLength(0);
- expect(postWorkerLogs).toHaveLength(0);
+ expect(postLogs).toHaveLength(0);
const { data, error } = await supabase_service
.from("firecrawl_jobs")
@@ -249,8 +186,7 @@ if (process.env.TEST_SUITE_SELF_HOSTED) {
}, 600000);
it("should clean up a batch scrape", async () => {
- const preServerLogs = await getServerLogs();
- const preWorkerLogs = await getWorkerLogs();
+ const preLogs = await getLogs();
let identity = await idmux({
name: `zdr/${scope}/batch-scrape`,
@@ -273,23 +209,13 @@ if (process.env.TEST_SUITE_SELF_HOSTED) {
identity,
);
- const postServerLogs = (await getServerLogs()).slice(
- preServerLogs.length,
- );
- const postWorkerLogs = (await getWorkerLogs()).slice(
- preWorkerLogs.length,
- );
+ const postLogs = (await getLogs()).slice(preLogs.length);
- if (postWorkerLogs.length > 0 || postServerLogs.length > 0) {
- console.warn(
- "Logs changed during batch scrape",
- postServerLogs,
- postWorkerLogs,
- );
+ if (postLogs.length > 0) {
+ console.warn("Logs changed during batch scrape", postLogs);
}
- expect(postServerLogs).toHaveLength(0);
- expect(postWorkerLogs).toHaveLength(0);
+ expect(postLogs).toHaveLength(0);
const { data, error } = await supabase_service
.from("firecrawl_jobs")
diff --git a/apps/api/src/harness.ts b/apps/api/src/harness.ts
index b9c9a4f76..dea00afb4 100644
--- a/apps/api/src/harness.ts
+++ b/apps/api/src/harness.ts
@@ -3,6 +3,7 @@ import { type ChildProcess, spawn } from "child_process";
import * as net from "net";
import { basename } from "path";
import { HTML_TO_MARKDOWN_PATH } from "./natives";
+import { createWriteStream } from "fs";
const childProcesses = new Set<ChildProcess>();
@@ -64,6 +65,8 @@ function formatDuration(nanoseconds: bigint): string {
return `${minutes}m ${remainingSeconds.toFixed(0)}s`;
}
+const stream = createWriteStream("firecrawl.log");
+
const logger = {
section(message: string) {
console.log(
@@ -110,6 +113,7 @@ const logger = {
const color = getProcessColor(name);
const label = `${color}${name.padEnd(12)}${colors.reset}`;
console.log(`${label} ${line}`);
+ stream.write(`${name.padEnd(12)} ${line}\n`);
},
};
diff --git a/apps/nuq-postgres/Dockerfile b/apps/nuq-postgres/Dockerfile
index 942b2fbe1..cbbe41cd9 100644
--- a/apps/nuq-postgres/Dockerfile
+++ b/apps/nuq-postgres/Dockerfile
@@ -17,8 +17,5 @@ RUN set -eux; \
sed -ri "s/^#?shared_preload_libraries\s*=.*/shared_preload_libraries = 'pg_cron'/" "$conf_sample"; \
printf "\n# Added for pg_cron\ncron.database_name = 'postgres'\n" >> "$conf_sample"
-# Create required extensions before executing our SQL
-RUN printf 'CREATE EXTENSION IF NOT EXISTS pgcrypto;\nCREATE EXTENSION IF NOT EXISTS pg_cron;\n' > /docker-entrypoint-initdb.d/010-extensions.sql
-
# Copy nuq.sql so it is executed as part of the initdb sequence
-COPY nuq.sql /docker-entrypoint-initdb.d/020-nuq.sql
\ No newline at end of file
+COPY nuq.sql /docker-entrypoint-initdb.d/010-nuq.sql
\ No newline at end of file
diff --git a/apps/nuq-postgres/nuq.sql b/apps/nuq-postgres/nuq.sql
index 5d3ad041c..96977fe3f 100644
--- a/apps/nuq-postgres/nuq.sql
+++ b/apps/nuq-postgres/nuq.sql
@@ -1,3 +1,6 @@
+CREATE EXTENSION IF NOT EXISTS pgcrypto;
+CREATE EXTENSION IF NOT EXISTS pg_cron;
+
CREATE SCHEMA IF NOT EXISTS nuq;
DO $$ BEGIN