Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): nuq (#1984)

Gergő Móriczfc7be6708f1c
diff --git a/.github/k8s/playwright.yml b/.github/k8s/playwright.yml
new file mode 100644
index 000000000..cd33ec6b5
--- /dev/null
+++ b/.github/k8s/playwright.yml
@@ -0,0 +1,62 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: firecrawl-playwright-service
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: firecrawl-playwright-service
+ template:
+ metadata:
+ labels:
+ app: firecrawl-playwright-service
+ spec:
+ containers:
+ - name: playwright-service
+ image: firecrawl/playwright-service:latest
+ imagePullPolicy: Never
+ envFrom:
+ - configMapRef:
+ name: firecrawl-config
+ - configMapRef:
+ name: firecrawl-config-dynamic
+ - secretRef:
+ name: firecrawl-secret-dynamic
+ env:
+ - name: PORT
+ value: "3003"
+ ports:
+ - containerPort: 3003
+ resources:
+ requests:
+ memory: "1G"
+ cpu: "500m"
+ limits:
+ memory: "3G"
+ cpu: "1000m"
+ livenessProbe:
+ httpGet:
+ path: /health
+ port: 3003
+ initialDelaySeconds: 30
+ periodSeconds: 10
+ readinessProbe:
+ httpGet:
+ path: /health
+ port: 3003
+ initialDelaySeconds: 30
+ periodSeconds: 10
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: firecrawl-playwright-service
+spec:
+ selector:
+ app: firecrawl-playwright-service
+ ports:
+ - name: http
+ protocol: TCP
+ port: 3003
+ targetPort: 3003
\ No newline at end of file
diff --git a/.github/k8s/searxng.yml b/.github/k8s/searxng.yml
new file mode 100644
index 000000000..03c9f88e6
--- /dev/null
+++ b/.github/k8s/searxng.yml
@@ -0,0 +1,59 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: searxng-settings
+data:
+ settings.yml: |
+ use_default_settings: true
+ search:
+ formats: [html, json, csv]
+ server:
+ secret_key: 'fcsecret'
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: firecrawl-searxng
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: firecrawl-searxng
+ template:
+ metadata:
+ labels:
+ app: firecrawl-searxng
+ spec:
+ containers:
+ - name: searxng
+ image: searxng/searxng:latest
+ imagePullPolicy: IfNotPresent
+ ports:
+ - containerPort: 8080
+ volumeMounts:
+ - name: settings
+ mountPath: /etc/searxng
+ resources:
+ requests:
+ memory: "256Mi"
+ cpu: "100m"
+ limits:
+ memory: "512Mi"
+ cpu: "200m"
+ volumes:
+ - name: settings
+ configMap:
+ name: searxng-settings
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: firecrawl-searxng
+spec:
+ selector:
+ app: firecrawl-searxng
+ ports:
+ - name: http
+ protocol: TCP
+ port: 8080
+ targetPort: 8080
diff --git a/.github/k8s/server-nsh.yml b/.github/k8s/server-nsh.yml
new file mode 100644
index 000000000..7778ed32a
--- /dev/null
+++ b/.github/k8s/server-nsh.yml
@@ -0,0 +1,114 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: tailscale-proxy
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: tailscale-proxy
+ template:
+ metadata:
+ labels:
+ app: tailscale-proxy
+ spec:
+ containers:
+ - name: proxy
+ image: firecrawl/tailscale-proxy:latest
+ imagePullPolicy: Never
+ ports:
+ - containerPort: 8080
+ securityContext:
+ capabilities:
+ add: ["NET_ADMIN", "NET_RAW"]
+ allowPrivilegeEscalation: false
+ volumeMounts:
+ - name: dev-net-tun
+ mountPath: /dev/net/tun
+ readinessProbe:
+ tcpSocket:
+ port: 8080
+ initialDelaySeconds: 3
+ periodSeconds: 5
+ timeoutSeconds: 3
+ successThreshold: 1
+ failureThreshold: 3
+ env:
+ - name: TS_AUTHKEY
+ valueFrom:
+ secretKeyRef:
+ name: tailscale
+ key: TS_AUTHKEY
+ - name: TS_ADVERTISE_TAGS
+ valueFrom:
+ secretKeyRef:
+ name: tailscale
+ key: TS_ADVERTISE_TAGS
+ - name: FIRE_ENGINE_BETA_URL
+ valueFrom:
+ secretKeyRef:
+ name: tailscale
+ key: FIRE_ENGINE_BETA_URL
+ resources:
+ requests:
+ memory: "128Mi"
+ cpu: "100m"
+ limits:
+ memory: "256Mi"
+ cpu: "200m"
+ volumes:
+ - name: dev-net-tun
+ hostPath:
+ path: /dev/net/tun
+ type: CharDevice
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: tailscale-proxy
+spec:
+ selector:
+ app: tailscale-proxy
+ ports:
+ - name: http
+ protocol: TCP
+ port: 8080
+ targetPort: 8080
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: firecrawl-index-worker
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: firecrawl-index-worker
+ template:
+ metadata:
+ labels:
+ app: firecrawl-index-worker
+ spec:
+ containers:
+ - name: index-worker
+ image: firecrawl/firecrawl:latest
+ imagePullPolicy: Never
+ command: [ "node" ]
+ args: [ "--max-old-space-size=2048", "dist/src/services/indexing/index-worker.js" ]
+ envFrom:
+ - configMapRef:
+ name: firecrawl-config
+ - configMapRef:
+ name: firecrawl-config-dynamic
+ - secretRef:
+ name: firecrawl-secret-dynamic
+ env:
+ - name: FLY_PROCESS_GROUP
+ value: "index-worker"
+ resources:
+ requests:
+ memory: "1G"
+ cpu: "500m"
+ limits:
+ memory: "3G"
+ cpu: "1000m"
\ No newline at end of file
diff --git a/.github/k8s/server.yml b/.github/k8s/server.yml
new file mode 100644
index 000000000..5d55bb080
--- /dev/null
+++ b/.github/k8s/server.yml
@@ -0,0 +1,305 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: firecrawl-config
+data:
+ HOST: "0.0.0.0"
+ REDIS_URL: "redis://firecrawl-dragonfly:6379"
+ REDIS_RATE_LIMIT_URL: "redis://firecrawl-dragonfly:6379"
+ REDIS_EVICT_URL: "redis://firecrawl-dragonfly:6379"
+ SENTRY_ENVIRONMENT: "dev"
+ ENV: "production"
+ LOGGING_LEVEL: "DEBUG"
+ IS_KUBERNETES: "true"
+ USE_GO_MARKDOWN_PARSER: "true"
+ NUQ_DATABASE_URL: "postgres://postgres:postgres@nuq-postgres:5432/postgres"
+ NUQ_DATABASE_URL_LISTEN: "postgres://postgres:postgres@nuq-postgres:5432/postgres"
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: nuq-postgres
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: nuq-postgres
+ template:
+ metadata:
+ labels:
+ app: nuq-postgres
+ spec:
+ containers:
+ - name: postgres
+ image: firecrawl/nuq-postgres:latest
+ imagePullPolicy: Never
+ ports:
+ - containerPort: 5432
+ env:
+ - name: POSTGRES_USER
+ value: "postgres"
+ - name: POSTGRES_PASSWORD
+ value: "postgres"
+ - name: POSTGRES_DB
+ value: "postgres"
+ resources:
+ requests:
+ memory: "512Mi"
+ cpu: "250m"
+ limits:
+ memory: "1Gi"
+ cpu: "500m"
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: nuq-postgres
+spec:
+ selector:
+ app: nuq-postgres
+ ports:
+ - name: postgres
+ protocol: TCP
+ port: 5432
+ targetPort: 5432
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: firecrawl-dragonfly
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: firecrawl-dragonfly
+ template:
+ metadata:
+ labels:
+ app: firecrawl-dragonfly
+ spec:
+ containers:
+ - name: dragonfly
+ image: ghcr.io/dragonflydb/dragonfly:v1.31.2
+ args:
+ - "--logtostderr"
+ - "--cluster_mode=emulated"
+ - "--lock_on_hashtags"
+ - "--bind=::"
+ - "--maxmemory=4gb"
+ - "--dir=/data"
+ ports:
+ - containerPort: 6379
+ readinessProbe:
+ tcpSocket:
+ port: 6379
+ initialDelaySeconds: 3
+ periodSeconds: 5
+ timeoutSeconds: 3
+ successThreshold: 1
+ failureThreshold: 3
+ resources:
+ requests:
+ memory: "1Gi"
+ cpu: "500m"
+ limits:
+ memory: "4Gi"
+ cpu: "1000m"
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: firecrawl-dragonfly
+spec:
+ selector:
+ app: firecrawl-dragonfly
+ ports:
+ - name: redis
+ protocol: TCP
+ port: 6379
+ targetPort: 6379
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: firecrawl-app
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: firecrawl-app
+ template:
+ metadata:
+ labels:
+ app: firecrawl-app
+ spec:
+ containers:
+ - name: firecrawl-app
+ image: firecrawl/firecrawl:latest
+ imagePullPolicy: Never
+ command: ["node"]
+ args: ["dist/src/index.js"]
+ ports:
+ - containerPort: 3002
+ envFrom:
+ - configMapRef:
+ name: firecrawl-config
+ - configMapRef:
+ name: firecrawl-config-dynamic
+ - secretRef:
+ name: firecrawl-secret-dynamic
+ env:
+ - name: FLY_PROCESS_GROUP
+ value: "app"
+ - name: PORT
+ value: "3002"
+ - name: NODE_OPTIONS
+ value: "--no-node-snapshot --max-old-space-size=6144"
+ resources:
+ requests:
+ memory: "1G"
+ cpu: "500m"
+ limits:
+ memory: "6G"
+ cpu: "1000m"
+ livenessProbe:
+ httpGet:
+ path: /v0/health/liveness
+ port: 3002
+ initialDelaySeconds: 5
+ periodSeconds: 5
+ timeoutSeconds: 5
+ successThreshold: 1
+ failureThreshold: 3
+ readinessProbe:
+ httpGet:
+ path: /v0/health/readiness
+ port: 3002
+ initialDelaySeconds: 5
+ periodSeconds: 5
+ timeoutSeconds: 5
+ successThreshold: 1
+ failureThreshold: 3
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: firecrawl-app
+ namespace: default
+ labels:
+ app: firecrawl-app
+spec:
+ selector:
+ app: firecrawl-app
+ ports:
+ - name: http
+ protocol: TCP
+ port: 3002
+ targetPort: 3002
+ type: NodePort
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: firecrawl-worker
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: firecrawl-worker
+ template:
+ metadata:
+ labels:
+ app: firecrawl-worker
+ spec:
+ containers:
+ - name: worker
+ image: firecrawl/firecrawl:latest
+ imagePullPolicy: Never
+ command: [ "node" ]
+ args: [ "dist/src/services/queue-worker.js", "--max-old-space-size=2048" ]
+ envFrom:
+ - configMapRef:
+ name: firecrawl-config
+ - configMapRef:
+ name: firecrawl-config-dynamic
+ - secretRef:
+ name: firecrawl-secret-dynamic
+ env:
+ - name: FLY_PROCESS_GROUP
+ value: "worker"
+ - name: PORT
+ value: "3005"
+ - name: NODE_OPTIONS
+ value: "--no-node-snapshot"
+ - name: GOMEMLIMIT
+ value: "1024MiB"
+ - name: SCRAPE_WORKER_MAX_OLD_SPACE_SIZE
+ value: "768"
+ resources:
+ requests:
+ memory: "2G"
+ cpu: "500m"
+ limits:
+ memory: "4G"
+ cpu: "1000m"
+ livenessProbe:
+ httpGet:
+ path: /liveness
+ port: 3005
+ initialDelaySeconds: 5
+ periodSeconds: 5
+ timeoutSeconds: 5
+ successThreshold: 1
+ failureThreshold: 3
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: firecrawl-nuq-worker
+spec:
+ replicas: 5
+ selector:
+ matchLabels:
+ app: firecrawl-nuq-worker
+ template:
+ metadata:
+ labels:
+ app: firecrawl-nuq-worker
+ spec:
+ containers:
+ - name: nuq-worker
+ image: firecrawl/firecrawl:latest
+ imagePullPolicy: Never
+ command: [ "node" ]
+ args: [ "dist/src/services/worker/nuq-worker.js", "--max-old-space-size=2048" ]
+ envFrom:
+ - configMapRef:
+ name: firecrawl-config
+ - configMapRef:
+ name: firecrawl-config-dynamic
+ - secretRef:
+ name: firecrawl-secret-dynamic
+ env:
+ - name: PORT
+ value: "3005"
+ - name: NODE_OPTIONS
+ value: "--no-node-snapshot"
+ - name: GOMEMLIMIT
+ value: "1024MiB"
+ resources:
+ requests:
+ memory: "2G"
+ cpu: "500m"
+ limits:
+ memory: "4G"
+ cpu: "1000m"
+ livenessProbe:
+ httpGet:
+ path: /health
+ port: 3005
+ initialDelaySeconds: 10
+ periodSeconds: 10
+ timeoutSeconds: 10
+ successThreshold: 1
+ failureThreshold: 3
\ No newline at end of file
diff --git a/.github/k8s/tailscale-proxy/Dockerfile b/.github/k8s/tailscale-proxy/Dockerfile
new file mode 100644
index 000000000..77bd48e43
--- /dev/null
+++ b/.github/k8s/tailscale-proxy/Dockerfile
@@ -0,0 +1,28 @@
+FROM debian:bookworm-slim
+
+# Install prerequisites, iptables, nginx, and Tailscale
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends \
+ ca-certificates \
+ curl \
+ gnupg \
+ iptables \
+ iproute2 \
+ nginx \
+ && curl -fsSL https://pkgs.tailscale.com/stable/debian/bookworm.noarmor.gpg \
+ -o /usr/share/keyrings/tailscale-archive-keyring.gpg \
+ && curl -fsSL https://pkgs.tailscale.com/stable/debian/bookworm.tailscale-keyring.list \
+ -o /etc/apt/sources.list.d/tailscale.list \
+ && apt-get update \
+ && apt-get install -y --no-install-recommends tailscale \
+ && rm -rf /var/lib/apt/lists/*
+
+# Configure nginx as a reverse proxy
+COPY nginx.conf /etc/nginx/nginx.conf.template
+
+# Start script that sets up tailscale and nginx
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+EXPOSE 8080
+ENTRYPOINT ["/entrypoint.sh"]
\ No newline at end of file
diff --git a/.github/k8s/tailscale-proxy/entrypoint.sh b/.github/k8s/tailscale-proxy/entrypoint.sh
new file mode 100644
index 000000000..a1a81639f
--- /dev/null
+++ b/.github/k8s/tailscale-proxy/entrypoint.sh
@@ -0,0 +1,48 @@
+#!/bin/sh
+set -eu
+
+# Require auth key
+if [ -z "${TS_AUTHKEY:-}" ]; then
+ echo "[tailscale-proxy] TS_AUTHKEY is not set; refusing to start." >&2
+ exit 1
+fi
+
+# Compute hostname: allow override via TS_HOSTNAME, else default to pod name
+TS_HOSTNAME=${TS_HOSTNAME:-}
+if [ -z "${TS_HOSTNAME}" ]; then
+ # Kubernetes sets HOSTNAME to the pod name
+ TS_HOSTNAME="firecrawl-proxy-${HOSTNAME:-container}"
+fi
+
+# Start tailscaled
+tailscaled --state=mem: &
+
+# Wait for tailscaled control socket to be ready (up to ~15s)
+i=0
+until tailscale version >/dev/null 2>&1 || [ $i -ge 30 ]; do
+ i=$((i+1))
+ sleep 0.5
+done
+
+# Prepare optional flags
+ADVERTISE_FLAGS=""
+if [ -n "${TS_ADVERTISE_TAGS:-}" ]; then
+ ADVERTISE_FLAGS="--advertise-tags=${TS_ADVERTISE_TAGS}"
+fi
+
+set -x
+tailscale up \
+ --authkey="${TS_AUTHKEY}" \
+ --hostname="${TS_HOSTNAME}" \
+ --accept-routes=true \
+ --accept-dns=true \
+ --ssh=false \
+ ${ADVERTISE_FLAGS} \
+ --reset
+set +x
+
+# Replace FIRE_ENGINE_BETA_URL in nginx config
+sed "s|\$FIRE_ENGINE_BETA_URL|${FIRE_ENGINE_BETA_URL}|g" /etc/nginx/nginx.conf.template > /etc/nginx/nginx.conf
+
+# Start nginx
+exec nginx -g "daemon off;"
\ No newline at end of file
diff --git a/.github/k8s/tailscale-proxy/nginx.conf b/.github/k8s/tailscale-proxy/nginx.conf
new file mode 100644
index 000000000..ca48512ac
--- /dev/null
+++ b/.github/k8s/tailscale-proxy/nginx.conf
@@ -0,0 +1,24 @@
+user www-data;
+worker_processes auto;
+error_log /dev/stderr notice;
+pid /run/nginx.pid;
+
+events {
+ worker_connections 1024;
+}
+
+http {
+ access_log /dev/stdout combined;
+
+ server {
+ listen 8080;
+
+ location / {
+ proxy_pass $FIRE_ENGINE_BETA_URL;
+ proxy_set_header Host $host;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Proto $scheme;
+ }
+ }
+}
\ No newline at end of file
diff --git a/.github/workflows/test-server-self-host.yml b/.github/workflows/test-server-self-host.yml
deleted file mode 100644
index 9c97c8f00..000000000
--- a/.github/workflows/test-server-self-host.yml
+++ /dev/null
@@ -1,145 +0,0 @@
-name: Self-hosted Server Test Suite
-
-on:
- pull_request:
- branches:
- - main
- - nsc/v2
-
-env:
- REDIS_URL: redis://localhost:6379
- ENV: ${{ secrets.ENV }}
- TEST_SUITE_SELF_HOSTED: true
- USE_GO_MARKDOWN_PARSER: true
- FIRECRAWL_DEBUG_FILTER_LINKS: true
- SENTRY_ENVIRONMENT: dev
-
-jobs:
- test:
- name: Run tests
- strategy:
- matrix:
- ai: ['openai', 'no-ai']
- search: ['searxng', 'google']
- engine: ['playwright', 'fetch']
- # proxy: ["proxy", "no-proxy"]
- proxy: ['proxy']
- fail-fast: false
- runs-on: ubuntu-latest
- services:
- redis:
- image: redis
- ports:
- - 6379:6379
- env:
- OPENAI_API_KEY: ${{ matrix.ai == 'openai' && secrets.OPENAI_API_KEY || '' }}
- SEARXNG_ENDPOINT: ${{ matrix.search == 'searxng' && 'http://localhost:3434' || '' }}
- PLAYWRIGHT_MICROSERVICE_URL: ${{ matrix.engine == 'playwright' && 'http://localhost:3003/scrape' || '' }}
- PROXY_SERVER: ${{ matrix.proxy == 'proxy' && secrets.PROXY_SERVER || '' }}
- PROXY_USERNAME: ${{ matrix.proxy == 'proxy' && secrets.PROXY_USERNAME || '' }}
- PROXY_PASSWORD: ${{ matrix.proxy == 'proxy' && secrets.PROXY_PASSWORD || '' }}
- steps:
- - uses: actions/checkout@v5
- - name: Install pnpm
- uses: pnpm/action-setup@v4
- with:
- version: 10
- - name: Set up Node.js
- uses: actions/setup-node@v4
- with:
- node-version: '20'
- cache: 'pnpm'
- cache-dependency-path: './apps/api/pnpm-lock.yaml'
- - name: Install dependencies
- run: pnpm install
- working-directory: ./apps/api
- - name: Install Playwright dependencies
- if: matrix.engine == 'playwright'
- run: |
- pnpm install
- pnpm exec playwright install-deps
- pnpm exec playwright install
- working-directory: ./apps/playwright-service-ts
- - name: Set up Go
- uses: actions/setup-go@v5
- with:
- go-version: '1.24'
- cache-dependency-path: ./apps/api/sharedLibs/go-html-to-md/go.sum
- - name: Build go-html-to-md
- run: |
- go mod tidy
- go build -o libhtml-to-markdown.so -buildmode=c-shared html-to-markdown.go
- chmod +x libhtml-to-markdown.so
- working-directory: ./apps/api/sharedLibs/go-html-to-md
- - name: Set up SearXNG
- if: matrix.search == 'searxng'
- run: |
- mkdir searxng
-
- echo "use_default_settings: true
- search:
- formats: [html, json, csv]
- server:
- secret_key: 'fcsecret'" > searxng/settings.yml
-
- docker run -d -p 3434:8080 -v "${PWD}/searxng:/etc/searxng" --name searxng searxng/searxng
- pnpx wait-on tcp:3434 -t 30s
- working-directory: ./
- - name: Build the application
- run: pnpm run build
- working-directory: ./apps/api
- - name: Start server
- run: pnpm run start:production:nobuild > api.log 2>&1 &
- env:
- PORT: 3002
- HOST: 0.0.0.0
- working-directory: ./apps/api
- - name: Start worker
- run: pnpm run worker:production > worker.log 2>&1 &
- env:
- PORT: 3005
- HOST: 0.0.0.0
- working-directory: ./apps/api
- - name: Start playwright
- if: matrix.engine == 'playwright'
- run: pnpm run dev > playwright.log 2>&1 &
- working-directory: ./apps/playwright-service-ts
- env:
- PORT: 3003
- - name: Wait for server
- run: pnpx wait-on tcp:3002 -t 60s
- - name: Wait for playwright
- if: matrix.engine == 'playwright'
- run: pnpx wait-on tcp:3003 -t 15s
- - name: Run snippet tests
- run: |
- npm run test:snips
- working-directory: ./apps/api
- - name: Kill instances
- if: always()
- run: pkill -9 node
- - name: Kill SearXNG
- if: always() && matrix.search == 'searxng'
- run: |
- docker logs searxng > searxng.log 2>&1
- docker kill searxng
- working-directory: ./
- - uses: actions/upload-artifact@v4
- if: always()
- with:
- name: Logs (${{ matrix.ai }}, ${{ matrix.search }}, ${{ matrix.engine }}, ${{ matrix.proxy }})
- path: |
- ./apps/api/api.log
- ./apps/api/worker.log
- - uses: actions/upload-artifact@v4
- if: always() && matrix.playwright
- with:
- name: Playwright Logs (${{ matrix.ai }}, ${{ matrix.search }}, ${{ matrix.proxy }})
- path: |
- ./apps/playwright-service-ts/playwright.log
- - uses: actions/upload-artifact@v4
- if: always() && matrix.search == 'searxng'
- with:
- name: SearXNG (${{ matrix.ai }}, ${{ matrix.engine }}, ${{ matrix.proxy }})
- path: |
- ./searxng.log
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.yml
index 6278db813..66b9093b6 100644
--- a/.github/workflows/test-server.yml
+++ b/.github/workflows/test-server.yml
@@ -6,50 +6,74 @@ on:
- main
- nsc/v2
-env:
- BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }}
- POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}
- POSTHOG_HOST: ${{ secrets.POSTHOG_HOST }}
- OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
- REDIS_URL: ${{ secrets.REDIS_URL }}
- SUPABASE_ANON_TOKEN: ${{ secrets.SUPABASE_ANON_TOKEN }}
- SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }}
- SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
- SUPABASE_REPLICA_URL: ${{ secrets.SUPABASE_REPLICA_URL }}
- INDEX_SUPABASE_SERVICE_TOKEN: ${{ secrets.INDEX_SUPABASE_SERVICE_TOKEN }}
- INDEX_SUPABASE_ANON_TOKEN: ${{ secrets.INDEX_SUPABASE_ANON_TOKEN }}
- INDEX_SUPABASE_URL: ${{ secrets.INDEX_SUPABASE_URL }}
- TEST_API_KEY: ${{ secrets.TEST_API_KEY }}
- TEST_TEAM_ID: ${{ secrets.TEST_TEAM_ID }}
- TEST_API_KEY_CONCURRENCY: ${{ secrets.TEST_API_KEY_CONCURRENCY }}
- TEST_TEAM_ID_CONCURRENCY: ${{ secrets.TEST_TEAM_ID_CONCURRENCY }}
- TEST_API_KEY_ZDR: ${{ secrets.TEST_API_KEY_ZDR }}
- TEST_TEAM_ID_ZDR: ${{ secrets.TEST_TEAM_ID_ZDR }}
- FIRE_ENGINE_BETA_URL: ${{ secrets.FIRE_ENGINE_BETA_URL }}
- FIRE_ENGINE_STAGING_URL: ${{ secrets.FIRE_ENGINE_STAGING_URL }}
- USE_DB_AUTHENTICATION: true
- SERPER_API_KEY: ${{ secrets.SERPER_API_KEY }}
- ENV: ${{ secrets.ENV }}
- RUNPOD_MU_POD_ID: ${{ secrets.RUNPOD_MU_POD_ID }}
- RUNPOD_MUV2_POD_ID: ${{ secrets.RUNPOD_MUV2_POD_ID }}
- RUNPOD_MU_API_KEY: ${{ secrets.RUNPOD_MU_API_KEY }}
- GCS_CREDENTIALS: ${{ secrets.GCS_CREDENTIALS }}
- GCS_BUCKET_NAME: ${{ secrets.GCS_BUCKET_NAME }}
- GCS_INDEX_BUCKET_NAME: ${{ secrets.GCS_INDEX_BUCKET_NAME }}
- GCS_MEDIA_BUCKET_NAME: ${{ secrets.GCS_MEDIA_BUCKET_NAME }}
- GOOGLE_GENERATIVE_AI_API_KEY: ${{ secrets.GOOGLE_GENERATIVE_AI_API_KEY }}
- GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
- ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
- VERTEX_CREDENTIALS: ${{ secrets.VERTEX_CREDENTIALS }}
- USE_GO_MARKDOWN_PARSER: true
- SENTRY_ENVIRONMENT: dev
- IDMUX_URL: ${{ secrets.IDMUX_URL }}
- LOG_ENCRYPTION_KEY: ${{ secrets.LOG_ENCRYPTION_KEY }}
-
jobs:
+ build-images:
+ name: Build images
+ runs-on: big-runner
+ steps:
+ - uses: actions/checkout@v5
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v3
+ - name: Build API image
+ run: |
+ docker buildx build \
+ -t firecrawl/firecrawl:latest \
+ --cache-from type=gha \
+ --cache-to type=gha,mode=max \
+ --load \
+ ./apps/api
+ - name: Build nuq-postgres image
+ run: |
+ docker buildx build \
+ -t firecrawl/nuq-postgres:latest \
+ --cache-from type=gha \
+ --cache-to type=gha,mode=max \
+ --load \
+ ./apps/nuq-postgres
+ - name: Build tailscale-proxy image
+ run: |
+ docker buildx build \
+ -t firecrawl/tailscale-proxy:latest \
+ --cache-from type=gha \
+ --cache-to type=gha,mode=max \
+ --load \
+ .github/k8s/tailscale-proxy
+ - name: Build Playwright image
+ run: |
+ docker buildx build \
+ -t firecrawl/playwright-service:latest \
+ --cache-from type=gha \
+ --cache-to type=gha,mode=max \
+ --load \
+ ./apps/playwright-service-ts
+ - name: Export images as tarballs
+ run: |
+ docker save -o firecrawl-firecrawl.tar firecrawl/firecrawl:latest
+ docker save -o firecrawl-nuq-postgres.tar firecrawl/nuq-postgres:latest
+ docker save -o firecrawl-tailscale-proxy.tar firecrawl/tailscale-proxy:latest
+ docker save -o firecrawl-playwright-service.tar firecrawl/playwright-service:latest
+ - name: Upload images artifact
+ uses: actions/upload-artifact@v4
+ with:
+ name: built-images
+ path: |
+ firecrawl-firecrawl.tar
+ firecrawl-nuq-postgres.tar
+ firecrawl-tailscale-proxy.tar
+ firecrawl-playwright-service.tar
+
test:
- name: Run tests
- runs-on: ubuntu-latest
+ name: Self-hosted (Kubernetes) environment tests
+ needs: build-images
+ strategy:
+ matrix:
+ ai: ["openai", "no-ai"]
+ search: ["searxng", "google"]
+ engine: ["playwright", "fetch"]
+ # proxy: ["proxy", "no-proxy"]
+ proxy: ["proxy"]
+ fail-fast: false
+ runs-on: big-runner
services:
redis:
image: redis
@@ -57,13 +81,43 @@ jobs:
- 6379:6379
steps:
- uses: actions/checkout@v5
- - name: Tailscale
- uses: tailscale/github-action@v3
+ - name: Start minikube
+ uses: medyagh/[email protected]
+ - name: Download images artifact
+ uses: actions/download-artifact@v4
with:
- oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }}
- oauth-secret: ${{ secrets.TS_OAUTH_SECRET }}
- tags: tag:ci
- use-cache: 'true'
+ name: built-images
+ path: ./images
+ - name: Load images into minikube
+ run: |
+ eval $(minikube docker-env)
+ docker load -i ./images/firecrawl-firecrawl.tar
+ docker load -i ./images/firecrawl-nuq-postgres.tar
+ - name: Load Playwright image into minikube
+ if: matrix.engine == 'playwright'
+ run: |
+ eval $(minikube docker-env)
+ docker load -i ./images/firecrawl-playwright-service.tar
+ - name: Create configmap
+ run: |
+ kubectl create configmap firecrawl-config-dynamic -n default \
+ --from-literal=SEARXNG_ENDPOINT=${{ matrix.search == 'searxng' && 'http://firecrawl-searxng:8080/search' || '' }} \
+ --from-literal=PLAYWRIGHT_MICROSERVICE_URL=${{ matrix.engine == 'playwright' && 'http://firecrawl-playwright-service:3003/scrape' || '' }} \
+ --from-literal=PROXY_SERVER=${{ matrix.proxy == 'proxy' && secrets.PROXY_SERVER || '' }} \
+ --from-literal=PROXY_USERNAME=${{ matrix.proxy == 'proxy' && secrets.PROXY_USERNAME || '' }}
+ - name: Create secret
+ run: |
+ kubectl create secret generic firecrawl-secret-dynamic \
+ --from-literal=OPENAI_API_KEY=${{ matrix.ai == 'openai' && secrets.OPENAI_API_KEY || '' }} \
+ --from-literal=PROXY_PASSWORD=${{ matrix.proxy == 'proxy' && secrets.PROXY_PASSWORD || '' }}
+ - name: Deploy server
+ run: kubectl apply -f .github/k8s/server.yml
+ - name: Deploy playwright
+ if: matrix.engine == 'playwright'
+ run: kubectl apply -f .github/k8s/playwright.yml
+ - name: Deploy searxng
+ if: matrix.search == 'searxng'
+ run: kubectl apply -f .github/k8s/searxng.yml
- name: Install pnpm
uses: pnpm/action-setup@v4
with:
@@ -77,43 +131,277 @@ jobs:
- name: Install dependencies
run: pnpm install
working-directory: ./apps/api
- - name: Set up Go
- uses: actions/setup-go@v5
- with:
- go-version: '1.24'
- cache-dependency-path: ./apps/api/sharedLibs/go-html-to-md/go.sum
- - name: Build go-html-to-md
- run: |
- go mod tidy
- go build -o libhtml-to-markdown.so -buildmode=c-shared html-to-markdown.go
- chmod +x libhtml-to-markdown.so
- working-directory: ./apps/api/sharedLibs/go-html-to-md
- - name: Build the application
- run: pnpm run build
+ - name: Wait for server
+ run: kubectl wait --for=condition=ready pod -l app=firecrawl-app --timeout=60s -o json
+ - name: Debug wait failure (server)
+ if: failure()
+ run: |
+ echo "Pods with label app=firecrawl-app:" || true
+ kubectl get pods -l app=firecrawl-app -o wide || true
+ echo "\nDescribe pods:" || true
+ kubectl describe pods -l app=firecrawl-app || true
+ echo "\nRecent events:" || true
+ kubectl get events --sort-by=.lastTimestamp | tail -n 100 || true
+ echo "\nLogs from firecrawl-app deployment (if any):" || true
+ kubectl logs deployment/firecrawl-app --all-containers=true --tail=200 || true
+ - name: Wait for playwright
+ if: matrix.engine == 'playwright'
+ run: kubectl wait --for=condition=ready pod -l app=firecrawl-playwright-service --timeout=60s -o json
+ - name: Debug wait failure (playwright)
+ if: failure() && matrix.engine == 'playwright'
+ run: |
+ echo "Pods with label app=firecrawl-playwright-service:" || true
+ kubectl get pods -l app=firecrawl-playwright-service -o wide || true
+ echo "\nDescribe pods:" || true
+ kubectl describe pods -l app=firecrawl-playwright-service || true
+ echo "\nRecent events:" || true
+ kubectl get events --sort-by=.lastTimestamp | tail -n 100 || true
+ echo "\nLogs from playwright deployment (if any):" || true
+ kubectl logs deployment/firecrawl-playwright-service --all-containers=true --tail=200 || true
+ - name: Wait for searxng
+ if: matrix.search == 'searxng'
+ run: kubectl wait --for=condition=ready pod -l app=firecrawl-searxng --timeout=60s -o json
+ - name: Debug wait failure (searxng)
+ if: failure() && matrix.search == 'searxng'
+ run: |
+ echo "Pods with label app=firecrawl-searxng:" || true
+ kubectl get pods -l app=firecrawl-searxng -o wide || true
+ echo "\nDescribe pods:" || true
+ kubectl describe pods -l app=firecrawl-searxng || true
+ echo "\nRecent events:" || true
+ kubectl get events --sort-by=.lastTimestamp | tail -n 100 || true
+ echo "\nLogs from searxng deployment (if any):" || true
+ kubectl logs deployment/firecrawl-searxng --all-containers=true --tail=200 || true
+ - name: Run snippet tests
+ env:
+ TEST_SUITE_SELF_HOSTED: true
+ OPENAI_API_KEY: ${{ matrix.ai == 'openai' && '<IS SET>' || '' }}
+ PROXY_SERVER: ${{ matrix.proxy == 'proxy' && secrets.PROXY_SERVER || '' }}
+ PLAYWRIGHT_MICROSERVICE_URL: ${{ matrix.engine == 'playwright' && '<IS SET>' || '' }}
+ run: TEST_API_URL=$(minikube service firecrawl-app --url) pnpm run test:snips
working-directory: ./apps/api
- - name: Start the application
- run: pnpm run start:production:nobuild > api.log 2>&1 &
+ - name: Create logs directory
+ if: always()
+ run: mkdir -p logs
+ - name: Copy log files
+ if: always()
+ run: |
+ kubectl logs deployment/firecrawl-app > logs/api.log || true
+ kubectl logs deployment/firecrawl-worker > logs/worker.log || true
+ kubectl logs deployment/firecrawl-nuq-worker > logs/nuq-worker.log || true
+ kubectl logs deployment/firecrawl-dragonfly > logs/dragonfly.log || true
+ kubectl logs deployment/nuq-postgres > logs/postgres.log || true
+ - name: Copy SearXNG logs
+ if: always() && matrix.search == 'searxng'
+ run: kubectl logs deployment/firecrawl-searxng > logs/searxng.log || true
+ - name: Copy Playwright logs
+ if: always() && matrix.engine == 'playwright'
+ run: kubectl logs deployment/firecrawl-playwright-service > logs/playwright.log || true
+ - name: Zip logs
+ if: always()
+ run: |
+ cd logs
+ zip -r logs.zip ./*
+ - uses: actions/upload-artifact@v4
+ if: always()
+ with:
+ name: Logs (kubernetes, ${{ matrix.ai }}, ${{ matrix.search }}, ${{ matrix.engine }}, ${{ matrix.proxy }})
+ path: logs/logs.zip
+
+ docker-test:
+ name: Self-hosted (Docker) environment tests
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v5
+ - name: Compose
env:
- PORT: 3002
- HOST: 0.0.0.0
+ PROXY_SERVER: ${{ secrets.PROXY_SERVER }}
+ PROXY_USERNAME: ${{ secrets.PROXY_USERNAME }}
+ PROXY_PASSWORD: ${{ secrets.PROXY_PASSWORD }}
+ run: |
+ docker compose up -d
+ - name: Install pnpm
+ uses: pnpm/action-setup@v4
+ with:
+ version: 10
+ - name: Set up Node.js
+ uses: actions/setup-node@v4
+ with:
+ node-version: "20"
+ cache: "pnpm"
+ cache-dependency-path: './apps/api/pnpm-lock.yaml'
+ - name: Install dependencies
+ run: pnpm install
working-directory: ./apps/api
- id: start_app
- - name: Start worker
- run: pnpm run worker:production > worker.log 2>&1 &
+ - name: Wait for server
+ run: pnpx wait-on http://localhost:3002
+ - name: Run snippet tests
env:
- PORT: 3005
- HOST: 0.0.0.0
+ PROXY_SERVER: ${{ secrets.PROXY_SERVER }}
+ PLAYWRIGHT_MICROSERVICE_URL: ${{ secrets.PLAYWRIGHT_MICROSERVICE_URL }}
+ run: pnpm run test:snips
working-directory: ./apps/api
- id: start_workers
- - name: Start index worker
- run: pnpm run index-worker:production > index-worker.log 2>&1 &
+ - name: Create logs directory
+ if: always()
+ run: mkdir -p logs
+ - name: Copy log files
+ if: always()
+ run: |
+ docker compose logs api > logs/api.log || true
+ docker compose logs playwright-service > logs/playwright.log || true
+ docker compose logs nuq-postgres > logs/postgres.log || true
+ - name: Zip logs
+ if: always()
+ run: |
+ cd logs
+ zip -r logs.zip ./*
+ - uses: actions/upload-artifact@v4
+ if: always()
+ with:
+ name: Logs (docker, no-ai, google, playwright, proxy)
+ path: logs/logs.zip
+
+ prod-test:
+ name: Production environment tests
+ needs: build-images
+ runs-on: big-runner
+ steps:
+ - uses: actions/checkout@v5
+ - name: Start minikube
+ uses: medyagh/[email protected]
+ - name: Download images artifact
+ uses: actions/download-artifact@v4
+ with:
+ name: built-images
+ path: ./images
+ - name: Tailscale
+ uses: tailscale/github-action@v3
+ with:
+ oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }}
+ oauth-secret: ${{ secrets.TS_OAUTH_SECRET }}
+ tags: tag:ci
+ - name: Load prebuilt images into minikube
+ run: |
+ eval $(minikube docker-env)
+ docker load -i ./images/firecrawl-firecrawl.tar
+ docker load -i ./images/firecrawl-nuq-postgres.tar
+ docker load -i ./images/firecrawl-tailscale-proxy.tar
+ - name: Create Tailscale secret
+ run: |
+ if [ -z "${{ secrets.TS_AUTHKEY }}" ]; then
+ echo "secrets.TS_AUTHKEY is empty; set a reusable/preauthorized or ephemeral key" >&2
+ exit 1
+ fi
+ kubectl create secret generic tailscale \
+ --from-literal=TS_AUTHKEY=${{ secrets.TS_AUTHKEY }} \
+ --from-literal=TS_ADVERTISE_TAGS=${{ secrets.TS_ADVERTISE_TAGS }} \
+ --from-literal=FIRE_ENGINE_BETA_URL=${{ secrets.FIRE_ENGINE_BETA_URL }} \
+ --dry-run=client -o yaml | kubectl apply -f -
+ - name: Create configmap
+ run: |
+ kubectl create configmap firecrawl-config-dynamic -n default \
+ --from-literal=USE_DB_AUTHENTICATION=true \
+ --from-literal=SUPABASE_ANON_TOKEN=${{ secrets.SUPABASE_ANON_TOKEN }} \
+ --from-literal=SUPABASE_URL=${{ secrets.SUPABASE_URL }} \
+ --from-literal=SUPABASE_REPLICA_URL=${{ secrets.SUPABASE_REPLICA_URL }} \
+ --from-literal=INDEX_SUPABASE_ANON_TOKEN=${{ secrets.INDEX_SUPABASE_ANON_TOKEN }} \
+ --from-literal=INDEX_SUPABASE_URL=${{ secrets.INDEX_SUPABASE_URL }} \
+ --from-literal=FIRE_ENGINE_BETA_URL=http://tailscale-proxy:8080 \
+ --from-literal=RUNPOD_MU_POD_ID=${{ secrets.RUNPOD_MU_POD_ID }} \
+ --from-literal=RUNPOD_MUV2_POD_ID=${{ secrets.RUNPOD_MUV2_POD_ID }} \
+ --from-literal=GCS_BUCKET_NAME=${{ secrets.GCS_BUCKET_NAME }} \
+ --from-literal=GCS_MEDIA_BUCKET_NAME=${{ secrets.GCS_MEDIA_BUCKET_NAME }} \
+ --from-literal=GCS_INDEX_BUCKET_NAME=${{ secrets.GCS_INDEX_BUCKET_NAME }}
+ - name: Create secret
+ run: |
+ kubectl create secret generic firecrawl-secret-dynamic \
+ --from-literal=BULL_AUTH_KEY=${{ secrets.BULL_AUTH_KEY }} \
+ --from-literal=OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }} \
+ --from-literal=SUPABASE_SERVICE_TOKEN=${{ secrets.SUPABASE_SERVICE_TOKEN }} \
+ --from-literal=INDEX_SUPABASE_SERVICE_TOKEN=${{ secrets.INDEX_SUPABASE_SERVICE_TOKEN }} \
+ --from-literal=SERPER_API_KEY=${{ secrets.SERPER_API_KEY }} \
+ --from-literal=RUNPOD_MU_API_KEY=${{ secrets.RUNPOD_MU_API_KEY }} \
+ --from-literal=GCS_CREDENTIALS=${{ secrets.GCS_CREDENTIALS }} \
+ --from-literal=GOOGLE_GENERATIVE_AI_API_KEY=${{ secrets.GOOGLE_GENERATIVE_AI_API_KEY }} \
+ --from-literal=GROQ_API_KEY=${{ secrets.GROQ_API_KEY }} \
+ --from-literal=ANTHROPIC_API_KEY=${{ secrets.ANTHROPIC_API_KEY }} \
+ --from-literal=VERTEX_CREDENTIALS=${{ secrets.VERTEX_CREDENTIALS }} \
+ --from-literal=LOG_ENCRYPTION_KEY=${{ secrets.LOG_ENCRYPTION_KEY }}
+ - name: Deploy server
+ run: |
+ kubectl apply -f .github/k8s/server.yml
+ kubectl apply -f .github/k8s/server-nsh.yml
+ - name: Install pnpm
+ uses: pnpm/action-setup@v4
+ with:
+ version: 10
+ - name: Set up Node.js
+ uses: actions/setup-node@v4
+ with:
+ node-version: "20"
+ cache: "pnpm"
+ cache-dependency-path: './apps/api/pnpm-lock.yaml'
+ - name: Install dependencies
+ run: pnpm install
working-directory: ./apps/api
- id: start_index_worker
- name: Wait for API
- run: pnpx wait-on tcp:3002 -t 60s
+ run: kubectl wait --for=condition=ready pod -l app=firecrawl-app --timeout=60s -o json
+ - name: Debug wait failure (API)
+ if: failure()
+ run: |
+ echo "Pods with label app=firecrawl-app:" || true
+ kubectl get pods -l app=firecrawl-app -o wide || true
+ echo "\nDescribe pods:" || true
+ kubectl describe pods -l app=firecrawl-app || true
+ echo "\nRecent events:" || true
+ kubectl get events --sort-by=.lastTimestamp | tail -n 100 || true
+ - name: Wait for nuq-postgres
+ run: kubectl wait --for=condition=ready pod -l app=nuq-postgres --timeout=60s -o json
+ - name: Debug wait failure (nuq-postgres)
+ if: failure()
+ run: |
+ echo "Pods with label app=nuq-postgres:" || true
+ kubectl get pods -l app=nuq-postgres -o wide || true
+ echo "\nDescribe pods:" || true
+ kubectl describe pods -l app=nuq-postgres || true
+ echo "\nRecent events:" || true
+ kubectl get events --sort-by=.lastTimestamp | tail -n 100 || true
+ echo "\nLogs from nuq-postgres deployment (if any):" || true
+ kubectl logs deployment/nuq-postgres --all-containers=true --tail=200 || true
+ - name: Wait for tailscale-proxy
+ run: kubectl wait --for=condition=ready pod -l app=tailscale-proxy --timeout=60s -o json
+ - name: Debug wait failure (tailscale-proxy)
+ if: failure()
+ run: |
+ echo "Pods with label app=tailscale-proxy:" || true
+ kubectl get pods -l app=tailscale-proxy -o wide || true
+ echo "\nDescribe pods:" || true
+ kubectl describe pods -l app=tailscale-proxy || true
+ echo "\nRecent events:" || true
+ kubectl get events --sort-by=.lastTimestamp | tail -n 100 || true
+ - name: Wait for dragonfly (redis)
+ run: kubectl wait --for=condition=ready pod -l app=firecrawl-dragonfly --timeout=60s -o json
+ - name: Debug wait failure (dragonfly)
+ if: failure()
+ run: |
+ echo "Pods with label app=firecrawl-dragonfly:" || true
+ kubectl get pods -l app=firecrawl-dragonfly -o wide || true
+ echo "\nDescribe pods:" || true
+ kubectl describe pods -l app=firecrawl-dragonfly || true
+ echo "\nRecent events:" || true
+ kubectl get events --sort-by=.lastTimestamp | tail -n 100 || true
- name: Run snippet tests
+ env:
+ IDMUX_URL: ${{ secrets.IDMUX_URL }}
+ USE_DB_AUTHENTICATION: true
+ SUPABASE_ANON_TOKEN: ${{ secrets.SUPABASE_ANON_TOKEN }}
+ SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
+ SUPABASE_REPLICA_URL: ${{ secrets.SUPABASE_REPLICA_URL }}
+ SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }}
+ GCS_BUCKET_NAME: ${{ secrets.GCS_BUCKET_NAME }}
+ GCS_CREDENTIALS: ${{ secrets.GCS_CREDENTIALS }}
run: |
- npm run test:snips
+ TEST_API_URL=$(minikube service firecrawl-app --url) pnpm run test:snips
working-directory: ./apps/api
- name: Kill instances
if: always()
@@ -124,9 +412,13 @@ jobs:
- name: Copy log files
if: always()
run: |
- cp ./apps/api/api.log logs/ || true
- cp ./apps/api/worker.log logs/ || true
- cp ./apps/api/index-worker.log logs/ || true
+ kubectl logs deployment/firecrawl-app > logs/api.log || true
+ kubectl logs deployment/firecrawl-worker > logs/worker.log || true
+ kubectl logs deployment/firecrawl-index-worker > logs/index-worker.log || true
+ kubectl logs deployment/firecrawl-nuq-worker > logs/nuq-worker.log || true
+ kubectl logs deployment/firecrawl-dragonfly > logs/dragonfly.log || true
+ kubectl logs deployment/nuq-postgres > logs/postgres.log || true
+ kubectl logs deployment/tailscale-proxy > logs/tailscale-proxy.log || true
- name: Zip and encrypt logs
if: always()
run: |
diff --git a/apps/api/Dockerfile b/apps/api/Dockerfile
index bf96a1615..5a2a9f62e 100644
--- a/apps/api/Dockerfile
+++ b/apps/api/Dockerfile
@@ -70,8 +70,4 @@ COPY --from=build /app/native ./native
# Copy Go shared library
COPY --from=go-build /app/sharedLibs/go-html-to-md/libhtml-to-markdown.so ./sharedLibs/go-html-to-md/
-# Copy and prepare entrypoint
-COPY docker-entrypoint.sh ./
-RUN sed -i 's/\r$//' ./docker-entrypoint.sh && chmod +x ./docker-entrypoint.sh
-
-ENTRYPOINT ["./docker-entrypoint.sh"]
\ No newline at end of file
+CMD ["node", "dist/src/harness.js", "--start-docker"]
diff --git a/apps/api/docker-entrypoint.sh b/apps/api/docker-entrypoint.sh
deleted file mode 100755
index 6b09e7053..000000000
--- a/apps/api/docker-entrypoint.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/bin/bash -e
-
-if [ "$UID" -eq 0 ]; then
- set +e # disable failing on errror
- ulimit -n 65535
- echo "NEW ULIMIT: $(ulimit -n)"
- set -e # enable failing on error
-else
- echo ENTRYPOINT DID NOT RUN AS ROOT
-fi
-
-if [ "$FLY_PROCESS_GROUP" = "app" ]; then
- echo "RUNNING app"
- node --max-old-space-size=8192 dist/src/index.js
-elif [ "$FLY_PROCESS_GROUP" = "worker" ]; then
- echo "RUNNING worker"
- node --max-old-space-size=8192 dist/src/services/queue-worker.js
-elif [ "$FLY_PROCESS_GROUP" = "index-worker" ]; then
- echo "RUNNING index worker"
- node --max-old-space-size=8192 dist/src/services/indexing/index-worker.js
-else
- echo "NO FLY PROCESS GROUP"
- node --max-old-space-size=8192 dist/src/index.js
-fi
diff --git a/apps/api/package.json b/apps/api/package.json
index be164054e..1716cd45a 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -4,9 +4,12 @@
"description": "",
"main": "index.js",
"scripts": {
- "start": "tsc-watch --onSuccess \"node dist/src/index.js\"",
- "start:production": "tsc && node dist/src/index.js",
- "start:production:nobuild": "node dist/src/index.js",
+ "start": "tsc-watch --onSuccess \"node dist/src/harness.js --start-built\"",
+ "start:production": "tsx src/harness.ts --start",
+ "server": "tsc-watch --onSuccess \"node dist/src/index.js\"",
+ "server:production": "tsc && node dist/src/index.js",
+ "server:production:nobuild": "node dist/src/index.js",
+ "format": "prettier --write \"src/**/*.(js|ts)\"",
"flyio": "node dist/src/index.js",
"start:dev": "tsc-watch --onSuccess \"node dist/src/index.js\"",
"build": "tsc",
@@ -19,6 +22,8 @@
"harness": "tsx src/harness.ts",
"workers": "tsc-watch --onSuccess \"node dist/src/services/queue-worker.js\"",
"worker:production": "node dist/src/services/queue-worker.js",
+ "nuq-worker": "tsc-watch --onSuccess \"node dist/src/services/worker/nuq-worker.js\"",
+ "nuq-worker:production": "node dist/src/services/worker/nuq-worker.js",
"index-worker": "tsc-watch --onSuccess \"node dist/src/services/indexing/index-worker.js\"",
"index-worker:production": "node dist/src/services/indexing/index-worker.js",
"mongo-docker": "docker run -d -p 2717:27017 -v ./mongo-data:/data/db --name mongodb mongo:latest",
@@ -43,6 +48,7 @@
"@types/lodash": "^4.17.14",
"@types/node": "^20.14.1",
"@types/pdf-parse": "^1.1.4",
+ "@types/pg": "^8.15.5",
"@types/supertest": "^6.0.2",
"@types/tough-cookie": "^4.0.5",
"husky": "^9.1.7",
@@ -136,6 +142,7 @@
"openai": "^5.12.0",
"parse-diff": "^0.11.1",
"pdf-parse": "^1.1.1",
+ "pg": "^8.16.3",
"pos": "^0.4.2",
"posthog-node": "^4.0.1",
"prettier": "^3.6.2",
@@ -172,6 +179,11 @@
"pnpm": {
"onlyBuiltDependencies": [
"@sentry-internal/node-cpu-profiler",
+ "bigint-buffer",
+ "bufferutil",
+ "keccak",
+ "libpq",
+ "utf-8-validate",
"supabase"
],
"overrides": {
diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yaml
index b04f60409..4642bd0ce 100644
--- a/apps/api/pnpm-lock.yaml
+++ b/apps/api/pnpm-lock.yaml
@@ -249,6 +249,9 @@ importers:
pdf-parse:
specifier: ^1.1.1
version: 1.1.1
+ pg:
+ specifier: ^8.16.3
+ version: 8.16.3([email protected])
pos:
specifier: ^0.4.2
version: 0.4.2
@@ -355,6 +358,9 @@ importers:
'@types/pdf-parse':
specifier: ^1.1.4
version: 1.1.4
+ '@types/pg':
+ specifier: ^8.15.5
+ version: 8.15.5
'@types/supertest':
specifier: ^6.0.2
version: 6.0.2
@@ -3476,6 +3482,9 @@ packages:
resolution: {integrity: sha512-I6UNVBAoYbvuWkkU3oosC8yxqH21f4/Jc4DK71JLG3dT2mdlGe1z+ep/LQGXaKaOgcvUrsQoPRqfgtMcvZiJhg==}
+ '@types/[email protected]':
+ resolution: {integrity: sha512-LF7lF6zWEKxuT3/OR8wAZGzkg4ENGXFNyiV/JeOt9z5B+0ZVwbql9McqX5c/WStFq1GaGso7H1AzP/qSzmlCKQ==}
+
resolution: {integrity: sha512-1Kc4oAGzAl7uqUStZCDvaLFqZrW9qWSjXOmBfdgyBP5La7Us6Mg4GBvRlSoaZMhQF/zSj1C8CtKMBkoiT8eL8w==}
@@ -5417,6 +5426,9 @@ packages:
resolution: {integrity: sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==}
engines: {node: '>=6'}
+ resolution: {integrity: sha512-4lSWmly2Nsj3LaTxxtFmJWuP3Kx+0hYHEd+aNrcXEWT0nKWaPd9/QZPiMkkC680zeALFGHQdQWjBvnilL+vgWA==}
+
resolution: {integrity: sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==}
@@ -5684,6 +5696,9 @@ packages:
resolution: {integrity: sha512-WWdIxpyjEn+FhQJQQv9aQAYlHoNVdzIzUySNV1gHUPDSdZJ3yZn7pAAbQcV7B56Mvu881q9FZV+0Vx2xC44VWA==}
engines: {node: ^18.17.0 || >=20.5.0}
+ resolution: {integrity: sha512-DANghxFkS1plDdRsX0X9pm0Z6SJNN6gBdtXfanwoZ8hooC5gosGFSBGRYHUVPz1asKA/kMRqDRdHrluZ61SpBQ==}
+
resolution: {integrity: sha512-21t+ozMQDAL/UGgQVBbZ/xXvNO10++ZPuTmKRO8k9V3AClVRht49ahtDjfY8l1q6nSHOrE5ASfthzH3ol6R/hg==}
engines: {node: '>=20.17'}
@@ -5975,10 +5990,27 @@ packages:
resolution: {integrity: sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==}
+ resolution: {integrity: sha512-YgCtzMH0ptvZJslLM1ffsY4EuGaU0cx4XSdXLRFae8bPP4dS5xL1tNB3k2o/N64cHJpwU7dxKli/nZ2lUa5fLg==}
+
+ resolution: {integrity: sha512-nkc6NpDcvPVpZXxrreI/FOtX3XemeLl8E0qFr6F2Lrm/I8WOnaWNhIPK2Z7OHpw7gh5XJThi6j6ppgNoaT1w4w==}
+
resolution: {integrity: sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==}
engines: {node: '>=4.0.0'}
+ resolution: {integrity: sha512-3oi+KVil86Vngo4H0IlhBaYSJWdcu8t2f1Y4TkQoQi5oZ9bNeYECGqW3oSGx69mjSZYHoC3h+3jYtqzRgndn5A==}
+
+ resolution: {integrity: sha512-Tu8jMlcX+9d8+QVzKIvM/uJtp07PKr82IUOYEphaWcoBhIYkoHpLXN3qO59nAI11ripznDsEzEv8nUxBVWajGg==}
+ peerDependencies:
+ pg: '>=8.0'
+
+ resolution: {integrity: sha512-6DIBgBQaTKDJyxnXaLiLR8wBpQQcGWuAESkRBX/t6OwA8YsqP+iVSiond2EDy6Y/dsGk8rh/jtax3js5NeV7JQ==}
+
resolution: {integrity: sha512-jPIlvgoD63hrEuihvIg+tJhoGjUsLPn6poJY9N5CnlPd91c2T18T/9zBtLxZSb1EhYxBRoZJtzScCaWlYLtktg==}
@@ -5986,6 +6018,18 @@ packages:
resolution: {integrity: sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==}
engines: {node: '>=4'}
+ resolution: {integrity: sha512-enxc1h0jA/aq5oSDMvqyW3q89ra6XIIDZgCX9vkMrnz5DFTw/Ny3Li2lFQ+pt3L6MCgm/5o2o8HW9hiJji+xvw==}
+ engines: {node: '>= 16.0.0'}
+ peerDependencies:
+ pg-native: '>=3.0.1'
+ peerDependenciesMeta:
+ pg-native:
+ optional: true
+
+ resolution: {integrity: sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==}
+
resolution: {integrity: sha512-anP1Z8qwhkbmu7MFP5iTt+wQKXgwzf7zTyGlcdzabySa9vd0Xt392U0rVmz9poOaBj0uHJKyyo9/upk0HrEQew==}
@@ -11585,7 +11629,7 @@ snapshots:
dependencies:
- '@types/pg': 8.6.1
+ '@types/pg': 8.15.5
dependencies:
@@ -11593,6 +11637,12 @@ snapshots:
pg-protocol: 1.6.1
pg-types: 2.2.0
+ '@types/[email protected]':
+ dependencies:
+ '@types/node': 20.14.1
+ pg-protocol: 1.10.3
+ pg-types: 2.2.0
+
dependencies:
'@types/node': 20.14.1
@@ -14356,6 +14406,12 @@ snapshots:
+ dependencies:
+ bindings: 1.5.0
+ nan: 2.22.2
+ optional: true
+
dependencies:
immediate: 3.0.6
@@ -14618,6 +14674,9 @@ snapshots:
+ optional: true
+
@@ -14931,8 +14990,25 @@ snapshots:
+ optional: true
+
+
+ dependencies:
+ libpq: 1.8.15
+ pg-types: 2.2.0
+ optional: true
+
+ dependencies:
+ pg: 8.16.3([email protected])
+
+
@@ -14943,6 +15019,21 @@ snapshots:
postgres-date: 1.0.7
postgres-interval: 1.2.0
+ dependencies:
+ pg-connection-string: 2.9.1
+ pg-protocol: 1.10.3
+ pg-types: 2.2.0
+ pgpass: 1.0.5
+ optionalDependencies:
+ pg-cloudflare: 1.2.7
+ pg-native: 3.5.2
+
+ dependencies:
+ split2: 4.2.0
+
diff --git a/apps/api/requests.http b/apps/api/requests.http
index 1507fd99a..5ffc11708 100644
--- a/apps/api/requests.http
+++ b/apps/api/requests.http
@@ -1,6 +1,6 @@
# Pick your baseUrl here:
-# @baseUrl = http://localhost:3002
-@baseUrl = https://api.firecrawl.dev
+@baseUrl = http://localhost:3002
+# @baseUrl = https://api.firecrawl.dev
### Scrape Website
# @name scrape
diff --git a/apps/api/sharedLibs/go-html-to-md/.gitignore b/apps/api/sharedLibs/go-html-to-md/.gitignore
index 9e40cf424..909db1866 100644
--- a/apps/api/sharedLibs/go-html-to-md/.gitignore
+++ b/apps/api/sharedLibs/go-html-to-md/.gitignore
@@ -1,2 +1,3 @@
html-to-markdown.*
-!html-to-markdown.go
\ No newline at end of file
+!html-to-markdown.go
+libhtml-to-markdown.*
\ No newline at end of file
diff --git a/apps/api/src/__tests__/queue-concurrency-integration.test.ts b/apps/api/src/__tests__/queue-concurrency-integration.test.ts
deleted file mode 100644
index 7e2141d07..000000000
--- a/apps/api/src/__tests__/queue-concurrency-integration.test.ts
+++ /dev/null
@@ -1,269 +0,0 @@
-import { redisEvictConnection } from "../services/redis";
-import { addScrapeJob, addScrapeJobs } from "../services/queue-jobs";
-import {
- cleanOldConcurrencyLimitEntries,
- pushConcurrencyLimitActiveJob,
- takeConcurrencyLimitedJob,
- removeConcurrencyLimitActiveJob,
-} from "../lib/concurrency-limit";
-import { WebScraperOptions } from "../types";
-import { getACUCTeam } from "../controllers/auth";
-
-// Mock all the dependencies
-const mockAdd = jest.fn();
-jest.mock("../services/queue-service", () => ({
- redisConnection: {
- zremrangebyscore: jest.fn(),
- zrangebyscore: jest.fn(),
- zadd: jest.fn(),
- zrem: jest.fn(),
- zmpop: jest.fn(),
- zcard: jest.fn(),
- smembers: jest.fn(),
- },
- getScrapeQueue: jest.fn(() => ({
- add: mockAdd,
- })),
-}));
-
-jest.mock("uuid", () => ({
- v4: jest.fn(() => "mock-uuid"),
-}));
-
-describe("Queue Concurrency Integration", () => {
- const mockTeamId = "test-team-id";
- const mockNow = Date.now();
-
- const defaultScrapeOptions = {
- formats: [{ type: "markdown" }],
- onlyMainContent: true,
- waitFor: 0,
- mobile: false,
- parsePDF: false,
- timeout: 30000,
- javascript: true,
- headers: {},
- cookies: [],
- blockResources: true,
- skipTlsVerification: false,
- removeBase64Images: true,
- fastMode: false,
- blockAds: true,
- maxAge: 0,
- storeInCache: true,
- proxy: "basic",
- };
-
- beforeEach(() => {
- jest.clearAllMocks();
- jest.spyOn(Date, "now").mockImplementation(() => mockNow);
- });
-
- describe("Single Job Addition", () => {
- const mockWebScraperOptions: WebScraperOptions = {
- url: "https://test.com",
- mode: "single_urls",
- team_id: mockTeamId,
- scrapeOptions: defaultScrapeOptions,
- crawlerOptions: null,
- zeroDataRetention: false,
- apiKeyId: null,
- } as WebScraperOptions;
-
- it("should add job directly to BullMQ when under concurrency limit", async () => {
- // Mock current active jobs to be under limit
- (redisEvictConnection.zrangebyscore as jest.Mock).mockResolvedValue([]);
-
- await addScrapeJob(mockWebScraperOptions);
-
- // Should have checked concurrency
- expect(redisEvictConnection.zrangebyscore).toHaveBeenCalled();
-
- // Should have added to BullMQ
- expect(mockAdd).toHaveBeenCalled();
-
- // Should have added to active jobs
- expect(redisEvictConnection.zadd).toHaveBeenCalledWith(
- expect.stringContaining("concurrency-limiter"),
- expect.any(Number),
- expect.any(String),
- );
- });
-
- it("should add job to concurrency queue when at concurrency limit", async () => {
- // Mock current active jobs to be at limit
- (getACUCTeam as jest.Mock).mockResolvedValue({
- concurrency: 15,
- } as any);
- const activeJobs = Array(15).fill("active-job");
- (redisEvictConnection.zrangebyscore as jest.Mock).mockResolvedValue(
- activeJobs,
- );
-
- await addScrapeJob(mockWebScraperOptions);
-
- // Should have checked concurrency
- expect(redisEvictConnection.zrangebyscore).toHaveBeenCalled();
-
- // Should NOT have added to BullMQ
- expect(mockAdd).not.toHaveBeenCalled();
-
- // Should have added to concurrency queue
- expect(redisEvictConnection.zadd).toHaveBeenCalledWith(
- expect.stringContaining("concurrency-limit-queue"),
- expect.any(Number),
- expect.stringContaining("mock-uuid"),
- );
- });
- });
-
- describe("Batch Job Addition", () => {
- const createMockJobs = (count: number) =>
- Array(count)
- .fill(null)
- .map((_, i) => ({
- data: {
- url: `https://test${i}.com`,
- mode: "single_urls",
- team_id: mockTeamId,
- scrapeOptions: defaultScrapeOptions,
- zeroDataRetention: false,
- } as any,
- opts: {
- jobId: `job-${i}`,
- priority: 1,
- },
- }));
-
- it("should handle batch jobs respecting concurrency limits", async () => {
- const maxConcurrency = 15;
- (getACUCTeam as jest.Mock).mockResolvedValue({
- concurrency: maxConcurrency,
- } as any);
- const totalJobs = maxConcurrency + 5; // Some jobs should go to queue
- const mockJobs = createMockJobs(totalJobs);
-
- // Mock current active jobs to be empty
- (redisEvictConnection.zrangebyscore as jest.Mock).mockResolvedValue([]);
-
- await addScrapeJobs(mockJobs);
-
- // Should have added maxConcurrency jobs to BullMQ
- expect(mockAdd).toHaveBeenCalledTimes(maxConcurrency);
-
- // Should have added remaining jobs to concurrency queue
- expect(redisEvictConnection.zadd).toHaveBeenCalledWith(
- expect.stringContaining("concurrency-limit-queue"),
- expect.any(Number),
- expect.any(String),
- );
- });
-
- it("should handle empty job array", async () => {
- const result = await addScrapeJobs([]);
- expect(result).toBe(true);
- expect(mockAdd).not.toHaveBeenCalled();
- expect(redisEvictConnection.zadd).not.toHaveBeenCalled();
- });
- });
-
- describe("Queue Worker Integration", () => {
- it("should process next queued job when active job completes", async () => {
- const mockJob = {
- id: "test-job",
- data: {
- team_id: mockTeamId,
- zeroDataRetention: false,
- },
- };
-
- // Mock a queued job
- const queuedJob = {
- id: "queued-job",
- data: { test: "data" },
- opts: {},
- };
- (redisEvictConnection.zmpop as jest.Mock).mockResolvedValueOnce([
- "key",
- [[JSON.stringify(queuedJob)]],
- ]);
-
- // Simulate job completion in worker
- await removeConcurrencyLimitActiveJob(mockTeamId, mockJob.id);
- await cleanOldConcurrencyLimitEntries(mockTeamId);
-
- const nextJob = await takeConcurrencyLimitedJob(mockTeamId);
-
- // Should have taken next job from queue
- expect(nextJob).toEqual(queuedJob);
-
- // Should have added new job to active jobs
- await pushConcurrencyLimitActiveJob(
- mockTeamId,
- nextJob!.id,
- 2 * 60 * 1000,
- );
- expect(redisEvictConnection.zadd).toHaveBeenCalledWith(
- expect.stringContaining("concurrency-limiter"),
- expect.any(Number),
- nextJob!.id,
- );
- });
-
- it("should handle job failure and cleanup", async () => {
- const mockJob = {
- id: "failing-job",
- data: {
- team_id: mockTeamId,
- },
- };
-
- // Add job to active jobs
- await pushConcurrencyLimitActiveJob(
- mockTeamId,
- mockJob.id,
- 2 * 60 * 1000,
- );
-
- // Simulate job failure and cleanup
- await removeConcurrencyLimitActiveJob(mockTeamId, mockJob.id);
- await cleanOldConcurrencyLimitEntries(mockTeamId);
-
- // Verify job was removed from active jobs
- expect(redisEvictConnection.zrem).toHaveBeenCalledWith(
- expect.stringContaining("concurrency-limiter"),
- mockJob.id,
- );
- });
- });
-
- describe("Edge Cases", () => {
- it("should handle stalled jobs cleanup", async () => {
- const stalledTime = mockNow - 3 * 60 * 1000; // 3 minutes ago
-
- // Mock stalled jobs in Redis
- (redisEvictConnection.zrangebyscore as jest.Mock).mockResolvedValueOnce([
- "stalled-job",
- ]);
-
- await cleanOldConcurrencyLimitEntries(mockTeamId, mockNow);
-
- // Should have cleaned up stalled jobs
- expect(redisEvictConnection.zremrangebyscore).toHaveBeenCalledWith(
- expect.stringContaining("concurrency-limiter"),
- -Infinity,
- mockNow,
- );
- });
-
- it("should handle race conditions in job queue processing", async () => {
- // Mock a race condition where job is taken by another worker
- (redisEvictConnection.zmpop as jest.Mock).mockResolvedValueOnce(null);
-
- const nextJob = await takeConcurrencyLimitedJob(mockTeamId);
-
- // Should handle gracefully when no job is available
- expect(nextJob).toBeNull();
- });
- });
-});
diff --git a/apps/api/src/__tests__/snips/lib.ts b/apps/api/src/__tests__/snips/lib.ts
index 43ef79ed7..ba9fec277 100644
--- a/apps/api/src/__tests__/snips/lib.ts
+++ b/apps/api/src/__tests__/snips/lib.ts
@@ -7,7 +7,7 @@ import { TeamFlags } from "../../controllers/v1/types";
// Configuration
// =========================================
-export const TEST_URL = "http://127.0.0.1:3002";
+export const TEST_URL = process.env.TEST_API_URL || "http://127.0.0.1:3002";
// Due to the limited resources of the CI runner, we need to set a longer timeout for the many many scrape tests
export const scrapeTimeout = 90000;
diff --git a/apps/api/src/__tests__/snips/v1/crawl.test.ts b/apps/api/src/__tests__/snips/v1/crawl.test.ts
index bb033d5cf..ce0550c66 100644
--- a/apps/api/src/__tests__/snips/v1/crawl.test.ts
+++ b/apps/api/src/__tests__/snips/v1/crawl.test.ts
@@ -387,110 +387,3 @@ describe("Crawl tests", () => {
},
);
});
-
-describe("Robots.txt FFI Integration tests", () => {
- it.concurrent(
- "handles normal robots.txt parsing via FFI",
- async () => {
- const result = await filterLinks({
- links: [
- "https://example.com/allowed",
- "https://example.com/disallowed",
- ],
- limit: 10,
- maxDepth: 10,
- baseUrl: "https://example.com",
- initialUrl: "https://example.com",
- regexOnFullUrl: false,
- excludes: [],
- includes: [],
- allowBackwardCrawling: true,
- ignoreRobotsTxt: false,
- robotsTxt: "User-agent: *\nDisallow: /disallowed",
- });
-
- expect(result.links).toHaveLength(1);
- expect(result.links[0]).toBe("https://example.com/allowed");
- expect("https://example.com/disallowed" in result.denialReasons).toBe(
- true,
- );
- expect(result.denialReasons["https://example.com/disallowed"]).toBe(
- "ROBOTS_TXT",
- );
- },
- 10000,
- );
-
- it.concurrent(
- "handles malformed robots.txt without crashing via FFI",
- async () => {
- const result = await filterLinks({
- links: ["https://example.com/test"],
- limit: 10,
- maxDepth: 10,
- baseUrl: "https://example.com",
- initialUrl: "https://example.com",
- regexOnFullUrl: false,
- excludes: [],
- includes: [],
- allowBackwardCrawling: true,
- ignoreRobotsTxt: false,
- robotsTxt:
- "Invalid robots.txt content with \x00 null bytes and malformed syntax",
- });
-
- expect(result.links).toHaveLength(1);
- expect(result.links[0]).toBe("https://example.com/test");
- },
- 10000,
- );
-
- it.concurrent(
- "handles non-UTF8 robots.txt content without crashing via FFI",
- async () => {
- const nonUtf8Content =
- String.fromCharCode(0xff, 0xfe) + "User-agent: *\nDisallow: /blocked";
- const result = await filterLinks({
- links: ["https://example.com/allowed"],
- limit: 10,
- maxDepth: 10,
- baseUrl: "https://example.com",
- initialUrl: "https://example.com",
- regexOnFullUrl: false,
- excludes: [],
- includes: [],
- allowBackwardCrawling: true,
- ignoreRobotsTxt: false,
- robotsTxt: nonUtf8Content,
- });
-
- expect(result.links).toHaveLength(1);
- expect(result.links[0]).toBe("https://example.com/allowed");
- },
- 10000,
- );
-
- it.concurrent(
- "handles char boundary issues without crashing via FFI",
- async () => {
- const problematicContent = "User-agent: *\nDisallow: /\u{a0}test";
- const result = await filterLinks({
- links: ["https://example.com/safe"],
- limit: 10,
- maxDepth: 10,
- baseUrl: "https://example.com",
- initialUrl: "https://example.com",
- regexOnFullUrl: false,
- excludes: [],
- includes: [],
- allowBackwardCrawling: true,
- ignoreRobotsTxt: false,
- robotsTxt: problematicContent,
- });
-
- expect(result.links).toHaveLength(1);
- expect(result.links[0]).toBe("https://example.com/safe");
- },
- 10000,
- );
-});
diff --git a/apps/api/src/__tests__/snips/v1/zdr.test.ts b/apps/api/src/__tests__/snips/v1/zdr.test.ts
index b23c49a02..27d7b81db 100644
--- a/apps/api/src/__tests__/snips/v1/zdr.test.ts
+++ b/apps/api/src/__tests__/snips/v1/zdr.test.ts
@@ -1,3 +1,4 @@
+import { exec, spawn } from "node:child_process";
import { getJobFromGCS } from "../../../lib/gcs-jobs";
import { supabase_service } from "../../../services/supabase";
import {
@@ -20,6 +21,9 @@ const logIgnoreList = [
"Billing batch processing",
"Processing batch of",
"Billing team",
+ "No jobs to process",
+ "nuqHealthCheck metrics",
+ "nuqGetJobToProcess metrics",
];
if (process.env.TEST_SUITE_SELF_HOSTED) {
@@ -27,7 +31,35 @@ if (process.env.TEST_SUITE_SELF_HOSTED) {
expect(true).toBe(true);
});
} else {
+ function getOutput(command: string): Promise<string> {
+ return new Promise((resolve, reject) => {
+ try {
+ const cmd = spawn(command, { shell: true });
+ let out = "";
+ cmd.stdout?.on("data", data => {
+ out += data;
+ });
+ cmd.stderr?.on("data", data => {
+ out += data;
+ });
+ cmd.on("error", e => {
+ reject(e);
+ });
+ cmd.on("close", code => {
+ if (code !== 0) {
+ reject(new Error(`Command ${command} failed with code ${code}`));
+ } else {
+ resolve(out);
+ }
+ });
+ } catch (e) {
+ reject(e);
+ }
+ });
+ }
+
async function getServerLogs() {
+ let logs: string;
if (!process.env.GITHUB_ACTIONS) {
try {
await stat("api.log");
@@ -35,8 +67,10 @@ if (process.env.TEST_SUITE_SELF_HOSTED) {
console.warn("No api.log file found");
return [];
}
+ logs = await readFile("api.log", "utf8");
+ } else {
+ logs = await getOutput("kubectl logs deployment/firecrawl-app");
}
- const logs = await readFile("api.log", "utf8");
return logs
.split("\n")
.filter(
@@ -45,6 +79,7 @@ if (process.env.TEST_SUITE_SELF_HOSTED) {
}
async function getWorkerLogs() {
+ let logs: string;
if (!process.env.GITHUB_ACTIONS) {
try {
await stat("worker.log");
@@ -52,8 +87,10 @@ if (process.env.TEST_SUITE_SELF_HOSTED) {
console.warn("No worker.log file found");
return [];
}
+ logs = await readFile("worker.log", "utf8");
+ } else {
+ logs = await getOutput("kubectl logs deployment/firecrawl-nuq-worker");
}
- const logs = await readFile("worker.log", "utf8");
return logs
.split("\n")
.filter(
diff --git a/apps/api/src/__tests__/snips/v2/crawl.test.ts b/apps/api/src/__tests__/snips/v2/crawl.test.ts
index 70c077abc..d81e7a74e 100644
--- a/apps/api/src/__tests__/snips/v2/crawl.test.ts
+++ b/apps/api/src/__tests__/snips/v2/crawl.test.ts
@@ -389,110 +389,3 @@ describe("Crawl tests", () => {
});
}
});
-
-describe("Robots.txt FFI Integration tests", () => {
- it.concurrent(
- "handles normal robots.txt parsing via FFI",
- async () => {
- const result = await filterLinks({
- links: [
- "https://example.com/allowed",
- "https://example.com/disallowed",
- ],
- limit: 10,
- maxDepth: 10,
- baseUrl: "https://example.com",
- initialUrl: "https://example.com",
- regexOnFullUrl: false,
- excludes: [],
- includes: [],
- allowBackwardCrawling: true,
- ignoreRobotsTxt: false,
- robotsTxt: "User-agent: *\nDisallow: /disallowed",
- });
-
- expect(result.links).toHaveLength(1);
- expect(result.links[0]).toBe("https://example.com/allowed");
- expect("https://example.com/disallowed" in result.denialReasons).toBe(
- true,
- );
- expect(result.denialReasons["https://example.com/disallowed"]).toBe(
- "ROBOTS_TXT",
- );
- },
- 10000,
- );
-
- it.concurrent(
- "handles malformed robots.txt without crashing via FFI",
- async () => {
- const result = await filterLinks({
- links: ["https://example.com/test"],
- limit: 10,
- maxDepth: 10,
- baseUrl: "https://example.com",
- initialUrl: "https://example.com",
- regexOnFullUrl: false,
- excludes: [],
- includes: [],
- allowBackwardCrawling: true,
- ignoreRobotsTxt: false,
- robotsTxt:
- "Invalid robots.txt content with \x00 null bytes and malformed syntax",
- });
-
- expect(result.links).toHaveLength(1);
- expect(result.links[0]).toBe("https://example.com/test");
- },
- 10000,
- );
-
- it.concurrent(
- "handles non-UTF8 robots.txt content without crashing via FFI",
- async () => {
- const nonUtf8Content =
- String.fromCharCode(0xff, 0xfe) + "User-agent: *\nDisallow: /blocked";
- const result = await filterLinks({
- links: ["https://example.com/allowed"],
- limit: 10,
- maxDepth: 10,
- baseUrl: "https://example.com",
- initialUrl: "https://example.com",
- regexOnFullUrl: false,
- excludes: [],
- includes: [],
- allowBackwardCrawling: true,
- ignoreRobotsTxt: false,
- robotsTxt: nonUtf8Content,
- });
-
- expect(result.links).toHaveLength(1);
- expect(result.links[0]).toBe("https://example.com/allowed");
- },
- 10000,
- );
-
- it.concurrent(
- "handles char boundary issues without crashing via FFI",
- async () => {
- const problematicContent = "User-agent: *\nDisallow: /\u{a0}test";
- const result = await filterLinks({
- links: ["https://example.com/safe"],
- limit: 10,
- maxDepth: 10,
- baseUrl: "https://example.com",
- initialUrl: "https://example.com",
- regexOnFullUrl: false,
- excludes: [],
- includes: [],
- allowBackwardCrawling: true,
- ignoreRobotsTxt: false,
- robotsTxt: problematicContent,
- });
-
- expect(result.links).toHaveLength(1);
- expect(result.links[0]).toBe("https://example.com/safe");
- },
- 10000,
- );
-});
diff --git a/apps/api/src/__tests__/snips/v2/zdr.test.ts b/apps/api/src/__tests__/snips/v2/zdr.test.ts
index 889b14e76..61937aa7c 100644
--- a/apps/api/src/__tests__/snips/v2/zdr.test.ts
+++ b/apps/api/src/__tests__/snips/v2/zdr.test.ts
@@ -9,6 +9,7 @@ import {
idmux,
} from "./lib";
import { readFile, stat } from "node:fs/promises";
+import { spawn } from "node:child_process";
const logIgnoreList = [
"Billing queue created",
@@ -20,6 +21,9 @@ const logIgnoreList = [
"Billing batch processing",
"Processing batch of",
"Billing team",
+ "No jobs to process",
+ "nuqHealthCheck metrics",
+ "nuqGetJobToProcess metrics",
];
if (process.env.TEST_SUITE_SELF_HOSTED) {
@@ -27,7 +31,35 @@ if (process.env.TEST_SUITE_SELF_HOSTED) {
expect(true).toBe(true);
});
} else {
+ function getOutput(command: string): Promise<string> {
+ return new Promise((resolve, reject) => {
+ try {
+ const cmd = spawn(command, { shell: true });
+ let out = "";
+ cmd.stdout?.on("data", data => {
+ out += data;
+ });
+ cmd.stderr?.on("data", data => {
+ out += data;
+ });
+ cmd.on("error", e => {
+ reject(e);
+ });
+ cmd.on("close", code => {
+ if (code !== 0) {
+ reject(new Error(`Command ${command} failed with code ${code}`));
+ } else {
+ resolve(out);
+ }
+ });
+ } catch (e) {
+ reject(e);
+ }
+ });
+ }
+
async function getServerLogs() {
+ let logs: string;
if (!process.env.GITHUB_ACTIONS) {
try {
await stat("api.log");
@@ -35,8 +67,10 @@ if (process.env.TEST_SUITE_SELF_HOSTED) {
console.warn("No api.log file found");
return [];
}
+ logs = await readFile("api.log", "utf8");
+ } else {
+ logs = await getOutput("kubectl logs deployment/firecrawl-app");
}
- const logs = await readFile("api.log", "utf8");
return logs
.split("\n")
.filter(
@@ -45,6 +79,7 @@ if (process.env.TEST_SUITE_SELF_HOSTED) {
}
async function getWorkerLogs() {
+ let logs: string;
if (!process.env.GITHUB_ACTIONS) {
try {
await stat("worker.log");
@@ -52,8 +87,10 @@ if (process.env.TEST_SUITE_SELF_HOSTED) {
console.warn("No worker.log file found");
return [];
}
+ logs = await readFile("worker.log", "utf8");
+ } else {
+ logs = await getOutput("kubectl logs deployment/firecrawl-nuq-worker");
}
- const logs = await readFile("worker.log", "utf8");
return logs
.split("\n")
.filter(
diff --git a/apps/api/src/controllers/v0/admin/metrics.ts b/apps/api/src/controllers/v0/admin/metrics.ts
index afe0c055b..54b12cdc9 100644
--- a/apps/api/src/controllers/v0/admin/metrics.ts
+++ b/apps/api/src/controllers/v0/admin/metrics.ts
@@ -1,5 +1,6 @@
import type { Request, Response } from "express";
import { redisEvictConnection } from "../../../services/redis";
+import { nuqGetLocalMetrics, scrapeQueue } from "../../../services/worker/nuq";
export async function metricsController(_: Request, res: Response) {
let cursor: string = "0";
@@ -34,5 +35,9 @@ ${Object.entries(metrics)
`concurrency_limit_queue_job_count{team_id="${key}"} ${value}`,
)
.join("\n")}
+
+${await scrapeQueue.getMetrics()}
+
+${nuqGetLocalMetrics()}
`);
}
diff --git a/apps/api/src/controllers/v0/admin/queue.ts b/apps/api/src/controllers/v0/admin/queue.ts
deleted file mode 100644
index c5a87cb61..000000000
--- a/apps/api/src/controllers/v0/admin/queue.ts
+++ /dev/null
@@ -1,201 +0,0 @@
-import { Request, Response } from "express";
-
-import { Job } from "bullmq";
-import { logger } from "../../../lib/logger";
-import { getScrapeQueue } from "../../../services/queue-service";
-import { checkAlerts } from "../../../services/alerts";
-import { sendSlackWebhook } from "../../../services/alerts/slack";
-
-export async function cleanBefore24hCompleteJobsController(
- req: Request,
- res: Response,
-) {
- logger.info("🐂 Cleaning jobs older than 24h");
- try {
- const scrapeQueue = getScrapeQueue();
- const batchSize = 10;
- const numberOfBatches = 9; // Adjust based on your needs
- const completedJobsPromises: Promise<Job[]>[] = [];
- for (let i = 0; i < numberOfBatches; i++) {
- completedJobsPromises.push(
- scrapeQueue.getJobs(
- ["completed"],
- i * batchSize,
- i * batchSize + batchSize,
- true,
- ),
- );
- }
- const completedJobs: Job[] = (
- await Promise.all(completedJobsPromises)
- ).flat();
- const before24hJobs =
- completedJobs.filter(
- job =>
- job.finishedOn !== undefined &&
- job.finishedOn < Date.now() - 24 * 60 * 60 * 1000,
- ) || [];
-
- let count = 0;
-
- if (!before24hJobs) {
- return res.status(200).send(`No jobs to remove.`);
- }
-
- for (const job of before24hJobs) {
- try {
- await job.remove();
- count++;
- } catch (jobError) {
- logger.error(`🐂 Failed to remove job with ID ${job.id}: ${jobError}`);
- }
- }
- return res.status(200).send(`Removed ${count} completed jobs.`);
- } catch (error) {
- logger.error(`🐂 Failed to clean last 24h complete jobs: ${error}`);
- return res.status(500).send("Failed to clean jobs");
- }
-}
-
-export async function checkQueuesController(req: Request, res: Response) {
- try {
- await checkAlerts();
- return res.status(200).send("Alerts initialized");
- } catch (error) {
- logger.debug(`Failed to initialize alerts: ${error}`);
- return res.status(500).send("Failed to initialize alerts");
- }
-}
-
-// Use this as a "health check" that way we dont destroy the server
-export async function queuesController(req: Request, res: Response) {
- try {
- const scrapeQueue = getScrapeQueue();
-
- const [webScraperActive] = await Promise.all([
- scrapeQueue.getActiveCount(),
- ]);
-
- const noActiveJobs = webScraperActive === 0;
- // 200 if no active jobs, 503 if there are active jobs
- return res.status(noActiveJobs ? 200 : 500).json({
- webScraperActive,
- noActiveJobs,
- });
- } catch (error) {
- logger.error(error);
- return res.status(500).json({ error: error.message });
- }
-}
-
-export async function autoscalerController(req: Request, res: Response) {
- try {
- const maxNumberOfMachines = 80;
- const minNumberOfMachines = 20;
-
- const scrapeQueue = getScrapeQueue();
-
- const [webScraperActive, webScraperWaiting, webScraperPriority] =
- await Promise.all([
- scrapeQueue.getActiveCount(),
- scrapeQueue.getWaitingCount(),
- scrapeQueue.getPrioritizedCount(),
- ]);
-
- let waitingAndPriorityCount = webScraperWaiting + webScraperPriority;
-
- // get number of machines active
- const request = await fetch(
- "https://api.machines.dev/v1/apps/firecrawl-scraper-js/machines",
- {
- headers: {
- Authorization: `Bearer ${process.env.FLY_API_TOKEN}`,
- },
- },
- );
- const machines = await request.json();
-
- // Only worker machines
- const activeMachines = machines.filter(
- machine =>
- (machine.state === "started" ||
- machine.state === "starting" ||
- machine.state === "replacing") &&
- machine.config.env["FLY_PROCESS_GROUP"] === "worker",
- ).length;
-
- let targetMachineCount = activeMachines;
-
- const baseScaleUp = 10;
- // Slow scale down
- const baseScaleDown = 2;
-
- // Scale up logic
- if (webScraperActive > 9000 || waitingAndPriorityCount > 2000) {
- targetMachineCount = Math.min(
- maxNumberOfMachines,
- activeMachines + baseScaleUp * 3,
- );
- } else if (webScraperActive > 5000 || waitingAndPriorityCount > 1000) {
- targetMachineCount = Math.min(
- maxNumberOfMachines,
- activeMachines + baseScaleUp * 2,
- );
- } else if (webScraperActive > 1000 || waitingAndPriorityCount > 500) {
- targetMachineCount = Math.min(
- maxNumberOfMachines,
- activeMachines + baseScaleUp,
- );
- }
-
- // Scale down logic
- if (webScraperActive < 100 && waitingAndPriorityCount < 50) {
- targetMachineCount = Math.max(
- minNumberOfMachines,
- activeMachines - baseScaleDown * 3,
- );
- } else if (webScraperActive < 500 && waitingAndPriorityCount < 200) {
- targetMachineCount = Math.max(
- minNumberOfMachines,
- activeMachines - baseScaleDown * 2,
- );
- } else if (webScraperActive < 1000 && waitingAndPriorityCount < 500) {
- targetMachineCount = Math.max(
- minNumberOfMachines,
- activeMachines - baseScaleDown,
- );
- }
-
- if (targetMachineCount !== activeMachines) {
- logger.info(
- `🐂 Scaling from ${activeMachines} to ${targetMachineCount} - ${webScraperActive} active, ${webScraperWaiting} waiting`,
- );
-
- if (targetMachineCount > activeMachines) {
- sendSlackWebhook(
- `🐂 Scaling from ${activeMachines} to ${targetMachineCount} - ${webScraperActive} active, ${webScraperWaiting} waiting - Current DateTime: ${new Date().toISOString()}`,
- false,
- process.env.SLACK_AUTOSCALER ?? "",
- );
- } else {
- sendSlackWebhook(
- `🐂 Scaling from ${activeMachines} to ${targetMachineCount} - ${webScraperActive} active, ${webScraperWaiting} waiting - Current DateTime: ${new Date().toISOString()}`,
- false,
- process.env.SLACK_AUTOSCALER ?? "",
- );
- }
- return res.status(200).json({
- mode: "scale-descale",
- count: targetMachineCount,
- });
- }
-
- return res.status(200).json({
- mode: "normal",
- count: activeMachines,
- });
- } catch (error) {
- logger.error(error);
- return res.status(500).send("Failed to initialize autoscaler");
- }
-}
diff --git a/apps/api/src/controllers/v0/crawl-status.ts b/apps/api/src/controllers/v0/crawl-status.ts
index d2fa6f30f..446e20f0c 100644
--- a/apps/api/src/controllers/v0/crawl-status.ts
+++ b/apps/api/src/controllers/v0/crawl-status.ts
@@ -1,27 +1,24 @@
import { Request, Response } from "express";
import { authenticateUser } from "../auth";
import { RateLimiterMode } from "../../../src/types";
-import { getScrapeQueue } from "../../../src/services/queue-service";
import { redisEvictConnection } from "../../../src/services/redis";
import { logger } from "../../../src/lib/logger";
import { getCrawl, getCrawlJobs } from "../../../src/lib/crawl-redis";
import { supabaseGetJobsByCrawlId } from "../../../src/lib/supabase-jobs";
import * as Sentry from "@sentry/node";
import { configDotenv } from "dotenv";
-import { Job } from "bullmq";
import { toLegacyDocument } from "../v1/types";
import type { DBJob, PseudoJob } from "../v1/crawl-status";
import { getJobFromGCS } from "../../lib/gcs-jobs";
+import { scrapeQueue, NuQJob } from "../../services/worker/nuq";
configDotenv();
export async function getJobs(
crawlId: string,
ids: string[],
): Promise<PseudoJob<any>[]> {
- const [bullJobs, dbJobs, gcsJobs] = await Promise.all([
- Promise.all(ids.map(x => getScrapeQueue().getJob(x))).then(x =>
- x.filter(x => x),
- ) as Promise<(Job<any, any, string> & { id: string })[]>,
+ const [nuqJobs, dbJobs, gcsJobs] = await Promise.all([
+ scrapeQueue.getJobs(ids),
process.env.USE_DB_AUTHENTICATION === "true"
? await supabaseGetJobsByCrawlId(crawlId)
: [],
@@ -34,12 +31,12 @@ export async function getJobs(
: [],
]);
- const bullJobMap = new Map<string, PseudoJob<any>>();
+ const nuqJobMap = new Map<string, NuQJob<any, any>>();
const dbJobMap = new Map<string, DBJob>();
const gcsJobMap = new Map<string, any>();
- for (const job of bullJobs) {
- bullJobMap.set(job.id, job);
+ for (const job of nuqJobs) {
+ nuqJobMap.set(job.id, job);
}
for (const job of dbJobs) {
@@ -53,13 +50,13 @@ export async function getJobs(
const jobs: PseudoJob<any>[] = [];
for (const id of ids) {
- const bullJob = bullJobMap.get(id);
+ const nuqJob = nuqJobMap.get(id);
const dbJob = dbJobMap.get(id);
const gcsJob = gcsJobMap.get(id);
- if (!bullJob && !dbJob) continue;
+ if (!nuqJob && !dbJob) continue;
- const data = gcsJob ?? dbJob?.docs ?? bullJob?.returnvalue;
+ const data = gcsJob ?? dbJob?.docs ?? nuqJob?.returnvalue;
if (gcsJob === null && data) {
logger.warn("GCS Job not found", {
jobId: id,
@@ -68,20 +65,16 @@ export async function getJobs(
const job: PseudoJob<any> = {
id,
- getState: dbJob
- ? () => (dbJob.success ? "completed" : "failed")
- : () => bullJob!.getState(),
+ status: dbJob ? (dbJob.success ? "completed" : "failed") : nuqJob!.status,
returnvalue: Array.isArray(data) ? data[0] : data,
data: {
- scrapeOptions: bullJob
- ? bullJob.data.scrapeOptions
- : dbJob!.page_options,
+ scrapeOptions: nuqJob ? nuqJob.data.scrapeOptions : dbJob!.page_options,
},
- timestamp: bullJob
- ? bullJob.timestamp
+ timestamp: nuqJob
+ ? nuqJob.createdAt.valueOf()
: new Date(dbJob!.date_added).valueOf(),
failedReason:
- (bullJob ? bullJob.failedReason : dbJob!.message) || undefined,
+ (nuqJob ? nuqJob.failedReason : dbJob!.message) || undefined,
};
jobs.push(job);
@@ -135,7 +128,7 @@ export async function crawlStatusController(req: Request, res: Response) {
}
let jobIDs = await getCrawlJobs(req.params.jobId);
let jobs = await getJobs(req.params.jobId, jobIDs);
- let jobStatuses = await Promise.all(jobs.map(x => x.getState()));
+ let jobStatuses = jobs.map(x => x.status);
// Combine jobs and jobStatuses into a single array of objects
let jobsWithStatuses = jobs.map((job, index) => ({
@@ -144,9 +137,7 @@ export async function crawlStatusController(req: Request, res: Response) {
}));
// Filter out failed jobs
- jobsWithStatuses = jobsWithStatuses.filter(
- x => x.status !== "failed" && x.status !== "unknown",
- );
+ jobsWithStatuses = jobsWithStatuses.filter(x => x.status !== "failed");
// Sort jobs by timestamp
jobsWithStatuses.sort((a, b) => a.job.timestamp - b.job.timestamp);
diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.ts
index 8cdcde057..287cd5a3e 100644
--- a/apps/api/src/controllers/v0/crawl.ts
+++ b/apps/api/src/controllers/v0/crawl.ts
@@ -27,7 +27,7 @@ import { redisEvictConnection } from "../../../src/services/redis";
import { checkAndUpdateURL } from "../../../src/lib/validateUrl";
import * as Sentry from "@sentry/node";
import { getJobPriority } from "../../lib/job-priority";
-import { fromLegacyScrapeOptions, url as urlSchema } from "../v1/types";
+import { url as urlSchema } from "../v1/types";
import { ZodError } from "zod";
import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
import { fromV0ScrapeOptions } from "../v2/types";
@@ -216,7 +216,7 @@ export async function crawlController(req: Request, res: Response) {
const jobs = urls.map(url => {
const uuid = uuidv4();
return {
- name: uuid,
+ jobId: uuid,
data: {
url,
mode: "single_urls" as const,
@@ -231,10 +231,7 @@ export async function crawlController(req: Request, res: Response) {
zeroDataRetention: false, // not supported on v0
apiKeyId: chunk?.api_key_id ?? null,
},
- opts: {
- jobId: uuid,
- priority: jobPriority,
- },
+ priority: jobPriority,
};
});
@@ -246,12 +243,12 @@ export async function crawlController(req: Request, res: Response) {
);
await addCrawlJobs(
id,
- jobs.map(x => x.opts.jobId),
+ jobs.map(x => x.jobId),
logger,
);
for (const job of jobs) {
// add with sentry instrumentation
- await addScrapeJob(job.data, {}, job.opts.jobId);
+ await addScrapeJob(job.data, job.jobId, job.priority);
}
});
@@ -276,10 +273,8 @@ export async function crawlController(req: Request, res: Response) {
zeroDataRetention: false, // not supported on v0
apiKeyId: chunk?.api_key_id ?? null,
},
- {
- priority: 15, // prioritize request 0 of crawl jobs same as scrape jobs
- },
jobId,
+ await getJobPriority({ team_id, basePriority: 15 }),
);
await addCrawlJob(id, jobId, logger);
}
diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.ts
index be901c96b..3d48d832b 100644
--- a/apps/api/src/controllers/v0/scrape.ts
+++ b/apps/api/src/controllers/v0/scrape.ts
@@ -12,7 +12,6 @@ import {
defaultOrigin,
} from "../../lib/default-values";
import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
-import { getScrapeQueue } from "../../services/queue-service";
import { redisEvictConnection } from "../../../src/services/redis";
import { v4 as uuidv4 } from "uuid";
import { logger } from "../../lib/logger";
@@ -23,6 +22,7 @@ import { Document as V0Document } from "./../../lib/entities";
import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
import { fromV0Combo } from "../v2/types";
import { ScrapeJobTimeoutError } from "../../lib/error";
+import { scrapeQueue } from "../../services/worker/nuq";
async function scrapeHelper(
jobId: string,
@@ -53,8 +53,6 @@ async function scrapeHelper(
};
}
- const jobPriority = await getJobPriority({ team_id, basePriority: 10 });
-
const { scrapeOptions, internalOptions } = fromV0Combo(
pageOptions,
extractorOptions,
@@ -81,15 +79,14 @@ async function scrapeHelper(
zeroDataRetention: false, // not supported on v0
apiKeyId,
},
- {},
jobId,
- jobPriority,
+ await getJobPriority({ team_id, basePriority: 10 }),
);
let doc;
try {
- doc = await waitForJob(jobId, timeout);
+ doc = await waitForJob(jobId, timeout, false);
} catch (e) {
if (e instanceof ScrapeJobTimeoutError) {
return {
@@ -120,7 +117,7 @@ async function scrapeHelper(
return err;
}
- await getScrapeQueue().remove(jobId);
+ await scrapeQueue.removeJob(jobId);
if (!doc) {
console.error("!!! PANIC DOC IS", doc);
diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.ts
index 086742a53..065c54c54 100644
--- a/apps/api/src/controllers/v0/search.ts
+++ b/apps/api/src/controllers/v0/search.ts
@@ -11,22 +11,18 @@ import { search } from "../../search";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
import { v4 as uuidv4 } from "uuid";
import { logger } from "../../lib/logger";
-import { getScrapeQueue } from "../../services/queue-service";
import { redisEvictConnection } from "../../../src/services/redis";
import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
import * as Sentry from "@sentry/node";
import { getJobPriority } from "../../lib/job-priority";
-import { Job } from "bullmq";
import {
- Document,
- fromLegacyCombo,
fromLegacyScrapeOptions,
TeamFlags,
toLegacyDocument,
} from "../v1/types";
-import { getJobFromGCS } from "../../lib/gcs-jobs";
import { fromV0Combo } from "../v2/types";
import { ScrapeJobTimeoutError } from "../../lib/error";
+import { scrapeQueue } from "../../services/worker/nuq";
export async function searchHelper(
jobId: string,
@@ -111,7 +107,7 @@ export async function searchHelper(
const url = x.url;
const uuid = uuidv4();
return {
- name: uuid,
+ jobId: uuid,
data: {
url,
mode: "single_urls" as const,
@@ -122,28 +118,23 @@ export async function searchHelper(
zeroDataRetention: false, // not supported on v0
apiKeyId: api_key_id,
},
- opts: {
- jobId: uuid,
- priority: jobPriority,
- },
};
});
// TODO: addScrapeJobs
for (const job of jobDatas) {
- await addScrapeJob(job.data, {}, job.opts.jobId, job.opts.priority);
+ await addScrapeJob(job.data, job.jobId, jobPriority);
}
const docs = (
- await Promise.all(jobDatas.map(x => waitForJob(x.opts.jobId, 60000)))
+ await Promise.all(jobDatas.map(x => waitForJob(x.jobId, 60000, false)))
).map(x => toLegacyDocument(x, internalOptions));
if (docs.length === 0) {
return { success: true, error: "No search results found", returnCode: 200 };
}
- const sq = getScrapeQueue();
- await Promise.all(jobDatas.map(x => sq.remove(x.opts.jobId)));
+ await scrapeQueue.removeJobs(jobDatas.map(x => x.jobId));
// make sure doc.content is not empty
const filteredDocs = docs.filter(
diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.ts
index 9cbd93716..bfad81b0b 100644
--- a/apps/api/src/controllers/v1/batch-scrape.ts
+++ b/apps/api/src/controllers/v1/batch-scrape.ts
@@ -145,6 +145,7 @@ export async function batchScrapeController(
logger.debug("Using job priority " + jobPriority, { jobPriority });
const jobs = urls.map(x => ({
+ jobId: uuidv4(),
data: {
url: x,
mode: "single_urls" as const,
@@ -161,10 +162,7 @@ export async function batchScrapeController(
zeroDataRetention: zeroDataRetention ?? false,
apiKeyId: req.acuc?.api_key_id ?? null,
},
- opts: {
- jobId: uuidv4(),
- priority: 20,
- },
+ priority: jobPriority,
}));
await finishCrawlKickoff(id);
@@ -179,7 +177,7 @@ export async function batchScrapeController(
logger.debug("Adding scrape jobs to Redis...");
await addCrawlJobs(
id,
- jobs.map(x => x.opts.jobId),
+ jobs.map(x => x.jobId),
logger,
);
logger.debug("Adding scrape jobs to BullMQ...");
diff --git a/apps/api/src/controllers/v1/crawl-errors.ts b/apps/api/src/controllers/v1/crawl-errors.ts
index e084310ea..a4b5cd2bb 100644
--- a/apps/api/src/controllers/v1/crawl-errors.ts
+++ b/apps/api/src/controllers/v1/crawl-errors.ts
@@ -5,31 +5,15 @@ import {
RequestWithAuth,
} from "./types";
import { getCrawl, getCrawlJobs } from "../../lib/crawl-redis";
-import { getScrapeQueue } from "../../services/queue-service";
import { redisEvictConnection } from "../../../src/services/redis";
import { configDotenv } from "dotenv";
-import { Job } from "bullmq";
import { supabase_rr_service } from "../../services/supabase";
-import { logger } from "../../lib/logger";
+import { logger as _logger } from "../../lib/logger";
import { deserializeTransportableError } from "../../lib/error-serde";
import { TransportableError } from "../../lib/error";
+import { scrapeQueue } from "../../services/worker/nuq";
configDotenv();
-export async function getJob(id: string) {
- const job = await getScrapeQueue().getJob(id);
- if (!job) return job;
-
- return job;
-}
-
-export async function getJobs(ids: string[]) {
- const jobs: (Job & { id: string })[] = (
- await Promise.all(ids.map(x => getScrapeQueue().getJob(x)))
- ).filter(x => x) as (Job & { id: string })[];
-
- return jobs;
-}
-
export async function crawlErrorsController(
req: RequestWithAuth<CrawlStatusParams, undefined, CrawlErrorsResponse>,
res: Response<CrawlErrorsResponse>,
@@ -41,25 +25,24 @@ export async function crawlErrorsController(
return res.status(403).json({ success: false, error: "Forbidden" });
}
- let jobStatuses = await Promise.all(
- (await getCrawlJobs(req.params.jobId)).map(
- async x => [x, await getScrapeQueue().getJobState(x)] as const,
- ),
- );
-
- const failedJobIDs: string[] = [];
+ const logger = _logger.child({
+ crawlId: req.params.jobId,
+ zeroDataRetention: sc.zeroDataRetention ?? false,
+ });
- for (const [id, status] of jobStatuses) {
- if (status === "failed") {
- failedJobIDs.push(id);
- }
- }
+ const failedJobs = (
+ await scrapeQueue.getJobsWithStatus(
+ await getCrawlJobs(req.params.jobId),
+ "failed",
+ logger,
+ )
+ ).filter(x => x.failedReason);
res.status(200).json({
- errors: (await getJobs(failedJobIDs))
+ errors: failedJobs
.map(x => {
const error = deserializeTransportableError(
- x.failedReason,
+ x.failedReason!,
) as TransportableError | null;
if (error?.code === "SCRAPE_RACED_REDIRECT_ERROR") {
return null;
@@ -67,8 +50,8 @@ export async function crawlErrorsController(
return {
id: x.id,
timestamp:
- x.finishedOn !== undefined
- ? new Date(x.finishedOn).toISOString()
+ x.finishedAt !== undefined
+ ? new Date(x.finishedAt).toISOString()
: undefined,
url: x.data.url,
...(error
@@ -77,7 +60,7 @@ export async function crawlErrorsController(
error: error.message,
}
: {
- error: x.failedReason,
+ error: x.failedReason!,
}),
};
})
@@ -95,7 +78,7 @@ export async function crawlErrorsController(
.throwOnError();
if (crawlJobError) {
- logger.error("Error getting crawl job", { error: crawlJobError });
+ _logger.error("Error getting crawl job", { error: crawlJobError });
throw crawlJobError;
}
@@ -130,7 +113,7 @@ export async function crawlErrorsController(
.throwOnError();
if (failedJobsError) {
- logger.error("Error getting failed jobs", { error: failedJobsError });
+ _logger.error("Error getting failed jobs", { error: failedJobsError });
throw failedJobsError;
}
diff --git a/apps/api/src/controllers/v1/crawl-status-ws.ts b/apps/api/src/controllers/v1/crawl-status-ws.ts
index 2313566fb..f1ec30606 100644
--- a/apps/api/src/controllers/v1/crawl-status-ws.ts
+++ b/apps/api/src/controllers/v1/crawl-status-ws.ts
@@ -15,15 +15,11 @@ import {
getCrawlExpiry,
getCrawlJobs,
getDoneJobsOrdered,
- getDoneJobsOrderedLength,
- isCrawlFinished,
- isCrawlFinishedLocked,
} from "../../lib/crawl-redis";
-import { getScrapeQueue } from "../../services/queue-service";
-import { getJob, getJobs } from "./crawl-status";
+import { getJobs, PseudoJob } from "./crawl-status";
import * as Sentry from "@sentry/node";
-import { Job, JobState } from "bullmq";
import { getConcurrencyLimitedJobs } from "../../lib/concurrency-limit";
+import { scrapeQueue, NuQJobStatus } from "../../services/worker/nuq";
type ErrorMessage = {
type: "error";
@@ -88,18 +84,14 @@ async function crawlStatusWS(
const notDoneJobIDs = jobIDs.filter(x => !doneJobIDs.includes(x));
- const queue = getScrapeQueue();
+ const newlyDoneJobIDs: string[] = (
+ await scrapeQueue.getJobsWithStatuses(notDoneJobIDs, [
+ "completed",
+ "failed",
+ ])
+ ).map(x => x.id);
- const jobStatuses = await Promise.all(
- notDoneJobIDs.map(async x => [x, await queue.getJobState(x)]),
- );
- const newlyDoneJobIDs: string[] = jobStatuses
- .filter(x => x[1] === "completed" || x[1] === "failed")
- .map(x => x[0]);
-
- const newlyDoneJobs: Job[] = (
- await Promise.all(newlyDoneJobIDs.map(x => getJob(x)))
- ).filter(x => x !== undefined) as Job[];
+ const newlyDoneJobs: PseudoJob<any>[] = await getJobs(newlyDoneJobIDs);
for (const job of newlyDoneJobs) {
if (job.returnvalue) {
@@ -118,28 +110,28 @@ async function crawlStatusWS(
setTimeout(loop, 1000);
- doneJobIDs = await getDoneJobsOrdered(req.params.jobId);
-
- let jobIDs = await getCrawlJobs(req.params.jobId);
-
- const queue = getScrapeQueue();
+ let [_doneJobIDs, jobIDs, throttledJobsSet] = await Promise.all([
+ getDoneJobsOrdered(req.params.jobId),
+ getCrawlJobs(req.params.jobId),
+ getConcurrencyLimitedJobs(req.auth.team_id),
+ ]);
- let jobStatuses = await Promise.all(
- jobIDs.map(async x => [x, await queue.getJobState(x)] as const),
- );
+ doneJobIDs = _doneJobIDs;
+ const jobs = new Map((await scrapeQueue.getJobs(jobIDs)).map(x => [x.id, x]));
- const throttledJobsSet = await getConcurrencyLimitedJobs(req.auth.team_id);
-
- const validJobStatuses: [string, JobState | "unknown"][] = [];
+ const validJobStatuses: [string, NuQJobStatus][] = [];
const validJobIDs: string[] = [];
- for (const [id, status] of jobStatuses) {
+ for (const id of jobIDs) {
if (throttledJobsSet.has(id)) {
- validJobStatuses.push([id, "prioritized"]);
- validJobIDs.push(id);
- } else if (status !== "failed" && status !== "unknown") {
- validJobStatuses.push([id, status]);
+ validJobStatuses.push([id, "queued"]);
validJobIDs.push(id);
+ } else {
+ const job = jobs.get(id);
+ if (job && job.status !== "failed") {
+ validJobStatuses.push([id, job.status]);
+ validJobIDs.push(id);
+ }
}
}
diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.ts
index 8380681ee..d856de2cf 100644
--- a/apps/api/src/controllers/v1/crawl-status.ts
+++ b/apps/api/src/controllers/v1/crawl-status.ts
@@ -13,21 +13,20 @@ import {
getCrawlQualifiedJobCount,
getDoneJobsOrderedUntil,
} from "../../lib/crawl-redis";
-import { getScrapeQueue } from "../../services/queue-service";
import {
supabaseGetJobById,
supabaseGetJobsById,
} from "../../lib/supabase-jobs";
import { configDotenv } from "dotenv";
-import type { Job, JobState } from "bullmq";
import { logger } from "../../lib/logger";
import { supabase_rr_service, supabase_service } from "../../services/supabase";
import { getJobFromGCS } from "../../lib/gcs-jobs";
+import { scrapeQueue, NuQJob, NuQJobStatus } from "../../services/worker/nuq";
configDotenv();
export type PseudoJob<T> = {
id: string;
- getState(): Promise<JobState | "unknown"> | JobState | "unknown";
+ status: NuQJobStatus;
returnvalue: T | null;
timestamp: number;
data: {
@@ -47,8 +46,8 @@ export type DBJob = {
};
export async function getJob(id: string): Promise<PseudoJob<any> | null> {
- const [bullJob, dbJob, gcsJob] = await Promise.all([
- getScrapeQueue().getJob(id),
+ const [nuqJob, dbJob, gcsJob] = await Promise.all([
+ scrapeQueue.getJob(id),
(process.env.USE_DB_AUTHENTICATION === "true"
? supabaseGetJobById(id)
: null) as Promise<DBJob | null>,
@@ -57,9 +56,9 @@ export async function getJob(id: string): Promise<PseudoJob<any> | null> {
>,
]);
- if (!bullJob && !dbJob) return null;
+ if (!nuqJob && !dbJob) return null;
- const data = gcsJob ?? dbJob?.docs ?? bullJob?.returnvalue;
+ const data = gcsJob ?? dbJob?.docs ?? nuqJob?.returnvalue;
if (gcsJob === null && data) {
logger.warn("GCS Job not found", {
jobId: id,
@@ -68,28 +67,23 @@ export async function getJob(id: string): Promise<PseudoJob<any> | null> {
const job: PseudoJob<any> = {
id,
- getState: dbJob
- ? () => (dbJob.success ? "completed" : "failed")
- : bullJob!.getState,
+ status: dbJob ? (dbJob.success ? "completed" : "failed") : nuqJob!.status,
returnvalue: Array.isArray(data) ? data[0] : data,
data: {
- scrapeOptions: bullJob ? bullJob.data.scrapeOptions : dbJob!.page_options,
+ scrapeOptions: nuqJob ? nuqJob.data.scrapeOptions : dbJob!.page_options,
},
- timestamp: bullJob
- ? bullJob.timestamp
+ timestamp: nuqJob
+ ? nuqJob.createdAt.valueOf()
: new Date(dbJob!.date_added).valueOf(),
- failedReason:
- (bullJob ? bullJob.failedReason : dbJob!.message) || undefined,
+ failedReason: (nuqJob ? nuqJob.failedReason : dbJob!.message) || undefined,
};
return job;
}
export async function getJobs(ids: string[]): Promise<PseudoJob<any>[]> {
- const [bullJobs, dbJobs, gcsJobs] = await Promise.all([
- Promise.all(ids.map(x => getScrapeQueue().getJob(x))).then(x =>
- x.filter(x => x),
- ) as Promise<(Job<any, any, string> & { id: string })[]>,
+ const [nuqJobs, dbJobs, gcsJobs] = await Promise.all([
+ scrapeQueue.getJobs(ids),
process.env.USE_DB_AUTHENTICATION === "true"
? supabaseGetJobsById(ids)
: [],
@@ -102,12 +96,12 @@ export async function getJobs(ids: string[]): Promise<PseudoJob<any>[]> {
: [],
]);
- const bullJobMap = new Map<string, PseudoJob<any>>();
+ const nuqJobMap = new Map<string, NuQJob<any, any>>();
const dbJobMap = new Map<string, DBJob>();
const gcsJobMap = new Map<string, any>();
- for (const job of bullJobs) {
- bullJobMap.set(job.id, job);
+ for (const job of nuqJobs) {
+ nuqJobMap.set(job.id, job);
}
for (const job of dbJobs) {
@@ -121,13 +115,13 @@ export async function getJobs(ids: string[]): Promise<PseudoJob<any>[]> {
const jobs: PseudoJob<any>[] = [];
for (const id of ids) {
- const bullJob = bullJobMap.get(id);
+ const nuqJob = nuqJobMap.get(id);
const dbJob = dbJobMap.get(id);
const gcsJob = gcsJobMap.get(id);
- if (!bullJob && !dbJob) continue;
+ if (!nuqJob && !dbJob) continue;
- const data = gcsJob ?? dbJob?.docs ?? bullJob?.returnvalue;
+ const data = gcsJob ?? dbJob?.docs ?? nuqJob?.returnvalue;
if (gcsJob === null && data) {
logger.warn("GCS Job not found", {
jobId: id,
@@ -136,20 +130,16 @@ export async function getJobs(ids: string[]): Promise<PseudoJob<any>[]> {
const job: PseudoJob<any> = {
id,
- getState: dbJob
- ? () => (dbJob.success ? "completed" : "failed")
- : () => bullJob!.getState(),
+ status: dbJob ? (dbJob.success ? "completed" : "failed") : nuqJob!.status,
returnvalue: Array.isArray(data) ? data[0] : data,
data: {
- scrapeOptions: bullJob
- ? bullJob.data.scrapeOptions
- : dbJob!.page_options,
+ scrapeOptions: nuqJob ? nuqJob.data.scrapeOptions : dbJob!.page_options,
},
- timestamp: bullJob
- ? bullJob.timestamp
+ timestamp: nuqJob
+ ? nuqJob.createdAt.valueOf()
: new Date(dbJob!.date_added).valueOf(),
failedReason:
- (bullJob ? bullJob.failedReason : dbJob!.message) || undefined,
+ (nuqJob ? nuqJob.failedReason : dbJob!.message) || undefined,
};
jobs.push(job);
@@ -390,30 +380,35 @@ export async function crawlStatusController(
let bytes = 0;
const bytesLimit = 10485760; // 10 MiB in bytes
- for (const jobId of doneJobs) {
- const job = await getScrapeQueue().getJob(jobId);
- const state = await job?.getState();
+ for (let i = 0; i < Math.ceil(doneJobs.length / 50); i++) {
+ const jobIds = doneJobs.slice(i * 50, (i + 1) * 50);
+ const jobs = await getJobs(jobIds);
- if (state === "failed") {
- // no iterated over, just ignore
- continue;
- } else {
- if (job?.returnvalue) {
- scrapes.push(job.returnvalue);
- bytes += JSON.stringify(job.returnvalue).length;
+ for (const job of jobs) {
+ if (job.status === "failed") {
+ continue;
} else {
- logger.warn(
- "Job was considered done, but returnvalue is undefined!",
- {
- scrapeId: jobId,
- crawlId: req.params.jobId,
- state,
- returnvalue: job?.returnvalue,
- },
- );
+ if (job?.returnvalue) {
+ scrapes.push(job.returnvalue);
+ bytes += JSON.stringify(job.returnvalue).length;
+ } else {
+ logger.warn(
+ "Job was considered done, but returnvalue is undefined!",
+ {
+ scrapeId: job.id,
+ crawlId: req.params.jobId,
+ state: job.status,
+ returnvalue: job?.returnvalue,
+ },
+ );
+ }
+
+ iteratedOver++;
}
- iteratedOver++;
+ if (bytes > bytesLimit) {
+ break;
+ }
}
if (bytes > bytesLimit) {
diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.ts
index 3e4399927..c04236470 100644
--- a/apps/api/src/controllers/v1/crawl.ts
+++ b/apps/api/src/controllers/v1/crawl.ts
@@ -148,9 +148,7 @@ export async function crawlController(
zeroDataRetention: zeroDataRetention || false,
apiKeyId: req.acuc?.api_key_id ?? null,
},
- {},
crypto.randomUUID(),
- 10,
);
const protocol = process.env.ENV === "local" ? req.protocol : "https";
diff --git a/apps/api/src/controllers/v1/extract-status.ts b/apps/api/src/controllers/v1/extract-status.ts
index d5bf6575b..cea0cc3bf 100644
--- a/apps/api/src/controllers/v1/extract-status.ts
+++ b/apps/api/src/controllers/v1/extract-status.ts
@@ -1,14 +1,27 @@
import { Response } from "express";
import { RequestWithAuth } from "./types";
import { getExtract, getExtractExpiry } from "../../lib/extract/extract-redis";
-import { DBJob, PseudoJob } from "./crawl-status";
+import { DBJob } from "./crawl-status";
import { getExtractQueue } from "../../services/queue-service";
import { ExtractResult } from "../../lib/extract/extraction-service";
import { supabaseGetJobByIdDirect } from "../../lib/supabase-jobs";
+import { JobState } from "bullmq";
+
+type ExtractPseudoJob<T> = {
+ id: string;
+ getState: () => Promise<JobState | "unknown"> | JobState | "unknown";
+ returnvalue: T | null;
+ timestamp: number;
+ data: {
+ scrapeOptions: any;
+ teamId?: string;
+ };
+ failedReason?: string;
+};
export async function getExtractJob(
id: string,
-): Promise<PseudoJob<ExtractResult> | null> {
+): Promise<ExtractPseudoJob<ExtractResult> | null> {
const [bullJob, dbJob] = await Promise.all([
getExtractQueue().getJob(id),
(process.env.USE_DB_AUTHENTICATION === "true"
@@ -20,7 +33,7 @@ export async function getExtractJob(
const data = dbJob?.docs ?? bullJob?.returnvalue?.data;
- const job: PseudoJob<any> = {
+ const job: ExtractPseudoJob<any> = {
id,
getState: dbJob
? () => (dbJob.success ? "completed" : "failed")
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.ts
index a8a85447e..baee96800 100644
--- a/apps/api/src/controllers/v1/scrape.ts
+++ b/apps/api/src/controllers/v1/scrape.ts
@@ -10,9 +10,9 @@ import {
import { v4 as uuidv4 } from "uuid";
import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
import { getJobPriority } from "../../lib/job-priority";
-import { getScrapeQueue } from "../../services/queue-service";
import { fromV1ScrapeOptions } from "../v2/types";
import { TransportableError } from "../../lib/error";
+import { scrapeQueue } from "../../services/worker/nuq";
import { checkPermissions } from "../../lib/permissions";
export async function scrapeController(
@@ -54,10 +54,6 @@ export async function scrapeController(
const timeout = req.body.timeout;
const startTime = new Date().getTime();
- const jobPriority = await getJobPriority({
- team_id: req.auth.team_id,
- basePriority: 10,
- });
const isDirectToBullMQ =
process.env.SEARCH_PREVIEW_TOKEN !== undefined &&
@@ -69,6 +65,11 @@ export async function scrapeController(
req.auth.team_id,
);
+ const jobPriority = await getJobPriority({
+ team_id: req.auth.team_id,
+ basePriority: 10,
+ });
+
const bullJob = await addScrapeJob(
{
url: req.body.url,
@@ -92,7 +93,6 @@ export async function scrapeController(
zeroDataRetention: zeroDataRetention ?? false,
apiKeyId: req.acuc?.api_key_id ?? null,
},
- {},
jobId,
jobPriority,
isDirectToBullMQ,
@@ -113,6 +113,7 @@ export async function scrapeController(
doc = await waitForJob(
bullJob ? bullJob : jobId,
timeout + totalWait,
+ zeroDataRetention ?? false,
logger,
);
} catch (e) {
@@ -122,7 +123,7 @@ export async function scrapeController(
});
if (zeroDataRetention) {
- await getScrapeQueue().remove(jobId);
+ await scrapeQueue.removeJob(jobId);
}
if (e instanceof TransportableError) {
@@ -142,7 +143,7 @@ export async function scrapeController(
logger.info("Done with waitForJob");
- await getScrapeQueue().remove(jobId);
+ await scrapeQueue.removeJob(jobId);
logger.info("Removed job from queue");
diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.ts
index 89a8a8e11..00b51ed26 100644
--- a/apps/api/src/controllers/v1/search.ts
+++ b/apps/api/src/controllers/v1/search.ts
@@ -7,26 +7,25 @@ import {
searchRequestSchema,
ScrapeOptions,
TeamFlags,
- scrapeOptions,
} from "./types";
import { billTeam } from "../../services/billing/credit_billing";
import { v4 as uuidv4 } from "uuid";
import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
import { logJob } from "../../services/logging/log_job";
-import { getJobPriority } from "../../lib/job-priority";
import { Mode } from "../../types";
-import { getScrapeQueue } from "../../services/queue-service";
import { search } from "../../search";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
import * as Sentry from "@sentry/node";
import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
import { logger as _logger } from "../../lib/logger";
import type { Logger } from "winston";
+import { getJobPriority } from "../../lib/job-priority";
import { CostTracking } from "../../lib/cost-tracking";
import { calculateCreditsToBeBilled } from "../../lib/scrape-billing";
import { supabase_service } from "../../services/supabase";
import { fromV1ScrapeOptions } from "../v2/types";
import { ScrapeJobTimeoutError } from "../../lib/error";
+import { scrapeQueue } from "../../services/worker/nuq";
interface DocumentWithCostTracking {
document: Document;
@@ -89,10 +88,6 @@ async function scrapeSearchResult(
isSearchPreview: boolean = false,
): Promise<DocumentWithCostTracking> {
const jobId = uuidv4();
- const jobPriority = await getJobPriority({
- team_id: options.teamId,
- basePriority: 10,
- });
const costTracking = new CostTracking();
@@ -116,6 +111,11 @@ async function scrapeSearchResult(
options.teamId,
);
+ const jobPriority = await getJobPriority({
+ team_id: options.teamId,
+ basePriority: 10,
+ });
+
await addScrapeJob(
{
url: searchResult.url,
@@ -140,13 +140,16 @@ async function scrapeSearchResult(
zeroDataRetention,
apiKeyId: options.apiKeyId,
},
- {},
jobId,
jobPriority,
directToBullMQ,
);
- const doc: Document = await waitForJob(jobId, options.timeout);
+ const doc: Document = await waitForJob(
+ jobId,
+ options.timeout,
+ zeroDataRetention,
+ );
logger.info("Scrape job completed", {
scrapeId: jobId,
@@ -154,7 +157,7 @@ async function scrapeSearchResult(
teamId: options.teamId,
origin: options.origin,
});
- await getScrapeQueue().remove(jobId);
+ await scrapeQueue.removeJob(jobId);
const document = {
title: searchResult.title,
diff --git a/apps/api/src/controllers/v1/x402-search.ts b/apps/api/src/controllers/v1/x402-search.ts
index 706fed9de..0e5aed0e0 100644
--- a/apps/api/src/controllers/v1/x402-search.ts
+++ b/apps/api/src/controllers/v1/x402-search.ts
@@ -11,9 +11,7 @@ import {
import { v4 as uuidv4 } from "uuid";
import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
import { logJob } from "../../services/logging/log_job";
-import { getJobPriority } from "../../lib/job-priority";
import { Mode } from "../../types";
-import { getScrapeQueue } from "../../services/queue-service";
import { search } from "../../search";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
import * as Sentry from "@sentry/node";
@@ -24,6 +22,8 @@ import { CostTracking } from "../../lib/cost-tracking";
import { supabase_service } from "../../services/supabase";
import { fromV1ScrapeOptions } from "../v2/types";
import { ScrapeJobTimeoutError } from "../../lib/error";
+import { scrapeQueue } from "../../services/worker/nuq";
+import { getJobPriority } from "../../lib/job-priority";
interface DocumentWithCostTracking {
document: Document;
@@ -45,10 +45,6 @@ async function scrapeX402SearchResult(
isSearchPreview: boolean = false,
): Promise<DocumentWithCostTracking> {
const jobId = uuidv4();
- const jobPriority = await getJobPriority({
- team_id: options.teamId,
- basePriority: 10,
- });
const costTracking = new CostTracking();
@@ -94,13 +90,19 @@ async function scrapeX402SearchResult(
zeroDataRetention,
apiKeyId: options.apiKeyId,
},
- {},
jobId,
- jobPriority,
+ await getJobPriority({
+ team_id: options.teamId,
+ basePriority: 10,
+ }),
directToBullMQ,
);
- const doc: Document = await waitForJob(jobId, options.timeout);
+ const doc: Document = await waitForJob(
+ jobId,
+ options.timeout,
+ zeroDataRetention,
+ );
logger.info("Scrape job [x402] completed", {
scrapeId: jobId,
@@ -108,7 +110,7 @@ async function scrapeX402SearchResult(
teamId: options.teamId,
origin: options.origin,
});
- await getScrapeQueue().remove(jobId);
+ await scrapeQueue.removeJob(jobId);
const document = {
title: searchResult.title,
diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.ts
index fe17023de..bdb704877 100644
--- a/apps/api/src/controllers/v2/batch-scrape.ts
+++ b/apps/api/src/controllers/v2/batch-scrape.ts
@@ -142,6 +142,7 @@ export async function batchScrapeController(
delete (scrapeOptions as any).appendToId;
const jobs = urls.map(x => ({
+ jobId: uuidv4(),
data: {
url: x,
mode: "single_urls" as const,
@@ -158,10 +159,7 @@ export async function batchScrapeController(
zeroDataRetention,
apiKeyId: req.acuc?.api_key_id ?? null,
},
- opts: {
- jobId: uuidv4(),
- priority: 20,
- },
+ priority: jobPriority,
}));
await finishCrawlKickoff(id);
@@ -176,7 +174,7 @@ export async function batchScrapeController(
logger.debug("Adding scrape jobs to Redis...");
await addCrawlJobs(
id,
- jobs.map(x => x.opts.jobId),
+ jobs.map(x => x.jobId),
logger,
);
logger.debug("Adding scrape jobs to BullMQ...");
diff --git a/apps/api/src/controllers/v2/crawl-errors.ts b/apps/api/src/controllers/v2/crawl-errors.ts
index e084310ea..a4b5cd2bb 100644
--- a/apps/api/src/controllers/v2/crawl-errors.ts
+++ b/apps/api/src/controllers/v2/crawl-errors.ts
@@ -5,31 +5,15 @@ import {
RequestWithAuth,
} from "./types";
import { getCrawl, getCrawlJobs } from "../../lib/crawl-redis";
-import { getScrapeQueue } from "../../services/queue-service";
import { redisEvictConnection } from "../../../src/services/redis";
import { configDotenv } from "dotenv";
-import { Job } from "bullmq";
import { supabase_rr_service } from "../../services/supabase";
-import { logger } from "../../lib/logger";
+import { logger as _logger } from "../../lib/logger";
import { deserializeTransportableError } from "../../lib/error-serde";
import { TransportableError } from "../../lib/error";
+import { scrapeQueue } from "../../services/worker/nuq";
configDotenv();
-export async function getJob(id: string) {
- const job = await getScrapeQueue().getJob(id);
- if (!job) return job;
-
- return job;
-}
-
-export async function getJobs(ids: string[]) {
- const jobs: (Job & { id: string })[] = (
- await Promise.all(ids.map(x => getScrapeQueue().getJob(x)))
- ).filter(x => x) as (Job & { id: string })[];
-
- return jobs;
-}
-
export async function crawlErrorsController(
req: RequestWithAuth<CrawlStatusParams, undefined, CrawlErrorsResponse>,
res: Response<CrawlErrorsResponse>,
@@ -41,25 +25,24 @@ export async function crawlErrorsController(
return res.status(403).json({ success: false, error: "Forbidden" });
}
- let jobStatuses = await Promise.all(
- (await getCrawlJobs(req.params.jobId)).map(
- async x => [x, await getScrapeQueue().getJobState(x)] as const,
- ),
- );
-
- const failedJobIDs: string[] = [];
+ const logger = _logger.child({
+ crawlId: req.params.jobId,
+ zeroDataRetention: sc.zeroDataRetention ?? false,
+ });
- for (const [id, status] of jobStatuses) {
- if (status === "failed") {
- failedJobIDs.push(id);
- }
- }
+ const failedJobs = (
+ await scrapeQueue.getJobsWithStatus(
+ await getCrawlJobs(req.params.jobId),
+ "failed",
+ logger,
+ )
+ ).filter(x => x.failedReason);
res.status(200).json({
- errors: (await getJobs(failedJobIDs))
+ errors: failedJobs
.map(x => {
const error = deserializeTransportableError(
- x.failedReason,
+ x.failedReason!,
) as TransportableError | null;
if (error?.code === "SCRAPE_RACED_REDIRECT_ERROR") {
return null;
@@ -67,8 +50,8 @@ export async function crawlErrorsController(
return {
id: x.id,
timestamp:
- x.finishedOn !== undefined
- ? new Date(x.finishedOn).toISOString()
+ x.finishedAt !== undefined
+ ? new Date(x.finishedAt).toISOString()
: undefined,
url: x.data.url,
...(error
@@ -77,7 +60,7 @@ export async function crawlErrorsController(
error: error.message,
}
: {
- error: x.failedReason,
+ error: x.failedReason!,
}),
};
})
@@ -95,7 +78,7 @@ export async function crawlErrorsController(
.throwOnError();
if (crawlJobError) {
- logger.error("Error getting crawl job", { error: crawlJobError });
+ _logger.error("Error getting crawl job", { error: crawlJobError });
throw crawlJobError;
}
@@ -130,7 +113,7 @@ export async function crawlErrorsController(
.throwOnError();
if (failedJobsError) {
- logger.error("Error getting failed jobs", { error: failedJobsError });
+ _logger.error("Error getting failed jobs", { error: failedJobsError });
throw failedJobsError;
}
diff --git a/apps/api/src/controllers/v2/crawl-status-ws.ts b/apps/api/src/controllers/v2/crawl-status-ws.ts
index 2313566fb..f1ec30606 100644
--- a/apps/api/src/controllers/v2/crawl-status-ws.ts
+++ b/apps/api/src/controllers/v2/crawl-status-ws.ts
@@ -15,15 +15,11 @@ import {
getCrawlExpiry,
getCrawlJobs,
getDoneJobsOrdered,
- getDoneJobsOrderedLength,
- isCrawlFinished,
- isCrawlFinishedLocked,
} from "../../lib/crawl-redis";
-import { getScrapeQueue } from "../../services/queue-service";
-import { getJob, getJobs } from "./crawl-status";
+import { getJobs, PseudoJob } from "./crawl-status";
import * as Sentry from "@sentry/node";
-import { Job, JobState } from "bullmq";
import { getConcurrencyLimitedJobs } from "../../lib/concurrency-limit";
+import { scrapeQueue, NuQJobStatus } from "../../services/worker/nuq";
type ErrorMessage = {
type: "error";
@@ -88,18 +84,14 @@ async function crawlStatusWS(
const notDoneJobIDs = jobIDs.filter(x => !doneJobIDs.includes(x));
- const queue = getScrapeQueue();
+ const newlyDoneJobIDs: string[] = (
+ await scrapeQueue.getJobsWithStatuses(notDoneJobIDs, [
+ "completed",
+ "failed",
+ ])
+ ).map(x => x.id);
- const jobStatuses = await Promise.all(
- notDoneJobIDs.map(async x => [x, await queue.getJobState(x)]),
- );
- const newlyDoneJobIDs: string[] = jobStatuses
- .filter(x => x[1] === "completed" || x[1] === "failed")
- .map(x => x[0]);
-
- const newlyDoneJobs: Job[] = (
- await Promise.all(newlyDoneJobIDs.map(x => getJob(x)))
- ).filter(x => x !== undefined) as Job[];
+ const newlyDoneJobs: PseudoJob<any>[] = await getJobs(newlyDoneJobIDs);
for (const job of newlyDoneJobs) {
if (job.returnvalue) {
@@ -118,28 +110,28 @@ async function crawlStatusWS(
setTimeout(loop, 1000);
- doneJobIDs = await getDoneJobsOrdered(req.params.jobId);
-
- let jobIDs = await getCrawlJobs(req.params.jobId);
-
- const queue = getScrapeQueue();
+ let [_doneJobIDs, jobIDs, throttledJobsSet] = await Promise.all([
+ getDoneJobsOrdered(req.params.jobId),
+ getCrawlJobs(req.params.jobId),
+ getConcurrencyLimitedJobs(req.auth.team_id),
+ ]);
- let jobStatuses = await Promise.all(
- jobIDs.map(async x => [x, await queue.getJobState(x)] as const),
- );
+ doneJobIDs = _doneJobIDs;
+ const jobs = new Map((await scrapeQueue.getJobs(jobIDs)).map(x => [x.id, x]));
- const throttledJobsSet = await getConcurrencyLimitedJobs(req.auth.team_id);
-
- const validJobStatuses: [string, JobState | "unknown"][] = [];
+ const validJobStatuses: [string, NuQJobStatus][] = [];
const validJobIDs: string[] = [];
- for (const [id, status] of jobStatuses) {
+ for (const id of jobIDs) {
if (throttledJobsSet.has(id)) {
- validJobStatuses.push([id, "prioritized"]);
- validJobIDs.push(id);
- } else if (status !== "failed" && status !== "unknown") {
- validJobStatuses.push([id, status]);
+ validJobStatuses.push([id, "queued"]);
validJobIDs.push(id);
+ } else {
+ const job = jobs.get(id);
+ if (job && job.status !== "failed") {
+ validJobStatuses.push([id, job.status]);
+ validJobIDs.push(id);
+ }
}
}
diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.ts
index 9c421d116..3520e2063 100644
--- a/apps/api/src/controllers/v2/crawl-status.ts
+++ b/apps/api/src/controllers/v2/crawl-status.ts
@@ -13,21 +13,20 @@ import {
getDoneJobsOrderedUntil,
isCrawlKickoffFinished,
} from "../../lib/crawl-redis";
-import { getScrapeQueue } from "../../services/queue-service";
import {
supabaseGetJobById,
supabaseGetJobsById,
} from "../../lib/supabase-jobs";
import { configDotenv } from "dotenv";
-import type { Job, JobState, Queue } from "bullmq";
import { logger } from "../../lib/logger";
import { supabase_rr_service, supabase_service } from "../../services/supabase";
import { getJobFromGCS } from "../../lib/gcs-jobs";
+import { scrapeQueue, NuQJob, NuQJobStatus } from "../../services/worker/nuq";
configDotenv();
export type PseudoJob<T> = {
id: string;
- getState(): Promise<JobState | "unknown"> | JobState | "unknown";
+ status: NuQJobStatus;
returnvalue: T | null;
timestamp: number;
data: {
@@ -47,8 +46,8 @@ export type DBJob = {
};
export async function getJob(id: string): Promise<PseudoJob<any> | null> {
- const [bullJob, dbJob, gcsJob] = await Promise.all([
- getScrapeQueue().getJob(id),
+ const [nuqJob, dbJob, gcsJob] = await Promise.all([
+ scrapeQueue.getJob(id),
(process.env.USE_DB_AUTHENTICATION === "true"
? supabaseGetJobById(id)
: null) as Promise<DBJob | null>,
@@ -57,9 +56,9 @@ export async function getJob(id: string): Promise<PseudoJob<any> | null> {
>,
]);
- if (!bullJob && !dbJob) return null;
+ if (!nuqJob && !dbJob) return null;
- const data = gcsJob ?? dbJob?.docs ?? bullJob?.returnvalue;
+ const data = gcsJob ?? dbJob?.docs ?? nuqJob?.returnvalue;
if (gcsJob === null && data) {
logger.warn("GCS Job not found", {
jobId: id,
@@ -68,28 +67,23 @@ export async function getJob(id: string): Promise<PseudoJob<any> | null> {
const job: PseudoJob<any> = {
id,
- getState: bullJob
- ? bullJob.getState
- : () => (dbJob!.success ? "completed" : "failed"),
+ status: dbJob ? (dbJob.success ? "completed" : "failed") : nuqJob!.status,
returnvalue: Array.isArray(data) ? data[0] : data,
data: {
- scrapeOptions: bullJob ? bullJob.data.scrapeOptions : dbJob!.page_options,
+ scrapeOptions: nuqJob ? nuqJob.data.scrapeOptions : dbJob!.page_options,
},
- timestamp: bullJob
- ? bullJob.timestamp
+ timestamp: nuqJob
+ ? nuqJob.createdAt.valueOf()
: new Date(dbJob!.date_added).valueOf(),
- failedReason:
- (bullJob ? bullJob.failedReason : dbJob!.message) || undefined,
+ failedReason: (nuqJob ? nuqJob.failedReason : dbJob!.message) || undefined,
};
return job;
}
export async function getJobs(ids: string[]): Promise<PseudoJob<any>[]> {
- const [bullJobs, dbJobs, gcsJobs] = await Promise.all([
- Promise.all(ids.map(x => getScrapeQueue().getJob(x))).then(x =>
- x.filter(x => x),
- ) as Promise<(Job<any, any, string> & { id: string })[]>,
+ const [nuqJobs, dbJobs, gcsJobs] = await Promise.all([
+ scrapeQueue.getJobs(ids),
process.env.USE_DB_AUTHENTICATION === "true"
? supabaseGetJobsById(ids)
: [],
@@ -102,12 +96,12 @@ export async function getJobs(ids: string[]): Promise<PseudoJob<any>[]> {
: [],
]);
- const bullJobMap = new Map<string, PseudoJob<any>>();
+ const nuqJobMap = new Map<string, NuQJob<any, any>>();
const dbJobMap = new Map<string, DBJob>();
const gcsJobMap = new Map<string, any>();
- for (const job of bullJobs) {
- bullJobMap.set(job.id, job);
+ for (const job of nuqJobs) {
+ nuqJobMap.set(job.id, job);
}
for (const job of dbJobs) {
@@ -121,13 +115,13 @@ export async function getJobs(ids: string[]): Promise<PseudoJob<any>[]> {
const jobs: PseudoJob<any>[] = [];
for (const id of ids) {
- const bullJob = bullJobMap.get(id);
+ const nuqJob = nuqJobMap.get(id);
const dbJob = dbJobMap.get(id);
const gcsJob = gcsJobMap.get(id);
- if (!bullJob && !dbJob) continue;
+ if (!nuqJob && !dbJob) continue;
- const data = gcsJob ?? dbJob?.docs ?? bullJob?.returnvalue;
+ const data = gcsJob ?? dbJob?.docs ?? nuqJob?.returnvalue;
if (gcsJob === null && data) {
logger.warn("GCS Job not found", {
jobId: id,
@@ -136,20 +130,16 @@ export async function getJobs(ids: string[]): Promise<PseudoJob<any>[]> {
const job: PseudoJob<any> = {
id,
- getState: bullJob
- ? () => bullJob.getState()
- : () => (dbJob!.success ? "completed" : "failed"),
+ status: dbJob ? (dbJob.success ? "completed" : "failed") : nuqJob!.status,
returnvalue: Array.isArray(data) ? data[0] : data,
data: {
- scrapeOptions: bullJob
- ? bullJob.data.scrapeOptions
- : dbJob!.page_options,
+ scrapeOptions: nuqJob ? nuqJob.data.scrapeOptions : dbJob!.page_options,
},
- timestamp: bullJob
- ? bullJob.timestamp
+ timestamp: nuqJob
+ ? nuqJob.createdAt.valueOf()
: new Date(dbJob!.date_added).valueOf(),
failedReason:
- (bullJob ? bullJob.failedReason : dbJob!.message) || undefined,
+ (nuqJob ? nuqJob.failedReason : dbJob!.message) || undefined,
};
jobs.push(job);
@@ -390,7 +380,6 @@ export async function crawlStatusController(
: undefined,
};
} else {
- // old BullMQ-based path
const doneJobs = await getDoneJobsOrderedUntil(
req.params.jobId,
djoCutoff,
@@ -403,33 +392,35 @@ export async function crawlStatusController(
let bytes = 0;
const bytesLimit = 10485760; // 10 MiB in bytes
- for (const jobId of doneJobs) {
- const job = await getScrapeQueue().getJob(jobId);
- const state = await job?.getState();
+ for (let i = 0; i < Math.ceil(doneJobs.length / 50); i++) {
+ const jobIds = doneJobs.slice(i * 50, (i + 1) * 50);
+ const jobs = await getJobs(jobIds);
- if (state === "failed") {
- continue;
- }
-
- let doc: Document | undefined;
- if (job?.returnvalue) {
- doc = job.returnvalue as Document;
- } else {
- const gcsDocs = await getJobFromGCS(jobId);
- if (gcsDocs && gcsDocs.length > 0) {
- doc = gcsDocs[0] as Document;
+ for (const job of jobs) {
+ if (job.status === "failed") {
+ continue;
} else {
- logger.warn(
- "Job was considered done, but neither BullMQ returnvalue nor GCS blob is available!",
- { scrapeId: jobId, crawlId: req.params.jobId, state },
- );
+ if (job?.returnvalue) {
+ scrapes.push(job.returnvalue);
+ bytes += JSON.stringify(job.returnvalue).length;
+ } else {
+ logger.warn(
+ "Job was considered done, but returnvalue is undefined!",
+ {
+ scrapeId: job.id,
+ crawlId: req.params.jobId,
+ state: job.status,
+ returnvalue: job?.returnvalue,
+ },
+ );
+ }
+
+ iteratedOver++;
}
- }
- if (doc) {
- scrapes.push(doc);
- bytes += JSON.stringify(doc).length;
- iteratedOver++;
+ if (bytes > bytesLimit) {
+ break;
+ }
}
if (bytes > bytesLimit) {
diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.ts
index ec0ff91b8..1982ad8b4 100644
--- a/apps/api/src/controllers/v2/crawl.ts
+++ b/apps/api/src/controllers/v2/crawl.ts
@@ -196,9 +196,7 @@ export async function crawlController(
zeroDataRetention: zeroDataRetention || false,
apiKeyId: req.acuc?.api_key_id ?? null,
},
- {},
crypto.randomUUID(),
- 10,
);
const protocol = process.env.ENV === "local" ? req.protocol : "https";
diff --git a/apps/api/src/controllers/v2/extract-status.ts b/apps/api/src/controllers/v2/extract-status.ts
index 08f24c784..bec3b62bc 100644
--- a/apps/api/src/controllers/v2/extract-status.ts
+++ b/apps/api/src/controllers/v2/extract-status.ts
@@ -1,14 +1,27 @@
import { Response } from "express";
import { RequestWithAuth } from "./types";
import { getExtract, getExtractExpiry } from "../../lib/extract/extract-redis";
-import { DBJob, PseudoJob } from "./crawl-status";
+import { DBJob } from "./crawl-status";
import { getExtractQueue } from "../../services/queue-service";
import { ExtractResult } from "../../lib/extract/extraction-service";
import { supabaseGetJobByIdDirect } from "../../lib/supabase-jobs";
+import { JobState } from "bullmq";
+
+type ExtractPseudoJob<T> = {
+ id: string;
+ getState: () => Promise<JobState | "unknown"> | JobState | "unknown";
+ returnvalue: T | null;
+ timestamp: number;
+ data: {
+ scrapeOptions: any;
+ teamId?: string;
+ };
+ failedReason?: string;
+};
export async function getExtractJob(
id: string,
-): Promise<PseudoJob<ExtractResult> | null> {
+): Promise<ExtractPseudoJob<ExtractResult> | null> {
const [bullJob, dbJob] = await Promise.all([
getExtractQueue().getJob(id),
(process.env.USE_DB_AUTHENTICATION === "true"
@@ -20,7 +33,7 @@ export async function getExtractJob(
const data = dbJob?.docs ?? bullJob?.returnvalue?.data;
- const job: PseudoJob<any> = {
+ const job: ExtractPseudoJob<any> = {
id,
getState: bullJob
? bullJob.getState.bind(bullJob)
diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.ts
index 7200f0865..3663f0135 100644
--- a/apps/api/src/controllers/v2/scrape.ts
+++ b/apps/api/src/controllers/v2/scrape.ts
@@ -10,9 +10,9 @@ import {
import { v4 as uuidv4 } from "uuid";
import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
import { getJobPriority } from "../../lib/job-priority";
-import { getScrapeQueue } from "../../services/queue-service";
import { hasFormatOfType } from "../../lib/format-utils";
import { TransportableError } from "../../lib/error";
+import { scrapeQueue } from "../../services/worker/nuq";
import { checkPermissions } from "../../lib/permissions";
export async function scrapeController(
@@ -54,16 +54,17 @@ export async function scrapeController(
const timeout = req.body.timeout;
const startTime = new Date().getTime();
- const jobPriority = await getJobPriority({
- team_id: req.auth.team_id,
- basePriority: 10,
- });
const isDirectToBullMQ =
process.env.SEARCH_PREVIEW_TOKEN !== undefined &&
process.env.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;
- await addScrapeJob(
+ const jobPriority = await getJobPriority({
+ team_id: req.auth.team_id,
+ basePriority: 10,
+ });
+
+ const job = await addScrapeJob(
{
url: req.body.url,
mode: "single_urls",
@@ -92,7 +93,6 @@ export async function scrapeController(
zeroDataRetention,
apiKeyId: req.acuc?.api_key_id ?? null,
},
- {},
jobId,
jobPriority,
isDirectToBullMQ,
@@ -108,8 +108,10 @@ export async function scrapeController(
let doc: Document;
try {
doc = await waitForJob(
- jobId,
+ job ?? jobId,
timeout !== undefined ? timeout + totalWait : null,
+ zeroDataRetention,
+ logger,
);
} catch (e) {
logger.error(`Error in scrapeController`, {
@@ -118,7 +120,7 @@ export async function scrapeController(
});
if (zeroDataRetention) {
- await getScrapeQueue().remove(jobId);
+ await scrapeQueue.removeJob(jobId);
}
if (e instanceof TransportableError) {
@@ -135,7 +137,7 @@ export async function scrapeController(
}
}
- await getScrapeQueue().remove(jobId);
+ await scrapeQueue.removeJob(jobId);
if (!hasFormatOfType(req.body.formats, "rawHtml")) {
if (doc && doc.rawHtml) {
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.ts
index c958cf63b..8989abfa6 100644
--- a/apps/api/src/controllers/v2/search.ts
+++ b/apps/api/src/controllers/v2/search.ts
@@ -12,19 +12,19 @@ import { billTeam } from "../../services/billing/credit_billing";
import { v4 as uuidv4 } from "uuid";
import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
import { logJob } from "../../services/logging/log_job";
-import { getJobPriority } from "../../lib/job-priority";
import { Mode } from "../../types";
-import { getScrapeQueue } from "../../services/queue-service";
import { search } from "../../search/v2";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
import * as Sentry from "@sentry/node";
import { logger as _logger } from "../../lib/logger";
import type { Logger } from "winston";
+import { getJobPriority } from "../../lib/job-priority";
import { CostTracking } from "../../lib/cost-tracking";
import { calculateCreditsToBeBilled } from "../../lib/scrape-billing";
import { supabase_service } from "../../services/supabase";
import { SearchResult, SearchV2Response } from "../../lib/entities";
import { ScrapeJobTimeoutError } from "../../lib/error";
+import { scrapeQueue } from "../../services/worker/nuq";
import { z } from "zod";
import {
buildSearchQuery,
@@ -59,10 +59,6 @@ async function startScrapeJob(
isSearchPreview: boolean = false,
): Promise<string> {
const jobId = uuidv4();
- const jobPriority = await getJobPriority({
- team_id: options.teamId,
- basePriority: 10,
- });
const zeroDataRetention = flags?.forceZDR ?? false;
@@ -74,6 +70,11 @@ async function startScrapeJob(
zeroDataRetention,
});
+ const jobPriority = await getJobPriority({
+ team_id: options.teamId,
+ basePriority: 10,
+ });
+
await addScrapeJob(
{
url: searchResult.url,
@@ -96,7 +97,6 @@ async function startScrapeJob(
zeroDataRetention,
apiKeyId: options.apiKeyId,
},
- {},
jobId,
jobPriority,
directToBullMQ,
@@ -131,8 +131,7 @@ async function scrapeSearchResult(
isSearchPreview,
);
- // Wait for the job to complete
- const doc: Document = await waitForJob(jobId, options.timeout);
+ const doc: Document = await waitForJob(jobId, options.timeout, false);
logger.info("Scrape job completed", {
scrapeId: jobId,
@@ -140,7 +139,8 @@ async function scrapeSearchResult(
teamId: options.teamId,
origin: options.origin,
});
- await getScrapeQueue().remove(jobId);
+
+ await scrapeQueue.removeJob(jobId);
const document = {
title: searchResult.title,
diff --git a/apps/api/src/harness.ts b/apps/api/src/harness.ts
index c59f4edbb..cc3b1d4b5 100644
--- a/apps/api/src/harness.ts
+++ b/apps/api/src/harness.ts
@@ -1,3 +1,4 @@
+import "dotenv/config";
import { exec } from "child_process";
import * as net from "net";
import { basename } from "path";
@@ -77,50 +78,64 @@ function execForward(fancyName: string, command: string): Promise<void> {
}
(async () => {
- console.log("=== Installing dependencies and building all components...");
- await Promise.all([
- (async () => {
- const install = execForward("api@install", "pnpm install");
- await install;
-
- const build = execForward("api@build", "pnpm build");
- await build;
- })(),
- execForward(
- "sharedLibs/crawler@build",
- "cd sharedLibs/crawler && cargo build --release",
- ),
- execForward(
- "sharedLibs/html-transformer@build",
- "cd sharedLibs/html-transformer && cargo build --release",
- ),
- execForward(
- "sharedLibs/pdf-parser@build",
- "cd sharedLibs/pdf-parser && cargo build --release",
- ),
- (async () => {
- const install = execForward(
- "sharedLibs/go-html-to-md@install",
- "cd sharedLibs/go-html-to-md && go mod tidy",
- );
- await install;
+ if (process.argv[2] !== "--start-docker") {
+ console.log("=== Installing dependencies and building all components...");
+ await Promise.all([
+ (async () => {
+ if (process.argv[2] !== "--start-built") {
+ const install = execForward("api@install", "pnpm install");
+ await install;
+
+ const build = execForward("api@build", "pnpm build");
+ await build;
+ } else {
+ console.log("=== Skipping install and build, using built files...");
+ }
+ })(),
+ execForward(
+ "sharedLibs/crawler@build",
+ "cd sharedLibs/crawler && cargo build --release",
+ ),
+ execForward(
+ "sharedLibs/html-transformer@build",
+ "cd sharedLibs/html-transformer && cargo build --release",
+ ),
+ execForward(
+ "sharedLibs/pdf-parser@build",
+ "cd sharedLibs/pdf-parser && cargo build --release",
+ ),
+ (async () => {
+ const install = execForward(
+ "sharedLibs/go-html-to-md@install",
+ "cd sharedLibs/go-html-to-md && go mod tidy",
+ );
+ await install;
- const build = execForward(
- "sharedLibs/go-html-to-md@build",
- `cd sharedLibs/go-html-to-md && go build -o ${basename(HTML_TO_MARKDOWN_PATH)} -buildmode=c-shared html-to-markdown.go`,
- );
- await build;
- })(),
- ]);
+ const build = execForward(
+ "sharedLibs/go-html-to-md@build",
+ `cd sharedLibs/go-html-to-md && go build -o ${basename(HTML_TO_MARKDOWN_PATH)} -buildmode=c-shared html-to-markdown.go`,
+ );
+ await build;
+ })(),
+ ]);
+ }
console.log("=== Starting API, Worker, and Index Worker...");
- const api = execForward("api", "pnpm start:production:nobuild");
+ const api = execForward("api", "pnpm server:production:nobuild");
const worker = execForward("worker", "pnpm worker:production");
- const indexWorker = execForward(
- "index-worker",
- "pnpm index-worker:production",
- );
+ const nuqWorkers = new Array(5)
+ .fill(0)
+ .map((_, i) =>
+ execForward(
+ `nuq-worker-${i}`,
+ `NUQ_WORKER_PORT=${3006 + i} NUQ_REDUCE_NOISE=true pnpm nuq-worker:production`,
+ ),
+ );
+ const indexWorker =
+ process.env.USE_DB_AUTHENTICATION === "true"
+ ? execForward("index-worker", "pnpm index-worker:production")
+ : null;
try {
await Promise.race([
@@ -130,17 +145,49 @@ function execForward(fancyName: string, command: string): Promise<void> {
),
]);
- console.log("=== Running command...");
-
- const cmd = execForward("command", command.join(" "));
-
- await Promise.race([cmd, api, worker, indexWorker]);
+ if (
+ process.argv[2] === "--start" ||
+ process.argv[2] === "--start-built" ||
+ process.argv[2] === "--start-docker"
+ ) {
+ console.log(
+ "=== Everything is up and running, waiting for termination or failure...",
+ );
+ await Promise.race([
+ new Promise(resolve => {
+ process.on("SIGINT", resolve);
+ process.on("SIGTERM", resolve);
+ }),
+ api,
+ worker,
+ ...nuqWorkers,
+ ...(indexWorker ? [indexWorker] : []),
+ ]);
+ } else {
+ console.log("=== Running command...");
+ const cmd = execForward("command", command.join(" "));
+ await Promise.race([
+ cmd,
+ api,
+ worker,
+ ...nuqWorkers,
+ ...(indexWorker ? [indexWorker] : []),
+ ]);
+ }
} finally {
console.log("=== Tearing down API, Worker, and Index Worker...");
exec("pkill -f 'queue-worker.js'");
exec("pkill -f 'index.js'");
- exec("pkill -f 'index-worker.js'");
- await Promise.all([api, worker, indexWorker]);
+ if (indexWorker) {
+ exec("pkill -f 'index-worker.js'");
+ }
+ exec("pkill -f 'nuq-worker.js'");
+ await Promise.all([
+ api,
+ worker,
+ ...nuqWorkers,
+ ...(indexWorker ? [indexWorker] : []),
+ ]);
}
console.log("=== Goodbye!");
diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts
index db0848b8d..c8ca7fd54 100644
--- a/apps/api/src/index.ts
+++ b/apps/api/src/index.ts
@@ -6,7 +6,6 @@ import bodyParser from "body-parser";
import cors from "cors";
import {
getExtractQueue,
- getScrapeQueue,
getGenerateLlmsTxtQueue,
getDeepResearchQueue,
getBillingQueue,
@@ -27,7 +26,6 @@ import {
} from "./controllers/v1/types";
import { ZodError } from "zod";
import { v4 as uuidv4 } from "uuid";
-import { RateLimiterMode } from "./types";
import { attachWsProxy } from "./services/agentLivecastWS";
import { cacheableLookup } from "./scraper/scrapeURL/lib/cacheableLookup";
import { v2Router } from "./routes/v2";
@@ -39,6 +37,7 @@ import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";
import { resourceFromAttributes } from "@opentelemetry/resources";
import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-node";
import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-grpc";
+import { nuqShutdown } from "./services/worker/nuq";
const { createBullBoard } = require("@bull-board/api");
const { BullMQAdapter } = require("@bull-board/api/bullMQAdapter");
@@ -105,7 +104,6 @@ serverAdapter.setBasePath(`/admin/${process.env.BULL_AUTH_KEY}/queues`);
const { addQueue, removeQueue, setQueues, replaceQueues } = createBullBoard({
queues: [
- new BullMQAdapter(getScrapeQueue()),
new BullMQAdapter(getExtractQueue()),
new BullMQAdapter(getGenerateLlmsTxtQueue()),
new BullMQAdapter(getDeepResearchQueue()),
@@ -156,14 +154,17 @@ function startServer(port = DEFAULT_PORT) {
}
server.close(() => {
logger.info("Server closed.");
- if (otelSdk) {
- otelSdk.shutdown().then(() => {
- logger.info("OTEL shutdown");
+ nuqShutdown().finally(() => {
+ logger.info("NUQ shutdown complete");
+ if (otelSdk) {
+ otelSdk.shutdown().finally(() => {
+ logger.info("OTEL shutdown");
+ process.exit(0);
+ });
+ } else {
process.exit(0);
- });
- } else {
- process.exit(0);
- }
+ }
+ });
});
};
@@ -176,77 +177,6 @@ if (require.main === module) {
startServer();
}
-app.get(`/serverHealthCheck`, async (req, res) => {
- try {
- const scrapeQueue = getScrapeQueue();
- const [waitingJobs] = await Promise.all([scrapeQueue.getWaitingCount()]);
- const noWaitingJobs = waitingJobs === 0;
- // 200 if no active jobs, 503 if there are active jobs
- return res.status(noWaitingJobs ? 200 : 500).json({
- waitingJobs,
- });
- } catch (error) {
- Sentry.captureException(error);
- logger.error(error);
- return res.status(500).json({ error: error.message });
- }
-});
-
-app.get("/serverHealthCheck/notify", async (req, res) => {
- if (process.env.SLACK_WEBHOOK_URL) {
- const treshold = 1; // The treshold value for the active jobs
- const timeout = 60000; // 1 minute // The timeout value for the check in milliseconds
-
- const getWaitingJobsCount = async () => {
- const scrapeQueue = getScrapeQueue();
- const [waitingJobsCount] = await Promise.all([
- scrapeQueue.getWaitingCount(),
- ]);
-
- return waitingJobsCount;
- };
-
- res.status(200).json({ message: "Check initiated" });
-
- const checkWaitingJobs = async () => {
- try {
- let waitingJobsCount = await getWaitingJobsCount();
- if (waitingJobsCount >= treshold) {
- setTimeout(async () => {
- // Re-check the waiting jobs count after the timeout
- waitingJobsCount = await getWaitingJobsCount();
- if (waitingJobsCount >= treshold) {
- const slackWebhookUrl = process.env.SLACK_WEBHOOK_URL!;
- const message = {
- text: `⚠️ Warning: The number of active jobs (${waitingJobsCount}) has exceeded the threshold (${treshold}) for more than ${
- timeout / 60000
- } minute(s).`,
- };
-
- const response = await fetch(slackWebhookUrl, {
- method: "POST",
- headers: {
- "Content-Type": "application/json",
- },
- body: JSON.stringify(message),
- });
-
- if (!response.ok) {
- logger.error("Failed to send Slack notification");
- }
- }
- }, timeout);
- }
- } catch (error) {
- Sentry.captureException(error);
- logger.debug(error);
- }
- };
-
- checkWaitingJobs();
- }
-});
-
app.get("/is-production", (req, res) => {
res.send({ isProduction: global.isProduction });
});
@@ -323,12 +253,3 @@ app.use(
);
logger.info(`Worker ${process.pid} started`);
-// const sq = getScrapeQueue();
-
-// sq.on("waiting", j => ScrapeEvents.logJobEvent(j, "waiting"));
-// sq.on("active", j => ScrapeEvents.logJobEvent(j, "active"));
-// sq.on("completed", j => ScrapeEvents.logJobEvent(j, "completed"));
-// sq.on("paused", j => ScrapeEvents.logJobEvent(j, "paused"));
-// sq.on("resumed", j => ScrapeEvents.logJobEvent(j, "resumed"));
-// sq.on("removed", j => ScrapeEvents.logJobEvent(j, "removed"));
-//
diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.ts
index 42527634b..e56721769 100644
--- a/apps/api/src/lib/concurrency-limit.ts
+++ b/apps/api/src/lib/concurrency-limit.ts
@@ -1,11 +1,10 @@
import { RateLimiterMode } from "../types";
import { redisEvictConnection } from "../services/redis";
-import type { Job, JobsOptions } from "bullmq";
import { getACUCTeam } from "../controllers/auth";
import { getCrawl, StoredCrawl } from "./crawl-redis";
-import { getScrapeQueue } from "../services/queue-service";
import { logger } from "./logger";
import { abTestJob } from "../services/ab-test";
+import { scrapeQueue, type NuQJob } from "../services/worker/nuq";
const constructKey = (team_id: string) => "concurrency-limiter:" + team_id;
const constructQueueKey = (team_id: string) =>
@@ -65,8 +64,7 @@ export async function removeConcurrencyLimitActiveJob(
export type ConcurrencyLimitedJob = {
id: string;
data: any;
- opts: JobsOptions;
- priority?: number;
+ priority: number;
};
export async function cleanOldConcurrencyLimitedJobs(
@@ -311,7 +309,7 @@ async function getNextConcurrentJob(
*
* @param job The BullMQ job that is done.
*/
-export async function concurrentJobDone(job: Job) {
+export async function concurrentJobDone(job: NuQJob<any>) {
if (job.id && job.data && job.data.team_id) {
await removeConcurrencyLimitActiveJob(job.data.team_id, job.id);
await cleanOldConcurrencyLimitEntries(job.data.team_id);
@@ -360,17 +358,13 @@ export async function concurrentJobDone(job: Job) {
abTestJob(nextJob.job.data);
- (await getScrapeQueue()).add(
+ await scrapeQueue.addJob(
nextJob.job.id,
{
...nextJob.job.data,
concurrencyLimitHit: true,
},
- {
- ...nextJob.job.opts,
- jobId: nextJob.job.id,
- priority: nextJob.job.priority,
- },
+ nextJob.job.priority,
);
}
}
diff --git a/apps/api/src/lib/extract/document-scraper.ts b/apps/api/src/lib/extract/document-scraper.ts
index becd6ea9e..d7b70618e 100644
--- a/apps/api/src/lib/extract/document-scraper.ts
+++ b/apps/api/src/lib/extract/document-scraper.ts
@@ -5,12 +5,12 @@ import {
URLTrace,
scrapeOptions as scrapeOptionsSchema,
} from "../../controllers/v2/types";
-import { getScrapeQueue } from "../../services/queue-service";
import { waitForJob } from "../../services/queue-jobs";
import { addScrapeJob } from "../../services/queue-jobs";
import { getJobPriority } from "../job-priority";
import type { Logger } from "winston";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
+import { scrapeQueue } from "../../services/worker/nuq";
interface ScrapeDocumentOptions {
url: string;
@@ -69,14 +69,16 @@ export async function scrapeDocument(
zeroDataRetention: false, // not supported
apiKeyId: options.apiKeyId,
},
- {},
jobId,
jobPriority,
);
- const doc = await waitForJob(jobId, timeout);
-
- await getScrapeQueue().remove(jobId);
+ const doc = await waitForJob(jobId, timeout, false, logger);
+ try {
+ await scrapeQueue.removeJob(jobId);
+ } catch (error) {
+ logger.warn("Error removing job from queue", { error, scrapeId: jobId });
+ }
if (trace) {
trace.timing.completedAt = new Date().toISOString();
diff --git a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
index cf9d04661..df7319b8e 100644
--- a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
+++ b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
@@ -5,12 +5,12 @@ import {
URLTrace,
scrapeOptions as scrapeOptionsSchema,
} from "../../../controllers/v2/types";
-import { getScrapeQueue } from "../../../services/queue-service";
import { waitForJob } from "../../../services/queue-jobs";
import { addScrapeJob } from "../../../services/queue-jobs";
import { getJobPriority } from "../../job-priority";
import type { Logger } from "winston";
import { isUrlBlocked } from "../../../scraper/WebScraper/utils/blocklist";
+import { scrapeQueue } from "../../../services/worker/nuq";
interface ScrapeDocumentOptions {
url: string;
@@ -68,13 +68,16 @@ export async function scrapeDocument_F0(
zeroDataRetention: false, // not supported
apiKeyId: options.apiKeyId,
},
- {},
jobId,
jobPriority,
);
- const doc = await waitForJob(jobId, timeout);
- await getScrapeQueue().remove(jobId);
+ const doc = await waitForJob(jobId, timeout, false, logger);
+ try {
+ await scrapeQueue.removeJob(jobId);
+ } catch (error) {
+ logger.warn("Error removing job from queue", { error, scrapeId: jobId });
+ }
if (trace) {
trace.timing.completedAt = new Date().toISOString();
diff --git a/apps/api/src/lib/logger.ts b/apps/api/src/lib/logger.ts
index 10a1e11a5..29d78e2d5 100644
--- a/apps/api/src/lib/logger.ts
+++ b/apps/api/src/lib/logger.ts
@@ -35,6 +35,17 @@ const zeroDataRetentionFilter = winston.format(info => {
return info;
})();
+const reduceNoiseFilter = winston.format(info => {
+ if (
+ (info.metadata?.module === "nuq/metrics" ||
+ info.module === "nuq/metrics") &&
+ process.env.NUQ_REDUCE_NOISE === "true"
+ ) {
+ return false; // Don't log this message
+ }
+ return info;
+})();
+
export const logger = winston.createLogger({
level: process.env.LOGGING_LEVEL?.toLowerCase() ?? "debug",
format: winston.format.json({
@@ -71,6 +82,7 @@ export const logger = winston.createLogger({
: []),
new winston.transports.Console({
format: winston.format.combine(
+ reduceNoiseFilter,
zeroDataRetentionFilter,
winston.format.timestamp({ format: "YYYY-MM-DD HH:mm:ss" }),
winston.format.metadata({
diff --git a/apps/api/src/lib/scrape-events.ts b/apps/api/src/lib/scrape-events.ts
deleted file mode 100644
index a50f8fba1..000000000
--- a/apps/api/src/lib/scrape-events.ts
+++ /dev/null
@@ -1,109 +0,0 @@
-// import { Job } from "bullmq";
-// import { supabase_service as supabase } from "../services/supabase";
-// import { logger } from "./logger";
-// import { configDotenv } from "dotenv";
-// import { Engine } from "../scraper/scrapeURL/engines";
-// configDotenv();
-
-// export type ScrapeErrorEvent = {
-// type: "error";
-// message: string;
-// stack?: string;
-// };
-
-// export type ScrapeScrapeEvent = {
-// type: "scrape";
-// url: string;
-// worker?: string;
-// method: Engine;
-// result: null | {
-// success: boolean;
-// response_code?: number;
-// response_size?: number;
-// error?: string | object;
-// // proxy?: string,
-// time_taken: number;
-// };
-// };
-
-// export type ScrapeQueueEvent = {
-// type: "queue";
-// event:
-// | "waiting"
-// | "active"
-// | "completed"
-// | "paused"
-// | "resumed"
-// | "removed"
-// | "failed";
-// worker?: string;
-// };
-
-// export type ScrapeEvent =
-// | ScrapeErrorEvent
-// | ScrapeScrapeEvent
-// | ScrapeQueueEvent;
-
-// export class ScrapeEvents {
-// static async insert(jobId: string, content: ScrapeEvent) {
-// if (jobId === "TEST") return null;
-
-// const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";
-// if (useDbAuthentication) {
-// try {
-// const result = await supabase
-// .from("scrape_events")
-// .insert({
-// job_id: jobId,
-// type: content.type,
-// content: content,
-// // created_at
-// })
-// .select()
-// .single();
-// return (result.data as any).id;
-// } catch (error) {
-// // logger.error(`Error inserting scrape event: ${error}`);
-// return null;
-// }
-// }
-
-// return null;
-// }
-
-// static async updateScrapeResult(
-// logId: number | null,
-// result: ScrapeScrapeEvent["result"],
-// ) {
-// if (logId === null) return;
-
-// try {
-// const previousLog = (
-// await supabase.from("scrape_events").select().eq("id", logId).single()
-// ).data as any;
-// await supabase
-// .from("scrape_events")
-// .update({
-// content: {
-// ...previousLog.content,
-// result,
-// },
-// })
-// .eq("id", logId);
-// } catch (error) {
-// logger.error(`Error updating scrape result: ${error}`);
-// }
-// }
-
-// static async logJobEvent(job: Job | any, event: ScrapeQueueEvent["event"]) {
-// try {
-// await this.insert(((job as any).id ? (job as any).id : job) as string, {
-// type: "queue",
-// event,
-// worker: process.env.FLY_MACHINE_ID,
-// });
-// } catch (error) {
-// logger.error(`Error logging job event: ${error}`);
-// }
-// }
-// }
diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.ts
index e500ebd4f..7d90faffe 100644
--- a/apps/api/src/main/runWebScraper.ts
+++ b/apps/api/src/main/runWebScraper.ts
@@ -1,9 +1,8 @@
-import { Job } from "bullmq";
import { WebScraperOptions, RunWebScraperParams } from "../types";
-import { supabase_service } from "../services/supabase";
import { logger as _logger } from "../lib/logger";
import { configDotenv } from "dotenv";
import { scrapeURL, ScrapeUrlResponse } from "../scraper/scrapeURL";
+import type { NuQJob } from "../services/worker/nuq";
import { CostTracking } from "../lib/cost-tracking";
configDotenv();
@@ -11,7 +10,7 @@ export async function startWebScraperPipeline({
job,
costTracking,
}: {
- job: Job<WebScraperOptions> & { id: string };
+ job: NuQJob<WebScraperOptions>;
costTracking: CostTracking;
}) {
return await runWebScraper({
@@ -33,8 +32,8 @@ export async function startWebScraperPipeline({
...job.data.internalOptions,
},
team_id: job.data.team_id,
- bull_job_id: job.id.toString(),
- priority: job.opts.priority,
+ bull_job_id: job.id,
+ priority: job.priority,
is_scrape: job.data.is_scrape ?? false,
is_crawl: !!(job.data.crawl_id && job.data.crawlerOptions !== null),
urlInvisibleInCurrentCrawl:
@@ -172,40 +171,3 @@ export async function runWebScraper({
}
}
}
-
-const saveJob = async (job: Job, result: any, mode: string) => {
- try {
- const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";
- if (useDbAuthentication) {
- const { data, error } = await supabase_service
- .from("firecrawl_jobs")
- .update({ docs: result })
- .eq("job_id", job.id);
-
- if (error) throw new Error(error.message);
- // try {
- // if (mode === "crawl") {
- // await job.moveToCompleted(null, token, false);
- // } else {
- // await job.moveToCompleted(result, token, false);
- // }
- // } catch (error) {
- // // I think the job won't exist here anymore
- // }
- // } else {
- // try {
- // await job.moveToCompleted(result, token, false);
- // } catch (error) {
- // // I think the job won't exist here anymore
- // }
- }
- // ScrapeEvents.logJobEvent(job, "completed");
- } catch (error) {
- _logger.error(`🐂 Failed to update job status`, {
- module: "runWebScraper",
- method: "saveJob",
- jobId: job.id,
- scrapeId: job.id,
- });
- }
-};
diff --git a/apps/api/src/routes/admin.ts b/apps/api/src/routes/admin.ts
index 5a8a83b8f..c455aa549 100644
--- a/apps/api/src/routes/admin.ts
+++ b/apps/api/src/routes/admin.ts
@@ -1,11 +1,5 @@
import express from "express";
import { redisHealthController } from "../controllers/v0/admin/redis-health";
-import {
- autoscalerController,
- checkQueuesController,
- cleanBefore24hCompleteJobsController,
- queuesController,
-} from "../controllers/v0/admin/queue";
import { wrap } from "./shared";
import { acucCacheClearController } from "../controllers/v0/admin/acuc-cache-clear";
import { checkFireEngine } from "../controllers/v0/admin/check-fire-engine";
@@ -22,23 +16,6 @@ adminRouter.get(
redisHealthController,
);
-adminRouter.get(
- `/admin/${process.env.BULL_AUTH_KEY}/clean-before-24h-complete-jobs`,
- cleanBefore24hCompleteJobsController,
-);
-
-adminRouter.get(
- `/admin/${process.env.BULL_AUTH_KEY}/check-queues`,
- checkQueuesController,
-);
-
-adminRouter.get(`/admin/${process.env.BULL_AUTH_KEY}/queues`, queuesController);
-
-adminRouter.get(
- `/admin/${process.env.BULL_AUTH_KEY}/autoscaler`,
- autoscalerController,
-);
-
adminRouter.post(
`/admin/${process.env.BULL_AUTH_KEY}/acuc-cache-clear`,
wrap(acucCacheClearController),
diff --git a/apps/api/src/services/alerts/index.ts b/apps/api/src/services/alerts/index.ts
deleted file mode 100644
index 44f2b8a04..000000000
--- a/apps/api/src/services/alerts/index.ts
+++ /dev/null
@@ -1,62 +0,0 @@
-import { logger } from "../../../src/lib/logger";
-import { getScrapeQueue } from "../queue-service";
-import { sendSlackWebhook } from "./slack";
-
-export async function checkAlerts() {
- try {
- if (
- process.env.SLACK_WEBHOOK_URL &&
- process.env.ENV === "production" &&
- process.env.ALERT_NUM_ACTIVE_JOBS &&
- process.env.ALERT_NUM_WAITING_JOBS
- ) {
- logger.info("Initializing alerts");
- const checkActiveJobs = async () => {
- try {
- const scrapeQueue = getScrapeQueue();
- const activeJobs = await scrapeQueue.getActiveCount();
- if (activeJobs > Number(process.env.ALERT_NUM_ACTIVE_JOBS)) {
- logger.warn(
- `Alert: Number of active jobs is over ${process.env.ALERT_NUM_ACTIVE_JOBS}. Current active jobs: ${activeJobs}.`,
- );
- sendSlackWebhook(
- `Alert: Number of active jobs is over ${process.env.ALERT_NUM_ACTIVE_JOBS}. Current active jobs: ${activeJobs}`,
- true,
- );
- } else {
- logger.info(
- `Number of active jobs is under ${process.env.ALERT_NUM_ACTIVE_JOBS}. Current active jobs: ${activeJobs}`,
- );
- }
- } catch (error) {
- logger.error(`Failed to check active jobs: ${error}`);
- }
- };
-
- const checkWaitingQueue = async () => {
- const scrapeQueue = getScrapeQueue();
- const waitingJobs = await scrapeQueue.getWaitingCount();
-
- if (waitingJobs > Number(process.env.ALERT_NUM_WAITING_JOBS)) {
- logger.warn(
- `Alert: Number of waiting jobs is over ${process.env.ALERT_NUM_WAITING_JOBS}. Current waiting jobs: ${waitingJobs}.`,
- );
- sendSlackWebhook(
- `Alert: Number of waiting jobs is over ${process.env.ALERT_NUM_WAITING_JOBS}. Current waiting jobs: ${waitingJobs}. Scale up the number of workers with fly scale count worker=20`,
- true,
- );
- }
- };
-
- const checkAll = async () => {
- await checkActiveJobs();
- await checkWaitingQueue();
- };
-
- await checkAll();
- // setInterval(checkAll, 10000); // Run every
- }
- } catch (error) {
- logger.error(`Failed to initialize alerts: ${error}`);
- }
-}
diff --git a/apps/api/src/services/billing/batch_billing.ts b/apps/api/src/services/billing/batch_billing.ts
index 5962f192a..8e675129b 100644
--- a/apps/api/src/services/billing/batch_billing.ts
+++ b/apps/api/src/services/billing/batch_billing.ts
@@ -2,13 +2,8 @@ import { logger } from "../../lib/logger";
import { getRedisConnection } from "../queue-service";
import { supabase_service } from "../supabase";
import * as Sentry from "@sentry/node";
-import { Queue } from "bullmq";
import { withAuth } from "../../lib/withAuth";
-import {
- getACUC,
- setCachedACUC,
- setCachedACUCTeam,
-} from "../../controllers/auth";
+import { setCachedACUC, setCachedACUCTeam } from "../../controllers/auth";
// Configuration constants
const BATCH_KEY = "billing_batch";
diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.ts
index c85f101ff..a70100f7a 100644
--- a/apps/api/src/services/indexing/index-worker.ts
+++ b/apps/api/src/services/indexing/index-worker.ts
@@ -218,9 +218,7 @@ const processPrecrawlJobInternal = async (token: string, job: Job) => {
zeroDataRetention: false,
apiKeyId: null,
},
- {},
crypto.randomUUID(),
- 10,
);
} catch (e) {
logger.error("Error processing one cycle of the precrawl job", {
diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.ts
index 8fda3d39e..cd0907205 100644
--- a/apps/api/src/services/logging/log_job.ts
+++ b/apps/api/src/services/logging/log_job.ts
@@ -55,6 +55,10 @@ export async function logJob(
});
try {
+ if (process.env.GCS_BUCKET_NAME) {
+ await saveJobToGCS(job);
+ }
+
const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";
if (!useDbAuthentication) {
return;
@@ -116,10 +120,6 @@ export async function logJob(
: null,
};
- if (process.env.GCS_BUCKET_NAME) {
- await saveJobToGCS(job);
- }
-
if (bypassLogging) {
return;
}
diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.ts
index 986ee1818..67141224a 100644
--- a/apps/api/src/services/queue-jobs.ts
+++ b/apps/api/src/services/queue-jobs.ts
@@ -1,7 +1,5 @@
-import { getScrapeQueue, getScrapeQueueEvents } from "./queue-service";
import { v4 as uuidv4 } from "uuid";
import { NotificationType, RateLimiterMode, WebScraperOptions } from "../types";
-import * as Sentry from "@sentry/node";
import {
cleanOldConcurrencyLimitEntries,
getConcurrencyLimitActiveJobs,
@@ -14,16 +12,15 @@ import {
import { logger as _logger } from "../lib/logger";
import { sendNotificationWithCustomDays } from "./notification/email_notification";
import { shouldSendConcurrencyLimitNotification } from "./notification/notification-check";
-import { getACUC, getACUCTeam } from "../controllers/auth";
+import { getACUCTeam } from "../controllers/auth";
import { getJobFromGCS, removeJobFromGCS } from "../lib/gcs-jobs";
import { Document } from "../controllers/v1/types";
import { getCrawl } from "../lib/crawl-redis";
import { Logger } from "winston";
-import { Job } from "bullmq";
import { ScrapeJobTimeoutError, TransportableError } from "../lib/error";
import { deserializeTransportableError } from "../lib/error-serde";
-import { robustFetch } from "../scraper/scrapeURL/lib/fetch";
import { abTestJob } from "./ab-test";
+import { NuQJob, scrapeQueue } from "./worker/nuq";
/**
* Checks if a job is a crawl or batch scrape based on its options
@@ -41,21 +38,15 @@ function isCrawlOrBatchScrape(options: {
async function _addScrapeJobToConcurrencyQueue(
webScraperOptions: any,
- options: any,
jobId: string,
- jobPriority: number,
+ priority: number = 0,
) {
await pushConcurrencyLimitedJob(
webScraperOptions.team_id,
{
id: jobId,
data: webScraperOptions,
- opts: {
- ...options,
- priority: jobPriority,
- jobId: jobId,
- },
- priority: jobPriority,
+ priority,
},
webScraperOptions.crawl_id
? Infinity
@@ -65,10 +56,9 @@ async function _addScrapeJobToConcurrencyQueue(
export async function _addScrapeJobToBullMQ(
webScraperOptions: WebScraperOptions,
- options: any,
jobId: string,
- jobPriority: number,
-): Promise<Job> {
+ priority: number = 0,
+): Promise<NuQJob<WebScraperOptions>> {
abTestJob(webScraperOptions);
if (webScraperOptions && webScraperOptions.team_id) {
@@ -90,20 +80,15 @@ export async function _addScrapeJobToBullMQ(
}
}
- return await getScrapeQueue().add(jobId, webScraperOptions, {
- ...options,
- priority: jobPriority,
- jobId,
- });
+ return await scrapeQueue.addJob(jobId, webScraperOptions, priority);
}
async function addScrapeJobRaw(
webScraperOptions: WebScraperOptions,
- options: any,
jobId: string,
- jobPriority: number,
+ priority: number = 0,
directToBullMQ: boolean = false,
-): Promise<Job | null> {
+): Promise<NuQJob<WebScraperOptions> | null> {
let concurrencyLimited: "yes" | "yes-crawl" | "no" | null = null;
let currentActiveConcurrency = 0;
let maxConcurrency = 0;
@@ -188,46 +173,32 @@ async function addScrapeJobRaw(
webScraperOptions.concurrencyLimited = true;
- await _addScrapeJobToConcurrencyQueue(
- webScraperOptions,
- options,
- jobId,
- jobPriority,
- );
+ await _addScrapeJobToConcurrencyQueue(webScraperOptions, jobId);
return null;
} else {
- return await _addScrapeJobToBullMQ(
- webScraperOptions,
- options,
- jobId,
- jobPriority,
- );
+ return await _addScrapeJobToBullMQ(webScraperOptions, jobId, priority);
}
}
export async function addScrapeJob(
webScraperOptions: WebScraperOptions,
- options: any = {},
jobId: string = uuidv4(),
- jobPriority: number = 10,
+ priority: number = 0,
directToBullMQ: boolean = false,
-): Promise<Job | null> {
+): Promise<NuQJob<WebScraperOptions> | null> {
return await addScrapeJobRaw(
webScraperOptions,
- options,
jobId,
- jobPriority,
+ priority,
directToBullMQ,
);
}
export async function addScrapeJobs(
jobs: {
+ jobId: string;
data: WebScraperOptions;
- opts: {
- jobId: string;
- priority: number;
- };
+ priority: number;
}[],
) {
if (jobs.length === 0) return true;
@@ -235,11 +206,9 @@ export async function addScrapeJobs(
const jobsByTeam = new Map<
string,
{
+ jobId: string;
data: WebScraperOptions;
- opts: {
- jobId: string;
- priority: number;
- };
+ priority: number;
}[]
>();
@@ -254,18 +223,14 @@ export async function addScrapeJobs(
// == Buckets for jobs ==
let jobsForcedToCQ: {
data: WebScraperOptions;
- opts: {
- jobId: string;
- priority: number;
- };
+ jobId: string;
+ priority: number;
}[] = [];
let jobsPotentiallyInCQ: {
data: WebScraperOptions;
- opts: {
- jobId: string;
- priority: number;
- };
+ jobId: string;
+ priority: number;
}[] = [];
// == Select jobs by crawl ID ==
@@ -273,19 +238,15 @@ export async function addScrapeJobs(
string,
{
data: WebScraperOptions;
- opts: {
- jobId: string;
- priority: number;
- };
+ jobId: string;
+ priority: number;
}[]
>();
const jobsWithoutCrawlID: {
data: WebScraperOptions;
- opts: {
- jobId: string;
- priority: number;
- };
+ jobId: string;
+ priority: number;
}[] = [];
for (const job of teamJobs) {
@@ -386,66 +347,46 @@ export async function addScrapeJobs(
const size = JSON.stringify(job.data).length;
await _addScrapeJobToConcurrencyQueue(
job.data,
- job.opts,
- job.opts.jobId,
- job.opts.priority,
+ job.jobId,
+ job.priority,
);
}),
);
await Promise.all(
addToBull.map(async job => {
- const size = JSON.stringify(job.data).length;
- await _addScrapeJobToBullMQ(
- job.data,
- job.opts,
- job.opts.jobId,
- job.opts.priority,
- );
+ await _addScrapeJobToBullMQ(job.data, job.jobId, job.priority);
}),
);
}
}
export async function waitForJob(
- _job: Job | string,
+ job: NuQJob<WebScraperOptions> | string,
timeout: number | null,
+ zeroDataRetention: boolean,
logger: Logger = _logger,
): Promise<Document> {
- const start = Date.now();
- const queue = getScrapeQueue();
- let job: Job | undefined =
- typeof _job == "string" ? await queue.getJob(_job) : _job;
- while (job === undefined) {
- logger.debug("Waiting for job to be created");
- await new Promise(resolve => setTimeout(resolve, 500));
- job = await queue.getJob(_job as string);
- if (Date.now() - start > (timeout ?? 180000)) {
- throw new ScrapeJobTimeoutError(
- "Scrape timed out while waiting in the concurrency limit queue",
- );
- }
- }
- let doc: Document;
+ const jobId = typeof job == "string" ? job : job.id;
+ const isConcurrencyLimited = !!(typeof job === "string");
+
+ let doc: Document | null = null;
try {
doc = await Promise.race(
[
- job.waitUntilFinished(getScrapeQueueEvents(), timeout ?? 180000),
+ scrapeQueue.waitForJob(jobId, timeout !== null ? timeout + 100 : null),
timeout !== null
- ? new Promise((resolve, reject) => {
- setTimeout(
- () => {
- reject(
- new ScrapeJobTimeoutError(
- "Scrape timed out" +
- (typeof _job === "string"
- ? " after waiting in the concurrency limit queue"
- : ""),
- ),
- );
- },
- Math.max(0, timeout - (Date.now() - start)),
- );
+ ? new Promise<Document>((_resolve, reject) => {
+ setTimeout(() => {
+ reject(
+ new ScrapeJobTimeoutError(
+ "Scrape timed out" +
+ (isConcurrencyLimited
+ ? " after waiting in the concurrency limit queue"
+ : ""),
+ ),
+ );
+ }, timeout);
})
: null,
].filter(x => x !== null),
@@ -467,15 +408,15 @@ export async function waitForJob(
logger.debug("Got job");
if (!doc) {
- const docs = await getJobFromGCS(job.id!);
+ const docs = await getJobFromGCS(jobId);
logger.debug("Got job from GCS");
if (!docs || docs.length === 0) {
throw new Error("Job not found in GCS");
}
- doc = docs[0];
+ doc = docs[0]!;
- if (job.data?.internalOptions?.zeroDataRetention) {
- await removeJobFromGCS(job.id!);
+ if (zeroDataRetention) {
+ await removeJobFromGCS(jobId);
}
}
diff --git a/apps/api/src/services/queue-service.ts b/apps/api/src/services/queue-service.ts
index 10d7c6474..5e0f68f47 100644
--- a/apps/api/src/services/queue-service.ts
+++ b/apps/api/src/services/queue-service.ts
@@ -1,4 +1,4 @@
-import { Queue, QueueEvents } from "bullmq";
+import { Queue } from "bullmq";
import { logger } from "../lib/logger";
import IORedis from "ioredis";
import { BullMQOtel } from "bullmq-otel";
@@ -6,8 +6,6 @@ import type { DeepResearchServiceOptions } from "../lib/deep-research/deep-resea
export type QueueFunction = () => Queue<any, any, string, any, any, string>;
-let scrapeQueue: Queue;
-let scrapeQueueEvents: QueueEvents;
let extractQueue: Queue;
let loggingQueue: Queue;
let indexQueue: Queue;
@@ -29,7 +27,6 @@ export function getRedisConnection(): IORedis {
return redisConnection;
}
-export const scrapeQueueName = "{scrapeQueue}";
export const extractQueueName = "{extractQueue}";
export const loggingQueueName = "{loggingQueue}";
export const indexQueueName = "{indexQueue}";
@@ -38,34 +35,6 @@ export const deepResearchQueueName = "{deepResearchQueue}";
export const billingQueueName = "{billingQueue}";
export const precrawlQueueName = "{precrawlQueue}";
-export function getScrapeQueue() {
- if (!scrapeQueue) {
- scrapeQueue = new Queue(scrapeQueueName, {
- connection: getRedisConnection(),
- defaultJobOptions: {
- removeOnComplete: {
- age: 3600, // 1 hour
- },
- removeOnFail: {
- age: 3600, // 1 hour
- },
- },
- telemetry: new BullMQOtel("firecrawl-bullmq"),
- });
- }
- return scrapeQueue;
-}
-
-export function getScrapeQueueEvents() {
- if (!scrapeQueueEvents) {
- scrapeQueueEvents = new QueueEvents(scrapeQueueName, {
- connection: getRedisConnection(),
- });
- }
-
- return scrapeQueueEvents;
-}
-
export function getExtractQueue() {
if (!extractQueue) {
extractQueue = new Queue(extractQueueName, {
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index 6b3fae215..83494420f 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -2,27 +2,16 @@ import "dotenv/config";
import "./sentry";
import * as Sentry from "@sentry/node";
import {
- getScrapeQueue,
getExtractQueue,
getDeepResearchQueue,
getGenerateLlmsTxtQueue,
- scrapeQueueName,
getRedisConnection,
} from "./queue-service";
-import { Job, Queue, QueueEvents } from "bullmq";
+import { Job, Queue, Worker } from "bullmq";
import { logger as _logger } from "../lib/logger";
-import { Worker } from "bullmq";
import systemMonitor from "./system-monitor";
import { v4 as uuidv4 } from "uuid";
-import {
- addCrawlJobDone,
- finishCrawlKickoff,
- getCrawl,
- normalizeURL,
-} from "../lib/crawl-redis";
-import { StoredCrawl } from "../lib/crawl-redis";
import { configDotenv } from "dotenv";
-import { concurrentJobDone } from "../lib/concurrency-limit";
import {
ExtractResult,
performExtraction,
@@ -39,9 +28,6 @@ import http from "http";
import https from "https";
import { cacheableLookup } from "../scraper/scrapeURL/lib/cacheableLookup";
import { robustFetch } from "../scraper/scrapeURL/lib/fetch";
-import { redisEvictConnection } from "./redis";
-import path from "path";
-import { finishCrawlIfNeeded } from "./worker/crawl-logic";
import { NodeSDK } from "@opentelemetry/sdk-node";
import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node";
import { LangfuseExporter } from "langfuse-vercel";
@@ -414,62 +400,6 @@ process.on("SIGTERM", () => {
let cantAcceptConnectionCount = 0;
-/**
- * Converts a file path to a proper URL for cross-platform compatibility.
- * On Windows, absolute paths need to be converted to file:// URLs for ESM imports.
- * @param filePath - The file path to convert
- * @returns A properly formatted path/URL for the current platform
- */
-function getWorkerPath(filePath: string): string | URL {
- if (process.platform === "win32" && path.isAbsolute(filePath)) {
- // On Windows, convert absolute paths to file:// URLs for ESM compatibility
- return pathToFileURL(filePath);
- }
- return filePath;
-}
-
-const separateWorkerFun = (queue: Queue, workerPath: string): Worker => {
- // Extract memory size from --max-old-space-size flag if present
- const maxOldSpaceSize =
- process.env.SCRAPE_WORKER_MAX_OLD_SPACE_SIZE ||
- process.execArgv
- .find(arg => arg.startsWith("--max-old-space-size="))
- ?.split("=")[1];
-
- // Filter out the invalid flag for worker threads
- const filteredExecArgv = process.execArgv.filter(
- arg => !arg.startsWith("--max-old-space-size"),
- );
-
- // Convert path to proper format for the current platform
- const platformWorkerPath = getWorkerPath(workerPath);
-
- const worker = new Worker(queue.name, platformWorkerPath, {
- connection: getRedisConnection(),
- lockDuration: 60 * 1000, // 60 seconds
- stalledInterval: 60 * 1000, // 60 seconds
- maxStalledCount: 10, // 10 times
- concurrency: 8,
- useWorkerThreads: false,
- workerForkOptions: {
- execArgv: filteredExecArgv.concat(
- maxOldSpaceSize ? ["--max-old-space-size=" + maxOldSpaceSize] : [],
- ),
- },
- workerThreadsOptions: {
- execArgv: filteredExecArgv,
- resourceLimits: maxOldSpaceSize
- ? {
- maxOldGenerationSizeMb: parseInt(maxOldSpaceSize),
- }
- : undefined,
- },
- telemetry: new BullMQOtel("firecrawl-bullmq"),
- });
-
- return worker;
-};
-
const workerFun = async (
queue: Queue,
processJobInternal: (token: string, job: Job) => Promise<any>,
@@ -526,17 +456,11 @@ const workerFun = async (
runningJobs.add(job.id);
}
- async function afterJobDone(job: Job<any, any, string>) {
- try {
- await concurrentJobDone(job);
- } finally {
- if (job.id) {
- runningJobs.delete(job.id);
- }
+ processJobInternal(token, job).finally(() => {
+ if (job.id) {
+ runningJobs.delete(job.id);
}
- }
-
- processJobInternal(token, job).finally(() => afterJobDone(job));
+ });
await sleep(gotJobInterval);
} else {
@@ -588,82 +512,7 @@ app.listen(workerPort, () => {
});
(async () => {
- async function failedListener(args: {
- jobId: string;
- failedReason: string;
- prev?: string | undefined;
- }) {
- const job = await getScrapeQueue().getJob(args.jobId);
-
- if (job && job.data.crawl_id) {
- await redisEvictConnection.srem(
- "crawl:" + job.data.crawl_id + ":jobs_qualified",
- args.jobId,
- );
- await redisEvictConnection.expire(
- "crawl:" + job.data.crawl_id + ":jobs_qualified",
- 24 * 60 * 60,
- );
- }
-
- if (args.failedReason === "job stalled more than allowable limit") {
- const set = await redisEvictConnection.set(
- "stalled-job-cleaner:" + args.jobId,
- "1",
- "EX",
- 60 * 60 * 24,
- "NX",
- );
- if (!set) {
- return;
- }
-
- let logger = _logger.child({
- jobId: args.jobId,
- scrapeId: args.jobId,
- module: "queue-worker",
- method: "failedListener",
- zeroDataRetention: job?.data.zeroDataRetention,
- });
- if (job && job.data.crawl_id) {
- logger = logger.child({ crawlId: job.data.crawl_id });
- logger.warn("Job stalled more than allowable limit");
-
- const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;
-
- if (job.data.mode === "kickoff") {
- await finishCrawlKickoff(job.data.crawl_id);
- if (sc) {
- await finishCrawlIfNeeded(job, sc);
- }
- } else {
- const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;
-
- logger.debug("Declaring job as done...");
- await addCrawlJobDone(job.data.crawl_id, job.id, false, logger);
- await redisEvictConnection.srem(
- "crawl:" + job.data.crawl_id + ":visited_unique",
- normalizeURL(job.data.url, sc),
- );
-
- await finishCrawlIfNeeded(job, sc);
- }
- } else {
- logger.warn("Job stalled more than allowable limit");
- }
- }
- }
-
- const scrapeQueueEvents = new QueueEvents(scrapeQueueName, {
- connection: getRedisConnection(),
- });
- scrapeQueueEvents.on("failed", failedListener);
-
- const results = await Promise.all([
- separateWorkerFun(
- getScrapeQueue(),
- path.join(__dirname, "worker", "scrape-worker.js"),
- ),
+ await Promise.all([
workerFun(getExtractQueue(), processExtractJobInternal),
workerFun(getDeepResearchQueue(), processDeepResearchJobInternal),
workerFun(getGenerateLlmsTxtQueue(), processGenerateLlmsTxtJobInternal),
@@ -671,26 +520,10 @@ app.listen(workerPort, () => {
console.log("All workers exited. Waiting for all jobs to finish...");
- const workerResults = results.filter(x => x instanceof Worker);
- await Promise.all(workerResults.map(x => x.close()));
-
while (runningJobs.size > 0) {
await new Promise(resolve => setTimeout(resolve, 500));
}
- setInterval(async () => {
- _logger.debug("Currently running jobs", {
- jobs: (
- await Promise.all(
- [...runningJobs].map(async jobId => {
- return await getScrapeQueue().getJob(jobId);
- }),
- )
- ).filter(x => x && !x.data?.zeroDataRetention),
- });
- }, 1000);
-
- await scrapeQueueEvents.close();
console.log("All jobs finished. Worker out!");
if (otelSdk) {
await otelSdk.shutdown();
diff --git a/apps/api/src/services/worker/crawl-logic.ts b/apps/api/src/services/worker/crawl-logic.ts
index c3971e76d..ff1e7db0c 100644
--- a/apps/api/src/services/worker/crawl-logic.ts
+++ b/apps/api/src/services/worker/crawl-logic.ts
@@ -1,5 +1,4 @@
import { logger as _logger } from "../../lib/logger";
-import { Job } from "bullmq";
import {
addCrawlJobs,
finishCrawl,
@@ -21,11 +20,9 @@ import { getJobs } from "../../controllers/v1/crawl-status";
import { logJob } from "../logging/log_job";
import { createWebhookSender, WebhookEvent } from "../webhook";
import { hasFormatOfType } from "../../lib/format-utils";
+import type { NuQJob } from "./nuq";
-export async function finishCrawlIfNeeded(
- job: Job & { id: string },
- sc: StoredCrawl,
-) {
+export async function finishCrawlIfNeeded(job: NuQJob<any>, sc: StoredCrawl) {
const logger = _logger.child({
module: "queue-worker",
method: "finishCrawlIfNeeded",
@@ -113,7 +110,7 @@ export async function finishCrawlIfNeeded(
const jobs = univistedUrls.links.slice(0, addableJobCount).map(url => {
const uuid = uuidv4();
return {
- name: uuid,
+ jobId: uuid,
data: {
url,
mode: "single_urls" as const,
@@ -133,24 +130,21 @@ export async function finishCrawlIfNeeded(
zeroDataRetention: job.data.zeroDataRetention,
apiKeyId: job.data.apiKeyId,
},
- opts: {
- jobId: uuid,
- priority: 20,
- },
+ priority: 20, // TODO: make this dynamic
};
});
const lockedIds = await lockURLsIndividually(
job.data.crawl_id,
sc,
- jobs.map(x => ({ id: x.opts.jobId, url: x.data.url })),
+ jobs.map(x => ({ id: x.jobId, url: x.data.url })),
);
const lockedJobs = jobs.filter(x =>
- lockedIds.find(y => y.id === x.opts.jobId),
+ lockedIds.find(y => y.id === x.jobId),
);
await addCrawlJobs(
job.data.crawl_id,
- lockedJobs.map(x => x.opts.jobId),
+ lockedJobs.map(x => x.jobId),
logger,
);
await addScrapeJobs(lockedJobs);
diff --git a/apps/api/src/services/worker/nuq-worker.ts b/apps/api/src/services/worker/nuq-worker.ts
new file mode 100644
index 000000000..e5fc91d01
--- /dev/null
+++ b/apps/api/src/services/worker/nuq-worker.ts
@@ -0,0 +1,119 @@
+import "dotenv/config";
+import { logger as _logger } from "../../lib/logger";
+import { processJobInternal } from "./scrape-worker";
+import { scrapeQueue, nuqGetLocalMetrics, nuqHealthCheck } from "./nuq";
+import Express from "express";
+import { _ } from "ajv";
+
+(async () => {
+ let isShuttingDown = false;
+ const myLock = crypto.randomUUID();
+
+ const app = Express();
+
+ app.get("/metrics", (_, res) =>
+ res.contentType("text/plain").send(nuqGetLocalMetrics()),
+ );
+ app.get("/health", async (_, res) => {
+ if (await nuqHealthCheck()) {
+ res.status(200).send("OK");
+ } else {
+ res.status(500).send("Not OK");
+ }
+ });
+
+ const server = app.listen(
+ Number(process.env.NUQ_WORKER_PORT ?? process.env.PORT ?? 3000),
+ () => {
+ _logger.info("NuQ worker metrics server started");
+ },
+ );
+
+ function shutdown() {
+ isShuttingDown = true;
+ }
+
+ process.on("SIGINT", shutdown);
+ process.on("SIGTERM", shutdown);
+
+ let noJobTimeout = 500;
+
+ while (!isShuttingDown) {
+ const job = await scrapeQueue.getJobToProcess(myLock);
+
+ if (job === null) {
+ _logger.info("No jobs to process", { module: "nuq/metrics" });
+ await new Promise(resolve => setTimeout(resolve, noJobTimeout));
+ noJobTimeout = Math.min(noJobTimeout * 2, 10000);
+ continue;
+ }
+
+ noJobTimeout = 500;
+
+ const logger = _logger.child({
+ module: "nuq-worker",
+ scrapeId: job.id,
+ zeroDataRetention: job.data?.zeroDataRetention ?? false,
+ });
+
+ logger.info("Acquired job");
+
+ const lockRenewInterval = setInterval(async () => {
+ logger.info("Renewing lock");
+ if (!(await scrapeQueue.renewLock(job.id, myLock, logger))) {
+ logger.warn("Failed to renew lock");
+ clearInterval(lockRenewInterval);
+ return;
+ }
+ logger.info("Renewed lock");
+ }, 15000);
+
+ let processResult:
+ | { ok: true; data: Awaited<ReturnType<typeof processJobInternal>> }
+ | { ok: false; error: any };
+
+ try {
+ processResult = { ok: true, data: await processJobInternal(job) };
+ } catch (error) {
+ processResult = { ok: false, error };
+ }
+
+ clearInterval(lockRenewInterval);
+
+ if (processResult.ok) {
+ if (
+ !(await scrapeQueue.jobFinish(
+ job.id,
+ myLock,
+ processResult.data,
+ logger,
+ ))
+ ) {
+ logger.warn("Could not update job status");
+ }
+ } else {
+ if (
+ !(await scrapeQueue.jobFail(
+ job.id,
+ myLock,
+ processResult.error instanceof Error
+ ? processResult.error.message
+ : typeof processResult.error === "string"
+ ? processResult.error
+ : JSON.stringify(processResult.error),
+ logger,
+ ))
+ ) {
+ logger.warn("Could not update job status");
+ }
+ }
+ }
+
+ _logger.info("NuQ worker shutting down");
+
+ server.close(async () => {
+ await scrapeQueue.shutdown();
+ _logger.info("NuQ worker shut down");
+ process.exit(0);
+ });
+})();
diff --git a/apps/api/src/services/worker/nuq.ts b/apps/api/src/services/worker/nuq.ts
new file mode 100644
index 000000000..993603a7a
--- /dev/null
+++ b/apps/api/src/services/worker/nuq.ts
@@ -0,0 +1,568 @@
+import { Logger } from "winston";
+import { logger } from "../../lib/logger";
+import { Client, Pool } from "pg";
+
+// === Basics
+
+const nuqPool = new Pool({
+ connectionString: process.env.NUQ_DATABASE_URL, // may be a pgbouncer transaction pooler URL
+ application_name: "nuq",
+});
+
+nuqPool.on("error", err =>
+ logger.error("Error in NuQ idle client", { err, module: "nuq" }),
+);
+
+export type NuQJobStatus = "queued" | "active" | "completed" | "failed"; // must match nuq.job_status enum
+export type NuQJob<Data = any, ReturnValue = any> = {
+ id: string;
+ status: NuQJobStatus;
+ createdAt: Date;
+ priority: number;
+ data: Data;
+ finishedAt?: Date;
+ returnvalue?: ReturnValue;
+ failedReason?: string;
+};
+
+// === Queue
+
+class NuQ<JobData = any, JobReturnValue = any> {
+ constructor(public readonly queueName: string) {}
+
+ // === Listener
+
+ private listener: Client | null = null;
+ private listens: {
+ [key: string]: ((status: "completed" | "failed") => void)[];
+ } = {};
+ private shuttingDown = false;
+
+ async startListener() {
+ if (this.listener || this.shuttingDown) return;
+
+ this.listener = new Client({
+ connectionString:
+ process.env.NUQ_DATABASE_URL_LISTEN ?? process.env.NUQ_DATABASE_URL, // will always be a direct connection
+ application_name: "nuq_listener",
+ });
+
+ this.listener.on("notification", msg => {
+ const tok = (msg.payload ?? "unknown|unknown").split("|");
+ if (tok[0] in this.listens) {
+ this.listens[tok[0]].forEach(listener =>
+ listener(tok[1] as "completed" | "failed"),
+ );
+ delete this.listens[tok[0]];
+ }
+ });
+
+ this.listener.on("error", err =>
+ logger.error("Error in NuQ listener", { err, module: "nuq" }),
+ );
+
+ this.listener.on("end", () => {
+ logger.info("NuQ listener disconnected", { module: "nuq" });
+ this.listener = null;
+ setTimeout(
+ (() => {
+ this.startListener().catch(err =>
+ logger.error("Error in NuQ listener reconnect", {
+ err,
+ module: "nuq",
+ }),
+ );
+ }).bind(this),
+ 250,
+ );
+ });
+
+ await this.listener.connect();
+ await this.listener.query(`LISTEN "${this.queueName}";`);
+
+ (async () => {
+ const backedUpJobs = (
+ await this.getJobs(Object.keys(this.listens))
+ ).filter(job => ["completed", "failed"].includes(job.status));
+ for (const job of backedUpJobs) {
+ this.listens[job.id].forEach(listener =>
+ listener(job.status as "completed" | "failed"),
+ );
+ delete this.listens[job.id];
+ }
+ })();
+ }
+
+ async addListener(
+ id: string,
+ listener: (status: "completed" | "failed") => void,
+ ) {
+ await this.startListener();
+
+ if (!(id in this.listens)) this.listens[id] = [listener];
+ else this.listens[id].push(listener);
+ }
+
+ async removeListener(
+ id: string,
+ listener: (status: "completed" | "failed") => void,
+ ) {
+ if (id in this.listens) {
+ this.listens[id] = this.listens[id].filter(l => l !== listener);
+ if (this.listens[id].length === 0) delete this.listens[id];
+ }
+ }
+
+ // === Job management
+
+ private readonly jobReturning = [
+ "id",
+ "status",
+ "created_at",
+ "priority",
+ "data",
+ "finished_at",
+ "returnvalue",
+ "failedreason",
+ ];
+
+ private rowToJob(row: any): NuQJob<JobData, JobReturnValue> | null {
+ if (!row) return null;
+ return {
+ id: row.id,
+ status: row.status,
+ createdAt: new Date(row.created_at),
+ priority: row.priority,
+ data: row.data,
+ finishedAt: row.finished_at ? new Date(row.finished_at) : undefined,
+ returnvalue: row.returnvalue ?? undefined,
+ failedReason: row.failedreason ?? undefined,
+ };
+ }
+
+ public async getJob(
+ id: string,
+ ): Promise<NuQJob<JobData, JobReturnValue> | null> {
+ const start = Date.now();
+ try {
+ return this.rowToJob(
+ (
+ await nuqPool.query(
+ `SELECT ${this.jobReturning.join(", ")} FROM ${this.queueName} WHERE ${this.queueName}.id = $1;`,
+ [id],
+ )
+ ).rows[0],
+ );
+ } finally {
+ logger.info("nuqGetJob metrics", {
+ module: "nuq/metrics",
+ method: "nuqGetJob",
+ duration: Date.now() - start,
+ scrapeId: id,
+ });
+ }
+ }
+
+ public async getJobs(
+ ids: string[],
+ ): Promise<NuQJob<JobData, JobReturnValue>[]> {
+ if (ids.length === 0) return [];
+
+ const start = Date.now();
+ try {
+ return (
+ await nuqPool.query(
+ `SELECT ${this.jobReturning.join(", ")} FROM ${this.queueName} WHERE ${this.queueName}.id = ANY($1::uuid[]);`,
+ [ids],
+ )
+ ).rows.map(row => this.rowToJob(row)!);
+ } finally {
+ logger.info("nuqGetJobs metrics", {
+ module: "nuq/metrics",
+ method: "nuqGetJobs",
+ duration: Date.now() - start,
+ scrapeIds: ids.length,
+ });
+ }
+ }
+
+ public async getJobsWithStatus(
+ ids: string[],
+ status: NuQJobStatus,
+ _logger: Logger = logger,
+ ): Promise<NuQJob<JobData, JobReturnValue>[]> {
+ if (ids.length === 0) return [];
+
+ const start = Date.now();
+ try {
+ return (
+ await nuqPool.query(
+ `SELECT ${this.jobReturning.join(", ")} FROM ${this.queueName} WHERE ${this.queueName}.id = ANY($1::uuid[]) AND ${this.queueName}.status = $2::nuq.job_status;`,
+ [ids, status],
+ )
+ ).rows.map(row => this.rowToJob(row)!);
+ } finally {
+ _logger.info("nuqGetJobsWithStatus metrics", {
+ module: "nuq/metrics",
+ method: "nuqGetJobsWithStatus",
+ duration: Date.now() - start,
+ scrapeIds: ids.length,
+ status,
+ });
+ }
+ }
+
+ public async getJobsWithStatuses(
+ ids: string[],
+ statuses: NuQJobStatus[],
+ ): Promise<NuQJob<JobData, JobReturnValue>[]> {
+ if (ids.length === 0) return [];
+
+ const start = Date.now();
+ try {
+ return (
+ await nuqPool.query(
+ `SELECT ${this.jobReturning.join(", ")} FROM ${this.queueName} WHERE ${this.queueName}.id = ANY($1::uuid[]) AND ${this.queueName}.status = ANY($2::nuq.job_status[]);`,
+ [ids, statuses],
+ )
+ ).rows.map(row => this.rowToJob(row)!);
+ } finally {
+ logger.info("nuqGetJobsWithStatuses metrics", {
+ module: "nuq/metrics",
+ method: "nuqGetJobsWithStatuses",
+ duration: Date.now() - start,
+ scrapeIds: ids.length,
+ statuses,
+ });
+ }
+ }
+
+ public async removeJob(id: string): Promise<boolean> {
+ const start = Date.now();
+ try {
+ return (
+ (
+ await nuqPool.query(`DELETE FROM ${this.queueName} WHERE id = $1;`, [
+ id,
+ ])
+ ).rowCount !== 0
+ );
+ } finally {
+ logger.info("nuqRemoveJob metrics", {
+ module: "nuq/metrics",
+ method: "nuqRemoveJob",
+ duration: Date.now() - start,
+ scrapeId: id,
+ });
+ }
+ }
+
+ public async removeJobs(ids: string[]): Promise<number> {
+ if (ids.length === 0) return 0;
+
+ const start = Date.now();
+ try {
+ return (
+ (
+ await nuqPool.query(
+ `DELETE FROM ${this.queueName} WHERE id = ANY($1::uuid[]);`,
+ [ids],
+ )
+ ).rowCount ?? 0
+ );
+ } finally {
+ logger.info("nuqRemoveJobs metrics", {
+ module: "nuq/metrics",
+ method: "nuqRemoveJobs",
+ duration: Date.now() - start,
+ scrapeIds: ids.length,
+ });
+ }
+ }
+
+ // === Producer
+ public async addJob(
+ id: string,
+ data: JobData,
+ priority: number = 0,
+ ): Promise<NuQJob<JobData, JobReturnValue>> {
+ const start = Date.now();
+ try {
+ return this.rowToJob(
+ (
+ await nuqPool.query(
+ `INSERT INTO ${this.queueName} (id, data, priority) VALUES ($1, $2, $3) RETURNING ${this.jobReturning.join(", ")};`,
+ [id, data, priority],
+ )
+ ).rows[0],
+ )!;
+ } finally {
+ logger.info("nuqAddJob metrics", {
+ module: "nuq/metrics",
+ method: "nuqAddJob",
+ duration: Date.now() - start,
+ scrapeId: id,
+ zeroDataRetention: (data as any)?.zeroDataRetention ?? false,
+ });
+ }
+ }
+
+ private readonly nuqWaitMode =
+ process.env.NUQ_WAIT_MODE === "listen"
+ ? ("listen" as const)
+ : ("poll" as const);
+
+ public waitForJob(
+ id: string,
+ timeout: number | null,
+ ): Promise<JobReturnValue> {
+ const done = new Promise<JobReturnValue>(
+ (async (resolve, reject) => {
+ if (this.nuqWaitMode === "listen") {
+ let timer: NodeJS.Timeout | null = null;
+ if (timeout !== null) {
+ timer = setTimeout(
+ (() => {
+ this.removeListener(id, listener);
+ reject(new Error("Timed out"));
+ }).bind(this),
+ timeout,
+ );
+ }
+
+ const listener = async function (_msg: "completed" | "failed") {
+ if (timer) clearTimeout(timer);
+ const job = await this.getJob(id);
+ if (!job) {
+ reject(new Error("Job raced out while waiting for it"));
+ } else {
+ if (job.status === "completed") {
+ resolve(job.returnvalue!);
+ } else {
+ reject(new Error(job.failedReason!));
+ }
+ }
+ }.bind(this);
+
+ try {
+ await this.addListener(id, listener);
+ } catch (e) {
+ reject(e);
+ }
+
+ try {
+ const job = await this.getJob(id);
+ if (job && ["completed", "failed"].includes(job.status)) {
+ this.removeListener(id, listener);
+ if (timer) clearTimeout(timer);
+ if (job.status === "completed") {
+ resolve(job.returnvalue!);
+ } else {
+ reject(new Error(job.failedReason!));
+ }
+ return;
+ }
+ } catch (e) {
+ logger.warn("nuqGetJob ensure check failed", {
+ module: "nuq",
+ method: "nuqWaitForJob",
+ error: e,
+ scrapeId: id,
+ });
+ }
+ } else {
+ const timeoutAt = timeout !== null ? Date.now() + timeout : null;
+ const poll = async function poll() {
+ try {
+ const job = await this.getJob(id);
+ if (job && ["completed", "failed"].includes(job.status)) {
+ if (job.status === "completed") {
+ return resolve(job.returnvalue!);
+ } else {
+ return reject(new Error(job.failedReason!));
+ }
+ }
+ } catch (e) {
+ return reject(e);
+ }
+
+ if (timeoutAt && Date.now() > timeoutAt) {
+ return reject(new Error("Timed out"));
+ }
+
+ setTimeout(poll.bind(this), 250);
+ }.bind(this);
+
+ poll();
+ }
+ }).bind(this),
+ );
+
+ return done;
+ }
+
+ // === Consumer
+
+ public async getJobToProcess(lock: string): Promise<NuQJob<any, any> | null> {
+ const start = Date.now();
+ try {
+ return this.rowToJob(
+ (
+ await nuqPool.query(
+ `
+ WITH next AS (SELECT ${this.jobReturning.join(", ")} FROM ${this.queueName} WHERE ${this.queueName}.status = 'queued'::nuq.job_status ORDER BY ${this.queueName}.priority ASC, ${this.queueName}.created_at ASC FOR UPDATE SKIP LOCKED LIMIT 1)
+ UPDATE ${this.queueName} q SET status = 'active'::nuq.job_status, lock = $1, locked_at = now() FROM next WHERE q.id = next.id RETURNING ${this.jobReturning.map(x => `q.${x}`).join(", ")};
+ `,
+ [lock],
+ )
+ ).rows[0],
+ )!;
+ } finally {
+ logger.info("nuqGetJobToProcess metrics", {
+ module: "nuq/metrics",
+ method: "nuqGetJobToProcess",
+ duration: Date.now() - start,
+ });
+ }
+ }
+
+ public async renewLock(
+ id: string,
+ lock: string,
+ _logger: Logger = logger,
+ ): Promise<boolean> {
+ const start = Date.now();
+ try {
+ return (
+ (
+ await nuqPool.query(
+ `UPDATE ${this.queueName} SET locked_at = now() WHERE id = $1 AND lock = $2 AND status = 'active'::nuq.job_status;`,
+ [id, lock],
+ )
+ ).rowCount !== 0
+ );
+ } finally {
+ _logger.info("nuqRenewLock metrics", {
+ module: "nuq/metrics",
+ method: "nuqRenewLock",
+ duration: Date.now() - start,
+ scrapeId: id,
+ });
+ }
+ }
+
+ public async jobFinish(
+ id: string,
+ lock: string,
+ returnvalue: any | null,
+ _logger: Logger = logger,
+ ): Promise<boolean> {
+ const start = Date.now();
+ try {
+ return (
+ (
+ await nuqPool.query(
+ `
+ WITH updated AS (UPDATE ${this.queueName} SET status = 'completed'::nuq.job_status, lock = null, locked_at = null, finished_at = now(), returnvalue = $3 WHERE id = $1 AND lock = $2 RETURNING id)
+ SELECT pg_notify('${this.queueName}', (id::text || '|completed')) FROM updated;
+ `,
+ [id, lock, returnvalue],
+ )
+ ).rowCount !== 0
+ );
+ } finally {
+ _logger.info("nuqJobFinish metrics", {
+ module: "nuq/metrics",
+ method: "nuqJobFinish",
+ duration: Date.now() - start,
+ scrapeId: id,
+ });
+ }
+ }
+
+ public async jobFail(
+ id: string,
+ lock: string,
+ failedReason: string,
+ _logger: Logger = logger,
+ ): Promise<boolean> {
+ const start = Date.now();
+ try {
+ return (
+ (
+ await nuqPool.query(
+ `
+ WITH updated AS (UPDATE ${this.queueName} SET status = 'failed'::nuq.job_status, lock = null, locked_at = null, finished_at = now(), failedreason = $3 WHERE id = $1 AND lock = $2 RETURNING id)
+ SELECT pg_notify('${this.queueName}', (id::text || '|failed')) FROM updated;
+ `,
+ [id, lock, failedReason],
+ )
+ ).rowCount !== 0
+ );
+ } finally {
+ _logger.info("nuqJobFail metrics", {
+ module: "nuq/metrics",
+ method: "nuqJobFail",
+ duration: Date.now() - start,
+ scrapeId: id,
+ });
+ }
+ }
+
+ // === Metrics
+ public async getMetrics(): Promise<string> {
+ const start = Date.now();
+ const result = await nuqPool.query(
+ `SELECT status, COUNT(id) as count FROM ${this.queueName} GROUP BY status ORDER BY count DESC;`,
+ );
+ logger.info("nuqGetMetrics metrics", {
+ module: "nuq/metrics",
+ method: "nuqGetMetrics",
+ duration: Date.now() - start,
+ });
+ const prometheusQueueName = this.queueName.replace(".", "_");
+ return `# HELP ${prometheusQueueName}_job_count Number of jobs in each status\n# TYPE ${prometheusQueueName}_job_count gauge\n${result.rows.map(x => `${prometheusQueueName}_job_count{status="${x.status}"} ${x.count}`).join("\n")}\n`;
+ }
+
+ // === Cleanup
+ public async shutdown() {
+ this.shuttingDown = true;
+ if (this.listener) {
+ const nl = this.listener;
+ this.listener = null;
+ this.listens = {};
+ await nl.query(`UNLISTEN "${this.queueName}";`);
+ await nl.end();
+ }
+ }
+}
+
+export function nuqGetLocalMetrics(): string {
+ return `# HELP nuq_pool_waiting_count Number of requests waiting in the pool\n# TYPE nuq_pool_waiting_count gauge\nnuq_pool_waiting_count ${nuqPool.waitingCount}\n
+# HELP nuq_pool_idle_count Number of connections idle in the pool\n# TYPE nuq_pool_idle_count gauge\nnuq_pool_idle_count ${nuqPool.idleCount}\n
+# HELP nuq_pool_total_count Number of connections in the pool\n# TYPE nuq_pool_total_count gauge\nnuq_pool_total_count ${nuqPool.totalCount}\n`;
+}
+
+export async function nuqHealthCheck(): Promise<boolean> {
+ const start = Date.now();
+ try {
+ return (await nuqPool.query("SELECT 1;")).rowCount !== 0;
+ } finally {
+ logger.info("nuqHealthCheck metrics", {
+ module: "nuq/metrics",
+ method: "nuqHealthCheck",
+ duration: Date.now() - start,
+ });
+ }
+}
+
+// === Instances
+
+export const scrapeQueue = new NuQ("nuq.queue_scrape");
+
+// === Cleanup
+
+export async function nuqShutdown() {
+ await scrapeQueue.shutdown();
+ await nuqPool.end();
+}
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index 29ded1a92..83a69d95c 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -1,5 +1,4 @@
import { configDotenv } from "dotenv";
-import { Job } from "bullmq";
import * as Sentry from "@sentry/node";
import http from "http";
import https from "https";
@@ -60,6 +59,7 @@ import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-node";
import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-grpc";
import { resourceFromAttributes } from "@opentelemetry/resources";
import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";
+import type { NuQJob } from "./nuq";
const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
@@ -74,7 +74,7 @@ cacheableLookup.install(http.globalAgent);
cacheableLookup.install(https.globalAgent);
async function billScrapeJob(
- job: Job & { id: string },
+ job: NuQJob<any>,
document: Document | null,
logger: Logger,
costTracking: CostTracking,
@@ -138,7 +138,7 @@ async function billScrapeJob(
return creditsToBeBilled;
}
-async function processJob(job: Job & { id: string }) {
+async function processJob(job: NuQJob<any>) {
const logger = _logger.child({
module: "queue-worker",
method: "processJob",
@@ -157,13 +157,6 @@ async function processJob(job: Job & { id: string }) {
const costTracking = new CostTracking();
try {
- job.updateProgress({
- current: 1,
- total: 100,
- current_step: "SCRAPING",
- current_url: "",
- });
-
if (remainingTime !== undefined && remainingTime < 0) {
throw new ScrapeJobTimeoutError("Scrape timed out");
}
@@ -368,7 +361,6 @@ async function processJob(job: Job & { id: string }) {
zeroDataRetention: job.data.zeroDataRetention,
apiKeyId: job.data.apiKeyId,
},
- {},
jobId,
jobPriority,
);
@@ -699,7 +691,7 @@ async function kickoffGetIndexLinks(
return validIndexLinks;
}
-async function processKickoffJob(job: Job & { id: string }) {
+async function processKickoffJob(job: NuQJob<any>) {
const logger = _logger.child({
module: "queue-worker",
method: "processKickoffJob",
@@ -739,10 +731,8 @@ async function processKickoffJob(job: Job & { id: string }) {
zeroDataRetention: job.data.zeroDataRetention,
apiKeyId: job.data.apiKeyId,
},
- {
- priority: 15,
- },
jobId,
+ await getJobPriority({ team_id: job.data.team_id, basePriority: 15 }),
);
logger.debug("Adding scrape job to BullMQ...", { jobId });
await addCrawlJob(job.data.crawl_id, jobId, logger);
@@ -780,7 +770,6 @@ async function processKickoffJob(job: Job & { id: string }) {
const jobs = urls.map(url => {
const uuid = uuidv4();
return {
- name: uuid,
data: {
url,
mode: "single_urls" as const,
@@ -797,10 +786,8 @@ async function processKickoffJob(job: Job & { id: string }) {
zeroDataRetention: job.data.zeroDataRetention,
apiKeyId: job.data.apiKeyId,
},
- opts: {
- jobId: uuid,
- priority: 20,
- },
+ jobId: uuid,
+ priority: jobPriority,
};
});
@@ -808,15 +795,15 @@ async function processKickoffJob(job: Job & { id: string }) {
const lockedIds = await lockURLsIndividually(
job.data.crawl_id,
sc,
- jobs.map(x => ({ id: x.opts.jobId, url: x.data.url })),
+ jobs.map(x => ({ id: x.jobId, url: x.data.url })),
);
const lockedJobs = jobs.filter(x =>
- lockedIds.find(y => y.id === x.opts.jobId),
+ lockedIds.find(y => y.id === x.jobId),
);
logger.debug("Adding scrape jobs to Redis...");
await addCrawlJobs(
job.data.crawl_id,
- lockedJobs.map(x => x.opts.jobId),
+ lockedJobs.map(x => x.jobId),
logger,
);
logger.debug("Adding scrape jobs to BullMQ...");
@@ -845,7 +832,7 @@ async function processKickoffJob(job: Job & { id: string }) {
const jobs = indexLinks.map(url => {
const uuid = uuidv4();
return {
- name: uuid,
+ jobId: uuid,
data: {
url,
mode: "single_urls" as const,
@@ -862,10 +849,7 @@ async function processKickoffJob(job: Job & { id: string }) {
zeroDataRetention: job.data.zeroDataRetention,
apiKeyId: job.data.apiKeyId,
},
- opts: {
- jobId: uuid,
- priority: 20,
- },
+ priority: jobPriority,
};
});
@@ -873,15 +857,15 @@ async function processKickoffJob(job: Job & { id: string }) {
const lockedIds = await lockURLsIndividually(
job.data.crawl_id,
sc,
- jobs.map(x => ({ id: x.opts.jobId, url: x.data.url })),
+ jobs.map(x => ({ id: x.jobId, url: x.data.url })),
);
const lockedJobs = jobs.filter(x =>
- lockedIds.find(y => y.id === x.opts.jobId),
+ lockedIds.find(y => y.id === x.jobId),
);
logger.debug("Adding scrape jobs to Redis...");
await addCrawlJobs(
job.data.crawl_id,
- lockedJobs.map(x => x.opts.jobId),
+ lockedJobs.map(x => x.jobId),
logger,
);
logger.debug("Adding scrape jobs to BullMQ...");
@@ -905,7 +889,7 @@ async function processKickoffJob(job: Job & { id: string }) {
}
}
-export const processJobInternal = async (job: Job & { id: string }) => {
+export const processJobInternal = async (job: NuQJob<any>) => {
const logger = _logger.child({
module: "queue-worker",
method: "processJobInternal",
@@ -954,10 +938,7 @@ export const processJobInternal = async (job: Job & { id: string }) => {
}
try {
- if (
- process.env.USE_DB_AUTHENTICATION === "true" &&
- (job.data.crawl_id || process.env.GCS_BUCKET_NAME)
- ) {
+ if (process.env.GCS_BUCKET_NAME) {
logger.debug("Job succeeded -- putting null in Redis");
return null;
} else {
@@ -1018,11 +999,9 @@ if (otelSdk) {
otelSdk.start();
}
-module.exports = processJobInternal;
-
const exitHandler = () => {
if (otelSdk) {
- otelSdk.shutdown().then(() => {
+ otelSdk.shutdown().finally(() => {
_logger.debug("OTEL shutdown");
process.exit(0);
});
diff --git a/apps/nuq-postgres/Dockerfile b/apps/nuq-postgres/Dockerfile
new file mode 100644
index 000000000..942b2fbe1
--- /dev/null
+++ b/apps/nuq-postgres/Dockerfile
@@ -0,0 +1,24 @@
+# Build a Postgres image that runs nuq.sql during initdb
+
+ARG PG_MAJOR=17
+FROM postgres:${PG_MAJOR}
+
+# Install pg_cron for the specified Postgres major version
+RUN set -eux; \
+ apt-get update; \
+ apt-get install -y --no-install-recommends \
+ postgresql-${PG_MAJOR}-cron; \
+ rm -rf /var/lib/apt/lists/*
+
+# Ensure pg_cron is preloaded on first startup by modifying the initdb template
+# This must be set before the first server start (init scripts run after start)
+RUN set -eux; \
+ conf_sample="/usr/share/postgresql/${PG_MAJOR}/postgresql.conf.sample"; \
+ sed -ri "s/^#?shared_preload_libraries\s*=.*/shared_preload_libraries = 'pg_cron'/" "$conf_sample"; \
+ printf "\n# Added for pg_cron\ncron.database_name = 'postgres'\n" >> "$conf_sample"
+
+# Create required extensions before executing our SQL
+RUN printf 'CREATE EXTENSION IF NOT EXISTS pgcrypto;\nCREATE EXTENSION IF NOT EXISTS pg_cron;\n' > /docker-entrypoint-initdb.d/010-extensions.sql
+
+# Copy nuq.sql so it is executed as part of the initdb sequence
+COPY nuq.sql /docker-entrypoint-initdb.d/020-nuq.sql
\ No newline at end of file
diff --git a/apps/nuq-postgres/nuq.sql b/apps/nuq-postgres/nuq.sql
new file mode 100644
index 000000000..5d3ad041c
--- /dev/null
+++ b/apps/nuq-postgres/nuq.sql
@@ -0,0 +1,51 @@
+CREATE SCHEMA IF NOT EXISTS nuq;
+
+DO $$ BEGIN
+ CREATE TYPE nuq.job_status AS ENUM ('queued', 'active', 'completed', 'failed');
+EXCEPTION
+ WHEN duplicate_object THEN null;
+END $$;
+
+CREATE TABLE IF NOT EXISTS nuq.queue_scrape (
+ id uuid NOT NULL DEFAULT gen_random_uuid(),
+ status nuq.job_status NOT NULL DEFAULT 'queued'::nuq.job_status,
+ data jsonb,
+ created_at timestamp with time zone NOT NULL DEFAULT now(),
+ priority int NOT NULL DEFAULT 0,
+ lock uuid,
+ locked_at timestamp with time zone,
+ stalls integer,
+ finished_at timestamp with time zone,
+ returnvalue jsonb, -- only for selfhost
+ failedreason text, -- only for selfhost
+ CONSTRAINT queue_scrape_pkey PRIMARY KEY (id)
+);
+
+ALTER TABLE nuq.queue_scrape
+SET (autovacuum_vacuum_scale_factor = 0.01,
+ autovacuum_analyze_scale_factor = 0.01,
+ autovacuum_vacuum_cost_limit = 2000,
+ autovacuum_vacuum_cost_delay = 2);
+
+CREATE INDEX IF NOT EXISTS queue_scrape_active_locked_at_idx ON nuq.queue_scrape USING btree (locked_at) WHERE (status = 'active'::nuq.job_status);
+CREATE INDEX IF NOT EXISTS nuq_queue_scrape_queued_optimal_2_idx ON nuq.queue_scrape (priority ASC, created_at ASC, id) WHERE (status = 'queued'::nuq.job_status);
+CREATE INDEX IF NOT EXISTS nuq_queue_scrape_failed_created_at_idx ON nuq.queue_scrape USING btree (created_at) WHERE (status = 'failed'::nuq.job_status);
+CREATE INDEX IF NOT EXISTS nuq_queue_scrape_completed_created_at_idx ON nuq.queue_scrape USING btree (created_at) WHERE (status = 'completed'::nuq.job_status);
+
+SELECT cron.schedule('nuq_queue_scrape_clean_completed', '*/5 * * * *', $$
+ DELETE FROM nuq.queue_scrape WHERE nuq.queue_scrape.status = 'completed'::nuq.job_status AND nuq.queue_scrape.created_at < now() - interval '1 hour';
+$$);
+
+SELECT cron.schedule('nuq_queue_scrape_clean_failed', '*/5 * * * *', $$
+ DELETE FROM nuq.queue_scrape WHERE nuq.queue_scrape.status = 'failed'::nuq.job_status AND nuq.queue_scrape.created_at < now() - interval '6 hours';
+$$);
+
+SELECT cron.schedule('nuq_queue_scrape_lock_reaper', '15 seconds', $$
+ UPDATE nuq.queue_scrape SET status = 'queued'::nuq.job_status, lock = null, locked_at = null, stalls = COALESCE(stalls, 0) + 1 WHERE nuq.queue_scrape.locked_at <= now() - interval '1 minute' AND nuq.queue_scrape.status = 'active'::nuq.job_status AND COALESCE(nuq.queue_scrape.stalls, 0) < 9;
+ WITH stallfail AS (UPDATE nuq.queue_scrape SET status = 'failed'::nuq.job_status, lock = null, locked_at = null, stalls = COALESCE(stalls, 0) + 1 WHERE nuq.queue_scrape.locked_at <= now() - interval '1 minute' AND nuq.queue_scrape.status = 'active'::nuq.job_status AND COALESCE(nuq.queue_scrape.stalls, 0) >= 9 RETURNING id)
+ SELECT pg_notify('nuq.queue_scrape', (id::text || '|' || 'failed'::text)) FROM stallfail;
+$$);
+
+SELECT cron.schedule('nuq_queue_scrape_reindex', '0 9 * * *', $$
+ REINDEX TABLE CONCURRENTLY nuq.queue_scrape;
+$$);
diff --git a/docker-compose.yaml b/docker-compose.yaml
index c6714dd5a..5892fa0db 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -19,6 +19,7 @@ x-common-env: &common-env
REDIS_URL: ${REDIS_URL:-redis://redis:6379}
REDIS_RATE_LIMIT_URL: ${REDIS_URL:-redis://redis:6379}
PLAYWRIGHT_MICROSERVICE_URL: ${PLAYWRIGHT_MICROSERVICE_URL:-http://playwright-service:3000/scrape}
+ NUQ_DATABASE_URL: postgres://postgres:postgres@nuq-postgres:5432/postgres
USE_DB_AUTHENTICATION: ${USE_DB_AUTHENTICATION}
OPENAI_API_KEY: ${OPENAI_API_KEY}
OPENAI_BASE_URL: ${OPENAI_BASE_URL}
@@ -65,26 +66,14 @@ services:
<<: *common-env
HOST: "0.0.0.0"
PORT: ${INTERNAL_PORT:-3002}
- FLY_PROCESS_GROUP: app
+ WORKER_PORT: ${WORKER_PORT:-3005}
ENV: local
depends_on:
- redis
- playwright-service
ports:
- "${PORT:-3002}:${INTERNAL_PORT:-3002}"
- command: [ "pnpm", "run", "start:production" ]
-
- worker:
- <<: *common-service
- environment:
- <<: *common-env
- FLY_PROCESS_GROUP: worker
- ENV: local
- depends_on:
- - redis
- - playwright-service
- - api
- command: [ "pnpm", "run", "workers" ]
+ command: node dist/src/harness.js --start-docker
redis:
# NOTE: If you want to use Valkey (open source) instead of Redis (source available),
@@ -96,6 +85,17 @@ services:
networks:
- backend
command: redis-server --bind 0.0.0.0
+
+ nuq-postgres:
+ build: apps/nuq-postgres
+ environment:
+ POSTGRES_USER: postgres
+ POSTGRES_PASSWORD: postgres
+ POSTGRES_DB: postgres
+ networks:
+ - backend
+ ports:
+ - "5432:5432"
networks:
backend: