Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): nuq (#1984)
Gergő Móriczfc7be6708f1c
diff --git a/.github/k8s/playwright.yml b/.github/k8s/playwright.ymlnew file mode 100644index 000000000..cd33ec6b5--- /dev/null+++ b/.github/k8s/playwright.yml@@ -0,0 +1,62 @@+apiVersion: apps/v1+kind: Deployment+metadata:+ name: firecrawl-playwright-service+spec:+ replicas: 1+ selector:+ matchLabels:+ app: firecrawl-playwright-service+ template:+ metadata:+ labels:+ app: firecrawl-playwright-service+ spec:+ containers:+ - name: playwright-service+ image: firecrawl/playwright-service:latest+ imagePullPolicy: Never+ envFrom:+ - configMapRef:+ name: firecrawl-config+ - configMapRef:+ name: firecrawl-config-dynamic+ - secretRef:+ name: firecrawl-secret-dynamic+ env:+ - name: PORT+ value: "3003"+ ports:+ - containerPort: 3003+ resources:+ requests:+ memory: "1G"+ cpu: "500m"+ limits:+ memory: "3G"+ cpu: "1000m"+ livenessProbe:+ httpGet:+ path: /health+ port: 3003+ initialDelaySeconds: 30+ periodSeconds: 10+ readinessProbe:+ httpGet:+ path: /health+ port: 3003+ initialDelaySeconds: 30+ periodSeconds: 10+---+apiVersion: v1+kind: Service+metadata:+ name: firecrawl-playwright-service+spec:+ selector:+ app: firecrawl-playwright-service+ ports:+ - name: http+ protocol: TCP+ port: 3003+ targetPort: 3003\ No newline at end of filediff --git a/.github/k8s/searxng.yml b/.github/k8s/searxng.ymlnew file mode 100644index 000000000..03c9f88e6--- /dev/null+++ b/.github/k8s/searxng.yml@@ -0,0 +1,59 @@+apiVersion: v1+kind: ConfigMap+metadata:+ name: searxng-settings+data:+ settings.yml: |+ use_default_settings: true+ search:+ formats: [html, json, csv]+ server:+ secret_key: 'fcsecret'+---+apiVersion: apps/v1+kind: Deployment+metadata:+ name: firecrawl-searxng+spec:+ replicas: 1+ selector:+ matchLabels:+ app: firecrawl-searxng+ template:+ metadata:+ labels:+ app: firecrawl-searxng+ spec:+ containers:+ - name: searxng+ image: searxng/searxng:latest+ imagePullPolicy: IfNotPresent+ ports:+ - containerPort: 8080+ volumeMounts:+ - name: settings+ mountPath: /etc/searxng+ resources:+ requests:+ memory: "256Mi"+ cpu: "100m"+ limits:+ memory: "512Mi"+ cpu: "200m"+ volumes:+ - name: settings+ configMap:+ name: searxng-settings+---+apiVersion: v1+kind: Service+metadata:+ name: firecrawl-searxng+spec:+ selector:+ app: firecrawl-searxng+ ports:+ - name: http+ protocol: TCP+ port: 8080+ targetPort: 8080diff --git a/.github/k8s/server-nsh.yml b/.github/k8s/server-nsh.ymlnew file mode 100644index 000000000..7778ed32a--- /dev/null+++ b/.github/k8s/server-nsh.yml@@ -0,0 +1,114 @@+apiVersion: apps/v1+kind: Deployment+metadata:+ name: tailscale-proxy+spec:+ replicas: 1+ selector:+ matchLabels:+ app: tailscale-proxy+ template:+ metadata:+ labels:+ app: tailscale-proxy+ spec:+ containers:+ - name: proxy+ image: firecrawl/tailscale-proxy:latest+ imagePullPolicy: Never+ ports:+ - containerPort: 8080+ securityContext:+ capabilities:+ add: ["NET_ADMIN", "NET_RAW"]+ allowPrivilegeEscalation: false+ volumeMounts:+ - name: dev-net-tun+ mountPath: /dev/net/tun+ readinessProbe:+ tcpSocket:+ port: 8080+ initialDelaySeconds: 3+ periodSeconds: 5+ timeoutSeconds: 3+ successThreshold: 1+ failureThreshold: 3+ env:+ - name: TS_AUTHKEY+ valueFrom:+ secretKeyRef:+ name: tailscale+ key: TS_AUTHKEY+ - name: TS_ADVERTISE_TAGS+ valueFrom:+ secretKeyRef:+ name: tailscale+ key: TS_ADVERTISE_TAGS+ - name: FIRE_ENGINE_BETA_URL+ valueFrom:+ secretKeyRef:+ name: tailscale+ key: FIRE_ENGINE_BETA_URL+ resources:+ requests:+ memory: "128Mi"+ cpu: "100m"+ limits:+ memory: "256Mi"+ cpu: "200m"+ volumes:+ - name: dev-net-tun+ hostPath:+ path: /dev/net/tun+ type: CharDevice+---+apiVersion: v1+kind: Service+metadata:+ name: tailscale-proxy+spec:+ selector:+ app: tailscale-proxy+ ports:+ - name: http+ protocol: TCP+ port: 8080+ targetPort: 8080+---+apiVersion: apps/v1+kind: Deployment+metadata:+ name: firecrawl-index-worker+spec:+ replicas: 1+ selector:+ matchLabels:+ app: firecrawl-index-worker+ template:+ metadata:+ labels:+ app: firecrawl-index-worker+ spec:+ containers:+ - name: index-worker+ image: firecrawl/firecrawl:latest+ imagePullPolicy: Never+ command: [ "node" ]+ args: [ "--max-old-space-size=2048", "dist/src/services/indexing/index-worker.js" ]+ envFrom:+ - configMapRef:+ name: firecrawl-config+ - configMapRef:+ name: firecrawl-config-dynamic+ - secretRef:+ name: firecrawl-secret-dynamic+ env:+ - name: FLY_PROCESS_GROUP+ value: "index-worker"+ resources:+ requests:+ memory: "1G"+ cpu: "500m"+ limits:+ memory: "3G"+ cpu: "1000m"\ No newline at end of filediff --git a/.github/k8s/server.yml b/.github/k8s/server.ymlnew file mode 100644index 000000000..5d55bb080--- /dev/null+++ b/.github/k8s/server.yml@@ -0,0 +1,305 @@+apiVersion: v1+kind: ConfigMap+metadata:+ name: firecrawl-config+data:+ HOST: "0.0.0.0"+ REDIS_URL: "redis://firecrawl-dragonfly:6379"+ REDIS_RATE_LIMIT_URL: "redis://firecrawl-dragonfly:6379"+ REDIS_EVICT_URL: "redis://firecrawl-dragonfly:6379"+ SENTRY_ENVIRONMENT: "dev"+ ENV: "production"+ LOGGING_LEVEL: "DEBUG"+ IS_KUBERNETES: "true"+ USE_GO_MARKDOWN_PARSER: "true"+ NUQ_DATABASE_URL: "postgres://postgres:postgres@nuq-postgres:5432/postgres"+ NUQ_DATABASE_URL_LISTEN: "postgres://postgres:postgres@nuq-postgres:5432/postgres"+---+apiVersion: apps/v1+kind: Deployment+metadata:+ name: nuq-postgres+spec:+ replicas: 1+ selector:+ matchLabels:+ app: nuq-postgres+ template:+ metadata:+ labels:+ app: nuq-postgres+ spec:+ containers:+ - name: postgres+ image: firecrawl/nuq-postgres:latest+ imagePullPolicy: Never+ ports:+ - containerPort: 5432+ env:+ - name: POSTGRES_USER+ value: "postgres"+ - name: POSTGRES_PASSWORD+ value: "postgres"+ - name: POSTGRES_DB+ value: "postgres"+ resources:+ requests:+ memory: "512Mi"+ cpu: "250m"+ limits:+ memory: "1Gi"+ cpu: "500m"++---+apiVersion: v1+kind: Service+metadata:+ name: nuq-postgres+spec:+ selector:+ app: nuq-postgres+ ports:+ - name: postgres+ protocol: TCP+ port: 5432+ targetPort: 5432+---+apiVersion: apps/v1+kind: Deployment+metadata:+ name: firecrawl-dragonfly+spec:+ replicas: 1+ selector:+ matchLabels:+ app: firecrawl-dragonfly+ template:+ metadata:+ labels:+ app: firecrawl-dragonfly+ spec:+ containers:+ - name: dragonfly+ image: ghcr.io/dragonflydb/dragonfly:v1.31.2+ args:+ - "--logtostderr"+ - "--cluster_mode=emulated"+ - "--lock_on_hashtags"+ - "--bind=::"+ - "--maxmemory=4gb"+ - "--dir=/data"+ ports:+ - containerPort: 6379+ readinessProbe:+ tcpSocket:+ port: 6379+ initialDelaySeconds: 3+ periodSeconds: 5+ timeoutSeconds: 3+ successThreshold: 1+ failureThreshold: 3+ resources:+ requests:+ memory: "1Gi"+ cpu: "500m"+ limits:+ memory: "4Gi"+ cpu: "1000m"+---+apiVersion: v1+kind: Service+metadata:+ name: firecrawl-dragonfly+spec:+ selector:+ app: firecrawl-dragonfly+ ports:+ - name: redis+ protocol: TCP+ port: 6379+ targetPort: 6379+---+apiVersion: apps/v1+kind: Deployment+metadata:+ name: firecrawl-app+spec:+ replicas: 1+ selector:+ matchLabels:+ app: firecrawl-app+ template:+ metadata:+ labels:+ app: firecrawl-app+ spec:+ containers:+ - name: firecrawl-app+ image: firecrawl/firecrawl:latest+ imagePullPolicy: Never+ command: ["node"]+ args: ["dist/src/index.js"]+ ports:+ - containerPort: 3002+ envFrom:+ - configMapRef:+ name: firecrawl-config+ - configMapRef:+ name: firecrawl-config-dynamic+ - secretRef:+ name: firecrawl-secret-dynamic+ env:+ - name: FLY_PROCESS_GROUP+ value: "app"+ - name: PORT+ value: "3002"+ - name: NODE_OPTIONS+ value: "--no-node-snapshot --max-old-space-size=6144"+ resources:+ requests:+ memory: "1G"+ cpu: "500m"+ limits:+ memory: "6G"+ cpu: "1000m"+ livenessProbe:+ httpGet:+ path: /v0/health/liveness+ port: 3002+ initialDelaySeconds: 5+ periodSeconds: 5+ timeoutSeconds: 5+ successThreshold: 1+ failureThreshold: 3+ readinessProbe:+ httpGet:+ path: /v0/health/readiness+ port: 3002+ initialDelaySeconds: 5+ periodSeconds: 5+ timeoutSeconds: 5+ successThreshold: 1+ failureThreshold: 3+---+apiVersion: v1+kind: Service+metadata:+ name: firecrawl-app+ namespace: default+ labels:+ app: firecrawl-app+spec:+ selector:+ app: firecrawl-app+ ports:+ - name: http+ protocol: TCP+ port: 3002+ targetPort: 3002+ type: NodePort+---+apiVersion: apps/v1+kind: Deployment+metadata:+ name: firecrawl-worker+spec:+ replicas: 1+ selector:+ matchLabels:+ app: firecrawl-worker+ template:+ metadata:+ labels:+ app: firecrawl-worker+ spec:+ containers:+ - name: worker+ image: firecrawl/firecrawl:latest+ imagePullPolicy: Never+ command: [ "node" ]+ args: [ "dist/src/services/queue-worker.js", "--max-old-space-size=2048" ]+ envFrom:+ - configMapRef:+ name: firecrawl-config+ - configMapRef:+ name: firecrawl-config-dynamic+ - secretRef:+ name: firecrawl-secret-dynamic+ env:+ - name: FLY_PROCESS_GROUP+ value: "worker"+ - name: PORT+ value: "3005"+ - name: NODE_OPTIONS+ value: "--no-node-snapshot"+ - name: GOMEMLIMIT+ value: "1024MiB"+ - name: SCRAPE_WORKER_MAX_OLD_SPACE_SIZE+ value: "768"+ resources:+ requests:+ memory: "2G"+ cpu: "500m"+ limits:+ memory: "4G"+ cpu: "1000m"+ livenessProbe:+ httpGet:+ path: /liveness+ port: 3005+ initialDelaySeconds: 5+ periodSeconds: 5+ timeoutSeconds: 5+ successThreshold: 1+ failureThreshold: 3+---+apiVersion: apps/v1+kind: Deployment+metadata:+ name: firecrawl-nuq-worker+spec:+ replicas: 5+ selector:+ matchLabels:+ app: firecrawl-nuq-worker+ template:+ metadata:+ labels:+ app: firecrawl-nuq-worker+ spec:+ containers:+ - name: nuq-worker+ image: firecrawl/firecrawl:latest+ imagePullPolicy: Never+ command: [ "node" ]+ args: [ "dist/src/services/worker/nuq-worker.js", "--max-old-space-size=2048" ]+ envFrom:+ - configMapRef:+ name: firecrawl-config+ - configMapRef:+ name: firecrawl-config-dynamic+ - secretRef:+ name: firecrawl-secret-dynamic+ env:+ - name: PORT+ value: "3005"+ - name: NODE_OPTIONS+ value: "--no-node-snapshot"+ - name: GOMEMLIMIT+ value: "1024MiB"+ resources:+ requests:+ memory: "2G"+ cpu: "500m"+ limits:+ memory: "4G"+ cpu: "1000m"+ livenessProbe:+ httpGet:+ path: /health+ port: 3005+ initialDelaySeconds: 10+ periodSeconds: 10+ timeoutSeconds: 10+ successThreshold: 1+ failureThreshold: 3\ No newline at end of filediff --git a/.github/k8s/tailscale-proxy/Dockerfile b/.github/k8s/tailscale-proxy/Dockerfilenew file mode 100644index 000000000..77bd48e43--- /dev/null+++ b/.github/k8s/tailscale-proxy/Dockerfile@@ -0,0 +1,28 @@+FROM debian:bookworm-slim++# Install prerequisites, iptables, nginx, and Tailscale+RUN apt-get update \+ && apt-get install -y --no-install-recommends \+ ca-certificates \+ curl \+ gnupg \+ iptables \+ iproute2 \+ nginx \+ && curl -fsSL https://pkgs.tailscale.com/stable/debian/bookworm.noarmor.gpg \+ -o /usr/share/keyrings/tailscale-archive-keyring.gpg \+ && curl -fsSL https://pkgs.tailscale.com/stable/debian/bookworm.tailscale-keyring.list \+ -o /etc/apt/sources.list.d/tailscale.list \+ && apt-get update \+ && apt-get install -y --no-install-recommends tailscale \+ && rm -rf /var/lib/apt/lists/*++# Configure nginx as a reverse proxy+COPY nginx.conf /etc/nginx/nginx.conf.template++# Start script that sets up tailscale and nginx+COPY entrypoint.sh /entrypoint.sh+RUN chmod +x /entrypoint.sh++EXPOSE 8080+ENTRYPOINT ["/entrypoint.sh"]\ No newline at end of filediff --git a/.github/k8s/tailscale-proxy/entrypoint.sh b/.github/k8s/tailscale-proxy/entrypoint.shnew file mode 100644index 000000000..a1a81639f--- /dev/null+++ b/.github/k8s/tailscale-proxy/entrypoint.sh@@ -0,0 +1,48 @@+#!/bin/sh+set -eu++# Require auth key+if [ -z "${TS_AUTHKEY:-}" ]; then+ echo "[tailscale-proxy] TS_AUTHKEY is not set; refusing to start." >&2+ exit 1+fi++# Compute hostname: allow override via TS_HOSTNAME, else default to pod name+TS_HOSTNAME=${TS_HOSTNAME:-}+if [ -z "${TS_HOSTNAME}" ]; then+ # Kubernetes sets HOSTNAME to the pod name+ TS_HOSTNAME="firecrawl-proxy-${HOSTNAME:-container}"+fi++# Start tailscaled+tailscaled --state=mem: &++# Wait for tailscaled control socket to be ready (up to ~15s)+i=0+until tailscale version >/dev/null 2>&1 || [ $i -ge 30 ]; do+ i=$((i+1))+ sleep 0.5+done++# Prepare optional flags+ADVERTISE_FLAGS=""+if [ -n "${TS_ADVERTISE_TAGS:-}" ]; then+ ADVERTISE_FLAGS="--advertise-tags=${TS_ADVERTISE_TAGS}"+fi++set -x+tailscale up \+ --authkey="${TS_AUTHKEY}" \+ --hostname="${TS_HOSTNAME}" \+ --accept-routes=true \+ --accept-dns=true \+ --ssh=false \+ ${ADVERTISE_FLAGS} \+ --reset+set +x++# Replace FIRE_ENGINE_BETA_URL in nginx config+sed "s|\$FIRE_ENGINE_BETA_URL|${FIRE_ENGINE_BETA_URL}|g" /etc/nginx/nginx.conf.template > /etc/nginx/nginx.conf++# Start nginx+exec nginx -g "daemon off;"\ No newline at end of filediff --git a/.github/k8s/tailscale-proxy/nginx.conf b/.github/k8s/tailscale-proxy/nginx.confnew file mode 100644index 000000000..ca48512ac--- /dev/null+++ b/.github/k8s/tailscale-proxy/nginx.conf@@ -0,0 +1,24 @@+user www-data;+worker_processes auto;+error_log /dev/stderr notice;+pid /run/nginx.pid;++events {+ worker_connections 1024;+}++http {+ access_log /dev/stdout combined;++ server {+ listen 8080;++ location / {+ proxy_pass $FIRE_ENGINE_BETA_URL;+ proxy_set_header Host $host;+ proxy_set_header X-Real-IP $remote_addr;+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;+ proxy_set_header X-Forwarded-Proto $scheme;+ }+ }+}\ No newline at end of filediff --git a/.github/workflows/test-server-self-host.yml b/.github/workflows/test-server-self-host.ymldeleted file mode 100644index 9c97c8f00..000000000--- a/.github/workflows/test-server-self-host.yml+++ /dev/null@@ -1,145 +0,0 @@-name: Self-hosted Server Test Suite--on:- pull_request:- branches:- - main- - nsc/v2--env:- REDIS_URL: redis://localhost:6379- ENV: ${{ secrets.ENV }}- TEST_SUITE_SELF_HOSTED: true- USE_GO_MARKDOWN_PARSER: true- FIRECRAWL_DEBUG_FILTER_LINKS: true- SENTRY_ENVIRONMENT: dev--jobs:- test:- name: Run tests- strategy:- matrix:- ai: ['openai', 'no-ai']- search: ['searxng', 'google']- engine: ['playwright', 'fetch']- # proxy: ["proxy", "no-proxy"]- proxy: ['proxy']- fail-fast: false- runs-on: ubuntu-latest- services:- redis:- image: redis- ports:- - 6379:6379- env:- OPENAI_API_KEY: ${{ matrix.ai == 'openai' && secrets.OPENAI_API_KEY || '' }}- SEARXNG_ENDPOINT: ${{ matrix.search == 'searxng' && 'http://localhost:3434' || '' }}- PLAYWRIGHT_MICROSERVICE_URL: ${{ matrix.engine == 'playwright' && 'http://localhost:3003/scrape' || '' }}- PROXY_SERVER: ${{ matrix.proxy == 'proxy' && secrets.PROXY_SERVER || '' }}- PROXY_USERNAME: ${{ matrix.proxy == 'proxy' && secrets.PROXY_USERNAME || '' }}- PROXY_PASSWORD: ${{ matrix.proxy == 'proxy' && secrets.PROXY_PASSWORD || '' }}- steps:- - uses: actions/checkout@v5- - name: Install pnpm- uses: pnpm/action-setup@v4- with:- version: 10- - name: Set up Node.js- uses: actions/setup-node@v4- with:- node-version: '20'- cache: 'pnpm'- cache-dependency-path: './apps/api/pnpm-lock.yaml'- - name: Install dependencies- run: pnpm install- working-directory: ./apps/api- - name: Install Playwright dependencies- if: matrix.engine == 'playwright'- run: |- pnpm install- pnpm exec playwright install-deps- pnpm exec playwright install- working-directory: ./apps/playwright-service-ts- - name: Set up Go- uses: actions/setup-go@v5- with:- go-version: '1.24'- cache-dependency-path: ./apps/api/sharedLibs/go-html-to-md/go.sum- - name: Build go-html-to-md- run: |- go mod tidy- go build -o libhtml-to-markdown.so -buildmode=c-shared html-to-markdown.go- chmod +x libhtml-to-markdown.so- working-directory: ./apps/api/sharedLibs/go-html-to-md- - name: Set up SearXNG- if: matrix.search == 'searxng'- run: |- mkdir searxng-- echo "use_default_settings: true- search:- formats: [html, json, csv]- server:- secret_key: 'fcsecret'" > searxng/settings.yml-- docker run -d -p 3434:8080 -v "${PWD}/searxng:/etc/searxng" --name searxng searxng/searxng- pnpx wait-on tcp:3434 -t 30s- working-directory: ./- - name: Build the application- run: pnpm run build- working-directory: ./apps/api- - name: Start server- run: pnpm run start:production:nobuild > api.log 2>&1 &- env:- PORT: 3002- HOST: 0.0.0.0- working-directory: ./apps/api- - name: Start worker- run: pnpm run worker:production > worker.log 2>&1 &- env:- PORT: 3005- HOST: 0.0.0.0- working-directory: ./apps/api- - name: Start playwright- if: matrix.engine == 'playwright'- run: pnpm run dev > playwright.log 2>&1 &- working-directory: ./apps/playwright-service-ts- env:- PORT: 3003- - name: Wait for server- run: pnpx wait-on tcp:3002 -t 60s- - name: Wait for playwright- if: matrix.engine == 'playwright'- run: pnpx wait-on tcp:3003 -t 15s- - name: Run snippet tests- run: |- npm run test:snips- working-directory: ./apps/api- - name: Kill instances- if: always()- run: pkill -9 node- - name: Kill SearXNG- if: always() && matrix.search == 'searxng'- run: |- docker logs searxng > searxng.log 2>&1- docker kill searxng- working-directory: ./- - uses: actions/upload-artifact@v4- if: always()- with:- name: Logs (${{ matrix.ai }}, ${{ matrix.search }}, ${{ matrix.engine }}, ${{ matrix.proxy }})- path: |- ./apps/api/api.log- ./apps/api/worker.log- - uses: actions/upload-artifact@v4- if: always() && matrix.playwright- with:- name: Playwright Logs (${{ matrix.ai }}, ${{ matrix.search }}, ${{ matrix.proxy }})- path: |- ./apps/playwright-service-ts/playwright.log- - uses: actions/upload-artifact@v4- if: always() && matrix.search == 'searxng'- with:- name: SearXNG (${{ matrix.ai }}, ${{ matrix.engine }}, ${{ matrix.proxy }})- path: |- ./searxng.logdiff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.ymlindex 6278db813..66b9093b6 100644--- a/.github/workflows/test-server.yml+++ b/.github/workflows/test-server.yml@@ -6,50 +6,74 @@ on:- main- nsc/v2-env:- BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }}- POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}- POSTHOG_HOST: ${{ secrets.POSTHOG_HOST }}- OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}- REDIS_URL: ${{ secrets.REDIS_URL }}- SUPABASE_ANON_TOKEN: ${{ secrets.SUPABASE_ANON_TOKEN }}- SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }}- SUPABASE_URL: ${{ secrets.SUPABASE_URL }}- SUPABASE_REPLICA_URL: ${{ secrets.SUPABASE_REPLICA_URL }}- INDEX_SUPABASE_SERVICE_TOKEN: ${{ secrets.INDEX_SUPABASE_SERVICE_TOKEN }}- INDEX_SUPABASE_ANON_TOKEN: ${{ secrets.INDEX_SUPABASE_ANON_TOKEN }}- INDEX_SUPABASE_URL: ${{ secrets.INDEX_SUPABASE_URL }}- TEST_API_KEY: ${{ secrets.TEST_API_KEY }}- TEST_TEAM_ID: ${{ secrets.TEST_TEAM_ID }}- TEST_API_KEY_CONCURRENCY: ${{ secrets.TEST_API_KEY_CONCURRENCY }}- TEST_TEAM_ID_CONCURRENCY: ${{ secrets.TEST_TEAM_ID_CONCURRENCY }}- TEST_API_KEY_ZDR: ${{ secrets.TEST_API_KEY_ZDR }}- TEST_TEAM_ID_ZDR: ${{ secrets.TEST_TEAM_ID_ZDR }}- FIRE_ENGINE_BETA_URL: ${{ secrets.FIRE_ENGINE_BETA_URL }}- FIRE_ENGINE_STAGING_URL: ${{ secrets.FIRE_ENGINE_STAGING_URL }}- USE_DB_AUTHENTICATION: true- SERPER_API_KEY: ${{ secrets.SERPER_API_KEY }}- ENV: ${{ secrets.ENV }}- RUNPOD_MU_POD_ID: ${{ secrets.RUNPOD_MU_POD_ID }}- RUNPOD_MUV2_POD_ID: ${{ secrets.RUNPOD_MUV2_POD_ID }}- RUNPOD_MU_API_KEY: ${{ secrets.RUNPOD_MU_API_KEY }}- GCS_CREDENTIALS: ${{ secrets.GCS_CREDENTIALS }}- GCS_BUCKET_NAME: ${{ secrets.GCS_BUCKET_NAME }}- GCS_INDEX_BUCKET_NAME: ${{ secrets.GCS_INDEX_BUCKET_NAME }}- GCS_MEDIA_BUCKET_NAME: ${{ secrets.GCS_MEDIA_BUCKET_NAME }}- GOOGLE_GENERATIVE_AI_API_KEY: ${{ secrets.GOOGLE_GENERATIVE_AI_API_KEY }}- GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}- ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}- VERTEX_CREDENTIALS: ${{ secrets.VERTEX_CREDENTIALS }}- USE_GO_MARKDOWN_PARSER: true- SENTRY_ENVIRONMENT: dev- IDMUX_URL: ${{ secrets.IDMUX_URL }}- LOG_ENCRYPTION_KEY: ${{ secrets.LOG_ENCRYPTION_KEY }}-jobs:+ build-images:+ name: Build images+ runs-on: big-runner+ steps:+ - uses: actions/checkout@v5+ - name: Set up Docker Buildx+ uses: docker/setup-buildx-action@v3+ - name: Build API image+ run: |+ docker buildx build \+ -t firecrawl/firecrawl:latest \+ --cache-from type=gha \+ --cache-to type=gha,mode=max \+ --load \+ ./apps/api+ - name: Build nuq-postgres image+ run: |+ docker buildx build \+ -t firecrawl/nuq-postgres:latest \+ --cache-from type=gha \+ --cache-to type=gha,mode=max \+ --load \+ ./apps/nuq-postgres+ - name: Build tailscale-proxy image+ run: |+ docker buildx build \+ -t firecrawl/tailscale-proxy:latest \+ --cache-from type=gha \+ --cache-to type=gha,mode=max \+ --load \+ .github/k8s/tailscale-proxy+ - name: Build Playwright image+ run: |+ docker buildx build \+ -t firecrawl/playwright-service:latest \+ --cache-from type=gha \+ --cache-to type=gha,mode=max \+ --load \+ ./apps/playwright-service-ts+ - name: Export images as tarballs+ run: |+ docker save -o firecrawl-firecrawl.tar firecrawl/firecrawl:latest+ docker save -o firecrawl-nuq-postgres.tar firecrawl/nuq-postgres:latest+ docker save -o firecrawl-tailscale-proxy.tar firecrawl/tailscale-proxy:latest+ docker save -o firecrawl-playwright-service.tar firecrawl/playwright-service:latest+ - name: Upload images artifact+ uses: actions/upload-artifact@v4+ with:+ name: built-images+ path: |+ firecrawl-firecrawl.tar+ firecrawl-nuq-postgres.tar+ firecrawl-tailscale-proxy.tar+ firecrawl-playwright-service.tar+test:- name: Run tests- runs-on: ubuntu-latest+ name: Self-hosted (Kubernetes) environment tests+ needs: build-images+ strategy:+ matrix:+ ai: ["openai", "no-ai"]+ search: ["searxng", "google"]+ engine: ["playwright", "fetch"]+ # proxy: ["proxy", "no-proxy"]+ proxy: ["proxy"]+ fail-fast: false+ runs-on: big-runnerservices:redis:image: redis@@ -57,13 +81,43 @@ jobs:- 6379:6379steps:- uses: actions/checkout@v5- - name: Tailscale- uses: tailscale/github-action@v3+ - name: Start minikube+ uses: medyagh/[email protected]+ - name: Download images artifact+ uses: actions/download-artifact@v4with:- oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }}- oauth-secret: ${{ secrets.TS_OAUTH_SECRET }}- tags: tag:ci- use-cache: 'true'+ name: built-images+ path: ./images+ - name: Load images into minikube+ run: |+ eval $(minikube docker-env)+ docker load -i ./images/firecrawl-firecrawl.tar+ docker load -i ./images/firecrawl-nuq-postgres.tar+ - name: Load Playwright image into minikube+ if: matrix.engine == 'playwright'+ run: |+ eval $(minikube docker-env)+ docker load -i ./images/firecrawl-playwright-service.tar+ - name: Create configmap+ run: |+ kubectl create configmap firecrawl-config-dynamic -n default \+ --from-literal=SEARXNG_ENDPOINT=${{ matrix.search == 'searxng' && 'http://firecrawl-searxng:8080/search' || '' }} \+ --from-literal=PLAYWRIGHT_MICROSERVICE_URL=${{ matrix.engine == 'playwright' && 'http://firecrawl-playwright-service:3003/scrape' || '' }} \+ --from-literal=PROXY_SERVER=${{ matrix.proxy == 'proxy' && secrets.PROXY_SERVER || '' }} \+ --from-literal=PROXY_USERNAME=${{ matrix.proxy == 'proxy' && secrets.PROXY_USERNAME || '' }}+ - name: Create secret+ run: |+ kubectl create secret generic firecrawl-secret-dynamic \+ --from-literal=OPENAI_API_KEY=${{ matrix.ai == 'openai' && secrets.OPENAI_API_KEY || '' }} \+ --from-literal=PROXY_PASSWORD=${{ matrix.proxy == 'proxy' && secrets.PROXY_PASSWORD || '' }}+ - name: Deploy server+ run: kubectl apply -f .github/k8s/server.yml+ - name: Deploy playwright+ if: matrix.engine == 'playwright'+ run: kubectl apply -f .github/k8s/playwright.yml+ - name: Deploy searxng+ if: matrix.search == 'searxng'+ run: kubectl apply -f .github/k8s/searxng.yml- name: Install pnpmuses: pnpm/action-setup@v4with:@@ -77,43 +131,277 @@ jobs:- name: Install dependenciesrun: pnpm installworking-directory: ./apps/api- - name: Set up Go- uses: actions/setup-go@v5- with:- go-version: '1.24'- cache-dependency-path: ./apps/api/sharedLibs/go-html-to-md/go.sum- - name: Build go-html-to-md- run: |- go mod tidy- go build -o libhtml-to-markdown.so -buildmode=c-shared html-to-markdown.go- chmod +x libhtml-to-markdown.so- working-directory: ./apps/api/sharedLibs/go-html-to-md- - name: Build the application- run: pnpm run build+ - name: Wait for server+ run: kubectl wait --for=condition=ready pod -l app=firecrawl-app --timeout=60s -o json+ - name: Debug wait failure (server)+ if: failure()+ run: |+ echo "Pods with label app=firecrawl-app:" || true+ kubectl get pods -l app=firecrawl-app -o wide || true+ echo "\nDescribe pods:" || true+ kubectl describe pods -l app=firecrawl-app || true+ echo "\nRecent events:" || true+ kubectl get events --sort-by=.lastTimestamp | tail -n 100 || true+ echo "\nLogs from firecrawl-app deployment (if any):" || true+ kubectl logs deployment/firecrawl-app --all-containers=true --tail=200 || true+ - name: Wait for playwright+ if: matrix.engine == 'playwright'+ run: kubectl wait --for=condition=ready pod -l app=firecrawl-playwright-service --timeout=60s -o json+ - name: Debug wait failure (playwright)+ if: failure() && matrix.engine == 'playwright'+ run: |+ echo "Pods with label app=firecrawl-playwright-service:" || true+ kubectl get pods -l app=firecrawl-playwright-service -o wide || true+ echo "\nDescribe pods:" || true+ kubectl describe pods -l app=firecrawl-playwright-service || true+ echo "\nRecent events:" || true+ kubectl get events --sort-by=.lastTimestamp | tail -n 100 || true+ echo "\nLogs from playwright deployment (if any):" || true+ kubectl logs deployment/firecrawl-playwright-service --all-containers=true --tail=200 || true+ - name: Wait for searxng+ if: matrix.search == 'searxng'+ run: kubectl wait --for=condition=ready pod -l app=firecrawl-searxng --timeout=60s -o json+ - name: Debug wait failure (searxng)+ if: failure() && matrix.search == 'searxng'+ run: |+ echo "Pods with label app=firecrawl-searxng:" || true+ kubectl get pods -l app=firecrawl-searxng -o wide || true+ echo "\nDescribe pods:" || true+ kubectl describe pods -l app=firecrawl-searxng || true+ echo "\nRecent events:" || true+ kubectl get events --sort-by=.lastTimestamp | tail -n 100 || true+ echo "\nLogs from searxng deployment (if any):" || true+ kubectl logs deployment/firecrawl-searxng --all-containers=true --tail=200 || true+ - name: Run snippet tests+ env:+ TEST_SUITE_SELF_HOSTED: true+ OPENAI_API_KEY: ${{ matrix.ai == 'openai' && '<IS SET>' || '' }}+ PROXY_SERVER: ${{ matrix.proxy == 'proxy' && secrets.PROXY_SERVER || '' }}+ PLAYWRIGHT_MICROSERVICE_URL: ${{ matrix.engine == 'playwright' && '<IS SET>' || '' }}+ run: TEST_API_URL=$(minikube service firecrawl-app --url) pnpm run test:snipsworking-directory: ./apps/api- - name: Start the application- run: pnpm run start:production:nobuild > api.log 2>&1 &+ - name: Create logs directory+ if: always()+ run: mkdir -p logs+ - name: Copy log files+ if: always()+ run: |+ kubectl logs deployment/firecrawl-app > logs/api.log || true+ kubectl logs deployment/firecrawl-worker > logs/worker.log || true+ kubectl logs deployment/firecrawl-nuq-worker > logs/nuq-worker.log || true+ kubectl logs deployment/firecrawl-dragonfly > logs/dragonfly.log || true+ kubectl logs deployment/nuq-postgres > logs/postgres.log || true+ - name: Copy SearXNG logs+ if: always() && matrix.search == 'searxng'+ run: kubectl logs deployment/firecrawl-searxng > logs/searxng.log || true+ - name: Copy Playwright logs+ if: always() && matrix.engine == 'playwright'+ run: kubectl logs deployment/firecrawl-playwright-service > logs/playwright.log || true+ - name: Zip logs+ if: always()+ run: |+ cd logs+ zip -r logs.zip ./*+ - uses: actions/upload-artifact@v4+ if: always()+ with:+ name: Logs (kubernetes, ${{ matrix.ai }}, ${{ matrix.search }}, ${{ matrix.engine }}, ${{ matrix.proxy }})+ path: logs/logs.zip++ docker-test:+ name: Self-hosted (Docker) environment tests+ runs-on: ubuntu-latest+ steps:+ - uses: actions/checkout@v5+ - name: Composeenv:- PORT: 3002- HOST: 0.0.0.0+ PROXY_SERVER: ${{ secrets.PROXY_SERVER }}+ PROXY_USERNAME: ${{ secrets.PROXY_USERNAME }}+ PROXY_PASSWORD: ${{ secrets.PROXY_PASSWORD }}+ run: |+ docker compose up -d+ - name: Install pnpm+ uses: pnpm/action-setup@v4+ with:+ version: 10+ - name: Set up Node.js+ uses: actions/setup-node@v4+ with:+ node-version: "20"+ cache: "pnpm"+ cache-dependency-path: './apps/api/pnpm-lock.yaml'+ - name: Install dependencies+ run: pnpm installworking-directory: ./apps/api- id: start_app- - name: Start worker- run: pnpm run worker:production > worker.log 2>&1 &+ - name: Wait for server+ run: pnpx wait-on http://localhost:3002+ - name: Run snippet testsenv:- PORT: 3005- HOST: 0.0.0.0+ PROXY_SERVER: ${{ secrets.PROXY_SERVER }}+ PLAYWRIGHT_MICROSERVICE_URL: ${{ secrets.PLAYWRIGHT_MICROSERVICE_URL }}+ run: pnpm run test:snipsworking-directory: ./apps/api- id: start_workers- - name: Start index worker- run: pnpm run index-worker:production > index-worker.log 2>&1 &+ - name: Create logs directory+ if: always()+ run: mkdir -p logs+ - name: Copy log files+ if: always()+ run: |+ docker compose logs api > logs/api.log || true+ docker compose logs playwright-service > logs/playwright.log || true+ docker compose logs nuq-postgres > logs/postgres.log || true+ - name: Zip logs+ if: always()+ run: |+ cd logs+ zip -r logs.zip ./*+ - uses: actions/upload-artifact@v4+ if: always()+ with:+ name: Logs (docker, no-ai, google, playwright, proxy)+ path: logs/logs.zip++ prod-test:+ name: Production environment tests+ needs: build-images+ runs-on: big-runner+ steps:+ - uses: actions/checkout@v5+ - name: Start minikube+ uses: medyagh/[email protected]+ - name: Download images artifact+ uses: actions/download-artifact@v4+ with:+ name: built-images+ path: ./images+ - name: Tailscale+ uses: tailscale/github-action@v3+ with:+ oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }}+ oauth-secret: ${{ secrets.TS_OAUTH_SECRET }}+ tags: tag:ci+ - name: Load prebuilt images into minikube+ run: |+ eval $(minikube docker-env)+ docker load -i ./images/firecrawl-firecrawl.tar+ docker load -i ./images/firecrawl-nuq-postgres.tar+ docker load -i ./images/firecrawl-tailscale-proxy.tar+ - name: Create Tailscale secret+ run: |+ if [ -z "${{ secrets.TS_AUTHKEY }}" ]; then+ echo "secrets.TS_AUTHKEY is empty; set a reusable/preauthorized or ephemeral key" >&2+ exit 1+ fi+ kubectl create secret generic tailscale \+ --from-literal=TS_AUTHKEY=${{ secrets.TS_AUTHKEY }} \+ --from-literal=TS_ADVERTISE_TAGS=${{ secrets.TS_ADVERTISE_TAGS }} \+ --from-literal=FIRE_ENGINE_BETA_URL=${{ secrets.FIRE_ENGINE_BETA_URL }} \+ --dry-run=client -o yaml | kubectl apply -f -+ - name: Create configmap+ run: |+ kubectl create configmap firecrawl-config-dynamic -n default \+ --from-literal=USE_DB_AUTHENTICATION=true \+ --from-literal=SUPABASE_ANON_TOKEN=${{ secrets.SUPABASE_ANON_TOKEN }} \+ --from-literal=SUPABASE_URL=${{ secrets.SUPABASE_URL }} \+ --from-literal=SUPABASE_REPLICA_URL=${{ secrets.SUPABASE_REPLICA_URL }} \+ --from-literal=INDEX_SUPABASE_ANON_TOKEN=${{ secrets.INDEX_SUPABASE_ANON_TOKEN }} \+ --from-literal=INDEX_SUPABASE_URL=${{ secrets.INDEX_SUPABASE_URL }} \+ --from-literal=FIRE_ENGINE_BETA_URL=http://tailscale-proxy:8080 \+ --from-literal=RUNPOD_MU_POD_ID=${{ secrets.RUNPOD_MU_POD_ID }} \+ --from-literal=RUNPOD_MUV2_POD_ID=${{ secrets.RUNPOD_MUV2_POD_ID }} \+ --from-literal=GCS_BUCKET_NAME=${{ secrets.GCS_BUCKET_NAME }} \+ --from-literal=GCS_MEDIA_BUCKET_NAME=${{ secrets.GCS_MEDIA_BUCKET_NAME }} \+ --from-literal=GCS_INDEX_BUCKET_NAME=${{ secrets.GCS_INDEX_BUCKET_NAME }}+ - name: Create secret+ run: |+ kubectl create secret generic firecrawl-secret-dynamic \+ --from-literal=BULL_AUTH_KEY=${{ secrets.BULL_AUTH_KEY }} \+ --from-literal=OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }} \+ --from-literal=SUPABASE_SERVICE_TOKEN=${{ secrets.SUPABASE_SERVICE_TOKEN }} \+ --from-literal=INDEX_SUPABASE_SERVICE_TOKEN=${{ secrets.INDEX_SUPABASE_SERVICE_TOKEN }} \+ --from-literal=SERPER_API_KEY=${{ secrets.SERPER_API_KEY }} \+ --from-literal=RUNPOD_MU_API_KEY=${{ secrets.RUNPOD_MU_API_KEY }} \+ --from-literal=GCS_CREDENTIALS=${{ secrets.GCS_CREDENTIALS }} \+ --from-literal=GOOGLE_GENERATIVE_AI_API_KEY=${{ secrets.GOOGLE_GENERATIVE_AI_API_KEY }} \+ --from-literal=GROQ_API_KEY=${{ secrets.GROQ_API_KEY }} \+ --from-literal=ANTHROPIC_API_KEY=${{ secrets.ANTHROPIC_API_KEY }} \+ --from-literal=VERTEX_CREDENTIALS=${{ secrets.VERTEX_CREDENTIALS }} \+ --from-literal=LOG_ENCRYPTION_KEY=${{ secrets.LOG_ENCRYPTION_KEY }}+ - name: Deploy server+ run: |+ kubectl apply -f .github/k8s/server.yml+ kubectl apply -f .github/k8s/server-nsh.yml+ - name: Install pnpm+ uses: pnpm/action-setup@v4+ with:+ version: 10+ - name: Set up Node.js+ uses: actions/setup-node@v4+ with:+ node-version: "20"+ cache: "pnpm"+ cache-dependency-path: './apps/api/pnpm-lock.yaml'+ - name: Install dependencies+ run: pnpm installworking-directory: ./apps/api- id: start_index_worker- name: Wait for API- run: pnpx wait-on tcp:3002 -t 60s+ run: kubectl wait --for=condition=ready pod -l app=firecrawl-app --timeout=60s -o json+ - name: Debug wait failure (API)+ if: failure()+ run: |+ echo "Pods with label app=firecrawl-app:" || true+ kubectl get pods -l app=firecrawl-app -o wide || true+ echo "\nDescribe pods:" || true+ kubectl describe pods -l app=firecrawl-app || true+ echo "\nRecent events:" || true+ kubectl get events --sort-by=.lastTimestamp | tail -n 100 || true+ - name: Wait for nuq-postgres+ run: kubectl wait --for=condition=ready pod -l app=nuq-postgres --timeout=60s -o json+ - name: Debug wait failure (nuq-postgres)+ if: failure()+ run: |+ echo "Pods with label app=nuq-postgres:" || true+ kubectl get pods -l app=nuq-postgres -o wide || true+ echo "\nDescribe pods:" || true+ kubectl describe pods -l app=nuq-postgres || true+ echo "\nRecent events:" || true+ kubectl get events --sort-by=.lastTimestamp | tail -n 100 || true+ echo "\nLogs from nuq-postgres deployment (if any):" || true+ kubectl logs deployment/nuq-postgres --all-containers=true --tail=200 || true+ - name: Wait for tailscale-proxy+ run: kubectl wait --for=condition=ready pod -l app=tailscale-proxy --timeout=60s -o json+ - name: Debug wait failure (tailscale-proxy)+ if: failure()+ run: |+ echo "Pods with label app=tailscale-proxy:" || true+ kubectl get pods -l app=tailscale-proxy -o wide || true+ echo "\nDescribe pods:" || true+ kubectl describe pods -l app=tailscale-proxy || true+ echo "\nRecent events:" || true+ kubectl get events --sort-by=.lastTimestamp | tail -n 100 || true+ - name: Wait for dragonfly (redis)+ run: kubectl wait --for=condition=ready pod -l app=firecrawl-dragonfly --timeout=60s -o json+ - name: Debug wait failure (dragonfly)+ if: failure()+ run: |+ echo "Pods with label app=firecrawl-dragonfly:" || true+ kubectl get pods -l app=firecrawl-dragonfly -o wide || true+ echo "\nDescribe pods:" || true+ kubectl describe pods -l app=firecrawl-dragonfly || true+ echo "\nRecent events:" || true+ kubectl get events --sort-by=.lastTimestamp | tail -n 100 || true- name: Run snippet tests+ env:+ IDMUX_URL: ${{ secrets.IDMUX_URL }}+ USE_DB_AUTHENTICATION: true+ SUPABASE_ANON_TOKEN: ${{ secrets.SUPABASE_ANON_TOKEN }}+ SUPABASE_URL: ${{ secrets.SUPABASE_URL }}+ SUPABASE_REPLICA_URL: ${{ secrets.SUPABASE_REPLICA_URL }}+ SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }}+ GCS_BUCKET_NAME: ${{ secrets.GCS_BUCKET_NAME }}+ GCS_CREDENTIALS: ${{ secrets.GCS_CREDENTIALS }}run: |- npm run test:snips+ TEST_API_URL=$(minikube service firecrawl-app --url) pnpm run test:snipsworking-directory: ./apps/api- name: Kill instancesif: always()@@ -124,9 +412,13 @@ jobs:- name: Copy log filesif: always()run: |- cp ./apps/api/api.log logs/ || true- cp ./apps/api/worker.log logs/ || true- cp ./apps/api/index-worker.log logs/ || true+ kubectl logs deployment/firecrawl-app > logs/api.log || true+ kubectl logs deployment/firecrawl-worker > logs/worker.log || true+ kubectl logs deployment/firecrawl-index-worker > logs/index-worker.log || true+ kubectl logs deployment/firecrawl-nuq-worker > logs/nuq-worker.log || true+ kubectl logs deployment/firecrawl-dragonfly > logs/dragonfly.log || true+ kubectl logs deployment/nuq-postgres > logs/postgres.log || true+ kubectl logs deployment/tailscale-proxy > logs/tailscale-proxy.log || true- name: Zip and encrypt logsif: always()run: |diff --git a/apps/api/Dockerfile b/apps/api/Dockerfileindex bf96a1615..5a2a9f62e 100644--- a/apps/api/Dockerfile+++ b/apps/api/Dockerfile@@ -70,8 +70,4 @@ COPY --from=build /app/native ./native# Copy Go shared libraryCOPY --from=go-build /app/sharedLibs/go-html-to-md/libhtml-to-markdown.so ./sharedLibs/go-html-to-md/-# Copy and prepare entrypoint-COPY docker-entrypoint.sh ./-RUN sed -i 's/\r$//' ./docker-entrypoint.sh && chmod +x ./docker-entrypoint.sh--ENTRYPOINT ["./docker-entrypoint.sh"]\ No newline at end of file+CMD ["node", "dist/src/harness.js", "--start-docker"]diff --git a/apps/api/docker-entrypoint.sh b/apps/api/docker-entrypoint.shdeleted file mode 100755index 6b09e7053..000000000--- a/apps/api/docker-entrypoint.sh+++ /dev/null@@ -1,24 +0,0 @@-#!/bin/bash -e--if [ "$UID" -eq 0 ]; then- set +e # disable failing on errror- ulimit -n 65535- echo "NEW ULIMIT: $(ulimit -n)"- set -e # enable failing on error-else- echo ENTRYPOINT DID NOT RUN AS ROOT-fi--if [ "$FLY_PROCESS_GROUP" = "app" ]; then- echo "RUNNING app"- node --max-old-space-size=8192 dist/src/index.js-elif [ "$FLY_PROCESS_GROUP" = "worker" ]; then- echo "RUNNING worker"- node --max-old-space-size=8192 dist/src/services/queue-worker.js-elif [ "$FLY_PROCESS_GROUP" = "index-worker" ]; then- echo "RUNNING index worker"- node --max-old-space-size=8192 dist/src/services/indexing/index-worker.js-else- echo "NO FLY PROCESS GROUP"- node --max-old-space-size=8192 dist/src/index.js-fidiff --git a/apps/api/package.json b/apps/api/package.jsonindex be164054e..1716cd45a 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -4,9 +4,12 @@"description": "","main": "index.js","scripts": {- "start": "tsc-watch --onSuccess \"node dist/src/index.js\"",- "start:production": "tsc && node dist/src/index.js",- "start:production:nobuild": "node dist/src/index.js",+ "start": "tsc-watch --onSuccess \"node dist/src/harness.js --start-built\"",+ "start:production": "tsx src/harness.ts --start",+ "server": "tsc-watch --onSuccess \"node dist/src/index.js\"",+ "server:production": "tsc && node dist/src/index.js",+ "server:production:nobuild": "node dist/src/index.js",+ "format": "prettier --write \"src/**/*.(js|ts)\"","flyio": "node dist/src/index.js","start:dev": "tsc-watch --onSuccess \"node dist/src/index.js\"","build": "tsc",@@ -19,6 +22,8 @@"harness": "tsx src/harness.ts","workers": "tsc-watch --onSuccess \"node dist/src/services/queue-worker.js\"","worker:production": "node dist/src/services/queue-worker.js",+ "nuq-worker": "tsc-watch --onSuccess \"node dist/src/services/worker/nuq-worker.js\"",+ "nuq-worker:production": "node dist/src/services/worker/nuq-worker.js","index-worker": "tsc-watch --onSuccess \"node dist/src/services/indexing/index-worker.js\"","index-worker:production": "node dist/src/services/indexing/index-worker.js","mongo-docker": "docker run -d -p 2717:27017 -v ./mongo-data:/data/db --name mongodb mongo:latest",@@ -43,6 +48,7 @@"@types/lodash": "^4.17.14","@types/node": "^20.14.1","@types/pdf-parse": "^1.1.4",+ "@types/pg": "^8.15.5","@types/supertest": "^6.0.2","@types/tough-cookie": "^4.0.5","husky": "^9.1.7",@@ -136,6 +142,7 @@"openai": "^5.12.0","parse-diff": "^0.11.1","pdf-parse": "^1.1.1",+ "pg": "^8.16.3","pos": "^0.4.2","posthog-node": "^4.0.1","prettier": "^3.6.2",@@ -172,6 +179,11 @@"pnpm": {"onlyBuiltDependencies": ["@sentry-internal/node-cpu-profiler",+ "bigint-buffer",+ "bufferutil",+ "keccak",+ "libpq",+ "utf-8-validate","supabase"],"overrides": {diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yamlindex b04f60409..4642bd0ce 100644--- a/apps/api/pnpm-lock.yaml+++ b/apps/api/pnpm-lock.yaml@@ -249,6 +249,9 @@ importers:pdf-parse:specifier: ^1.1.1version: 1.1.1+ pg:+ specifier: ^8.16.3+ version: 8.16.3([email protected])pos:specifier: ^0.4.2version: 0.4.2@@ -355,6 +358,9 @@ importers:'@types/pdf-parse':specifier: ^1.1.4version: 1.1.4+ '@types/pg':+ specifier: ^8.15.5+ version: 8.15.5'@types/supertest':specifier: ^6.0.2version: 6.0.2@@ -3476,6 +3482,9 @@ packages:'@types/[email protected]':resolution: {integrity: sha512-I6UNVBAoYbvuWkkU3oosC8yxqH21f4/Jc4DK71JLG3dT2mdlGe1z+ep/LQGXaKaOgcvUrsQoPRqfgtMcvZiJhg==}+ '@types/[email protected]':+ resolution: {integrity: sha512-LF7lF6zWEKxuT3/OR8wAZGzkg4ENGXFNyiV/JeOt9z5B+0ZVwbql9McqX5c/WStFq1GaGso7H1AzP/qSzmlCKQ==}+'@types/[email protected]':resolution: {integrity: sha512-1Kc4oAGzAl7uqUStZCDvaLFqZrW9qWSjXOmBfdgyBP5La7Us6Mg4GBvRlSoaZMhQF/zSj1C8CtKMBkoiT8eL8w==}@@ -5417,6 +5426,9 @@ packages:resolution: {integrity: sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==}engines: {node: '>=6'}+ resolution: {integrity: sha512-4lSWmly2Nsj3LaTxxtFmJWuP3Kx+0hYHEd+aNrcXEWT0nKWaPd9/QZPiMkkC680zeALFGHQdQWjBvnilL+vgWA==}+resolution: {integrity: sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==}@@ -5684,6 +5696,9 @@ packages:resolution: {integrity: sha512-WWdIxpyjEn+FhQJQQv9aQAYlHoNVdzIzUySNV1gHUPDSdZJ3yZn7pAAbQcV7B56Mvu881q9FZV+0Vx2xC44VWA==}engines: {node: ^18.17.0 || >=20.5.0}+ resolution: {integrity: sha512-DANghxFkS1plDdRsX0X9pm0Z6SJNN6gBdtXfanwoZ8hooC5gosGFSBGRYHUVPz1asKA/kMRqDRdHrluZ61SpBQ==}+resolution: {integrity: sha512-21t+ozMQDAL/UGgQVBbZ/xXvNO10++ZPuTmKRO8k9V3AClVRht49ahtDjfY8l1q6nSHOrE5ASfthzH3ol6R/hg==}engines: {node: '>=20.17'}@@ -5975,10 +5990,27 @@ packages:resolution: {integrity: sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==}+ resolution: {integrity: sha512-YgCtzMH0ptvZJslLM1ffsY4EuGaU0cx4XSdXLRFae8bPP4dS5xL1tNB3k2o/N64cHJpwU7dxKli/nZ2lUa5fLg==}++ resolution: {integrity: sha512-nkc6NpDcvPVpZXxrreI/FOtX3XemeLl8E0qFr6F2Lrm/I8WOnaWNhIPK2Z7OHpw7gh5XJThi6j6ppgNoaT1w4w==}+resolution: {integrity: sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==}engines: {node: '>=4.0.0'}+ resolution: {integrity: sha512-3oi+KVil86Vngo4H0IlhBaYSJWdcu8t2f1Y4TkQoQi5oZ9bNeYECGqW3oSGx69mjSZYHoC3h+3jYtqzRgndn5A==}++ resolution: {integrity: sha512-Tu8jMlcX+9d8+QVzKIvM/uJtp07PKr82IUOYEphaWcoBhIYkoHpLXN3qO59nAI11ripznDsEzEv8nUxBVWajGg==}+ peerDependencies:+ pg: '>=8.0'++ resolution: {integrity: sha512-6DIBgBQaTKDJyxnXaLiLR8wBpQQcGWuAESkRBX/t6OwA8YsqP+iVSiond2EDy6Y/dsGk8rh/jtax3js5NeV7JQ==}+resolution: {integrity: sha512-jPIlvgoD63hrEuihvIg+tJhoGjUsLPn6poJY9N5CnlPd91c2T18T/9zBtLxZSb1EhYxBRoZJtzScCaWlYLtktg==}@@ -5986,6 +6018,18 @@ packages:resolution: {integrity: sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==}engines: {node: '>=4'}+ resolution: {integrity: sha512-enxc1h0jA/aq5oSDMvqyW3q89ra6XIIDZgCX9vkMrnz5DFTw/Ny3Li2lFQ+pt3L6MCgm/5o2o8HW9hiJji+xvw==}+ engines: {node: '>= 16.0.0'}+ peerDependencies:+ pg-native: '>=3.0.1'+ peerDependenciesMeta:+ pg-native:+ optional: true++ resolution: {integrity: sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==}+resolution: {integrity: sha512-anP1Z8qwhkbmu7MFP5iTt+wQKXgwzf7zTyGlcdzabySa9vd0Xt392U0rVmz9poOaBj0uHJKyyo9/upk0HrEQew==}@@ -11585,7 +11629,7 @@ snapshots:'@types/[email protected]':dependencies:- '@types/pg': 8.6.1+ '@types/pg': 8.15.5'@types/[email protected]':dependencies:@@ -11593,6 +11637,12 @@ snapshots:pg-protocol: 1.6.1pg-types: 2.2.0+ '@types/[email protected]':+ dependencies:+ '@types/node': 20.14.1+ pg-protocol: 1.10.3+ pg-types: 2.2.0+'@types/[email protected]':dependencies:'@types/node': 20.14.1@@ -14356,6 +14406,12 @@ snapshots:+ dependencies:+ bindings: 1.5.0+ nan: 2.22.2+ optional: true+dependencies:immediate: 3.0.6@@ -14618,6 +14674,9 @@ snapshots:+ optional: true+@@ -14931,8 +14990,25 @@ snapshots:+ optional: true++ [email protected]: {}++ dependencies:+ libpq: 1.8.15+ pg-types: 2.2.0+ optional: true++ dependencies:+ pg: 8.16.3([email protected])++ [email protected]: {}+@@ -14943,6 +15019,21 @@ snapshots:postgres-date: 1.0.7postgres-interval: 1.2.0+ dependencies:+ pg-connection-string: 2.9.1+ pg-pool: 3.10.1([email protected]([email protected]))+ pg-protocol: 1.10.3+ pg-types: 2.2.0+ pgpass: 1.0.5+ optionalDependencies:+ pg-cloudflare: 1.2.7+ pg-native: 3.5.2++ dependencies:+ split2: 4.2.0+diff --git a/apps/api/requests.http b/apps/api/requests.httpindex 1507fd99a..5ffc11708 100644--- a/apps/api/requests.http+++ b/apps/api/requests.http@@ -1,6 +1,6 @@# Pick your baseUrl here:-# @baseUrl = http://localhost:3002-@baseUrl = https://api.firecrawl.dev+@baseUrl = http://localhost:3002+# @baseUrl = https://api.firecrawl.dev### Scrape Website# @name scrapediff --git a/apps/api/sharedLibs/go-html-to-md/.gitignore b/apps/api/sharedLibs/go-html-to-md/.gitignoreindex 9e40cf424..909db1866 100644--- a/apps/api/sharedLibs/go-html-to-md/.gitignore+++ b/apps/api/sharedLibs/go-html-to-md/.gitignore@@ -1,2 +1,3 @@html-to-markdown.*-!html-to-markdown.go\ No newline at end of file+!html-to-markdown.go+libhtml-to-markdown.*\ No newline at end of filediff --git a/apps/api/src/__tests__/queue-concurrency-integration.test.ts b/apps/api/src/__tests__/queue-concurrency-integration.test.tsdeleted file mode 100644index 7e2141d07..000000000--- a/apps/api/src/__tests__/queue-concurrency-integration.test.ts+++ /dev/null@@ -1,269 +0,0 @@-import { redisEvictConnection } from "../services/redis";-import { addScrapeJob, addScrapeJobs } from "../services/queue-jobs";-import {- cleanOldConcurrencyLimitEntries,- pushConcurrencyLimitActiveJob,- takeConcurrencyLimitedJob,- removeConcurrencyLimitActiveJob,-} from "../lib/concurrency-limit";-import { WebScraperOptions } from "../types";-import { getACUCTeam } from "../controllers/auth";--// Mock all the dependencies-const mockAdd = jest.fn();-jest.mock("../services/queue-service", () => ({- redisConnection: {- zremrangebyscore: jest.fn(),- zrangebyscore: jest.fn(),- zadd: jest.fn(),- zrem: jest.fn(),- zmpop: jest.fn(),- zcard: jest.fn(),- smembers: jest.fn(),- },- getScrapeQueue: jest.fn(() => ({- add: mockAdd,- })),-}));--jest.mock("uuid", () => ({- v4: jest.fn(() => "mock-uuid"),-}));--describe("Queue Concurrency Integration", () => {- const mockTeamId = "test-team-id";- const mockNow = Date.now();-- const defaultScrapeOptions = {- formats: [{ type: "markdown" }],- onlyMainContent: true,- waitFor: 0,- mobile: false,- parsePDF: false,- timeout: 30000,- javascript: true,- headers: {},- cookies: [],- blockResources: true,- skipTlsVerification: false,- removeBase64Images: true,- fastMode: false,- blockAds: true,- maxAge: 0,- storeInCache: true,- proxy: "basic",- };-- beforeEach(() => {- jest.clearAllMocks();- jest.spyOn(Date, "now").mockImplementation(() => mockNow);- });-- describe("Single Job Addition", () => {- const mockWebScraperOptions: WebScraperOptions = {- url: "https://test.com",- mode: "single_urls",- team_id: mockTeamId,- scrapeOptions: defaultScrapeOptions,- crawlerOptions: null,- zeroDataRetention: false,- apiKeyId: null,- } as WebScraperOptions;-- it("should add job directly to BullMQ when under concurrency limit", async () => {- // Mock current active jobs to be under limit- (redisEvictConnection.zrangebyscore as jest.Mock).mockResolvedValue([]);-- await addScrapeJob(mockWebScraperOptions);-- // Should have checked concurrency- expect(redisEvictConnection.zrangebyscore).toHaveBeenCalled();-- // Should have added to BullMQ- expect(mockAdd).toHaveBeenCalled();-- // Should have added to active jobs- expect(redisEvictConnection.zadd).toHaveBeenCalledWith(- expect.stringContaining("concurrency-limiter"),- expect.any(Number),- expect.any(String),- );- });-- it("should add job to concurrency queue when at concurrency limit", async () => {- // Mock current active jobs to be at limit- (getACUCTeam as jest.Mock).mockResolvedValue({- concurrency: 15,- } as any);- const activeJobs = Array(15).fill("active-job");- (redisEvictConnection.zrangebyscore as jest.Mock).mockResolvedValue(- activeJobs,- );-- await addScrapeJob(mockWebScraperOptions);-- // Should have checked concurrency- expect(redisEvictConnection.zrangebyscore).toHaveBeenCalled();-- // Should NOT have added to BullMQ- expect(mockAdd).not.toHaveBeenCalled();-- // Should have added to concurrency queue- expect(redisEvictConnection.zadd).toHaveBeenCalledWith(- expect.stringContaining("concurrency-limit-queue"),- expect.any(Number),- expect.stringContaining("mock-uuid"),- );- });- });-- describe("Batch Job Addition", () => {- const createMockJobs = (count: number) =>- Array(count)- .fill(null)- .map((_, i) => ({- data: {- url: `https://test${i}.com`,- mode: "single_urls",- team_id: mockTeamId,- scrapeOptions: defaultScrapeOptions,- zeroDataRetention: false,- } as any,- opts: {- jobId: `job-${i}`,- priority: 1,- },- }));-- it("should handle batch jobs respecting concurrency limits", async () => {- const maxConcurrency = 15;- (getACUCTeam as jest.Mock).mockResolvedValue({- concurrency: maxConcurrency,- } as any);- const totalJobs = maxConcurrency + 5; // Some jobs should go to queue- const mockJobs = createMockJobs(totalJobs);-- // Mock current active jobs to be empty- (redisEvictConnection.zrangebyscore as jest.Mock).mockResolvedValue([]);-- await addScrapeJobs(mockJobs);-- // Should have added maxConcurrency jobs to BullMQ- expect(mockAdd).toHaveBeenCalledTimes(maxConcurrency);-- // Should have added remaining jobs to concurrency queue- expect(redisEvictConnection.zadd).toHaveBeenCalledWith(- expect.stringContaining("concurrency-limit-queue"),- expect.any(Number),- expect.any(String),- );- });-- it("should handle empty job array", async () => {- const result = await addScrapeJobs([]);- expect(result).toBe(true);- expect(mockAdd).not.toHaveBeenCalled();- expect(redisEvictConnection.zadd).not.toHaveBeenCalled();- });- });-- describe("Queue Worker Integration", () => {- it("should process next queued job when active job completes", async () => {- const mockJob = {- id: "test-job",- data: {- team_id: mockTeamId,- zeroDataRetention: false,- },- };-- // Mock a queued job- const queuedJob = {- id: "queued-job",- data: { test: "data" },- opts: {},- };- (redisEvictConnection.zmpop as jest.Mock).mockResolvedValueOnce([- "key",- [[JSON.stringify(queuedJob)]],- ]);-- // Simulate job completion in worker- await removeConcurrencyLimitActiveJob(mockTeamId, mockJob.id);- await cleanOldConcurrencyLimitEntries(mockTeamId);-- const nextJob = await takeConcurrencyLimitedJob(mockTeamId);-- // Should have taken next job from queue- expect(nextJob).toEqual(queuedJob);-- // Should have added new job to active jobs- await pushConcurrencyLimitActiveJob(- mockTeamId,- nextJob!.id,- 2 * 60 * 1000,- );- expect(redisEvictConnection.zadd).toHaveBeenCalledWith(- expect.stringContaining("concurrency-limiter"),- expect.any(Number),- nextJob!.id,- );- });-- it("should handle job failure and cleanup", async () => {- const mockJob = {- id: "failing-job",- data: {- team_id: mockTeamId,- },- };-- // Add job to active jobs- await pushConcurrencyLimitActiveJob(- mockTeamId,- mockJob.id,- 2 * 60 * 1000,- );-- // Simulate job failure and cleanup- await removeConcurrencyLimitActiveJob(mockTeamId, mockJob.id);- await cleanOldConcurrencyLimitEntries(mockTeamId);-- // Verify job was removed from active jobs- expect(redisEvictConnection.zrem).toHaveBeenCalledWith(- expect.stringContaining("concurrency-limiter"),- mockJob.id,- );- });- });-- describe("Edge Cases", () => {- it("should handle stalled jobs cleanup", async () => {- const stalledTime = mockNow - 3 * 60 * 1000; // 3 minutes ago-- // Mock stalled jobs in Redis- (redisEvictConnection.zrangebyscore as jest.Mock).mockResolvedValueOnce([- "stalled-job",- ]);-- await cleanOldConcurrencyLimitEntries(mockTeamId, mockNow);-- // Should have cleaned up stalled jobs- expect(redisEvictConnection.zremrangebyscore).toHaveBeenCalledWith(- expect.stringContaining("concurrency-limiter"),- -Infinity,- mockNow,- );- });-- it("should handle race conditions in job queue processing", async () => {- // Mock a race condition where job is taken by another worker- (redisEvictConnection.zmpop as jest.Mock).mockResolvedValueOnce(null);-- const nextJob = await takeConcurrencyLimitedJob(mockTeamId);-- // Should handle gracefully when no job is available- expect(nextJob).toBeNull();- });- });-});diff --git a/apps/api/src/__tests__/snips/lib.ts b/apps/api/src/__tests__/snips/lib.tsindex 43ef79ed7..ba9fec277 100644--- a/apps/api/src/__tests__/snips/lib.ts+++ b/apps/api/src/__tests__/snips/lib.ts@@ -7,7 +7,7 @@ import { TeamFlags } from "../../controllers/v1/types";// Configuration// =========================================-export const TEST_URL = "http://127.0.0.1:3002";+export const TEST_URL = process.env.TEST_API_URL || "http://127.0.0.1:3002";// Due to the limited resources of the CI runner, we need to set a longer timeout for the many many scrape testsexport const scrapeTimeout = 90000;diff --git a/apps/api/src/__tests__/snips/v1/crawl.test.ts b/apps/api/src/__tests__/snips/v1/crawl.test.tsindex bb033d5cf..ce0550c66 100644--- a/apps/api/src/__tests__/snips/v1/crawl.test.ts+++ b/apps/api/src/__tests__/snips/v1/crawl.test.ts@@ -387,110 +387,3 @@ describe("Crawl tests", () => {},);});--describe("Robots.txt FFI Integration tests", () => {- it.concurrent(- "handles normal robots.txt parsing via FFI",- async () => {- const result = await filterLinks({- links: [- "https://example.com/allowed",- "https://example.com/disallowed",- ],- limit: 10,- maxDepth: 10,- baseUrl: "https://example.com",- initialUrl: "https://example.com",- regexOnFullUrl: false,- excludes: [],- includes: [],- allowBackwardCrawling: true,- ignoreRobotsTxt: false,- robotsTxt: "User-agent: *\nDisallow: /disallowed",- });-- expect(result.links).toHaveLength(1);- expect(result.links[0]).toBe("https://example.com/allowed");- expect("https://example.com/disallowed" in result.denialReasons).toBe(- true,- );- expect(result.denialReasons["https://example.com/disallowed"]).toBe(- "ROBOTS_TXT",- );- },- 10000,- );-- it.concurrent(- "handles malformed robots.txt without crashing via FFI",- async () => {- const result = await filterLinks({- links: ["https://example.com/test"],- limit: 10,- maxDepth: 10,- baseUrl: "https://example.com",- initialUrl: "https://example.com",- regexOnFullUrl: false,- excludes: [],- includes: [],- allowBackwardCrawling: true,- ignoreRobotsTxt: false,- robotsTxt:- "Invalid robots.txt content with \x00 null bytes and malformed syntax",- });-- expect(result.links).toHaveLength(1);- expect(result.links[0]).toBe("https://example.com/test");- },- 10000,- );-- it.concurrent(- "handles non-UTF8 robots.txt content without crashing via FFI",- async () => {- const nonUtf8Content =- String.fromCharCode(0xff, 0xfe) + "User-agent: *\nDisallow: /blocked";- const result = await filterLinks({- links: ["https://example.com/allowed"],- limit: 10,- maxDepth: 10,- baseUrl: "https://example.com",- initialUrl: "https://example.com",- regexOnFullUrl: false,- excludes: [],- includes: [],- allowBackwardCrawling: true,- ignoreRobotsTxt: false,- robotsTxt: nonUtf8Content,- });-- expect(result.links).toHaveLength(1);- expect(result.links[0]).toBe("https://example.com/allowed");- },- 10000,- );-- it.concurrent(- "handles char boundary issues without crashing via FFI",- async () => {- const problematicContent = "User-agent: *\nDisallow: /\u{a0}test";- const result = await filterLinks({- links: ["https://example.com/safe"],- limit: 10,- maxDepth: 10,- baseUrl: "https://example.com",- initialUrl: "https://example.com",- regexOnFullUrl: false,- excludes: [],- includes: [],- allowBackwardCrawling: true,- ignoreRobotsTxt: false,- robotsTxt: problematicContent,- });-- expect(result.links).toHaveLength(1);- expect(result.links[0]).toBe("https://example.com/safe");- },- 10000,- );-});diff --git a/apps/api/src/__tests__/snips/v1/zdr.test.ts b/apps/api/src/__tests__/snips/v1/zdr.test.tsindex b23c49a02..27d7b81db 100644--- a/apps/api/src/__tests__/snips/v1/zdr.test.ts+++ b/apps/api/src/__tests__/snips/v1/zdr.test.ts@@ -1,3 +1,4 @@+import { exec, spawn } from "node:child_process";import { getJobFromGCS } from "../../../lib/gcs-jobs";import { supabase_service } from "../../../services/supabase";import {@@ -20,6 +21,9 @@ const logIgnoreList = ["Billing batch processing","Processing batch of","Billing team",+ "No jobs to process",+ "nuqHealthCheck metrics",+ "nuqGetJobToProcess metrics",];if (process.env.TEST_SUITE_SELF_HOSTED) {@@ -27,7 +31,35 @@ if (process.env.TEST_SUITE_SELF_HOSTED) {expect(true).toBe(true);});} else {+ function getOutput(command: string): Promise<string> {+ return new Promise((resolve, reject) => {+ try {+ const cmd = spawn(command, { shell: true });+ let out = "";+ cmd.stdout?.on("data", data => {+ out += data;+ });+ cmd.stderr?.on("data", data => {+ out += data;+ });+ cmd.on("error", e => {+ reject(e);+ });+ cmd.on("close", code => {+ if (code !== 0) {+ reject(new Error(`Command ${command} failed with code ${code}`));+ } else {+ resolve(out);+ }+ });+ } catch (e) {+ reject(e);+ }+ });+ }+async function getServerLogs() {+ let logs: string;if (!process.env.GITHUB_ACTIONS) {try {await stat("api.log");@@ -35,8 +67,10 @@ if (process.env.TEST_SUITE_SELF_HOSTED) {console.warn("No api.log file found");return [];}+ logs = await readFile("api.log", "utf8");+ } else {+ logs = await getOutput("kubectl logs deployment/firecrawl-app");}- const logs = await readFile("api.log", "utf8");return logs.split("\n").filter(@@ -45,6 +79,7 @@ if (process.env.TEST_SUITE_SELF_HOSTED) {}async function getWorkerLogs() {+ let logs: string;if (!process.env.GITHUB_ACTIONS) {try {await stat("worker.log");@@ -52,8 +87,10 @@ if (process.env.TEST_SUITE_SELF_HOSTED) {console.warn("No worker.log file found");return [];}+ logs = await readFile("worker.log", "utf8");+ } else {+ logs = await getOutput("kubectl logs deployment/firecrawl-nuq-worker");}- const logs = await readFile("worker.log", "utf8");return logs.split("\n").filter(diff --git a/apps/api/src/__tests__/snips/v2/crawl.test.ts b/apps/api/src/__tests__/snips/v2/crawl.test.tsindex 70c077abc..d81e7a74e 100644--- a/apps/api/src/__tests__/snips/v2/crawl.test.ts+++ b/apps/api/src/__tests__/snips/v2/crawl.test.ts@@ -389,110 +389,3 @@ describe("Crawl tests", () => {});}});--describe("Robots.txt FFI Integration tests", () => {- it.concurrent(- "handles normal robots.txt parsing via FFI",- async () => {- const result = await filterLinks({- links: [- "https://example.com/allowed",- "https://example.com/disallowed",- ],- limit: 10,- maxDepth: 10,- baseUrl: "https://example.com",- initialUrl: "https://example.com",- regexOnFullUrl: false,- excludes: [],- includes: [],- allowBackwardCrawling: true,- ignoreRobotsTxt: false,- robotsTxt: "User-agent: *\nDisallow: /disallowed",- });-- expect(result.links).toHaveLength(1);- expect(result.links[0]).toBe("https://example.com/allowed");- expect("https://example.com/disallowed" in result.denialReasons).toBe(- true,- );- expect(result.denialReasons["https://example.com/disallowed"]).toBe(- "ROBOTS_TXT",- );- },- 10000,- );-- it.concurrent(- "handles malformed robots.txt without crashing via FFI",- async () => {- const result = await filterLinks({- links: ["https://example.com/test"],- limit: 10,- maxDepth: 10,- baseUrl: "https://example.com",- initialUrl: "https://example.com",- regexOnFullUrl: false,- excludes: [],- includes: [],- allowBackwardCrawling: true,- ignoreRobotsTxt: false,- robotsTxt:- "Invalid robots.txt content with \x00 null bytes and malformed syntax",- });-- expect(result.links).toHaveLength(1);- expect(result.links[0]).toBe("https://example.com/test");- },- 10000,- );-- it.concurrent(- "handles non-UTF8 robots.txt content without crashing via FFI",- async () => {- const nonUtf8Content =- String.fromCharCode(0xff, 0xfe) + "User-agent: *\nDisallow: /blocked";- const result = await filterLinks({- links: ["https://example.com/allowed"],- limit: 10,- maxDepth: 10,- baseUrl: "https://example.com",- initialUrl: "https://example.com",- regexOnFullUrl: false,- excludes: [],- includes: [],- allowBackwardCrawling: true,- ignoreRobotsTxt: false,- robotsTxt: nonUtf8Content,- });-- expect(result.links).toHaveLength(1);- expect(result.links[0]).toBe("https://example.com/allowed");- },- 10000,- );-- it.concurrent(- "handles char boundary issues without crashing via FFI",- async () => {- const problematicContent = "User-agent: *\nDisallow: /\u{a0}test";- const result = await filterLinks({- links: ["https://example.com/safe"],- limit: 10,- maxDepth: 10,- baseUrl: "https://example.com",- initialUrl: "https://example.com",- regexOnFullUrl: false,- excludes: [],- includes: [],- allowBackwardCrawling: true,- ignoreRobotsTxt: false,- robotsTxt: problematicContent,- });-- expect(result.links).toHaveLength(1);- expect(result.links[0]).toBe("https://example.com/safe");- },- 10000,- );-});diff --git a/apps/api/src/__tests__/snips/v2/zdr.test.ts b/apps/api/src/__tests__/snips/v2/zdr.test.tsindex 889b14e76..61937aa7c 100644--- a/apps/api/src/__tests__/snips/v2/zdr.test.ts+++ b/apps/api/src/__tests__/snips/v2/zdr.test.ts@@ -9,6 +9,7 @@ import {idmux,} from "./lib";import { readFile, stat } from "node:fs/promises";+import { spawn } from "node:child_process";const logIgnoreList = ["Billing queue created",@@ -20,6 +21,9 @@ const logIgnoreList = ["Billing batch processing","Processing batch of","Billing team",+ "No jobs to process",+ "nuqHealthCheck metrics",+ "nuqGetJobToProcess metrics",];if (process.env.TEST_SUITE_SELF_HOSTED) {@@ -27,7 +31,35 @@ if (process.env.TEST_SUITE_SELF_HOSTED) {expect(true).toBe(true);});} else {+ function getOutput(command: string): Promise<string> {+ return new Promise((resolve, reject) => {+ try {+ const cmd = spawn(command, { shell: true });+ let out = "";+ cmd.stdout?.on("data", data => {+ out += data;+ });+ cmd.stderr?.on("data", data => {+ out += data;+ });+ cmd.on("error", e => {+ reject(e);+ });+ cmd.on("close", code => {+ if (code !== 0) {+ reject(new Error(`Command ${command} failed with code ${code}`));+ } else {+ resolve(out);+ }+ });+ } catch (e) {+ reject(e);+ }+ });+ }+async function getServerLogs() {+ let logs: string;if (!process.env.GITHUB_ACTIONS) {try {await stat("api.log");@@ -35,8 +67,10 @@ if (process.env.TEST_SUITE_SELF_HOSTED) {console.warn("No api.log file found");return [];}+ logs = await readFile("api.log", "utf8");+ } else {+ logs = await getOutput("kubectl logs deployment/firecrawl-app");}- const logs = await readFile("api.log", "utf8");return logs.split("\n").filter(@@ -45,6 +79,7 @@ if (process.env.TEST_SUITE_SELF_HOSTED) {}async function getWorkerLogs() {+ let logs: string;if (!process.env.GITHUB_ACTIONS) {try {await stat("worker.log");@@ -52,8 +87,10 @@ if (process.env.TEST_SUITE_SELF_HOSTED) {console.warn("No worker.log file found");return [];}+ logs = await readFile("worker.log", "utf8");+ } else {+ logs = await getOutput("kubectl logs deployment/firecrawl-nuq-worker");}- const logs = await readFile("worker.log", "utf8");return logs.split("\n").filter(diff --git a/apps/api/src/controllers/v0/admin/metrics.ts b/apps/api/src/controllers/v0/admin/metrics.tsindex afe0c055b..54b12cdc9 100644--- a/apps/api/src/controllers/v0/admin/metrics.ts+++ b/apps/api/src/controllers/v0/admin/metrics.ts@@ -1,5 +1,6 @@import type { Request, Response } from "express";import { redisEvictConnection } from "../../../services/redis";+import { nuqGetLocalMetrics, scrapeQueue } from "../../../services/worker/nuq";export async function metricsController(_: Request, res: Response) {let cursor: string = "0";@@ -34,5 +35,9 @@ ${Object.entries(metrics)`concurrency_limit_queue_job_count{team_id="${key}"} ${value}`,).join("\n")}++${await scrapeQueue.getMetrics()}++${nuqGetLocalMetrics()}`);}diff --git a/apps/api/src/controllers/v0/admin/queue.ts b/apps/api/src/controllers/v0/admin/queue.tsdeleted file mode 100644index c5a87cb61..000000000--- a/apps/api/src/controllers/v0/admin/queue.ts+++ /dev/null@@ -1,201 +0,0 @@-import { Request, Response } from "express";--import { Job } from "bullmq";-import { logger } from "../../../lib/logger";-import { getScrapeQueue } from "../../../services/queue-service";-import { checkAlerts } from "../../../services/alerts";-import { sendSlackWebhook } from "../../../services/alerts/slack";--export async function cleanBefore24hCompleteJobsController(- req: Request,- res: Response,-) {- logger.info("🐂 Cleaning jobs older than 24h");- try {- const scrapeQueue = getScrapeQueue();- const batchSize = 10;- const numberOfBatches = 9; // Adjust based on your needs- const completedJobsPromises: Promise<Job[]>[] = [];- for (let i = 0; i < numberOfBatches; i++) {- completedJobsPromises.push(- scrapeQueue.getJobs(- ["completed"],- i * batchSize,- i * batchSize + batchSize,- true,- ),- );- }- const completedJobs: Job[] = (- await Promise.all(completedJobsPromises)- ).flat();- const before24hJobs =- completedJobs.filter(- job =>- job.finishedOn !== undefined &&- job.finishedOn < Date.now() - 24 * 60 * 60 * 1000,- ) || [];-- let count = 0;-- if (!before24hJobs) {- return res.status(200).send(`No jobs to remove.`);- }-- for (const job of before24hJobs) {- try {- await job.remove();- count++;- } catch (jobError) {- logger.error(`🐂 Failed to remove job with ID ${job.id}: ${jobError}`);- }- }- return res.status(200).send(`Removed ${count} completed jobs.`);- } catch (error) {- logger.error(`🐂 Failed to clean last 24h complete jobs: ${error}`);- return res.status(500).send("Failed to clean jobs");- }-}--export async function checkQueuesController(req: Request, res: Response) {- try {- await checkAlerts();- return res.status(200).send("Alerts initialized");- } catch (error) {- logger.debug(`Failed to initialize alerts: ${error}`);- return res.status(500).send("Failed to initialize alerts");- }-}--// Use this as a "health check" that way we dont destroy the server-export async function queuesController(req: Request, res: Response) {- try {- const scrapeQueue = getScrapeQueue();-- const [webScraperActive] = await Promise.all([- scrapeQueue.getActiveCount(),- ]);-- const noActiveJobs = webScraperActive === 0;- // 200 if no active jobs, 503 if there are active jobs- return res.status(noActiveJobs ? 200 : 500).json({- webScraperActive,- noActiveJobs,- });- } catch (error) {- logger.error(error);- return res.status(500).json({ error: error.message });- }-}--export async function autoscalerController(req: Request, res: Response) {- try {- const maxNumberOfMachines = 80;- const minNumberOfMachines = 20;-- const scrapeQueue = getScrapeQueue();-- const [webScraperActive, webScraperWaiting, webScraperPriority] =- await Promise.all([- scrapeQueue.getActiveCount(),- scrapeQueue.getWaitingCount(),- scrapeQueue.getPrioritizedCount(),- ]);-- let waitingAndPriorityCount = webScraperWaiting + webScraperPriority;-- // get number of machines active- const request = await fetch(- "https://api.machines.dev/v1/apps/firecrawl-scraper-js/machines",- {- headers: {- Authorization: `Bearer ${process.env.FLY_API_TOKEN}`,- },- },- );- const machines = await request.json();-- // Only worker machines- const activeMachines = machines.filter(- machine =>- (machine.state === "started" ||- machine.state === "starting" ||- machine.state === "replacing") &&- machine.config.env["FLY_PROCESS_GROUP"] === "worker",- ).length;-- let targetMachineCount = activeMachines;-- const baseScaleUp = 10;- // Slow scale down- const baseScaleDown = 2;-- // Scale up logic- if (webScraperActive > 9000 || waitingAndPriorityCount > 2000) {- targetMachineCount = Math.min(- maxNumberOfMachines,- activeMachines + baseScaleUp * 3,- );- } else if (webScraperActive > 5000 || waitingAndPriorityCount > 1000) {- targetMachineCount = Math.min(- maxNumberOfMachines,- activeMachines + baseScaleUp * 2,- );- } else if (webScraperActive > 1000 || waitingAndPriorityCount > 500) {- targetMachineCount = Math.min(- maxNumberOfMachines,- activeMachines + baseScaleUp,- );- }-- // Scale down logic- if (webScraperActive < 100 && waitingAndPriorityCount < 50) {- targetMachineCount = Math.max(- minNumberOfMachines,- activeMachines - baseScaleDown * 3,- );- } else if (webScraperActive < 500 && waitingAndPriorityCount < 200) {- targetMachineCount = Math.max(- minNumberOfMachines,- activeMachines - baseScaleDown * 2,- );- } else if (webScraperActive < 1000 && waitingAndPriorityCount < 500) {- targetMachineCount = Math.max(- minNumberOfMachines,- activeMachines - baseScaleDown,- );- }-- if (targetMachineCount !== activeMachines) {- logger.info(- `🐂 Scaling from ${activeMachines} to ${targetMachineCount} - ${webScraperActive} active, ${webScraperWaiting} waiting`,- );-- if (targetMachineCount > activeMachines) {- sendSlackWebhook(- `🐂 Scaling from ${activeMachines} to ${targetMachineCount} - ${webScraperActive} active, ${webScraperWaiting} waiting - Current DateTime: ${new Date().toISOString()}`,- false,- process.env.SLACK_AUTOSCALER ?? "",- );- } else {- sendSlackWebhook(- `🐂 Scaling from ${activeMachines} to ${targetMachineCount} - ${webScraperActive} active, ${webScraperWaiting} waiting - Current DateTime: ${new Date().toISOString()}`,- false,- process.env.SLACK_AUTOSCALER ?? "",- );- }- return res.status(200).json({- mode: "scale-descale",- count: targetMachineCount,- });- }-- return res.status(200).json({- mode: "normal",- count: activeMachines,- });- } catch (error) {- logger.error(error);- return res.status(500).send("Failed to initialize autoscaler");- }-}diff --git a/apps/api/src/controllers/v0/crawl-status.ts b/apps/api/src/controllers/v0/crawl-status.tsindex d2fa6f30f..446e20f0c 100644--- a/apps/api/src/controllers/v0/crawl-status.ts+++ b/apps/api/src/controllers/v0/crawl-status.ts@@ -1,27 +1,24 @@import { Request, Response } from "express";import { authenticateUser } from "../auth";import { RateLimiterMode } from "../../../src/types";-import { getScrapeQueue } from "../../../src/services/queue-service";import { redisEvictConnection } from "../../../src/services/redis";import { logger } from "../../../src/lib/logger";import { getCrawl, getCrawlJobs } from "../../../src/lib/crawl-redis";import { supabaseGetJobsByCrawlId } from "../../../src/lib/supabase-jobs";import * as Sentry from "@sentry/node";import { configDotenv } from "dotenv";-import { Job } from "bullmq";import { toLegacyDocument } from "../v1/types";import type { DBJob, PseudoJob } from "../v1/crawl-status";import { getJobFromGCS } from "../../lib/gcs-jobs";+import { scrapeQueue, NuQJob } from "../../services/worker/nuq";configDotenv();export async function getJobs(crawlId: string,ids: string[],): Promise<PseudoJob<any>[]> {- const [bullJobs, dbJobs, gcsJobs] = await Promise.all([- Promise.all(ids.map(x => getScrapeQueue().getJob(x))).then(x =>- x.filter(x => x),- ) as Promise<(Job<any, any, string> & { id: string })[]>,+ const [nuqJobs, dbJobs, gcsJobs] = await Promise.all([+ scrapeQueue.getJobs(ids),process.env.USE_DB_AUTHENTICATION === "true"? await supabaseGetJobsByCrawlId(crawlId): [],@@ -34,12 +31,12 @@ export async function getJobs(: [],]);- const bullJobMap = new Map<string, PseudoJob<any>>();+ const nuqJobMap = new Map<string, NuQJob<any, any>>();const dbJobMap = new Map<string, DBJob>();const gcsJobMap = new Map<string, any>();- for (const job of bullJobs) {- bullJobMap.set(job.id, job);+ for (const job of nuqJobs) {+ nuqJobMap.set(job.id, job);}for (const job of dbJobs) {@@ -53,13 +50,13 @@ export async function getJobs(const jobs: PseudoJob<any>[] = [];for (const id of ids) {- const bullJob = bullJobMap.get(id);+ const nuqJob = nuqJobMap.get(id);const dbJob = dbJobMap.get(id);const gcsJob = gcsJobMap.get(id);- if (!bullJob && !dbJob) continue;+ if (!nuqJob && !dbJob) continue;- const data = gcsJob ?? dbJob?.docs ?? bullJob?.returnvalue;+ const data = gcsJob ?? dbJob?.docs ?? nuqJob?.returnvalue;if (gcsJob === null && data) {logger.warn("GCS Job not found", {jobId: id,@@ -68,20 +65,16 @@ export async function getJobs(const job: PseudoJob<any> = {id,- getState: dbJob- ? () => (dbJob.success ? "completed" : "failed")- : () => bullJob!.getState(),+ status: dbJob ? (dbJob.success ? "completed" : "failed") : nuqJob!.status,returnvalue: Array.isArray(data) ? data[0] : data,data: {- scrapeOptions: bullJob- ? bullJob.data.scrapeOptions- : dbJob!.page_options,+ scrapeOptions: nuqJob ? nuqJob.data.scrapeOptions : dbJob!.page_options,},- timestamp: bullJob- ? bullJob.timestamp+ timestamp: nuqJob+ ? nuqJob.createdAt.valueOf(): new Date(dbJob!.date_added).valueOf(),failedReason:- (bullJob ? bullJob.failedReason : dbJob!.message) || undefined,+ (nuqJob ? nuqJob.failedReason : dbJob!.message) || undefined,};jobs.push(job);@@ -135,7 +128,7 @@ export async function crawlStatusController(req: Request, res: Response) {}let jobIDs = await getCrawlJobs(req.params.jobId);let jobs = await getJobs(req.params.jobId, jobIDs);- let jobStatuses = await Promise.all(jobs.map(x => x.getState()));+ let jobStatuses = jobs.map(x => x.status);// Combine jobs and jobStatuses into a single array of objectslet jobsWithStatuses = jobs.map((job, index) => ({@@ -144,9 +137,7 @@ export async function crawlStatusController(req: Request, res: Response) {}));// Filter out failed jobs- jobsWithStatuses = jobsWithStatuses.filter(- x => x.status !== "failed" && x.status !== "unknown",- );+ jobsWithStatuses = jobsWithStatuses.filter(x => x.status !== "failed");// Sort jobs by timestampjobsWithStatuses.sort((a, b) => a.job.timestamp - b.job.timestamp);diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.tsindex 8cdcde057..287cd5a3e 100644--- a/apps/api/src/controllers/v0/crawl.ts+++ b/apps/api/src/controllers/v0/crawl.ts@@ -27,7 +27,7 @@ import { redisEvictConnection } from "../../../src/services/redis";import { checkAndUpdateURL } from "../../../src/lib/validateUrl";import * as Sentry from "@sentry/node";import { getJobPriority } from "../../lib/job-priority";-import { fromLegacyScrapeOptions, url as urlSchema } from "../v1/types";+import { url as urlSchema } from "../v1/types";import { ZodError } from "zod";import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";import { fromV0ScrapeOptions } from "../v2/types";@@ -216,7 +216,7 @@ export async function crawlController(req: Request, res: Response) {const jobs = urls.map(url => {const uuid = uuidv4();return {- name: uuid,+ jobId: uuid,data: {url,mode: "single_urls" as const,@@ -231,10 +231,7 @@ export async function crawlController(req: Request, res: Response) {zeroDataRetention: false, // not supported on v0apiKeyId: chunk?.api_key_id ?? null,},- opts: {- jobId: uuid,- priority: jobPriority,- },+ priority: jobPriority,};});@@ -246,12 +243,12 @@ export async function crawlController(req: Request, res: Response) {);await addCrawlJobs(id,- jobs.map(x => x.opts.jobId),+ jobs.map(x => x.jobId),logger,);for (const job of jobs) {// add with sentry instrumentation- await addScrapeJob(job.data, {}, job.opts.jobId);+ await addScrapeJob(job.data, job.jobId, job.priority);}});@@ -276,10 +273,8 @@ export async function crawlController(req: Request, res: Response) {zeroDataRetention: false, // not supported on v0apiKeyId: chunk?.api_key_id ?? null,},- {- priority: 15, // prioritize request 0 of crawl jobs same as scrape jobs- },jobId,+ await getJobPriority({ team_id, basePriority: 15 }),);await addCrawlJob(id, jobId, logger);}diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.tsindex be901c96b..3d48d832b 100644--- a/apps/api/src/controllers/v0/scrape.ts+++ b/apps/api/src/controllers/v0/scrape.ts@@ -12,7 +12,6 @@ import {defaultOrigin,} from "../../lib/default-values";import { addScrapeJob, waitForJob } from "../../services/queue-jobs";-import { getScrapeQueue } from "../../services/queue-service";import { redisEvictConnection } from "../../../src/services/redis";import { v4 as uuidv4 } from "uuid";import { logger } from "../../lib/logger";@@ -23,6 +22,7 @@ import { Document as V0Document } from "./../../lib/entities";import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";import { fromV0Combo } from "../v2/types";import { ScrapeJobTimeoutError } from "../../lib/error";+import { scrapeQueue } from "../../services/worker/nuq";async function scrapeHelper(jobId: string,@@ -53,8 +53,6 @@ async function scrapeHelper(};}- const jobPriority = await getJobPriority({ team_id, basePriority: 10 });-const { scrapeOptions, internalOptions } = fromV0Combo(pageOptions,extractorOptions,@@ -81,15 +79,14 @@ async function scrapeHelper(zeroDataRetention: false, // not supported on v0apiKeyId,},- {},jobId,- jobPriority,+ await getJobPriority({ team_id, basePriority: 10 }),);let doc;try {- doc = await waitForJob(jobId, timeout);+ doc = await waitForJob(jobId, timeout, false);} catch (e) {if (e instanceof ScrapeJobTimeoutError) {return {@@ -120,7 +117,7 @@ async function scrapeHelper(return err;}- await getScrapeQueue().remove(jobId);+ await scrapeQueue.removeJob(jobId);if (!doc) {console.error("!!! PANIC DOC IS", doc);diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.tsindex 086742a53..065c54c54 100644--- a/apps/api/src/controllers/v0/search.ts+++ b/apps/api/src/controllers/v0/search.ts@@ -11,22 +11,18 @@ import { search } from "../../search";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";import { v4 as uuidv4 } from "uuid";import { logger } from "../../lib/logger";-import { getScrapeQueue } from "../../services/queue-service";import { redisEvictConnection } from "../../../src/services/redis";import { addScrapeJob, waitForJob } from "../../services/queue-jobs";import * as Sentry from "@sentry/node";import { getJobPriority } from "../../lib/job-priority";-import { Job } from "bullmq";import {- Document,- fromLegacyCombo,fromLegacyScrapeOptions,TeamFlags,toLegacyDocument,} from "../v1/types";-import { getJobFromGCS } from "../../lib/gcs-jobs";import { fromV0Combo } from "../v2/types";import { ScrapeJobTimeoutError } from "../../lib/error";+import { scrapeQueue } from "../../services/worker/nuq";export async function searchHelper(jobId: string,@@ -111,7 +107,7 @@ export async function searchHelper(const url = x.url;const uuid = uuidv4();return {- name: uuid,+ jobId: uuid,data: {url,mode: "single_urls" as const,@@ -122,28 +118,23 @@ export async function searchHelper(zeroDataRetention: false, // not supported on v0apiKeyId: api_key_id,},- opts: {- jobId: uuid,- priority: jobPriority,- },};});// TODO: addScrapeJobsfor (const job of jobDatas) {- await addScrapeJob(job.data, {}, job.opts.jobId, job.opts.priority);+ await addScrapeJob(job.data, job.jobId, jobPriority);}const docs = (- await Promise.all(jobDatas.map(x => waitForJob(x.opts.jobId, 60000)))+ await Promise.all(jobDatas.map(x => waitForJob(x.jobId, 60000, false)))).map(x => toLegacyDocument(x, internalOptions));if (docs.length === 0) {return { success: true, error: "No search results found", returnCode: 200 };}- const sq = getScrapeQueue();- await Promise.all(jobDatas.map(x => sq.remove(x.opts.jobId)));+ await scrapeQueue.removeJobs(jobDatas.map(x => x.jobId));// make sure doc.content is not emptyconst filteredDocs = docs.filter(diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.tsindex 9cbd93716..bfad81b0b 100644--- a/apps/api/src/controllers/v1/batch-scrape.ts+++ b/apps/api/src/controllers/v1/batch-scrape.ts@@ -145,6 +145,7 @@ export async function batchScrapeController(logger.debug("Using job priority " + jobPriority, { jobPriority });const jobs = urls.map(x => ({+ jobId: uuidv4(),data: {url: x,mode: "single_urls" as const,@@ -161,10 +162,7 @@ export async function batchScrapeController(zeroDataRetention: zeroDataRetention ?? false,apiKeyId: req.acuc?.api_key_id ?? null,},- opts: {- jobId: uuidv4(),- priority: 20,- },+ priority: jobPriority,}));await finishCrawlKickoff(id);@@ -179,7 +177,7 @@ export async function batchScrapeController(logger.debug("Adding scrape jobs to Redis...");await addCrawlJobs(id,- jobs.map(x => x.opts.jobId),+ jobs.map(x => x.jobId),logger,);logger.debug("Adding scrape jobs to BullMQ...");diff --git a/apps/api/src/controllers/v1/crawl-errors.ts b/apps/api/src/controllers/v1/crawl-errors.tsindex e084310ea..a4b5cd2bb 100644--- a/apps/api/src/controllers/v1/crawl-errors.ts+++ b/apps/api/src/controllers/v1/crawl-errors.ts@@ -5,31 +5,15 @@ import {RequestWithAuth,} from "./types";import { getCrawl, getCrawlJobs } from "../../lib/crawl-redis";-import { getScrapeQueue } from "../../services/queue-service";import { redisEvictConnection } from "../../../src/services/redis";import { configDotenv } from "dotenv";-import { Job } from "bullmq";import { supabase_rr_service } from "../../services/supabase";-import { logger } from "../../lib/logger";+import { logger as _logger } from "../../lib/logger";import { deserializeTransportableError } from "../../lib/error-serde";import { TransportableError } from "../../lib/error";+import { scrapeQueue } from "../../services/worker/nuq";configDotenv();-export async function getJob(id: string) {- const job = await getScrapeQueue().getJob(id);- if (!job) return job;-- return job;-}--export async function getJobs(ids: string[]) {- const jobs: (Job & { id: string })[] = (- await Promise.all(ids.map(x => getScrapeQueue().getJob(x)))- ).filter(x => x) as (Job & { id: string })[];-- return jobs;-}-export async function crawlErrorsController(req: RequestWithAuth<CrawlStatusParams, undefined, CrawlErrorsResponse>,res: Response<CrawlErrorsResponse>,@@ -41,25 +25,24 @@ export async function crawlErrorsController(return res.status(403).json({ success: false, error: "Forbidden" });}- let jobStatuses = await Promise.all(- (await getCrawlJobs(req.params.jobId)).map(- async x => [x, await getScrapeQueue().getJobState(x)] as const,- ),- );-- const failedJobIDs: string[] = [];+ const logger = _logger.child({+ crawlId: req.params.jobId,+ zeroDataRetention: sc.zeroDataRetention ?? false,+ });- for (const [id, status] of jobStatuses) {- if (status === "failed") {- failedJobIDs.push(id);- }- }+ const failedJobs = (+ await scrapeQueue.getJobsWithStatus(+ await getCrawlJobs(req.params.jobId),+ "failed",+ logger,+ )+ ).filter(x => x.failedReason);res.status(200).json({- errors: (await getJobs(failedJobIDs))+ errors: failedJobs.map(x => {const error = deserializeTransportableError(- x.failedReason,+ x.failedReason!,) as TransportableError | null;if (error?.code === "SCRAPE_RACED_REDIRECT_ERROR") {return null;@@ -67,8 +50,8 @@ export async function crawlErrorsController(return {id: x.id,timestamp:- x.finishedOn !== undefined- ? new Date(x.finishedOn).toISOString()+ x.finishedAt !== undefined+ ? new Date(x.finishedAt).toISOString(): undefined,url: x.data.url,...(error@@ -77,7 +60,7 @@ export async function crawlErrorsController(error: error.message,}: {- error: x.failedReason,+ error: x.failedReason!,}),};})@@ -95,7 +78,7 @@ export async function crawlErrorsController(.throwOnError();if (crawlJobError) {- logger.error("Error getting crawl job", { error: crawlJobError });+ _logger.error("Error getting crawl job", { error: crawlJobError });throw crawlJobError;}@@ -130,7 +113,7 @@ export async function crawlErrorsController(.throwOnError();if (failedJobsError) {- logger.error("Error getting failed jobs", { error: failedJobsError });+ _logger.error("Error getting failed jobs", { error: failedJobsError });throw failedJobsError;}diff --git a/apps/api/src/controllers/v1/crawl-status-ws.ts b/apps/api/src/controllers/v1/crawl-status-ws.tsindex 2313566fb..f1ec30606 100644--- a/apps/api/src/controllers/v1/crawl-status-ws.ts+++ b/apps/api/src/controllers/v1/crawl-status-ws.ts@@ -15,15 +15,11 @@ import {getCrawlExpiry,getCrawlJobs,getDoneJobsOrdered,- getDoneJobsOrderedLength,- isCrawlFinished,- isCrawlFinishedLocked,} from "../../lib/crawl-redis";-import { getScrapeQueue } from "../../services/queue-service";-import { getJob, getJobs } from "./crawl-status";+import { getJobs, PseudoJob } from "./crawl-status";import * as Sentry from "@sentry/node";-import { Job, JobState } from "bullmq";import { getConcurrencyLimitedJobs } from "../../lib/concurrency-limit";+import { scrapeQueue, NuQJobStatus } from "../../services/worker/nuq";type ErrorMessage = {type: "error";@@ -88,18 +84,14 @@ async function crawlStatusWS(const notDoneJobIDs = jobIDs.filter(x => !doneJobIDs.includes(x));- const queue = getScrapeQueue();+ const newlyDoneJobIDs: string[] = (+ await scrapeQueue.getJobsWithStatuses(notDoneJobIDs, [+ "completed",+ "failed",+ ])+ ).map(x => x.id);- const jobStatuses = await Promise.all(- notDoneJobIDs.map(async x => [x, await queue.getJobState(x)]),- );- const newlyDoneJobIDs: string[] = jobStatuses- .filter(x => x[1] === "completed" || x[1] === "failed")- .map(x => x[0]);-- const newlyDoneJobs: Job[] = (- await Promise.all(newlyDoneJobIDs.map(x => getJob(x)))- ).filter(x => x !== undefined) as Job[];+ const newlyDoneJobs: PseudoJob<any>[] = await getJobs(newlyDoneJobIDs);for (const job of newlyDoneJobs) {if (job.returnvalue) {@@ -118,28 +110,28 @@ async function crawlStatusWS(setTimeout(loop, 1000);- doneJobIDs = await getDoneJobsOrdered(req.params.jobId);-- let jobIDs = await getCrawlJobs(req.params.jobId);-- const queue = getScrapeQueue();+ let [_doneJobIDs, jobIDs, throttledJobsSet] = await Promise.all([+ getDoneJobsOrdered(req.params.jobId),+ getCrawlJobs(req.params.jobId),+ getConcurrencyLimitedJobs(req.auth.team_id),+ ]);- let jobStatuses = await Promise.all(- jobIDs.map(async x => [x, await queue.getJobState(x)] as const),- );+ doneJobIDs = _doneJobIDs;+ const jobs = new Map((await scrapeQueue.getJobs(jobIDs)).map(x => [x.id, x]));- const throttledJobsSet = await getConcurrencyLimitedJobs(req.auth.team_id);-- const validJobStatuses: [string, JobState | "unknown"][] = [];+ const validJobStatuses: [string, NuQJobStatus][] = [];const validJobIDs: string[] = [];- for (const [id, status] of jobStatuses) {+ for (const id of jobIDs) {if (throttledJobsSet.has(id)) {- validJobStatuses.push([id, "prioritized"]);- validJobIDs.push(id);- } else if (status !== "failed" && status !== "unknown") {- validJobStatuses.push([id, status]);+ validJobStatuses.push([id, "queued"]);validJobIDs.push(id);+ } else {+ const job = jobs.get(id);+ if (job && job.status !== "failed") {+ validJobStatuses.push([id, job.status]);+ validJobIDs.push(id);+ }}}diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.tsindex 8380681ee..d856de2cf 100644--- a/apps/api/src/controllers/v1/crawl-status.ts+++ b/apps/api/src/controllers/v1/crawl-status.ts@@ -13,21 +13,20 @@ import {getCrawlQualifiedJobCount,getDoneJobsOrderedUntil,} from "../../lib/crawl-redis";-import { getScrapeQueue } from "../../services/queue-service";import {supabaseGetJobById,supabaseGetJobsById,} from "../../lib/supabase-jobs";import { configDotenv } from "dotenv";-import type { Job, JobState } from "bullmq";import { logger } from "../../lib/logger";import { supabase_rr_service, supabase_service } from "../../services/supabase";import { getJobFromGCS } from "../../lib/gcs-jobs";+import { scrapeQueue, NuQJob, NuQJobStatus } from "../../services/worker/nuq";configDotenv();export type PseudoJob<T> = {id: string;- getState(): Promise<JobState | "unknown"> | JobState | "unknown";+ status: NuQJobStatus;returnvalue: T | null;timestamp: number;data: {@@ -47,8 +46,8 @@ export type DBJob = {};export async function getJob(id: string): Promise<PseudoJob<any> | null> {- const [bullJob, dbJob, gcsJob] = await Promise.all([- getScrapeQueue().getJob(id),+ const [nuqJob, dbJob, gcsJob] = await Promise.all([+ scrapeQueue.getJob(id),(process.env.USE_DB_AUTHENTICATION === "true"? supabaseGetJobById(id): null) as Promise<DBJob | null>,@@ -57,9 +56,9 @@ export async function getJob(id: string): Promise<PseudoJob<any> | null> {>,]);- if (!bullJob && !dbJob) return null;+ if (!nuqJob && !dbJob) return null;- const data = gcsJob ?? dbJob?.docs ?? bullJob?.returnvalue;+ const data = gcsJob ?? dbJob?.docs ?? nuqJob?.returnvalue;if (gcsJob === null && data) {logger.warn("GCS Job not found", {jobId: id,@@ -68,28 +67,23 @@ export async function getJob(id: string): Promise<PseudoJob<any> | null> {const job: PseudoJob<any> = {id,- getState: dbJob- ? () => (dbJob.success ? "completed" : "failed")- : bullJob!.getState,+ status: dbJob ? (dbJob.success ? "completed" : "failed") : nuqJob!.status,returnvalue: Array.isArray(data) ? data[0] : data,data: {- scrapeOptions: bullJob ? bullJob.data.scrapeOptions : dbJob!.page_options,+ scrapeOptions: nuqJob ? nuqJob.data.scrapeOptions : dbJob!.page_options,},- timestamp: bullJob- ? bullJob.timestamp+ timestamp: nuqJob+ ? nuqJob.createdAt.valueOf(): new Date(dbJob!.date_added).valueOf(),- failedReason:- (bullJob ? bullJob.failedReason : dbJob!.message) || undefined,+ failedReason: (nuqJob ? nuqJob.failedReason : dbJob!.message) || undefined,};return job;}export async function getJobs(ids: string[]): Promise<PseudoJob<any>[]> {- const [bullJobs, dbJobs, gcsJobs] = await Promise.all([- Promise.all(ids.map(x => getScrapeQueue().getJob(x))).then(x =>- x.filter(x => x),- ) as Promise<(Job<any, any, string> & { id: string })[]>,+ const [nuqJobs, dbJobs, gcsJobs] = await Promise.all([+ scrapeQueue.getJobs(ids),process.env.USE_DB_AUTHENTICATION === "true"? supabaseGetJobsById(ids): [],@@ -102,12 +96,12 @@ export async function getJobs(ids: string[]): Promise<PseudoJob<any>[]> {: [],]);- const bullJobMap = new Map<string, PseudoJob<any>>();+ const nuqJobMap = new Map<string, NuQJob<any, any>>();const dbJobMap = new Map<string, DBJob>();const gcsJobMap = new Map<string, any>();- for (const job of bullJobs) {- bullJobMap.set(job.id, job);+ for (const job of nuqJobs) {+ nuqJobMap.set(job.id, job);}for (const job of dbJobs) {@@ -121,13 +115,13 @@ export async function getJobs(ids: string[]): Promise<PseudoJob<any>[]> {const jobs: PseudoJob<any>[] = [];for (const id of ids) {- const bullJob = bullJobMap.get(id);+ const nuqJob = nuqJobMap.get(id);const dbJob = dbJobMap.get(id);const gcsJob = gcsJobMap.get(id);- if (!bullJob && !dbJob) continue;+ if (!nuqJob && !dbJob) continue;- const data = gcsJob ?? dbJob?.docs ?? bullJob?.returnvalue;+ const data = gcsJob ?? dbJob?.docs ?? nuqJob?.returnvalue;if (gcsJob === null && data) {logger.warn("GCS Job not found", {jobId: id,@@ -136,20 +130,16 @@ export async function getJobs(ids: string[]): Promise<PseudoJob<any>[]> {const job: PseudoJob<any> = {id,- getState: dbJob- ? () => (dbJob.success ? "completed" : "failed")- : () => bullJob!.getState(),+ status: dbJob ? (dbJob.success ? "completed" : "failed") : nuqJob!.status,returnvalue: Array.isArray(data) ? data[0] : data,data: {- scrapeOptions: bullJob- ? bullJob.data.scrapeOptions- : dbJob!.page_options,+ scrapeOptions: nuqJob ? nuqJob.data.scrapeOptions : dbJob!.page_options,},- timestamp: bullJob- ? bullJob.timestamp+ timestamp: nuqJob+ ? nuqJob.createdAt.valueOf(): new Date(dbJob!.date_added).valueOf(),failedReason:- (bullJob ? bullJob.failedReason : dbJob!.message) || undefined,+ (nuqJob ? nuqJob.failedReason : dbJob!.message) || undefined,};jobs.push(job);@@ -390,30 +380,35 @@ export async function crawlStatusController(let bytes = 0;const bytesLimit = 10485760; // 10 MiB in bytes- for (const jobId of doneJobs) {- const job = await getScrapeQueue().getJob(jobId);- const state = await job?.getState();+ for (let i = 0; i < Math.ceil(doneJobs.length / 50); i++) {+ const jobIds = doneJobs.slice(i * 50, (i + 1) * 50);+ const jobs = await getJobs(jobIds);- if (state === "failed") {- // no iterated over, just ignore- continue;- } else {- if (job?.returnvalue) {- scrapes.push(job.returnvalue);- bytes += JSON.stringify(job.returnvalue).length;+ for (const job of jobs) {+ if (job.status === "failed") {+ continue;} else {- logger.warn(- "Job was considered done, but returnvalue is undefined!",- {- scrapeId: jobId,- crawlId: req.params.jobId,- state,- returnvalue: job?.returnvalue,- },- );+ if (job?.returnvalue) {+ scrapes.push(job.returnvalue);+ bytes += JSON.stringify(job.returnvalue).length;+ } else {+ logger.warn(+ "Job was considered done, but returnvalue is undefined!",+ {+ scrapeId: job.id,+ crawlId: req.params.jobId,+ state: job.status,+ returnvalue: job?.returnvalue,+ },+ );+ }++ iteratedOver++;}- iteratedOver++;+ if (bytes > bytesLimit) {+ break;+ }}if (bytes > bytesLimit) {diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.tsindex 3e4399927..c04236470 100644--- a/apps/api/src/controllers/v1/crawl.ts+++ b/apps/api/src/controllers/v1/crawl.ts@@ -148,9 +148,7 @@ export async function crawlController(zeroDataRetention: zeroDataRetention || false,apiKeyId: req.acuc?.api_key_id ?? null,},- {},crypto.randomUUID(),- 10,);const protocol = process.env.ENV === "local" ? req.protocol : "https";diff --git a/apps/api/src/controllers/v1/extract-status.ts b/apps/api/src/controllers/v1/extract-status.tsindex d5bf6575b..cea0cc3bf 100644--- a/apps/api/src/controllers/v1/extract-status.ts+++ b/apps/api/src/controllers/v1/extract-status.ts@@ -1,14 +1,27 @@import { Response } from "express";import { RequestWithAuth } from "./types";import { getExtract, getExtractExpiry } from "../../lib/extract/extract-redis";-import { DBJob, PseudoJob } from "./crawl-status";+import { DBJob } from "./crawl-status";import { getExtractQueue } from "../../services/queue-service";import { ExtractResult } from "../../lib/extract/extraction-service";import { supabaseGetJobByIdDirect } from "../../lib/supabase-jobs";+import { JobState } from "bullmq";++type ExtractPseudoJob<T> = {+ id: string;+ getState: () => Promise<JobState | "unknown"> | JobState | "unknown";+ returnvalue: T | null;+ timestamp: number;+ data: {+ scrapeOptions: any;+ teamId?: string;+ };+ failedReason?: string;+};export async function getExtractJob(id: string,-): Promise<PseudoJob<ExtractResult> | null> {+): Promise<ExtractPseudoJob<ExtractResult> | null> {const [bullJob, dbJob] = await Promise.all([getExtractQueue().getJob(id),(process.env.USE_DB_AUTHENTICATION === "true"@@ -20,7 +33,7 @@ export async function getExtractJob(const data = dbJob?.docs ?? bullJob?.returnvalue?.data;- const job: PseudoJob<any> = {+ const job: ExtractPseudoJob<any> = {id,getState: dbJob? () => (dbJob.success ? "completed" : "failed")diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex a8a85447e..baee96800 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -10,9 +10,9 @@ import {import { v4 as uuidv4 } from "uuid";import { addScrapeJob, waitForJob } from "../../services/queue-jobs";import { getJobPriority } from "../../lib/job-priority";-import { getScrapeQueue } from "../../services/queue-service";import { fromV1ScrapeOptions } from "../v2/types";import { TransportableError } from "../../lib/error";+import { scrapeQueue } from "../../services/worker/nuq";import { checkPermissions } from "../../lib/permissions";export async function scrapeController(@@ -54,10 +54,6 @@ export async function scrapeController(const timeout = req.body.timeout;const startTime = new Date().getTime();- const jobPriority = await getJobPriority({- team_id: req.auth.team_id,- basePriority: 10,- });const isDirectToBullMQ =process.env.SEARCH_PREVIEW_TOKEN !== undefined &&@@ -69,6 +65,11 @@ export async function scrapeController(req.auth.team_id,);+ const jobPriority = await getJobPriority({+ team_id: req.auth.team_id,+ basePriority: 10,+ });+const bullJob = await addScrapeJob({url: req.body.url,@@ -92,7 +93,6 @@ export async function scrapeController(zeroDataRetention: zeroDataRetention ?? false,apiKeyId: req.acuc?.api_key_id ?? null,},- {},jobId,jobPriority,isDirectToBullMQ,@@ -113,6 +113,7 @@ export async function scrapeController(doc = await waitForJob(bullJob ? bullJob : jobId,timeout + totalWait,+ zeroDataRetention ?? false,logger,);} catch (e) {@@ -122,7 +123,7 @@ export async function scrapeController(});if (zeroDataRetention) {- await getScrapeQueue().remove(jobId);+ await scrapeQueue.removeJob(jobId);}if (e instanceof TransportableError) {@@ -142,7 +143,7 @@ export async function scrapeController(logger.info("Done with waitForJob");- await getScrapeQueue().remove(jobId);+ await scrapeQueue.removeJob(jobId);logger.info("Removed job from queue");diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex 89a8a8e11..00b51ed26 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -7,26 +7,25 @@ import {searchRequestSchema,ScrapeOptions,TeamFlags,- scrapeOptions,} from "./types";import { billTeam } from "../../services/billing/credit_billing";import { v4 as uuidv4 } from "uuid";import { addScrapeJob, waitForJob } from "../../services/queue-jobs";import { logJob } from "../../services/logging/log_job";-import { getJobPriority } from "../../lib/job-priority";import { Mode } from "../../types";-import { getScrapeQueue } from "../../services/queue-service";import { search } from "../../search";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";import * as Sentry from "@sentry/node";import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";import { logger as _logger } from "../../lib/logger";import type { Logger } from "winston";+import { getJobPriority } from "../../lib/job-priority";import { CostTracking } from "../../lib/cost-tracking";import { calculateCreditsToBeBilled } from "../../lib/scrape-billing";import { supabase_service } from "../../services/supabase";import { fromV1ScrapeOptions } from "../v2/types";import { ScrapeJobTimeoutError } from "../../lib/error";+import { scrapeQueue } from "../../services/worker/nuq";interface DocumentWithCostTracking {document: Document;@@ -89,10 +88,6 @@ async function scrapeSearchResult(isSearchPreview: boolean = false,): Promise<DocumentWithCostTracking> {const jobId = uuidv4();- const jobPriority = await getJobPriority({- team_id: options.teamId,- basePriority: 10,- });const costTracking = new CostTracking();@@ -116,6 +111,11 @@ async function scrapeSearchResult(options.teamId,);+ const jobPriority = await getJobPriority({+ team_id: options.teamId,+ basePriority: 10,+ });+await addScrapeJob({url: searchResult.url,@@ -140,13 +140,16 @@ async function scrapeSearchResult(zeroDataRetention,apiKeyId: options.apiKeyId,},- {},jobId,jobPriority,directToBullMQ,);- const doc: Document = await waitForJob(jobId, options.timeout);+ const doc: Document = await waitForJob(+ jobId,+ options.timeout,+ zeroDataRetention,+ );logger.info("Scrape job completed", {scrapeId: jobId,@@ -154,7 +157,7 @@ async function scrapeSearchResult(teamId: options.teamId,origin: options.origin,});- await getScrapeQueue().remove(jobId);+ await scrapeQueue.removeJob(jobId);const document = {title: searchResult.title,diff --git a/apps/api/src/controllers/v1/x402-search.ts b/apps/api/src/controllers/v1/x402-search.tsindex 706fed9de..0e5aed0e0 100644--- a/apps/api/src/controllers/v1/x402-search.ts+++ b/apps/api/src/controllers/v1/x402-search.ts@@ -11,9 +11,7 @@ import {import { v4 as uuidv4 } from "uuid";import { addScrapeJob, waitForJob } from "../../services/queue-jobs";import { logJob } from "../../services/logging/log_job";-import { getJobPriority } from "../../lib/job-priority";import { Mode } from "../../types";-import { getScrapeQueue } from "../../services/queue-service";import { search } from "../../search";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";import * as Sentry from "@sentry/node";@@ -24,6 +22,8 @@ import { CostTracking } from "../../lib/cost-tracking";import { supabase_service } from "../../services/supabase";import { fromV1ScrapeOptions } from "../v2/types";import { ScrapeJobTimeoutError } from "../../lib/error";+import { scrapeQueue } from "../../services/worker/nuq";+import { getJobPriority } from "../../lib/job-priority";interface DocumentWithCostTracking {document: Document;@@ -45,10 +45,6 @@ async function scrapeX402SearchResult(isSearchPreview: boolean = false,): Promise<DocumentWithCostTracking> {const jobId = uuidv4();- const jobPriority = await getJobPriority({- team_id: options.teamId,- basePriority: 10,- });const costTracking = new CostTracking();@@ -94,13 +90,19 @@ async function scrapeX402SearchResult(zeroDataRetention,apiKeyId: options.apiKeyId,},- {},jobId,- jobPriority,+ await getJobPriority({+ team_id: options.teamId,+ basePriority: 10,+ }),directToBullMQ,);- const doc: Document = await waitForJob(jobId, options.timeout);+ const doc: Document = await waitForJob(+ jobId,+ options.timeout,+ zeroDataRetention,+ );logger.info("Scrape job [x402] completed", {scrapeId: jobId,@@ -108,7 +110,7 @@ async function scrapeX402SearchResult(teamId: options.teamId,origin: options.origin,});- await getScrapeQueue().remove(jobId);+ await scrapeQueue.removeJob(jobId);const document = {title: searchResult.title,diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.tsindex fe17023de..bdb704877 100644--- a/apps/api/src/controllers/v2/batch-scrape.ts+++ b/apps/api/src/controllers/v2/batch-scrape.ts@@ -142,6 +142,7 @@ export async function batchScrapeController(delete (scrapeOptions as any).appendToId;const jobs = urls.map(x => ({+ jobId: uuidv4(),data: {url: x,mode: "single_urls" as const,@@ -158,10 +159,7 @@ export async function batchScrapeController(zeroDataRetention,apiKeyId: req.acuc?.api_key_id ?? null,},- opts: {- jobId: uuidv4(),- priority: 20,- },+ priority: jobPriority,}));await finishCrawlKickoff(id);@@ -176,7 +174,7 @@ export async function batchScrapeController(logger.debug("Adding scrape jobs to Redis...");await addCrawlJobs(id,- jobs.map(x => x.opts.jobId),+ jobs.map(x => x.jobId),logger,);logger.debug("Adding scrape jobs to BullMQ...");diff --git a/apps/api/src/controllers/v2/crawl-errors.ts b/apps/api/src/controllers/v2/crawl-errors.tsindex e084310ea..a4b5cd2bb 100644--- a/apps/api/src/controllers/v2/crawl-errors.ts+++ b/apps/api/src/controllers/v2/crawl-errors.ts@@ -5,31 +5,15 @@ import {RequestWithAuth,} from "./types";import { getCrawl, getCrawlJobs } from "../../lib/crawl-redis";-import { getScrapeQueue } from "../../services/queue-service";import { redisEvictConnection } from "../../../src/services/redis";import { configDotenv } from "dotenv";-import { Job } from "bullmq";import { supabase_rr_service } from "../../services/supabase";-import { logger } from "../../lib/logger";+import { logger as _logger } from "../../lib/logger";import { deserializeTransportableError } from "../../lib/error-serde";import { TransportableError } from "../../lib/error";+import { scrapeQueue } from "../../services/worker/nuq";configDotenv();-export async function getJob(id: string) {- const job = await getScrapeQueue().getJob(id);- if (!job) return job;-- return job;-}--export async function getJobs(ids: string[]) {- const jobs: (Job & { id: string })[] = (- await Promise.all(ids.map(x => getScrapeQueue().getJob(x)))- ).filter(x => x) as (Job & { id: string })[];-- return jobs;-}-export async function crawlErrorsController(req: RequestWithAuth<CrawlStatusParams, undefined, CrawlErrorsResponse>,res: Response<CrawlErrorsResponse>,@@ -41,25 +25,24 @@ export async function crawlErrorsController(return res.status(403).json({ success: false, error: "Forbidden" });}- let jobStatuses = await Promise.all(- (await getCrawlJobs(req.params.jobId)).map(- async x => [x, await getScrapeQueue().getJobState(x)] as const,- ),- );-- const failedJobIDs: string[] = [];+ const logger = _logger.child({+ crawlId: req.params.jobId,+ zeroDataRetention: sc.zeroDataRetention ?? false,+ });- for (const [id, status] of jobStatuses) {- if (status === "failed") {- failedJobIDs.push(id);- }- }+ const failedJobs = (+ await scrapeQueue.getJobsWithStatus(+ await getCrawlJobs(req.params.jobId),+ "failed",+ logger,+ )+ ).filter(x => x.failedReason);res.status(200).json({- errors: (await getJobs(failedJobIDs))+ errors: failedJobs.map(x => {const error = deserializeTransportableError(- x.failedReason,+ x.failedReason!,) as TransportableError | null;if (error?.code === "SCRAPE_RACED_REDIRECT_ERROR") {return null;@@ -67,8 +50,8 @@ export async function crawlErrorsController(return {id: x.id,timestamp:- x.finishedOn !== undefined- ? new Date(x.finishedOn).toISOString()+ x.finishedAt !== undefined+ ? new Date(x.finishedAt).toISOString(): undefined,url: x.data.url,...(error@@ -77,7 +60,7 @@ export async function crawlErrorsController(error: error.message,}: {- error: x.failedReason,+ error: x.failedReason!,}),};})@@ -95,7 +78,7 @@ export async function crawlErrorsController(.throwOnError();if (crawlJobError) {- logger.error("Error getting crawl job", { error: crawlJobError });+ _logger.error("Error getting crawl job", { error: crawlJobError });throw crawlJobError;}@@ -130,7 +113,7 @@ export async function crawlErrorsController(.throwOnError();if (failedJobsError) {- logger.error("Error getting failed jobs", { error: failedJobsError });+ _logger.error("Error getting failed jobs", { error: failedJobsError });throw failedJobsError;}diff --git a/apps/api/src/controllers/v2/crawl-status-ws.ts b/apps/api/src/controllers/v2/crawl-status-ws.tsindex 2313566fb..f1ec30606 100644--- a/apps/api/src/controllers/v2/crawl-status-ws.ts+++ b/apps/api/src/controllers/v2/crawl-status-ws.ts@@ -15,15 +15,11 @@ import {getCrawlExpiry,getCrawlJobs,getDoneJobsOrdered,- getDoneJobsOrderedLength,- isCrawlFinished,- isCrawlFinishedLocked,} from "../../lib/crawl-redis";-import { getScrapeQueue } from "../../services/queue-service";-import { getJob, getJobs } from "./crawl-status";+import { getJobs, PseudoJob } from "./crawl-status";import * as Sentry from "@sentry/node";-import { Job, JobState } from "bullmq";import { getConcurrencyLimitedJobs } from "../../lib/concurrency-limit";+import { scrapeQueue, NuQJobStatus } from "../../services/worker/nuq";type ErrorMessage = {type: "error";@@ -88,18 +84,14 @@ async function crawlStatusWS(const notDoneJobIDs = jobIDs.filter(x => !doneJobIDs.includes(x));- const queue = getScrapeQueue();+ const newlyDoneJobIDs: string[] = (+ await scrapeQueue.getJobsWithStatuses(notDoneJobIDs, [+ "completed",+ "failed",+ ])+ ).map(x => x.id);- const jobStatuses = await Promise.all(- notDoneJobIDs.map(async x => [x, await queue.getJobState(x)]),- );- const newlyDoneJobIDs: string[] = jobStatuses- .filter(x => x[1] === "completed" || x[1] === "failed")- .map(x => x[0]);-- const newlyDoneJobs: Job[] = (- await Promise.all(newlyDoneJobIDs.map(x => getJob(x)))- ).filter(x => x !== undefined) as Job[];+ const newlyDoneJobs: PseudoJob<any>[] = await getJobs(newlyDoneJobIDs);for (const job of newlyDoneJobs) {if (job.returnvalue) {@@ -118,28 +110,28 @@ async function crawlStatusWS(setTimeout(loop, 1000);- doneJobIDs = await getDoneJobsOrdered(req.params.jobId);-- let jobIDs = await getCrawlJobs(req.params.jobId);-- const queue = getScrapeQueue();+ let [_doneJobIDs, jobIDs, throttledJobsSet] = await Promise.all([+ getDoneJobsOrdered(req.params.jobId),+ getCrawlJobs(req.params.jobId),+ getConcurrencyLimitedJobs(req.auth.team_id),+ ]);- let jobStatuses = await Promise.all(- jobIDs.map(async x => [x, await queue.getJobState(x)] as const),- );+ doneJobIDs = _doneJobIDs;+ const jobs = new Map((await scrapeQueue.getJobs(jobIDs)).map(x => [x.id, x]));- const throttledJobsSet = await getConcurrencyLimitedJobs(req.auth.team_id);-- const validJobStatuses: [string, JobState | "unknown"][] = [];+ const validJobStatuses: [string, NuQJobStatus][] = [];const validJobIDs: string[] = [];- for (const [id, status] of jobStatuses) {+ for (const id of jobIDs) {if (throttledJobsSet.has(id)) {- validJobStatuses.push([id, "prioritized"]);- validJobIDs.push(id);- } else if (status !== "failed" && status !== "unknown") {- validJobStatuses.push([id, status]);+ validJobStatuses.push([id, "queued"]);validJobIDs.push(id);+ } else {+ const job = jobs.get(id);+ if (job && job.status !== "failed") {+ validJobStatuses.push([id, job.status]);+ validJobIDs.push(id);+ }}}diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.tsindex 9c421d116..3520e2063 100644--- a/apps/api/src/controllers/v2/crawl-status.ts+++ b/apps/api/src/controllers/v2/crawl-status.ts@@ -13,21 +13,20 @@ import {getDoneJobsOrderedUntil,isCrawlKickoffFinished,} from "../../lib/crawl-redis";-import { getScrapeQueue } from "../../services/queue-service";import {supabaseGetJobById,supabaseGetJobsById,} from "../../lib/supabase-jobs";import { configDotenv } from "dotenv";-import type { Job, JobState, Queue } from "bullmq";import { logger } from "../../lib/logger";import { supabase_rr_service, supabase_service } from "../../services/supabase";import { getJobFromGCS } from "../../lib/gcs-jobs";+import { scrapeQueue, NuQJob, NuQJobStatus } from "../../services/worker/nuq";configDotenv();export type PseudoJob<T> = {id: string;- getState(): Promise<JobState | "unknown"> | JobState | "unknown";+ status: NuQJobStatus;returnvalue: T | null;timestamp: number;data: {@@ -47,8 +46,8 @@ export type DBJob = {};export async function getJob(id: string): Promise<PseudoJob<any> | null> {- const [bullJob, dbJob, gcsJob] = await Promise.all([- getScrapeQueue().getJob(id),+ const [nuqJob, dbJob, gcsJob] = await Promise.all([+ scrapeQueue.getJob(id),(process.env.USE_DB_AUTHENTICATION === "true"? supabaseGetJobById(id): null) as Promise<DBJob | null>,@@ -57,9 +56,9 @@ export async function getJob(id: string): Promise<PseudoJob<any> | null> {>,]);- if (!bullJob && !dbJob) return null;+ if (!nuqJob && !dbJob) return null;- const data = gcsJob ?? dbJob?.docs ?? bullJob?.returnvalue;+ const data = gcsJob ?? dbJob?.docs ?? nuqJob?.returnvalue;if (gcsJob === null && data) {logger.warn("GCS Job not found", {jobId: id,@@ -68,28 +67,23 @@ export async function getJob(id: string): Promise<PseudoJob<any> | null> {const job: PseudoJob<any> = {id,- getState: bullJob- ? bullJob.getState- : () => (dbJob!.success ? "completed" : "failed"),+ status: dbJob ? (dbJob.success ? "completed" : "failed") : nuqJob!.status,returnvalue: Array.isArray(data) ? data[0] : data,data: {- scrapeOptions: bullJob ? bullJob.data.scrapeOptions : dbJob!.page_options,+ scrapeOptions: nuqJob ? nuqJob.data.scrapeOptions : dbJob!.page_options,},- timestamp: bullJob- ? bullJob.timestamp+ timestamp: nuqJob+ ? nuqJob.createdAt.valueOf(): new Date(dbJob!.date_added).valueOf(),- failedReason:- (bullJob ? bullJob.failedReason : dbJob!.message) || undefined,+ failedReason: (nuqJob ? nuqJob.failedReason : dbJob!.message) || undefined,};return job;}export async function getJobs(ids: string[]): Promise<PseudoJob<any>[]> {- const [bullJobs, dbJobs, gcsJobs] = await Promise.all([- Promise.all(ids.map(x => getScrapeQueue().getJob(x))).then(x =>- x.filter(x => x),- ) as Promise<(Job<any, any, string> & { id: string })[]>,+ const [nuqJobs, dbJobs, gcsJobs] = await Promise.all([+ scrapeQueue.getJobs(ids),process.env.USE_DB_AUTHENTICATION === "true"? supabaseGetJobsById(ids): [],@@ -102,12 +96,12 @@ export async function getJobs(ids: string[]): Promise<PseudoJob<any>[]> {: [],]);- const bullJobMap = new Map<string, PseudoJob<any>>();+ const nuqJobMap = new Map<string, NuQJob<any, any>>();const dbJobMap = new Map<string, DBJob>();const gcsJobMap = new Map<string, any>();- for (const job of bullJobs) {- bullJobMap.set(job.id, job);+ for (const job of nuqJobs) {+ nuqJobMap.set(job.id, job);}for (const job of dbJobs) {@@ -121,13 +115,13 @@ export async function getJobs(ids: string[]): Promise<PseudoJob<any>[]> {const jobs: PseudoJob<any>[] = [];for (const id of ids) {- const bullJob = bullJobMap.get(id);+ const nuqJob = nuqJobMap.get(id);const dbJob = dbJobMap.get(id);const gcsJob = gcsJobMap.get(id);- if (!bullJob && !dbJob) continue;+ if (!nuqJob && !dbJob) continue;- const data = gcsJob ?? dbJob?.docs ?? bullJob?.returnvalue;+ const data = gcsJob ?? dbJob?.docs ?? nuqJob?.returnvalue;if (gcsJob === null && data) {logger.warn("GCS Job not found", {jobId: id,@@ -136,20 +130,16 @@ export async function getJobs(ids: string[]): Promise<PseudoJob<any>[]> {const job: PseudoJob<any> = {id,- getState: bullJob- ? () => bullJob.getState()- : () => (dbJob!.success ? "completed" : "failed"),+ status: dbJob ? (dbJob.success ? "completed" : "failed") : nuqJob!.status,returnvalue: Array.isArray(data) ? data[0] : data,data: {- scrapeOptions: bullJob- ? bullJob.data.scrapeOptions- : dbJob!.page_options,+ scrapeOptions: nuqJob ? nuqJob.data.scrapeOptions : dbJob!.page_options,},- timestamp: bullJob- ? bullJob.timestamp+ timestamp: nuqJob+ ? nuqJob.createdAt.valueOf(): new Date(dbJob!.date_added).valueOf(),failedReason:- (bullJob ? bullJob.failedReason : dbJob!.message) || undefined,+ (nuqJob ? nuqJob.failedReason : dbJob!.message) || undefined,};jobs.push(job);@@ -390,7 +380,6 @@ export async function crawlStatusController(: undefined,};} else {- // old BullMQ-based pathconst doneJobs = await getDoneJobsOrderedUntil(req.params.jobId,djoCutoff,@@ -403,33 +392,35 @@ export async function crawlStatusController(let bytes = 0;const bytesLimit = 10485760; // 10 MiB in bytes- for (const jobId of doneJobs) {- const job = await getScrapeQueue().getJob(jobId);- const state = await job?.getState();+ for (let i = 0; i < Math.ceil(doneJobs.length / 50); i++) {+ const jobIds = doneJobs.slice(i * 50, (i + 1) * 50);+ const jobs = await getJobs(jobIds);- if (state === "failed") {- continue;- }-- let doc: Document | undefined;- if (job?.returnvalue) {- doc = job.returnvalue as Document;- } else {- const gcsDocs = await getJobFromGCS(jobId);- if (gcsDocs && gcsDocs.length > 0) {- doc = gcsDocs[0] as Document;+ for (const job of jobs) {+ if (job.status === "failed") {+ continue;} else {- logger.warn(- "Job was considered done, but neither BullMQ returnvalue nor GCS blob is available!",- { scrapeId: jobId, crawlId: req.params.jobId, state },- );+ if (job?.returnvalue) {+ scrapes.push(job.returnvalue);+ bytes += JSON.stringify(job.returnvalue).length;+ } else {+ logger.warn(+ "Job was considered done, but returnvalue is undefined!",+ {+ scrapeId: job.id,+ crawlId: req.params.jobId,+ state: job.status,+ returnvalue: job?.returnvalue,+ },+ );+ }++ iteratedOver++;}- }- if (doc) {- scrapes.push(doc);- bytes += JSON.stringify(doc).length;- iteratedOver++;+ if (bytes > bytesLimit) {+ break;+ }}if (bytes > bytesLimit) {diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.tsindex ec0ff91b8..1982ad8b4 100644--- a/apps/api/src/controllers/v2/crawl.ts+++ b/apps/api/src/controllers/v2/crawl.ts@@ -196,9 +196,7 @@ export async function crawlController(zeroDataRetention: zeroDataRetention || false,apiKeyId: req.acuc?.api_key_id ?? null,},- {},crypto.randomUUID(),- 10,);const protocol = process.env.ENV === "local" ? req.protocol : "https";diff --git a/apps/api/src/controllers/v2/extract-status.ts b/apps/api/src/controllers/v2/extract-status.tsindex 08f24c784..bec3b62bc 100644--- a/apps/api/src/controllers/v2/extract-status.ts+++ b/apps/api/src/controllers/v2/extract-status.ts@@ -1,14 +1,27 @@import { Response } from "express";import { RequestWithAuth } from "./types";import { getExtract, getExtractExpiry } from "../../lib/extract/extract-redis";-import { DBJob, PseudoJob } from "./crawl-status";+import { DBJob } from "./crawl-status";import { getExtractQueue } from "../../services/queue-service";import { ExtractResult } from "../../lib/extract/extraction-service";import { supabaseGetJobByIdDirect } from "../../lib/supabase-jobs";+import { JobState } from "bullmq";++type ExtractPseudoJob<T> = {+ id: string;+ getState: () => Promise<JobState | "unknown"> | JobState | "unknown";+ returnvalue: T | null;+ timestamp: number;+ data: {+ scrapeOptions: any;+ teamId?: string;+ };+ failedReason?: string;+};export async function getExtractJob(id: string,-): Promise<PseudoJob<ExtractResult> | null> {+): Promise<ExtractPseudoJob<ExtractResult> | null> {const [bullJob, dbJob] = await Promise.all([getExtractQueue().getJob(id),(process.env.USE_DB_AUTHENTICATION === "true"@@ -20,7 +33,7 @@ export async function getExtractJob(const data = dbJob?.docs ?? bullJob?.returnvalue?.data;- const job: PseudoJob<any> = {+ const job: ExtractPseudoJob<any> = {id,getState: bullJob? bullJob.getState.bind(bullJob)diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex 7200f0865..3663f0135 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -10,9 +10,9 @@ import {import { v4 as uuidv4 } from "uuid";import { addScrapeJob, waitForJob } from "../../services/queue-jobs";import { getJobPriority } from "../../lib/job-priority";-import { getScrapeQueue } from "../../services/queue-service";import { hasFormatOfType } from "../../lib/format-utils";import { TransportableError } from "../../lib/error";+import { scrapeQueue } from "../../services/worker/nuq";import { checkPermissions } from "../../lib/permissions";export async function scrapeController(@@ -54,16 +54,17 @@ export async function scrapeController(const timeout = req.body.timeout;const startTime = new Date().getTime();- const jobPriority = await getJobPriority({- team_id: req.auth.team_id,- basePriority: 10,- });const isDirectToBullMQ =process.env.SEARCH_PREVIEW_TOKEN !== undefined &&process.env.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;- await addScrapeJob(+ const jobPriority = await getJobPriority({+ team_id: req.auth.team_id,+ basePriority: 10,+ });++ const job = await addScrapeJob({url: req.body.url,mode: "single_urls",@@ -92,7 +93,6 @@ export async function scrapeController(zeroDataRetention,apiKeyId: req.acuc?.api_key_id ?? null,},- {},jobId,jobPriority,isDirectToBullMQ,@@ -108,8 +108,10 @@ export async function scrapeController(let doc: Document;try {doc = await waitForJob(- jobId,+ job ?? jobId,timeout !== undefined ? timeout + totalWait : null,+ zeroDataRetention,+ logger,);} catch (e) {logger.error(`Error in scrapeController`, {@@ -118,7 +120,7 @@ export async function scrapeController(});if (zeroDataRetention) {- await getScrapeQueue().remove(jobId);+ await scrapeQueue.removeJob(jobId);}if (e instanceof TransportableError) {@@ -135,7 +137,7 @@ export async function scrapeController(}}- await getScrapeQueue().remove(jobId);+ await scrapeQueue.removeJob(jobId);if (!hasFormatOfType(req.body.formats, "rawHtml")) {if (doc && doc.rawHtml) {diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex c958cf63b..8989abfa6 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -12,19 +12,19 @@ import { billTeam } from "../../services/billing/credit_billing";import { v4 as uuidv4 } from "uuid";import { addScrapeJob, waitForJob } from "../../services/queue-jobs";import { logJob } from "../../services/logging/log_job";-import { getJobPriority } from "../../lib/job-priority";import { Mode } from "../../types";-import { getScrapeQueue } from "../../services/queue-service";import { search } from "../../search/v2";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";import * as Sentry from "@sentry/node";import { logger as _logger } from "../../lib/logger";import type { Logger } from "winston";+import { getJobPriority } from "../../lib/job-priority";import { CostTracking } from "../../lib/cost-tracking";import { calculateCreditsToBeBilled } from "../../lib/scrape-billing";import { supabase_service } from "../../services/supabase";import { SearchResult, SearchV2Response } from "../../lib/entities";import { ScrapeJobTimeoutError } from "../../lib/error";+import { scrapeQueue } from "../../services/worker/nuq";import { z } from "zod";import {buildSearchQuery,@@ -59,10 +59,6 @@ async function startScrapeJob(isSearchPreview: boolean = false,): Promise<string> {const jobId = uuidv4();- const jobPriority = await getJobPriority({- team_id: options.teamId,- basePriority: 10,- });const zeroDataRetention = flags?.forceZDR ?? false;@@ -74,6 +70,11 @@ async function startScrapeJob(zeroDataRetention,});+ const jobPriority = await getJobPriority({+ team_id: options.teamId,+ basePriority: 10,+ });+await addScrapeJob({url: searchResult.url,@@ -96,7 +97,6 @@ async function startScrapeJob(zeroDataRetention,apiKeyId: options.apiKeyId,},- {},jobId,jobPriority,directToBullMQ,@@ -131,8 +131,7 @@ async function scrapeSearchResult(isSearchPreview,);- // Wait for the job to complete- const doc: Document = await waitForJob(jobId, options.timeout);+ const doc: Document = await waitForJob(jobId, options.timeout, false);logger.info("Scrape job completed", {scrapeId: jobId,@@ -140,7 +139,8 @@ async function scrapeSearchResult(teamId: options.teamId,origin: options.origin,});- await getScrapeQueue().remove(jobId);++ await scrapeQueue.removeJob(jobId);const document = {title: searchResult.title,diff --git a/apps/api/src/harness.ts b/apps/api/src/harness.tsindex c59f4edbb..cc3b1d4b5 100644--- a/apps/api/src/harness.ts+++ b/apps/api/src/harness.ts@@ -1,3 +1,4 @@+import "dotenv/config";import { exec } from "child_process";import * as net from "net";import { basename } from "path";@@ -77,50 +78,64 @@ function execForward(fancyName: string, command: string): Promise<void> {}(async () => {- console.log("=== Installing dependencies and building all components...");- await Promise.all([- (async () => {- const install = execForward("api@install", "pnpm install");- await install;-- const build = execForward("api@build", "pnpm build");- await build;- })(),- execForward(- "sharedLibs/crawler@build",- "cd sharedLibs/crawler && cargo build --release",- ),- execForward(- "sharedLibs/html-transformer@build",- "cd sharedLibs/html-transformer && cargo build --release",- ),- execForward(- "sharedLibs/pdf-parser@build",- "cd sharedLibs/pdf-parser && cargo build --release",- ),- (async () => {- const install = execForward(- "sharedLibs/go-html-to-md@install",- "cd sharedLibs/go-html-to-md && go mod tidy",- );- await install;+ if (process.argv[2] !== "--start-docker") {+ console.log("=== Installing dependencies and building all components...");+ await Promise.all([+ (async () => {+ if (process.argv[2] !== "--start-built") {+ const install = execForward("api@install", "pnpm install");+ await install;++ const build = execForward("api@build", "pnpm build");+ await build;+ } else {+ console.log("=== Skipping install and build, using built files...");+ }+ })(),+ execForward(+ "sharedLibs/crawler@build",+ "cd sharedLibs/crawler && cargo build --release",+ ),+ execForward(+ "sharedLibs/html-transformer@build",+ "cd sharedLibs/html-transformer && cargo build --release",+ ),+ execForward(+ "sharedLibs/pdf-parser@build",+ "cd sharedLibs/pdf-parser && cargo build --release",+ ),+ (async () => {+ const install = execForward(+ "sharedLibs/go-html-to-md@install",+ "cd sharedLibs/go-html-to-md && go mod tidy",+ );+ await install;- const build = execForward(- "sharedLibs/go-html-to-md@build",- `cd sharedLibs/go-html-to-md && go build -o ${basename(HTML_TO_MARKDOWN_PATH)} -buildmode=c-shared html-to-markdown.go`,- );- await build;- })(),- ]);+ const build = execForward(+ "sharedLibs/go-html-to-md@build",+ `cd sharedLibs/go-html-to-md && go build -o ${basename(HTML_TO_MARKDOWN_PATH)} -buildmode=c-shared html-to-markdown.go`,+ );+ await build;+ })(),+ ]);+ }console.log("=== Starting API, Worker, and Index Worker...");- const api = execForward("api", "pnpm start:production:nobuild");+ const api = execForward("api", "pnpm server:production:nobuild");const worker = execForward("worker", "pnpm worker:production");- const indexWorker = execForward(- "index-worker",- "pnpm index-worker:production",- );+ const nuqWorkers = new Array(5)+ .fill(0)+ .map((_, i) =>+ execForward(+ `nuq-worker-${i}`,+ `NUQ_WORKER_PORT=${3006 + i} NUQ_REDUCE_NOISE=true pnpm nuq-worker:production`,+ ),+ );+ const indexWorker =+ process.env.USE_DB_AUTHENTICATION === "true"+ ? execForward("index-worker", "pnpm index-worker:production")+ : null;try {await Promise.race([@@ -130,17 +145,49 @@ function execForward(fancyName: string, command: string): Promise<void> {),]);- console.log("=== Running command...");-- const cmd = execForward("command", command.join(" "));-- await Promise.race([cmd, api, worker, indexWorker]);+ if (+ process.argv[2] === "--start" ||+ process.argv[2] === "--start-built" ||+ process.argv[2] === "--start-docker"+ ) {+ console.log(+ "=== Everything is up and running, waiting for termination or failure...",+ );+ await Promise.race([+ new Promise(resolve => {+ process.on("SIGINT", resolve);+ process.on("SIGTERM", resolve);+ }),+ api,+ worker,+ ...nuqWorkers,+ ...(indexWorker ? [indexWorker] : []),+ ]);+ } else {+ console.log("=== Running command...");+ const cmd = execForward("command", command.join(" "));+ await Promise.race([+ cmd,+ api,+ worker,+ ...nuqWorkers,+ ...(indexWorker ? [indexWorker] : []),+ ]);+ }} finally {console.log("=== Tearing down API, Worker, and Index Worker...");exec("pkill -f 'queue-worker.js'");exec("pkill -f 'index.js'");- exec("pkill -f 'index-worker.js'");- await Promise.all([api, worker, indexWorker]);+ if (indexWorker) {+ exec("pkill -f 'index-worker.js'");+ }+ exec("pkill -f 'nuq-worker.js'");+ await Promise.all([+ api,+ worker,+ ...nuqWorkers,+ ...(indexWorker ? [indexWorker] : []),+ ]);}console.log("=== Goodbye!");diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex db0848b8d..c8ca7fd54 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -6,7 +6,6 @@ import bodyParser from "body-parser";import cors from "cors";import {getExtractQueue,- getScrapeQueue,getGenerateLlmsTxtQueue,getDeepResearchQueue,getBillingQueue,@@ -27,7 +26,6 @@ import {} from "./controllers/v1/types";import { ZodError } from "zod";import { v4 as uuidv4 } from "uuid";-import { RateLimiterMode } from "./types";import { attachWsProxy } from "./services/agentLivecastWS";import { cacheableLookup } from "./scraper/scrapeURL/lib/cacheableLookup";import { v2Router } from "./routes/v2";@@ -39,6 +37,7 @@ import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";import { resourceFromAttributes } from "@opentelemetry/resources";import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-node";import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-grpc";+import { nuqShutdown } from "./services/worker/nuq";const { createBullBoard } = require("@bull-board/api");const { BullMQAdapter } = require("@bull-board/api/bullMQAdapter");@@ -105,7 +104,6 @@ serverAdapter.setBasePath(`/admin/${process.env.BULL_AUTH_KEY}/queues`);const { addQueue, removeQueue, setQueues, replaceQueues } = createBullBoard({queues: [- new BullMQAdapter(getScrapeQueue()),new BullMQAdapter(getExtractQueue()),new BullMQAdapter(getGenerateLlmsTxtQueue()),new BullMQAdapter(getDeepResearchQueue()),@@ -156,14 +154,17 @@ function startServer(port = DEFAULT_PORT) {}server.close(() => {logger.info("Server closed.");- if (otelSdk) {- otelSdk.shutdown().then(() => {- logger.info("OTEL shutdown");+ nuqShutdown().finally(() => {+ logger.info("NUQ shutdown complete");+ if (otelSdk) {+ otelSdk.shutdown().finally(() => {+ logger.info("OTEL shutdown");+ process.exit(0);+ });+ } else {process.exit(0);- });- } else {- process.exit(0);- }+ }+ });});};@@ -176,77 +177,6 @@ if (require.main === module) {startServer();}-app.get(`/serverHealthCheck`, async (req, res) => {- try {- const scrapeQueue = getScrapeQueue();- const [waitingJobs] = await Promise.all([scrapeQueue.getWaitingCount()]);- const noWaitingJobs = waitingJobs === 0;- // 200 if no active jobs, 503 if there are active jobs- return res.status(noWaitingJobs ? 200 : 500).json({- waitingJobs,- });- } catch (error) {- Sentry.captureException(error);- logger.error(error);- return res.status(500).json({ error: error.message });- }-});--app.get("/serverHealthCheck/notify", async (req, res) => {- if (process.env.SLACK_WEBHOOK_URL) {- const treshold = 1; // The treshold value for the active jobs- const timeout = 60000; // 1 minute // The timeout value for the check in milliseconds-- const getWaitingJobsCount = async () => {- const scrapeQueue = getScrapeQueue();- const [waitingJobsCount] = await Promise.all([- scrapeQueue.getWaitingCount(),- ]);-- return waitingJobsCount;- };-- res.status(200).json({ message: "Check initiated" });-- const checkWaitingJobs = async () => {- try {- let waitingJobsCount = await getWaitingJobsCount();- if (waitingJobsCount >= treshold) {- setTimeout(async () => {- // Re-check the waiting jobs count after the timeout- waitingJobsCount = await getWaitingJobsCount();- if (waitingJobsCount >= treshold) {- const slackWebhookUrl = process.env.SLACK_WEBHOOK_URL!;- const message = {- text: `⚠️ Warning: The number of active jobs (${waitingJobsCount}) has exceeded the threshold (${treshold}) for more than ${- timeout / 60000- } minute(s).`,- };-- const response = await fetch(slackWebhookUrl, {- method: "POST",- headers: {- "Content-Type": "application/json",- },- body: JSON.stringify(message),- });-- if (!response.ok) {- logger.error("Failed to send Slack notification");- }- }- }, timeout);- }- } catch (error) {- Sentry.captureException(error);- logger.debug(error);- }- };-- checkWaitingJobs();- }-});-app.get("/is-production", (req, res) => {res.send({ isProduction: global.isProduction });});@@ -323,12 +253,3 @@ app.use();logger.info(`Worker ${process.pid} started`);-// const sq = getScrapeQueue();--// sq.on("waiting", j => ScrapeEvents.logJobEvent(j, "waiting"));-// sq.on("active", j => ScrapeEvents.logJobEvent(j, "active"));-// sq.on("completed", j => ScrapeEvents.logJobEvent(j, "completed"));-// sq.on("paused", j => ScrapeEvents.logJobEvent(j, "paused"));-// sq.on("resumed", j => ScrapeEvents.logJobEvent(j, "resumed"));-// sq.on("removed", j => ScrapeEvents.logJobEvent(j, "removed"));-//diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.tsindex 42527634b..e56721769 100644--- a/apps/api/src/lib/concurrency-limit.ts+++ b/apps/api/src/lib/concurrency-limit.ts@@ -1,11 +1,10 @@import { RateLimiterMode } from "../types";import { redisEvictConnection } from "../services/redis";-import type { Job, JobsOptions } from "bullmq";import { getACUCTeam } from "../controllers/auth";import { getCrawl, StoredCrawl } from "./crawl-redis";-import { getScrapeQueue } from "../services/queue-service";import { logger } from "./logger";import { abTestJob } from "../services/ab-test";+import { scrapeQueue, type NuQJob } from "../services/worker/nuq";const constructKey = (team_id: string) => "concurrency-limiter:" + team_id;const constructQueueKey = (team_id: string) =>@@ -65,8 +64,7 @@ export async function removeConcurrencyLimitActiveJob(export type ConcurrencyLimitedJob = {id: string;data: any;- opts: JobsOptions;- priority?: number;+ priority: number;};export async function cleanOldConcurrencyLimitedJobs(@@ -311,7 +309,7 @@ async function getNextConcurrentJob(** @param job The BullMQ job that is done.*/-export async function concurrentJobDone(job: Job) {+export async function concurrentJobDone(job: NuQJob<any>) {if (job.id && job.data && job.data.team_id) {await removeConcurrencyLimitActiveJob(job.data.team_id, job.id);await cleanOldConcurrencyLimitEntries(job.data.team_id);@@ -360,17 +358,13 @@ export async function concurrentJobDone(job: Job) {abTestJob(nextJob.job.data);- (await getScrapeQueue()).add(+ await scrapeQueue.addJob(nextJob.job.id,{...nextJob.job.data,concurrencyLimitHit: true,},- {- ...nextJob.job.opts,- jobId: nextJob.job.id,- priority: nextJob.job.priority,- },+ nextJob.job.priority,);}}diff --git a/apps/api/src/lib/extract/document-scraper.ts b/apps/api/src/lib/extract/document-scraper.tsindex becd6ea9e..d7b70618e 100644--- a/apps/api/src/lib/extract/document-scraper.ts+++ b/apps/api/src/lib/extract/document-scraper.ts@@ -5,12 +5,12 @@ import {URLTrace,scrapeOptions as scrapeOptionsSchema,} from "../../controllers/v2/types";-import { getScrapeQueue } from "../../services/queue-service";import { waitForJob } from "../../services/queue-jobs";import { addScrapeJob } from "../../services/queue-jobs";import { getJobPriority } from "../job-priority";import type { Logger } from "winston";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";+import { scrapeQueue } from "../../services/worker/nuq";interface ScrapeDocumentOptions {url: string;@@ -69,14 +69,16 @@ export async function scrapeDocument(zeroDataRetention: false, // not supportedapiKeyId: options.apiKeyId,},- {},jobId,jobPriority,);- const doc = await waitForJob(jobId, timeout);-- await getScrapeQueue().remove(jobId);+ const doc = await waitForJob(jobId, timeout, false, logger);+ try {+ await scrapeQueue.removeJob(jobId);+ } catch (error) {+ logger.warn("Error removing job from queue", { error, scrapeId: jobId });+ }if (trace) {trace.timing.completedAt = new Date().toISOString();diff --git a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts b/apps/api/src/lib/extract/fire-0/document-scraper-f0.tsindex cf9d04661..df7319b8e 100644--- a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts+++ b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts@@ -5,12 +5,12 @@ import {URLTrace,scrapeOptions as scrapeOptionsSchema,} from "../../../controllers/v2/types";-import { getScrapeQueue } from "../../../services/queue-service";import { waitForJob } from "../../../services/queue-jobs";import { addScrapeJob } from "../../../services/queue-jobs";import { getJobPriority } from "../../job-priority";import type { Logger } from "winston";import { isUrlBlocked } from "../../../scraper/WebScraper/utils/blocklist";+import { scrapeQueue } from "../../../services/worker/nuq";interface ScrapeDocumentOptions {url: string;@@ -68,13 +68,16 @@ export async function scrapeDocument_F0(zeroDataRetention: false, // not supportedapiKeyId: options.apiKeyId,},- {},jobId,jobPriority,);- const doc = await waitForJob(jobId, timeout);- await getScrapeQueue().remove(jobId);+ const doc = await waitForJob(jobId, timeout, false, logger);+ try {+ await scrapeQueue.removeJob(jobId);+ } catch (error) {+ logger.warn("Error removing job from queue", { error, scrapeId: jobId });+ }if (trace) {trace.timing.completedAt = new Date().toISOString();diff --git a/apps/api/src/lib/logger.ts b/apps/api/src/lib/logger.tsindex 10a1e11a5..29d78e2d5 100644--- a/apps/api/src/lib/logger.ts+++ b/apps/api/src/lib/logger.ts@@ -35,6 +35,17 @@ const zeroDataRetentionFilter = winston.format(info => {return info;})();+const reduceNoiseFilter = winston.format(info => {+ if (+ (info.metadata?.module === "nuq/metrics" ||+ info.module === "nuq/metrics") &&+ process.env.NUQ_REDUCE_NOISE === "true"+ ) {+ return false; // Don't log this message+ }+ return info;+})();+export const logger = winston.createLogger({level: process.env.LOGGING_LEVEL?.toLowerCase() ?? "debug",format: winston.format.json({@@ -71,6 +82,7 @@ export const logger = winston.createLogger({: []),new winston.transports.Console({format: winston.format.combine(+ reduceNoiseFilter,zeroDataRetentionFilter,winston.format.timestamp({ format: "YYYY-MM-DD HH:mm:ss" }),winston.format.metadata({diff --git a/apps/api/src/lib/scrape-events.ts b/apps/api/src/lib/scrape-events.tsdeleted file mode 100644index a50f8fba1..000000000--- a/apps/api/src/lib/scrape-events.ts+++ /dev/null@@ -1,109 +0,0 @@-// import { Job } from "bullmq";-// import { supabase_service as supabase } from "../services/supabase";-// import { logger } from "./logger";-// import { configDotenv } from "dotenv";-// import { Engine } from "../scraper/scrapeURL/engines";-// configDotenv();--// export type ScrapeErrorEvent = {-// type: "error";-// message: string;-// stack?: string;-// };--// export type ScrapeScrapeEvent = {-// type: "scrape";-// url: string;-// worker?: string;-// method: Engine;-// result: null | {-// success: boolean;-// response_code?: number;-// response_size?: number;-// error?: string | object;-// // proxy?: string,-// time_taken: number;-// };-// };--// export type ScrapeQueueEvent = {-// type: "queue";-// event:-// | "waiting"-// | "active"-// | "completed"-// | "paused"-// | "resumed"-// | "removed"-// | "failed";-// worker?: string;-// };--// export type ScrapeEvent =-// | ScrapeErrorEvent-// | ScrapeScrapeEvent-// | ScrapeQueueEvent;--// export class ScrapeEvents {-// static async insert(jobId: string, content: ScrapeEvent) {-// if (jobId === "TEST") return null;--// const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";-// if (useDbAuthentication) {-// try {-// const result = await supabase-// .from("scrape_events")-// .insert({-// job_id: jobId,-// type: content.type,-// content: content,-// // created_at-// })-// .select()-// .single();-// return (result.data as any).id;-// } catch (error) {-// // logger.error(`Error inserting scrape event: ${error}`);-// return null;-// }-// }--// return null;-// }--// static async updateScrapeResult(-// logId: number | null,-// result: ScrapeScrapeEvent["result"],-// ) {-// if (logId === null) return;--// try {-// const previousLog = (-// await supabase.from("scrape_events").select().eq("id", logId).single()-// ).data as any;-// await supabase-// .from("scrape_events")-// .update({-// content: {-// ...previousLog.content,-// result,-// },-// })-// .eq("id", logId);-// } catch (error) {-// logger.error(`Error updating scrape result: ${error}`);-// }-// }--// static async logJobEvent(job: Job | any, event: ScrapeQueueEvent["event"]) {-// try {-// await this.insert(((job as any).id ? (job as any).id : job) as string, {-// type: "queue",-// event,-// worker: process.env.FLY_MACHINE_ID,-// });-// } catch (error) {-// logger.error(`Error logging job event: ${error}`);-// }-// }-// }diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.tsindex e500ebd4f..7d90faffe 100644--- a/apps/api/src/main/runWebScraper.ts+++ b/apps/api/src/main/runWebScraper.ts@@ -1,9 +1,8 @@-import { Job } from "bullmq";import { WebScraperOptions, RunWebScraperParams } from "../types";-import { supabase_service } from "../services/supabase";import { logger as _logger } from "../lib/logger";import { configDotenv } from "dotenv";import { scrapeURL, ScrapeUrlResponse } from "../scraper/scrapeURL";+import type { NuQJob } from "../services/worker/nuq";import { CostTracking } from "../lib/cost-tracking";configDotenv();@@ -11,7 +10,7 @@ export async function startWebScraperPipeline({job,costTracking,}: {- job: Job<WebScraperOptions> & { id: string };+ job: NuQJob<WebScraperOptions>;costTracking: CostTracking;}) {return await runWebScraper({@@ -33,8 +32,8 @@ export async function startWebScraperPipeline({...job.data.internalOptions,},team_id: job.data.team_id,- bull_job_id: job.id.toString(),- priority: job.opts.priority,+ bull_job_id: job.id,+ priority: job.priority,is_scrape: job.data.is_scrape ?? false,is_crawl: !!(job.data.crawl_id && job.data.crawlerOptions !== null),urlInvisibleInCurrentCrawl:@@ -172,40 +171,3 @@ export async function runWebScraper({}}}--const saveJob = async (job: Job, result: any, mode: string) => {- try {- const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";- if (useDbAuthentication) {- const { data, error } = await supabase_service- .from("firecrawl_jobs")- .update({ docs: result })- .eq("job_id", job.id);-- if (error) throw new Error(error.message);- // try {- // if (mode === "crawl") {- // await job.moveToCompleted(null, token, false);- // } else {- // await job.moveToCompleted(result, token, false);- // }- // } catch (error) {- // // I think the job won't exist here anymore- // }- // } else {- // try {- // await job.moveToCompleted(result, token, false);- // } catch (error) {- // // I think the job won't exist here anymore- // }- }- // ScrapeEvents.logJobEvent(job, "completed");- } catch (error) {- _logger.error(`🐂 Failed to update job status`, {- module: "runWebScraper",- method: "saveJob",- jobId: job.id,- scrapeId: job.id,- });- }-};diff --git a/apps/api/src/routes/admin.ts b/apps/api/src/routes/admin.tsindex 5a8a83b8f..c455aa549 100644--- a/apps/api/src/routes/admin.ts+++ b/apps/api/src/routes/admin.ts@@ -1,11 +1,5 @@import express from "express";import { redisHealthController } from "../controllers/v0/admin/redis-health";-import {- autoscalerController,- checkQueuesController,- cleanBefore24hCompleteJobsController,- queuesController,-} from "../controllers/v0/admin/queue";import { wrap } from "./shared";import { acucCacheClearController } from "../controllers/v0/admin/acuc-cache-clear";import { checkFireEngine } from "../controllers/v0/admin/check-fire-engine";@@ -22,23 +16,6 @@ adminRouter.get(redisHealthController,);-adminRouter.get(- `/admin/${process.env.BULL_AUTH_KEY}/clean-before-24h-complete-jobs`,- cleanBefore24hCompleteJobsController,-);--adminRouter.get(- `/admin/${process.env.BULL_AUTH_KEY}/check-queues`,- checkQueuesController,-);--adminRouter.get(`/admin/${process.env.BULL_AUTH_KEY}/queues`, queuesController);--adminRouter.get(- `/admin/${process.env.BULL_AUTH_KEY}/autoscaler`,- autoscalerController,-);-adminRouter.post(`/admin/${process.env.BULL_AUTH_KEY}/acuc-cache-clear`,wrap(acucCacheClearController),diff --git a/apps/api/src/services/alerts/index.ts b/apps/api/src/services/alerts/index.tsdeleted file mode 100644index 44f2b8a04..000000000--- a/apps/api/src/services/alerts/index.ts+++ /dev/null@@ -1,62 +0,0 @@-import { logger } from "../../../src/lib/logger";-import { getScrapeQueue } from "../queue-service";-import { sendSlackWebhook } from "./slack";--export async function checkAlerts() {- try {- if (- process.env.SLACK_WEBHOOK_URL &&- process.env.ENV === "production" &&- process.env.ALERT_NUM_ACTIVE_JOBS &&- process.env.ALERT_NUM_WAITING_JOBS- ) {- logger.info("Initializing alerts");- const checkActiveJobs = async () => {- try {- const scrapeQueue = getScrapeQueue();- const activeJobs = await scrapeQueue.getActiveCount();- if (activeJobs > Number(process.env.ALERT_NUM_ACTIVE_JOBS)) {- logger.warn(- `Alert: Number of active jobs is over ${process.env.ALERT_NUM_ACTIVE_JOBS}. Current active jobs: ${activeJobs}.`,- );- sendSlackWebhook(- `Alert: Number of active jobs is over ${process.env.ALERT_NUM_ACTIVE_JOBS}. Current active jobs: ${activeJobs}`,- true,- );- } else {- logger.info(- `Number of active jobs is under ${process.env.ALERT_NUM_ACTIVE_JOBS}. Current active jobs: ${activeJobs}`,- );- }- } catch (error) {- logger.error(`Failed to check active jobs: ${error}`);- }- };-- const checkWaitingQueue = async () => {- const scrapeQueue = getScrapeQueue();- const waitingJobs = await scrapeQueue.getWaitingCount();-- if (waitingJobs > Number(process.env.ALERT_NUM_WAITING_JOBS)) {- logger.warn(- `Alert: Number of waiting jobs is over ${process.env.ALERT_NUM_WAITING_JOBS}. Current waiting jobs: ${waitingJobs}.`,- );- sendSlackWebhook(- `Alert: Number of waiting jobs is over ${process.env.ALERT_NUM_WAITING_JOBS}. Current waiting jobs: ${waitingJobs}. Scale up the number of workers with fly scale count worker=20`,- true,- );- }- };-- const checkAll = async () => {- await checkActiveJobs();- await checkWaitingQueue();- };-- await checkAll();- // setInterval(checkAll, 10000); // Run every- }- } catch (error) {- logger.error(`Failed to initialize alerts: ${error}`);- }-}diff --git a/apps/api/src/services/billing/batch_billing.ts b/apps/api/src/services/billing/batch_billing.tsindex 5962f192a..8e675129b 100644--- a/apps/api/src/services/billing/batch_billing.ts+++ b/apps/api/src/services/billing/batch_billing.ts@@ -2,13 +2,8 @@ import { logger } from "../../lib/logger";import { getRedisConnection } from "../queue-service";import { supabase_service } from "../supabase";import * as Sentry from "@sentry/node";-import { Queue } from "bullmq";import { withAuth } from "../../lib/withAuth";-import {- getACUC,- setCachedACUC,- setCachedACUCTeam,-} from "../../controllers/auth";+import { setCachedACUC, setCachedACUCTeam } from "../../controllers/auth";// Configuration constantsconst BATCH_KEY = "billing_batch";diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.tsindex c85f101ff..a70100f7a 100644--- a/apps/api/src/services/indexing/index-worker.ts+++ b/apps/api/src/services/indexing/index-worker.ts@@ -218,9 +218,7 @@ const processPrecrawlJobInternal = async (token: string, job: Job) => {zeroDataRetention: false,apiKeyId: null,},- {},crypto.randomUUID(),- 10,);} catch (e) {logger.error("Error processing one cycle of the precrawl job", {diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.tsindex 8fda3d39e..cd0907205 100644--- a/apps/api/src/services/logging/log_job.ts+++ b/apps/api/src/services/logging/log_job.ts@@ -55,6 +55,10 @@ export async function logJob(});try {+ if (process.env.GCS_BUCKET_NAME) {+ await saveJobToGCS(job);+ }+const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";if (!useDbAuthentication) {return;@@ -116,10 +120,6 @@ export async function logJob(: null,};- if (process.env.GCS_BUCKET_NAME) {- await saveJobToGCS(job);- }-if (bypassLogging) {return;}diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.tsindex 986ee1818..67141224a 100644--- a/apps/api/src/services/queue-jobs.ts+++ b/apps/api/src/services/queue-jobs.ts@@ -1,7 +1,5 @@-import { getScrapeQueue, getScrapeQueueEvents } from "./queue-service";import { v4 as uuidv4 } from "uuid";import { NotificationType, RateLimiterMode, WebScraperOptions } from "../types";-import * as Sentry from "@sentry/node";import {cleanOldConcurrencyLimitEntries,getConcurrencyLimitActiveJobs,@@ -14,16 +12,15 @@ import {import { logger as _logger } from "../lib/logger";import { sendNotificationWithCustomDays } from "./notification/email_notification";import { shouldSendConcurrencyLimitNotification } from "./notification/notification-check";-import { getACUC, getACUCTeam } from "../controllers/auth";+import { getACUCTeam } from "../controllers/auth";import { getJobFromGCS, removeJobFromGCS } from "../lib/gcs-jobs";import { Document } from "../controllers/v1/types";import { getCrawl } from "../lib/crawl-redis";import { Logger } from "winston";-import { Job } from "bullmq";import { ScrapeJobTimeoutError, TransportableError } from "../lib/error";import { deserializeTransportableError } from "../lib/error-serde";-import { robustFetch } from "../scraper/scrapeURL/lib/fetch";import { abTestJob } from "./ab-test";+import { NuQJob, scrapeQueue } from "./worker/nuq";/*** Checks if a job is a crawl or batch scrape based on its options@@ -41,21 +38,15 @@ function isCrawlOrBatchScrape(options: {async function _addScrapeJobToConcurrencyQueue(webScraperOptions: any,- options: any,jobId: string,- jobPriority: number,+ priority: number = 0,) {await pushConcurrencyLimitedJob(webScraperOptions.team_id,{id: jobId,data: webScraperOptions,- opts: {- ...options,- priority: jobPriority,- jobId: jobId,- },- priority: jobPriority,+ priority,},webScraperOptions.crawl_id? Infinity@@ -65,10 +56,9 @@ async function _addScrapeJobToConcurrencyQueue(export async function _addScrapeJobToBullMQ(webScraperOptions: WebScraperOptions,- options: any,jobId: string,- jobPriority: number,-): Promise<Job> {+ priority: number = 0,+): Promise<NuQJob<WebScraperOptions>> {abTestJob(webScraperOptions);if (webScraperOptions && webScraperOptions.team_id) {@@ -90,20 +80,15 @@ export async function _addScrapeJobToBullMQ(}}- return await getScrapeQueue().add(jobId, webScraperOptions, {- ...options,- priority: jobPriority,- jobId,- });+ return await scrapeQueue.addJob(jobId, webScraperOptions, priority);}async function addScrapeJobRaw(webScraperOptions: WebScraperOptions,- options: any,jobId: string,- jobPriority: number,+ priority: number = 0,directToBullMQ: boolean = false,-): Promise<Job | null> {+): Promise<NuQJob<WebScraperOptions> | null> {let concurrencyLimited: "yes" | "yes-crawl" | "no" | null = null;let currentActiveConcurrency = 0;let maxConcurrency = 0;@@ -188,46 +173,32 @@ async function addScrapeJobRaw(webScraperOptions.concurrencyLimited = true;- await _addScrapeJobToConcurrencyQueue(- webScraperOptions,- options,- jobId,- jobPriority,- );+ await _addScrapeJobToConcurrencyQueue(webScraperOptions, jobId);return null;} else {- return await _addScrapeJobToBullMQ(- webScraperOptions,- options,- jobId,- jobPriority,- );+ return await _addScrapeJobToBullMQ(webScraperOptions, jobId, priority);}}export async function addScrapeJob(webScraperOptions: WebScraperOptions,- options: any = {},jobId: string = uuidv4(),- jobPriority: number = 10,+ priority: number = 0,directToBullMQ: boolean = false,-): Promise<Job | null> {+): Promise<NuQJob<WebScraperOptions> | null> {return await addScrapeJobRaw(webScraperOptions,- options,jobId,- jobPriority,+ priority,directToBullMQ,);}export async function addScrapeJobs(jobs: {+ jobId: string;data: WebScraperOptions;- opts: {- jobId: string;- priority: number;- };+ priority: number;}[],) {if (jobs.length === 0) return true;@@ -235,11 +206,9 @@ export async function addScrapeJobs(const jobsByTeam = new Map<string,{+ jobId: string;data: WebScraperOptions;- opts: {- jobId: string;- priority: number;- };+ priority: number;}[]>();@@ -254,18 +223,14 @@ export async function addScrapeJobs(// == Buckets for jobs ==let jobsForcedToCQ: {data: WebScraperOptions;- opts: {- jobId: string;- priority: number;- };+ jobId: string;+ priority: number;}[] = [];let jobsPotentiallyInCQ: {data: WebScraperOptions;- opts: {- jobId: string;- priority: number;- };+ jobId: string;+ priority: number;}[] = [];// == Select jobs by crawl ID ==@@ -273,19 +238,15 @@ export async function addScrapeJobs(string,{data: WebScraperOptions;- opts: {- jobId: string;- priority: number;- };+ jobId: string;+ priority: number;}[]>();const jobsWithoutCrawlID: {data: WebScraperOptions;- opts: {- jobId: string;- priority: number;- };+ jobId: string;+ priority: number;}[] = [];for (const job of teamJobs) {@@ -386,66 +347,46 @@ export async function addScrapeJobs(const size = JSON.stringify(job.data).length;await _addScrapeJobToConcurrencyQueue(job.data,- job.opts,- job.opts.jobId,- job.opts.priority,+ job.jobId,+ job.priority,);}),);await Promise.all(addToBull.map(async job => {- const size = JSON.stringify(job.data).length;- await _addScrapeJobToBullMQ(- job.data,- job.opts,- job.opts.jobId,- job.opts.priority,- );+ await _addScrapeJobToBullMQ(job.data, job.jobId, job.priority);}),);}}export async function waitForJob(- _job: Job | string,+ job: NuQJob<WebScraperOptions> | string,timeout: number | null,+ zeroDataRetention: boolean,logger: Logger = _logger,): Promise<Document> {- const start = Date.now();- const queue = getScrapeQueue();- let job: Job | undefined =- typeof _job == "string" ? await queue.getJob(_job) : _job;- while (job === undefined) {- logger.debug("Waiting for job to be created");- await new Promise(resolve => setTimeout(resolve, 500));- job = await queue.getJob(_job as string);- if (Date.now() - start > (timeout ?? 180000)) {- throw new ScrapeJobTimeoutError(- "Scrape timed out while waiting in the concurrency limit queue",- );- }- }- let doc: Document;+ const jobId = typeof job == "string" ? job : job.id;+ const isConcurrencyLimited = !!(typeof job === "string");++ let doc: Document | null = null;try {doc = await Promise.race([- job.waitUntilFinished(getScrapeQueueEvents(), timeout ?? 180000),+ scrapeQueue.waitForJob(jobId, timeout !== null ? timeout + 100 : null),timeout !== null- ? new Promise((resolve, reject) => {- setTimeout(- () => {- reject(- new ScrapeJobTimeoutError(- "Scrape timed out" +- (typeof _job === "string"- ? " after waiting in the concurrency limit queue"- : ""),- ),- );- },- Math.max(0, timeout - (Date.now() - start)),- );+ ? new Promise<Document>((_resolve, reject) => {+ setTimeout(() => {+ reject(+ new ScrapeJobTimeoutError(+ "Scrape timed out" ++ (isConcurrencyLimited+ ? " after waiting in the concurrency limit queue"+ : ""),+ ),+ );+ }, timeout);}): null,].filter(x => x !== null),@@ -467,15 +408,15 @@ export async function waitForJob(logger.debug("Got job");if (!doc) {- const docs = await getJobFromGCS(job.id!);+ const docs = await getJobFromGCS(jobId);logger.debug("Got job from GCS");if (!docs || docs.length === 0) {throw new Error("Job not found in GCS");}- doc = docs[0];+ doc = docs[0]!;- if (job.data?.internalOptions?.zeroDataRetention) {- await removeJobFromGCS(job.id!);+ if (zeroDataRetention) {+ await removeJobFromGCS(jobId);}}diff --git a/apps/api/src/services/queue-service.ts b/apps/api/src/services/queue-service.tsindex 10d7c6474..5e0f68f47 100644--- a/apps/api/src/services/queue-service.ts+++ b/apps/api/src/services/queue-service.ts@@ -1,4 +1,4 @@-import { Queue, QueueEvents } from "bullmq";+import { Queue } from "bullmq";import { logger } from "../lib/logger";import IORedis from "ioredis";import { BullMQOtel } from "bullmq-otel";@@ -6,8 +6,6 @@ import type { DeepResearchServiceOptions } from "../lib/deep-research/deep-reseaexport type QueueFunction = () => Queue<any, any, string, any, any, string>;-let scrapeQueue: Queue;-let scrapeQueueEvents: QueueEvents;let extractQueue: Queue;let loggingQueue: Queue;let indexQueue: Queue;@@ -29,7 +27,6 @@ export function getRedisConnection(): IORedis {return redisConnection;}-export const scrapeQueueName = "{scrapeQueue}";export const extractQueueName = "{extractQueue}";export const loggingQueueName = "{loggingQueue}";export const indexQueueName = "{indexQueue}";@@ -38,34 +35,6 @@ export const deepResearchQueueName = "{deepResearchQueue}";export const billingQueueName = "{billingQueue}";export const precrawlQueueName = "{precrawlQueue}";-export function getScrapeQueue() {- if (!scrapeQueue) {- scrapeQueue = new Queue(scrapeQueueName, {- connection: getRedisConnection(),- defaultJobOptions: {- removeOnComplete: {- age: 3600, // 1 hour- },- removeOnFail: {- age: 3600, // 1 hour- },- },- telemetry: new BullMQOtel("firecrawl-bullmq"),- });- }- return scrapeQueue;-}--export function getScrapeQueueEvents() {- if (!scrapeQueueEvents) {- scrapeQueueEvents = new QueueEvents(scrapeQueueName, {- connection: getRedisConnection(),- });- }-- return scrapeQueueEvents;-}-export function getExtractQueue() {if (!extractQueue) {extractQueue = new Queue(extractQueueName, {diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 6b3fae215..83494420f 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -2,27 +2,16 @@ import "dotenv/config";import "./sentry";import * as Sentry from "@sentry/node";import {- getScrapeQueue,getExtractQueue,getDeepResearchQueue,getGenerateLlmsTxtQueue,- scrapeQueueName,getRedisConnection,} from "./queue-service";-import { Job, Queue, QueueEvents } from "bullmq";+import { Job, Queue, Worker } from "bullmq";import { logger as _logger } from "../lib/logger";-import { Worker } from "bullmq";import systemMonitor from "./system-monitor";import { v4 as uuidv4 } from "uuid";-import {- addCrawlJobDone,- finishCrawlKickoff,- getCrawl,- normalizeURL,-} from "../lib/crawl-redis";-import { StoredCrawl } from "../lib/crawl-redis";import { configDotenv } from "dotenv";-import { concurrentJobDone } from "../lib/concurrency-limit";import {ExtractResult,performExtraction,@@ -39,9 +28,6 @@ import http from "http";import https from "https";import { cacheableLookup } from "../scraper/scrapeURL/lib/cacheableLookup";import { robustFetch } from "../scraper/scrapeURL/lib/fetch";-import { redisEvictConnection } from "./redis";-import path from "path";-import { finishCrawlIfNeeded } from "./worker/crawl-logic";import { NodeSDK } from "@opentelemetry/sdk-node";import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node";import { LangfuseExporter } from "langfuse-vercel";@@ -414,62 +400,6 @@ process.on("SIGTERM", () => {let cantAcceptConnectionCount = 0;-/**- * Converts a file path to a proper URL for cross-platform compatibility.- * On Windows, absolute paths need to be converted to file:// URLs for ESM imports.- * @param filePath - The file path to convert- * @returns A properly formatted path/URL for the current platform- */-function getWorkerPath(filePath: string): string | URL {- if (process.platform === "win32" && path.isAbsolute(filePath)) {- // On Windows, convert absolute paths to file:// URLs for ESM compatibility- return pathToFileURL(filePath);- }- return filePath;-}--const separateWorkerFun = (queue: Queue, workerPath: string): Worker => {- // Extract memory size from --max-old-space-size flag if present- const maxOldSpaceSize =- process.env.SCRAPE_WORKER_MAX_OLD_SPACE_SIZE ||- process.execArgv- .find(arg => arg.startsWith("--max-old-space-size="))- ?.split("=")[1];-- // Filter out the invalid flag for worker threads- const filteredExecArgv = process.execArgv.filter(- arg => !arg.startsWith("--max-old-space-size"),- );-- // Convert path to proper format for the current platform- const platformWorkerPath = getWorkerPath(workerPath);-- const worker = new Worker(queue.name, platformWorkerPath, {- connection: getRedisConnection(),- lockDuration: 60 * 1000, // 60 seconds- stalledInterval: 60 * 1000, // 60 seconds- maxStalledCount: 10, // 10 times- concurrency: 8,- useWorkerThreads: false,- workerForkOptions: {- execArgv: filteredExecArgv.concat(- maxOldSpaceSize ? ["--max-old-space-size=" + maxOldSpaceSize] : [],- ),- },- workerThreadsOptions: {- execArgv: filteredExecArgv,- resourceLimits: maxOldSpaceSize- ? {- maxOldGenerationSizeMb: parseInt(maxOldSpaceSize),- }- : undefined,- },- telemetry: new BullMQOtel("firecrawl-bullmq"),- });-- return worker;-};-const workerFun = async (queue: Queue,processJobInternal: (token: string, job: Job) => Promise<any>,@@ -526,17 +456,11 @@ const workerFun = async (runningJobs.add(job.id);}- async function afterJobDone(job: Job<any, any, string>) {- try {- await concurrentJobDone(job);- } finally {- if (job.id) {- runningJobs.delete(job.id);- }+ processJobInternal(token, job).finally(() => {+ if (job.id) {+ runningJobs.delete(job.id);}- }-- processJobInternal(token, job).finally(() => afterJobDone(job));+ });await sleep(gotJobInterval);} else {@@ -588,82 +512,7 @@ app.listen(workerPort, () => {});(async () => {- async function failedListener(args: {- jobId: string;- failedReason: string;- prev?: string | undefined;- }) {- const job = await getScrapeQueue().getJob(args.jobId);-- if (job && job.data.crawl_id) {- await redisEvictConnection.srem(- "crawl:" + job.data.crawl_id + ":jobs_qualified",- args.jobId,- );- await redisEvictConnection.expire(- "crawl:" + job.data.crawl_id + ":jobs_qualified",- 24 * 60 * 60,- );- }-- if (args.failedReason === "job stalled more than allowable limit") {- const set = await redisEvictConnection.set(- "stalled-job-cleaner:" + args.jobId,- "1",- "EX",- 60 * 60 * 24,- "NX",- );- if (!set) {- return;- }-- let logger = _logger.child({- jobId: args.jobId,- scrapeId: args.jobId,- module: "queue-worker",- method: "failedListener",- zeroDataRetention: job?.data.zeroDataRetention,- });- if (job && job.data.crawl_id) {- logger = logger.child({ crawlId: job.data.crawl_id });- logger.warn("Job stalled more than allowable limit");-- const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;-- if (job.data.mode === "kickoff") {- await finishCrawlKickoff(job.data.crawl_id);- if (sc) {- await finishCrawlIfNeeded(job, sc);- }- } else {- const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;-- logger.debug("Declaring job as done...");- await addCrawlJobDone(job.data.crawl_id, job.id, false, logger);- await redisEvictConnection.srem(- "crawl:" + job.data.crawl_id + ":visited_unique",- normalizeURL(job.data.url, sc),- );-- await finishCrawlIfNeeded(job, sc);- }- } else {- logger.warn("Job stalled more than allowable limit");- }- }- }-- const scrapeQueueEvents = new QueueEvents(scrapeQueueName, {- connection: getRedisConnection(),- });- scrapeQueueEvents.on("failed", failedListener);-- const results = await Promise.all([- separateWorkerFun(- getScrapeQueue(),- path.join(__dirname, "worker", "scrape-worker.js"),- ),+ await Promise.all([workerFun(getExtractQueue(), processExtractJobInternal),workerFun(getDeepResearchQueue(), processDeepResearchJobInternal),workerFun(getGenerateLlmsTxtQueue(), processGenerateLlmsTxtJobInternal),@@ -671,26 +520,10 @@ app.listen(workerPort, () => {console.log("All workers exited. Waiting for all jobs to finish...");- const workerResults = results.filter(x => x instanceof Worker);- await Promise.all(workerResults.map(x => x.close()));-while (runningJobs.size > 0) {await new Promise(resolve => setTimeout(resolve, 500));}- setInterval(async () => {- _logger.debug("Currently running jobs", {- jobs: (- await Promise.all(- [...runningJobs].map(async jobId => {- return await getScrapeQueue().getJob(jobId);- }),- )- ).filter(x => x && !x.data?.zeroDataRetention),- });- }, 1000);-- await scrapeQueueEvents.close();console.log("All jobs finished. Worker out!");if (otelSdk) {await otelSdk.shutdown();diff --git a/apps/api/src/services/worker/crawl-logic.ts b/apps/api/src/services/worker/crawl-logic.tsindex c3971e76d..ff1e7db0c 100644--- a/apps/api/src/services/worker/crawl-logic.ts+++ b/apps/api/src/services/worker/crawl-logic.ts@@ -1,5 +1,4 @@import { logger as _logger } from "../../lib/logger";-import { Job } from "bullmq";import {addCrawlJobs,finishCrawl,@@ -21,11 +20,9 @@ import { getJobs } from "../../controllers/v1/crawl-status";import { logJob } from "../logging/log_job";import { createWebhookSender, WebhookEvent } from "../webhook";import { hasFormatOfType } from "../../lib/format-utils";+import type { NuQJob } from "./nuq";-export async function finishCrawlIfNeeded(- job: Job & { id: string },- sc: StoredCrawl,-) {+export async function finishCrawlIfNeeded(job: NuQJob<any>, sc: StoredCrawl) {const logger = _logger.child({module: "queue-worker",method: "finishCrawlIfNeeded",@@ -113,7 +110,7 @@ export async function finishCrawlIfNeeded(const jobs = univistedUrls.links.slice(0, addableJobCount).map(url => {const uuid = uuidv4();return {- name: uuid,+ jobId: uuid,data: {url,mode: "single_urls" as const,@@ -133,24 +130,21 @@ export async function finishCrawlIfNeeded(zeroDataRetention: job.data.zeroDataRetention,apiKeyId: job.data.apiKeyId,},- opts: {- jobId: uuid,- priority: 20,- },+ priority: 20, // TODO: make this dynamic};});const lockedIds = await lockURLsIndividually(job.data.crawl_id,sc,- jobs.map(x => ({ id: x.opts.jobId, url: x.data.url })),+ jobs.map(x => ({ id: x.jobId, url: x.data.url })),);const lockedJobs = jobs.filter(x =>- lockedIds.find(y => y.id === x.opts.jobId),+ lockedIds.find(y => y.id === x.jobId),);await addCrawlJobs(job.data.crawl_id,- lockedJobs.map(x => x.opts.jobId),+ lockedJobs.map(x => x.jobId),logger,);await addScrapeJobs(lockedJobs);diff --git a/apps/api/src/services/worker/nuq-worker.ts b/apps/api/src/services/worker/nuq-worker.tsnew file mode 100644index 000000000..e5fc91d01--- /dev/null+++ b/apps/api/src/services/worker/nuq-worker.ts@@ -0,0 +1,119 @@+import "dotenv/config";+import { logger as _logger } from "../../lib/logger";+import { processJobInternal } from "./scrape-worker";+import { scrapeQueue, nuqGetLocalMetrics, nuqHealthCheck } from "./nuq";+import Express from "express";+import { _ } from "ajv";++(async () => {+ let isShuttingDown = false;+ const myLock = crypto.randomUUID();++ const app = Express();++ app.get("/metrics", (_, res) =>+ res.contentType("text/plain").send(nuqGetLocalMetrics()),+ );+ app.get("/health", async (_, res) => {+ if (await nuqHealthCheck()) {+ res.status(200).send("OK");+ } else {+ res.status(500).send("Not OK");+ }+ });++ const server = app.listen(+ Number(process.env.NUQ_WORKER_PORT ?? process.env.PORT ?? 3000),+ () => {+ _logger.info("NuQ worker metrics server started");+ },+ );++ function shutdown() {+ isShuttingDown = true;+ }++ process.on("SIGINT", shutdown);+ process.on("SIGTERM", shutdown);++ let noJobTimeout = 500;++ while (!isShuttingDown) {+ const job = await scrapeQueue.getJobToProcess(myLock);++ if (job === null) {+ _logger.info("No jobs to process", { module: "nuq/metrics" });+ await new Promise(resolve => setTimeout(resolve, noJobTimeout));+ noJobTimeout = Math.min(noJobTimeout * 2, 10000);+ continue;+ }++ noJobTimeout = 500;++ const logger = _logger.child({+ module: "nuq-worker",+ scrapeId: job.id,+ zeroDataRetention: job.data?.zeroDataRetention ?? false,+ });++ logger.info("Acquired job");++ const lockRenewInterval = setInterval(async () => {+ logger.info("Renewing lock");+ if (!(await scrapeQueue.renewLock(job.id, myLock, logger))) {+ logger.warn("Failed to renew lock");+ clearInterval(lockRenewInterval);+ return;+ }+ logger.info("Renewed lock");+ }, 15000);++ let processResult:+ | { ok: true; data: Awaited<ReturnType<typeof processJobInternal>> }+ | { ok: false; error: any };++ try {+ processResult = { ok: true, data: await processJobInternal(job) };+ } catch (error) {+ processResult = { ok: false, error };+ }++ clearInterval(lockRenewInterval);++ if (processResult.ok) {+ if (+ !(await scrapeQueue.jobFinish(+ job.id,+ myLock,+ processResult.data,+ logger,+ ))+ ) {+ logger.warn("Could not update job status");+ }+ } else {+ if (+ !(await scrapeQueue.jobFail(+ job.id,+ myLock,+ processResult.error instanceof Error+ ? processResult.error.message+ : typeof processResult.error === "string"+ ? processResult.error+ : JSON.stringify(processResult.error),+ logger,+ ))+ ) {+ logger.warn("Could not update job status");+ }+ }+ }++ _logger.info("NuQ worker shutting down");++ server.close(async () => {+ await scrapeQueue.shutdown();+ _logger.info("NuQ worker shut down");+ process.exit(0);+ });+})();diff --git a/apps/api/src/services/worker/nuq.ts b/apps/api/src/services/worker/nuq.tsnew file mode 100644index 000000000..993603a7a--- /dev/null+++ b/apps/api/src/services/worker/nuq.ts@@ -0,0 +1,568 @@+import { Logger } from "winston";+import { logger } from "../../lib/logger";+import { Client, Pool } from "pg";++// === Basics++const nuqPool = new Pool({+ connectionString: process.env.NUQ_DATABASE_URL, // may be a pgbouncer transaction pooler URL+ application_name: "nuq",+});++nuqPool.on("error", err =>+ logger.error("Error in NuQ idle client", { err, module: "nuq" }),+);++export type NuQJobStatus = "queued" | "active" | "completed" | "failed"; // must match nuq.job_status enum+export type NuQJob<Data = any, ReturnValue = any> = {+ id: string;+ status: NuQJobStatus;+ createdAt: Date;+ priority: number;+ data: Data;+ finishedAt?: Date;+ returnvalue?: ReturnValue;+ failedReason?: string;+};++// === Queue++class NuQ<JobData = any, JobReturnValue = any> {+ constructor(public readonly queueName: string) {}++ // === Listener++ private listener: Client | null = null;+ private listens: {+ [key: string]: ((status: "completed" | "failed") => void)[];+ } = {};+ private shuttingDown = false;++ async startListener() {+ if (this.listener || this.shuttingDown) return;++ this.listener = new Client({+ connectionString:+ process.env.NUQ_DATABASE_URL_LISTEN ?? process.env.NUQ_DATABASE_URL, // will always be a direct connection+ application_name: "nuq_listener",+ });++ this.listener.on("notification", msg => {+ const tok = (msg.payload ?? "unknown|unknown").split("|");+ if (tok[0] in this.listens) {+ this.listens[tok[0]].forEach(listener =>+ listener(tok[1] as "completed" | "failed"),+ );+ delete this.listens[tok[0]];+ }+ });++ this.listener.on("error", err =>+ logger.error("Error in NuQ listener", { err, module: "nuq" }),+ );++ this.listener.on("end", () => {+ logger.info("NuQ listener disconnected", { module: "nuq" });+ this.listener = null;+ setTimeout(+ (() => {+ this.startListener().catch(err =>+ logger.error("Error in NuQ listener reconnect", {+ err,+ module: "nuq",+ }),+ );+ }).bind(this),+ 250,+ );+ });++ await this.listener.connect();+ await this.listener.query(`LISTEN "${this.queueName}";`);++ (async () => {+ const backedUpJobs = (+ await this.getJobs(Object.keys(this.listens))+ ).filter(job => ["completed", "failed"].includes(job.status));+ for (const job of backedUpJobs) {+ this.listens[job.id].forEach(listener =>+ listener(job.status as "completed" | "failed"),+ );+ delete this.listens[job.id];+ }+ })();+ }++ async addListener(+ id: string,+ listener: (status: "completed" | "failed") => void,+ ) {+ await this.startListener();++ if (!(id in this.listens)) this.listens[id] = [listener];+ else this.listens[id].push(listener);+ }++ async removeListener(+ id: string,+ listener: (status: "completed" | "failed") => void,+ ) {+ if (id in this.listens) {+ this.listens[id] = this.listens[id].filter(l => l !== listener);+ if (this.listens[id].length === 0) delete this.listens[id];+ }+ }++ // === Job management++ private readonly jobReturning = [+ "id",+ "status",+ "created_at",+ "priority",+ "data",+ "finished_at",+ "returnvalue",+ "failedreason",+ ];++ private rowToJob(row: any): NuQJob<JobData, JobReturnValue> | null {+ if (!row) return null;+ return {+ id: row.id,+ status: row.status,+ createdAt: new Date(row.created_at),+ priority: row.priority,+ data: row.data,+ finishedAt: row.finished_at ? new Date(row.finished_at) : undefined,+ returnvalue: row.returnvalue ?? undefined,+ failedReason: row.failedreason ?? undefined,+ };+ }++ public async getJob(+ id: string,+ ): Promise<NuQJob<JobData, JobReturnValue> | null> {+ const start = Date.now();+ try {+ return this.rowToJob(+ (+ await nuqPool.query(+ `SELECT ${this.jobReturning.join(", ")} FROM ${this.queueName} WHERE ${this.queueName}.id = $1;`,+ [id],+ )+ ).rows[0],+ );+ } finally {+ logger.info("nuqGetJob metrics", {+ module: "nuq/metrics",+ method: "nuqGetJob",+ duration: Date.now() - start,+ scrapeId: id,+ });+ }+ }++ public async getJobs(+ ids: string[],+ ): Promise<NuQJob<JobData, JobReturnValue>[]> {+ if (ids.length === 0) return [];++ const start = Date.now();+ try {+ return (+ await nuqPool.query(+ `SELECT ${this.jobReturning.join(", ")} FROM ${this.queueName} WHERE ${this.queueName}.id = ANY($1::uuid[]);`,+ [ids],+ )+ ).rows.map(row => this.rowToJob(row)!);+ } finally {+ logger.info("nuqGetJobs metrics", {+ module: "nuq/metrics",+ method: "nuqGetJobs",+ duration: Date.now() - start,+ scrapeIds: ids.length,+ });+ }+ }++ public async getJobsWithStatus(+ ids: string[],+ status: NuQJobStatus,+ _logger: Logger = logger,+ ): Promise<NuQJob<JobData, JobReturnValue>[]> {+ if (ids.length === 0) return [];++ const start = Date.now();+ try {+ return (+ await nuqPool.query(+ `SELECT ${this.jobReturning.join(", ")} FROM ${this.queueName} WHERE ${this.queueName}.id = ANY($1::uuid[]) AND ${this.queueName}.status = $2::nuq.job_status;`,+ [ids, status],+ )+ ).rows.map(row => this.rowToJob(row)!);+ } finally {+ _logger.info("nuqGetJobsWithStatus metrics", {+ module: "nuq/metrics",+ method: "nuqGetJobsWithStatus",+ duration: Date.now() - start,+ scrapeIds: ids.length,+ status,+ });+ }+ }++ public async getJobsWithStatuses(+ ids: string[],+ statuses: NuQJobStatus[],+ ): Promise<NuQJob<JobData, JobReturnValue>[]> {+ if (ids.length === 0) return [];++ const start = Date.now();+ try {+ return (+ await nuqPool.query(+ `SELECT ${this.jobReturning.join(", ")} FROM ${this.queueName} WHERE ${this.queueName}.id = ANY($1::uuid[]) AND ${this.queueName}.status = ANY($2::nuq.job_status[]);`,+ [ids, statuses],+ )+ ).rows.map(row => this.rowToJob(row)!);+ } finally {+ logger.info("nuqGetJobsWithStatuses metrics", {+ module: "nuq/metrics",+ method: "nuqGetJobsWithStatuses",+ duration: Date.now() - start,+ scrapeIds: ids.length,+ statuses,+ });+ }+ }++ public async removeJob(id: string): Promise<boolean> {+ const start = Date.now();+ try {+ return (+ (+ await nuqPool.query(`DELETE FROM ${this.queueName} WHERE id = $1;`, [+ id,+ ])+ ).rowCount !== 0+ );+ } finally {+ logger.info("nuqRemoveJob metrics", {+ module: "nuq/metrics",+ method: "nuqRemoveJob",+ duration: Date.now() - start,+ scrapeId: id,+ });+ }+ }++ public async removeJobs(ids: string[]): Promise<number> {+ if (ids.length === 0) return 0;++ const start = Date.now();+ try {+ return (+ (+ await nuqPool.query(+ `DELETE FROM ${this.queueName} WHERE id = ANY($1::uuid[]);`,+ [ids],+ )+ ).rowCount ?? 0+ );+ } finally {+ logger.info("nuqRemoveJobs metrics", {+ module: "nuq/metrics",+ method: "nuqRemoveJobs",+ duration: Date.now() - start,+ scrapeIds: ids.length,+ });+ }+ }++ // === Producer+ public async addJob(+ id: string,+ data: JobData,+ priority: number = 0,+ ): Promise<NuQJob<JobData, JobReturnValue>> {+ const start = Date.now();+ try {+ return this.rowToJob(+ (+ await nuqPool.query(+ `INSERT INTO ${this.queueName} (id, data, priority) VALUES ($1, $2, $3) RETURNING ${this.jobReturning.join(", ")};`,+ [id, data, priority],+ )+ ).rows[0],+ )!;+ } finally {+ logger.info("nuqAddJob metrics", {+ module: "nuq/metrics",+ method: "nuqAddJob",+ duration: Date.now() - start,+ scrapeId: id,+ zeroDataRetention: (data as any)?.zeroDataRetention ?? false,+ });+ }+ }++ private readonly nuqWaitMode =+ process.env.NUQ_WAIT_MODE === "listen"+ ? ("listen" as const)+ : ("poll" as const);++ public waitForJob(+ id: string,+ timeout: number | null,+ ): Promise<JobReturnValue> {+ const done = new Promise<JobReturnValue>(+ (async (resolve, reject) => {+ if (this.nuqWaitMode === "listen") {+ let timer: NodeJS.Timeout | null = null;+ if (timeout !== null) {+ timer = setTimeout(+ (() => {+ this.removeListener(id, listener);+ reject(new Error("Timed out"));+ }).bind(this),+ timeout,+ );+ }++ const listener = async function (_msg: "completed" | "failed") {+ if (timer) clearTimeout(timer);+ const job = await this.getJob(id);+ if (!job) {+ reject(new Error("Job raced out while waiting for it"));+ } else {+ if (job.status === "completed") {+ resolve(job.returnvalue!);+ } else {+ reject(new Error(job.failedReason!));+ }+ }+ }.bind(this);++ try {+ await this.addListener(id, listener);+ } catch (e) {+ reject(e);+ }++ try {+ const job = await this.getJob(id);+ if (job && ["completed", "failed"].includes(job.status)) {+ this.removeListener(id, listener);+ if (timer) clearTimeout(timer);+ if (job.status === "completed") {+ resolve(job.returnvalue!);+ } else {+ reject(new Error(job.failedReason!));+ }+ return;+ }+ } catch (e) {+ logger.warn("nuqGetJob ensure check failed", {+ module: "nuq",+ method: "nuqWaitForJob",+ error: e,+ scrapeId: id,+ });+ }+ } else {+ const timeoutAt = timeout !== null ? Date.now() + timeout : null;+ const poll = async function poll() {+ try {+ const job = await this.getJob(id);+ if (job && ["completed", "failed"].includes(job.status)) {+ if (job.status === "completed") {+ return resolve(job.returnvalue!);+ } else {+ return reject(new Error(job.failedReason!));+ }+ }+ } catch (e) {+ return reject(e);+ }++ if (timeoutAt && Date.now() > timeoutAt) {+ return reject(new Error("Timed out"));+ }++ setTimeout(poll.bind(this), 250);+ }.bind(this);++ poll();+ }+ }).bind(this),+ );++ return done;+ }++ // === Consumer++ public async getJobToProcess(lock: string): Promise<NuQJob<any, any> | null> {+ const start = Date.now();+ try {+ return this.rowToJob(+ (+ await nuqPool.query(+ `+ WITH next AS (SELECT ${this.jobReturning.join(", ")} FROM ${this.queueName} WHERE ${this.queueName}.status = 'queued'::nuq.job_status ORDER BY ${this.queueName}.priority ASC, ${this.queueName}.created_at ASC FOR UPDATE SKIP LOCKED LIMIT 1)+ UPDATE ${this.queueName} q SET status = 'active'::nuq.job_status, lock = $1, locked_at = now() FROM next WHERE q.id = next.id RETURNING ${this.jobReturning.map(x => `q.${x}`).join(", ")};+ `,+ [lock],+ )+ ).rows[0],+ )!;+ } finally {+ logger.info("nuqGetJobToProcess metrics", {+ module: "nuq/metrics",+ method: "nuqGetJobToProcess",+ duration: Date.now() - start,+ });+ }+ }++ public async renewLock(+ id: string,+ lock: string,+ _logger: Logger = logger,+ ): Promise<boolean> {+ const start = Date.now();+ try {+ return (+ (+ await nuqPool.query(+ `UPDATE ${this.queueName} SET locked_at = now() WHERE id = $1 AND lock = $2 AND status = 'active'::nuq.job_status;`,+ [id, lock],+ )+ ).rowCount !== 0+ );+ } finally {+ _logger.info("nuqRenewLock metrics", {+ module: "nuq/metrics",+ method: "nuqRenewLock",+ duration: Date.now() - start,+ scrapeId: id,+ });+ }+ }++ public async jobFinish(+ id: string,+ lock: string,+ returnvalue: any | null,+ _logger: Logger = logger,+ ): Promise<boolean> {+ const start = Date.now();+ try {+ return (+ (+ await nuqPool.query(+ `+ WITH updated AS (UPDATE ${this.queueName} SET status = 'completed'::nuq.job_status, lock = null, locked_at = null, finished_at = now(), returnvalue = $3 WHERE id = $1 AND lock = $2 RETURNING id)+ SELECT pg_notify('${this.queueName}', (id::text || '|completed')) FROM updated;+ `,+ [id, lock, returnvalue],+ )+ ).rowCount !== 0+ );+ } finally {+ _logger.info("nuqJobFinish metrics", {+ module: "nuq/metrics",+ method: "nuqJobFinish",+ duration: Date.now() - start,+ scrapeId: id,+ });+ }+ }++ public async jobFail(+ id: string,+ lock: string,+ failedReason: string,+ _logger: Logger = logger,+ ): Promise<boolean> {+ const start = Date.now();+ try {+ return (+ (+ await nuqPool.query(+ `+ WITH updated AS (UPDATE ${this.queueName} SET status = 'failed'::nuq.job_status, lock = null, locked_at = null, finished_at = now(), failedreason = $3 WHERE id = $1 AND lock = $2 RETURNING id)+ SELECT pg_notify('${this.queueName}', (id::text || '|failed')) FROM updated;+ `,+ [id, lock, failedReason],+ )+ ).rowCount !== 0+ );+ } finally {+ _logger.info("nuqJobFail metrics", {+ module: "nuq/metrics",+ method: "nuqJobFail",+ duration: Date.now() - start,+ scrapeId: id,+ });+ }+ }++ // === Metrics+ public async getMetrics(): Promise<string> {+ const start = Date.now();+ const result = await nuqPool.query(+ `SELECT status, COUNT(id) as count FROM ${this.queueName} GROUP BY status ORDER BY count DESC;`,+ );+ logger.info("nuqGetMetrics metrics", {+ module: "nuq/metrics",+ method: "nuqGetMetrics",+ duration: Date.now() - start,+ });+ const prometheusQueueName = this.queueName.replace(".", "_");+ return `# HELP ${prometheusQueueName}_job_count Number of jobs in each status\n# TYPE ${prometheusQueueName}_job_count gauge\n${result.rows.map(x => `${prometheusQueueName}_job_count{status="${x.status}"} ${x.count}`).join("\n")}\n`;+ }++ // === Cleanup+ public async shutdown() {+ this.shuttingDown = true;+ if (this.listener) {+ const nl = this.listener;+ this.listener = null;+ this.listens = {};+ await nl.query(`UNLISTEN "${this.queueName}";`);+ await nl.end();+ }+ }+}++export function nuqGetLocalMetrics(): string {+ return `# HELP nuq_pool_waiting_count Number of requests waiting in the pool\n# TYPE nuq_pool_waiting_count gauge\nnuq_pool_waiting_count ${nuqPool.waitingCount}\n+# HELP nuq_pool_idle_count Number of connections idle in the pool\n# TYPE nuq_pool_idle_count gauge\nnuq_pool_idle_count ${nuqPool.idleCount}\n+# HELP nuq_pool_total_count Number of connections in the pool\n# TYPE nuq_pool_total_count gauge\nnuq_pool_total_count ${nuqPool.totalCount}\n`;+}++export async function nuqHealthCheck(): Promise<boolean> {+ const start = Date.now();+ try {+ return (await nuqPool.query("SELECT 1;")).rowCount !== 0;+ } finally {+ logger.info("nuqHealthCheck metrics", {+ module: "nuq/metrics",+ method: "nuqHealthCheck",+ duration: Date.now() - start,+ });+ }+}++// === Instances++export const scrapeQueue = new NuQ("nuq.queue_scrape");++// === Cleanup++export async function nuqShutdown() {+ await scrapeQueue.shutdown();+ await nuqPool.end();+}diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex 29ded1a92..83a69d95c 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -1,5 +1,4 @@import { configDotenv } from "dotenv";-import { Job } from "bullmq";import * as Sentry from "@sentry/node";import http from "http";import https from "https";@@ -60,6 +59,7 @@ import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-node";import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-grpc";import { resourceFromAttributes } from "@opentelemetry/resources";import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";+import type { NuQJob } from "./nuq";const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));@@ -74,7 +74,7 @@ cacheableLookup.install(http.globalAgent);cacheableLookup.install(https.globalAgent);async function billScrapeJob(- job: Job & { id: string },+ job: NuQJob<any>,document: Document | null,logger: Logger,costTracking: CostTracking,@@ -138,7 +138,7 @@ async function billScrapeJob(return creditsToBeBilled;}-async function processJob(job: Job & { id: string }) {+async function processJob(job: NuQJob<any>) {const logger = _logger.child({module: "queue-worker",method: "processJob",@@ -157,13 +157,6 @@ async function processJob(job: Job & { id: string }) {const costTracking = new CostTracking();try {- job.updateProgress({- current: 1,- total: 100,- current_step: "SCRAPING",- current_url: "",- });-if (remainingTime !== undefined && remainingTime < 0) {throw new ScrapeJobTimeoutError("Scrape timed out");}@@ -368,7 +361,6 @@ async function processJob(job: Job & { id: string }) {zeroDataRetention: job.data.zeroDataRetention,apiKeyId: job.data.apiKeyId,},- {},jobId,jobPriority,);@@ -699,7 +691,7 @@ async function kickoffGetIndexLinks(return validIndexLinks;}-async function processKickoffJob(job: Job & { id: string }) {+async function processKickoffJob(job: NuQJob<any>) {const logger = _logger.child({module: "queue-worker",method: "processKickoffJob",@@ -739,10 +731,8 @@ async function processKickoffJob(job: Job & { id: string }) {zeroDataRetention: job.data.zeroDataRetention,apiKeyId: job.data.apiKeyId,},- {- priority: 15,- },jobId,+ await getJobPriority({ team_id: job.data.team_id, basePriority: 15 }),);logger.debug("Adding scrape job to BullMQ...", { jobId });await addCrawlJob(job.data.crawl_id, jobId, logger);@@ -780,7 +770,6 @@ async function processKickoffJob(job: Job & { id: string }) {const jobs = urls.map(url => {const uuid = uuidv4();return {- name: uuid,data: {url,mode: "single_urls" as const,@@ -797,10 +786,8 @@ async function processKickoffJob(job: Job & { id: string }) {zeroDataRetention: job.data.zeroDataRetention,apiKeyId: job.data.apiKeyId,},- opts: {- jobId: uuid,- priority: 20,- },+ jobId: uuid,+ priority: jobPriority,};});@@ -808,15 +795,15 @@ async function processKickoffJob(job: Job & { id: string }) {const lockedIds = await lockURLsIndividually(job.data.crawl_id,sc,- jobs.map(x => ({ id: x.opts.jobId, url: x.data.url })),+ jobs.map(x => ({ id: x.jobId, url: x.data.url })),);const lockedJobs = jobs.filter(x =>- lockedIds.find(y => y.id === x.opts.jobId),+ lockedIds.find(y => y.id === x.jobId),);logger.debug("Adding scrape jobs to Redis...");await addCrawlJobs(job.data.crawl_id,- lockedJobs.map(x => x.opts.jobId),+ lockedJobs.map(x => x.jobId),logger,);logger.debug("Adding scrape jobs to BullMQ...");@@ -845,7 +832,7 @@ async function processKickoffJob(job: Job & { id: string }) {const jobs = indexLinks.map(url => {const uuid = uuidv4();return {- name: uuid,+ jobId: uuid,data: {url,mode: "single_urls" as const,@@ -862,10 +849,7 @@ async function processKickoffJob(job: Job & { id: string }) {zeroDataRetention: job.data.zeroDataRetention,apiKeyId: job.data.apiKeyId,},- opts: {- jobId: uuid,- priority: 20,- },+ priority: jobPriority,};});@@ -873,15 +857,15 @@ async function processKickoffJob(job: Job & { id: string }) {const lockedIds = await lockURLsIndividually(job.data.crawl_id,sc,- jobs.map(x => ({ id: x.opts.jobId, url: x.data.url })),+ jobs.map(x => ({ id: x.jobId, url: x.data.url })),);const lockedJobs = jobs.filter(x =>- lockedIds.find(y => y.id === x.opts.jobId),+ lockedIds.find(y => y.id === x.jobId),);logger.debug("Adding scrape jobs to Redis...");await addCrawlJobs(job.data.crawl_id,- lockedJobs.map(x => x.opts.jobId),+ lockedJobs.map(x => x.jobId),logger,);logger.debug("Adding scrape jobs to BullMQ...");@@ -905,7 +889,7 @@ async function processKickoffJob(job: Job & { id: string }) {}}-export const processJobInternal = async (job: Job & { id: string }) => {+export const processJobInternal = async (job: NuQJob<any>) => {const logger = _logger.child({module: "queue-worker",method: "processJobInternal",@@ -954,10 +938,7 @@ export const processJobInternal = async (job: Job & { id: string }) => {}try {- if (- process.env.USE_DB_AUTHENTICATION === "true" &&- (job.data.crawl_id || process.env.GCS_BUCKET_NAME)- ) {+ if (process.env.GCS_BUCKET_NAME) {logger.debug("Job succeeded -- putting null in Redis");return null;} else {@@ -1018,11 +999,9 @@ if (otelSdk) {otelSdk.start();}-module.exports = processJobInternal;-const exitHandler = () => {if (otelSdk) {- otelSdk.shutdown().then(() => {+ otelSdk.shutdown().finally(() => {_logger.debug("OTEL shutdown");process.exit(0);});diff --git a/apps/nuq-postgres/Dockerfile b/apps/nuq-postgres/Dockerfilenew file mode 100644index 000000000..942b2fbe1--- /dev/null+++ b/apps/nuq-postgres/Dockerfile@@ -0,0 +1,24 @@+# Build a Postgres image that runs nuq.sql during initdb++ARG PG_MAJOR=17+FROM postgres:${PG_MAJOR}++# Install pg_cron for the specified Postgres major version+RUN set -eux; \+ apt-get update; \+ apt-get install -y --no-install-recommends \+ postgresql-${PG_MAJOR}-cron; \+ rm -rf /var/lib/apt/lists/*++# Ensure pg_cron is preloaded on first startup by modifying the initdb template+# This must be set before the first server start (init scripts run after start)+RUN set -eux; \+ conf_sample="/usr/share/postgresql/${PG_MAJOR}/postgresql.conf.sample"; \+ sed -ri "s/^#?shared_preload_libraries\s*=.*/shared_preload_libraries = 'pg_cron'/" "$conf_sample"; \+ printf "\n# Added for pg_cron\ncron.database_name = 'postgres'\n" >> "$conf_sample"++# Create required extensions before executing our SQL+RUN printf 'CREATE EXTENSION IF NOT EXISTS pgcrypto;\nCREATE EXTENSION IF NOT EXISTS pg_cron;\n' > /docker-entrypoint-initdb.d/010-extensions.sql++# Copy nuq.sql so it is executed as part of the initdb sequence+COPY nuq.sql /docker-entrypoint-initdb.d/020-nuq.sql\ No newline at end of filediff --git a/apps/nuq-postgres/nuq.sql b/apps/nuq-postgres/nuq.sqlnew file mode 100644index 000000000..5d3ad041c--- /dev/null+++ b/apps/nuq-postgres/nuq.sql@@ -0,0 +1,51 @@+CREATE SCHEMA IF NOT EXISTS nuq;++DO $$ BEGIN+ CREATE TYPE nuq.job_status AS ENUM ('queued', 'active', 'completed', 'failed');+EXCEPTION+ WHEN duplicate_object THEN null;+END $$;++CREATE TABLE IF NOT EXISTS nuq.queue_scrape (+ id uuid NOT NULL DEFAULT gen_random_uuid(),+ status nuq.job_status NOT NULL DEFAULT 'queued'::nuq.job_status,+ data jsonb,+ created_at timestamp with time zone NOT NULL DEFAULT now(),+ priority int NOT NULL DEFAULT 0,+ lock uuid,+ locked_at timestamp with time zone,+ stalls integer,+ finished_at timestamp with time zone,+ returnvalue jsonb, -- only for selfhost+ failedreason text, -- only for selfhost+ CONSTRAINT queue_scrape_pkey PRIMARY KEY (id)+);++ALTER TABLE nuq.queue_scrape+SET (autovacuum_vacuum_scale_factor = 0.01,+ autovacuum_analyze_scale_factor = 0.01,+ autovacuum_vacuum_cost_limit = 2000,+ autovacuum_vacuum_cost_delay = 2);++CREATE INDEX IF NOT EXISTS queue_scrape_active_locked_at_idx ON nuq.queue_scrape USING btree (locked_at) WHERE (status = 'active'::nuq.job_status);+CREATE INDEX IF NOT EXISTS nuq_queue_scrape_queued_optimal_2_idx ON nuq.queue_scrape (priority ASC, created_at ASC, id) WHERE (status = 'queued'::nuq.job_status);+CREATE INDEX IF NOT EXISTS nuq_queue_scrape_failed_created_at_idx ON nuq.queue_scrape USING btree (created_at) WHERE (status = 'failed'::nuq.job_status);+CREATE INDEX IF NOT EXISTS nuq_queue_scrape_completed_created_at_idx ON nuq.queue_scrape USING btree (created_at) WHERE (status = 'completed'::nuq.job_status);++SELECT cron.schedule('nuq_queue_scrape_clean_completed', '*/5 * * * *', $$+ DELETE FROM nuq.queue_scrape WHERE nuq.queue_scrape.status = 'completed'::nuq.job_status AND nuq.queue_scrape.created_at < now() - interval '1 hour';+$$);++SELECT cron.schedule('nuq_queue_scrape_clean_failed', '*/5 * * * *', $$+ DELETE FROM nuq.queue_scrape WHERE nuq.queue_scrape.status = 'failed'::nuq.job_status AND nuq.queue_scrape.created_at < now() - interval '6 hours';+$$);++SELECT cron.schedule('nuq_queue_scrape_lock_reaper', '15 seconds', $$+ UPDATE nuq.queue_scrape SET status = 'queued'::nuq.job_status, lock = null, locked_at = null, stalls = COALESCE(stalls, 0) + 1 WHERE nuq.queue_scrape.locked_at <= now() - interval '1 minute' AND nuq.queue_scrape.status = 'active'::nuq.job_status AND COALESCE(nuq.queue_scrape.stalls, 0) < 9;+ WITH stallfail AS (UPDATE nuq.queue_scrape SET status = 'failed'::nuq.job_status, lock = null, locked_at = null, stalls = COALESCE(stalls, 0) + 1 WHERE nuq.queue_scrape.locked_at <= now() - interval '1 minute' AND nuq.queue_scrape.status = 'active'::nuq.job_status AND COALESCE(nuq.queue_scrape.stalls, 0) >= 9 RETURNING id)+ SELECT pg_notify('nuq.queue_scrape', (id::text || '|' || 'failed'::text)) FROM stallfail;+$$);++SELECT cron.schedule('nuq_queue_scrape_reindex', '0 9 * * *', $$+ REINDEX TABLE CONCURRENTLY nuq.queue_scrape;+$$);diff --git a/docker-compose.yaml b/docker-compose.yamlindex c6714dd5a..5892fa0db 100644--- a/docker-compose.yaml+++ b/docker-compose.yaml@@ -19,6 +19,7 @@ x-common-env: &common-envREDIS_URL: ${REDIS_URL:-redis://redis:6379}REDIS_RATE_LIMIT_URL: ${REDIS_URL:-redis://redis:6379}PLAYWRIGHT_MICROSERVICE_URL: ${PLAYWRIGHT_MICROSERVICE_URL:-http://playwright-service:3000/scrape}+ NUQ_DATABASE_URL: postgres://postgres:postgres@nuq-postgres:5432/postgresUSE_DB_AUTHENTICATION: ${USE_DB_AUTHENTICATION}OPENAI_API_KEY: ${OPENAI_API_KEY}OPENAI_BASE_URL: ${OPENAI_BASE_URL}@@ -65,26 +66,14 @@ services:<<: *common-envHOST: "0.0.0.0"PORT: ${INTERNAL_PORT:-3002}- FLY_PROCESS_GROUP: app+ WORKER_PORT: ${WORKER_PORT:-3005}ENV: localdepends_on:- redis- playwright-serviceports:- "${PORT:-3002}:${INTERNAL_PORT:-3002}"- command: [ "pnpm", "run", "start:production" ]-- worker:- <<: *common-service- environment:- <<: *common-env- FLY_PROCESS_GROUP: worker- ENV: local- depends_on:- - redis- - playwright-service- - api- command: [ "pnpm", "run", "workers" ]+ command: node dist/src/harness.js --start-dockerredis:# NOTE: If you want to use Valkey (open source) instead of Redis (source available),@@ -96,6 +85,17 @@ services:networks:- backendcommand: redis-server --bind 0.0.0.0++ nuq-postgres:+ build: apps/nuq-postgres+ environment:+ POSTGRES_USER: postgres+ POSTGRES_PASSWORD: postgres+ POSTGRES_DB: postgres+ networks:+ - backend+ ports:+ - "5432:5432"networks:backend: