Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat: add NuQ worker deployment to Kubernetes examples (#2163)

devin-ai-integration[bot]26f843802d2b
diff --git a/.github/workflows/deploy-nuq-postgres.yml b/.github/workflows/deploy-nuq-postgres.yml
new file mode 100644
index 000000000..200ea814d
--- /dev/null
+++ b/.github/workflows/deploy-nuq-postgres.yml
@@ -0,0 +1,31 @@
+name: Deploy NuQ Postgres to GHCR
+
+on:
+ push:
+ branches:
+ - main
+ paths:
+ - apps/nuq-postgres/**
+ workflow_dispatch:
+
+jobs:
+ push-app-image:
+ runs-on: ubuntu-latest
+ defaults:
+ run:
+ working-directory: './apps/nuq-postgres'
+ steps:
+ - name: 'Checkout GitHub Action'
+ uses: actions/checkout@main
+
+ - name: 'Login to GitHub Container Registry'
+ uses: docker/login-action@v3
+ with:
+ registry: ghcr.io
+ username: ${{github.actor}}
+ password: ${{secrets.GITHUB_TOKEN}}
+
+ - name: 'Build NuQ Postgres Image'
+ run: |
+ docker build . --tag ghcr.io/firecrawl/nuq-postgres:latest
+ docker push ghcr.io/firecrawl/nuq-postgres:latest
diff --git a/examples/kubernetes/cluster-install/README.md b/examples/kubernetes/cluster-install/README.md
index d820d7c92..909dba6bb 100644
--- a/examples/kubernetes/cluster-install/README.md
+++ b/examples/kubernetes/cluster-install/README.md
@@ -15,6 +15,8 @@ kubectl apply -f secret.yaml
kubectl apply -f playwright-service.yaml
kubectl apply -f api.yaml
kubectl apply -f worker.yaml
+kubectl apply -f nuq-worker.yaml
+kubectl apply -f nuq-postgres.yaml
kubectl apply -f redis.yaml
```
@@ -31,5 +33,7 @@ kubectl delete -f secret.yaml
kubectl delete -f playwright-service.yaml
kubectl delete -f api.yaml
kubectl delete -f worker.yaml
+kubectl delete -f nuq-worker.yaml
+kubectl delete -f nuq-postgres.yaml
kubectl delete -f redis.yaml
```
diff --git a/examples/kubernetes/cluster-install/configmap.yaml b/examples/kubernetes/cluster-install/configmap.yaml
index b60dbacfc..bb1072d46 100644
--- a/examples/kubernetes/cluster-install/configmap.yaml
+++ b/examples/kubernetes/cluster-install/configmap.yaml
@@ -12,3 +12,4 @@ data:
ENV: "production"
LOGGING_LEVEL: "DEBUG"
IS_KUBERNETES: "true"
+ NUQ_DATABASE_URL: "postgresql://postgres:password@nuq-postgres:5432/postgres"
diff --git a/examples/kubernetes/cluster-install/nuq-postgres.yaml b/examples/kubernetes/cluster-install/nuq-postgres.yaml
new file mode 100644
index 000000000..16cf38002
--- /dev/null
+++ b/examples/kubernetes/cluster-install/nuq-postgres.yaml
@@ -0,0 +1,53 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: nuq-postgres
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: nuq-postgres
+ template:
+ metadata:
+ labels:
+ app: nuq-postgres
+ spec:
+ containers:
+ - name: nuq-postgres
+ image: ghcr.io/firecrawl/nuq-postgres:latest
+ imagePullPolicy: Always
+ env:
+ - name: POSTGRES_USER
+ value: "postgres"
+ - name: POSTGRES_PASSWORD
+ value: "password"
+ - name: POSTGRES_DB
+ value: "postgres"
+ ports:
+ - containerPort: 5432
+ volumeMounts:
+ - name: postgres-storage
+ mountPath: /var/lib/postgresql/data
+ resources:
+ requests:
+ memory: "512Mi"
+ cpu: "250m"
+ limits:
+ memory: "1Gi"
+ cpu: "500m"
+ volumes:
+ - name: postgres-storage
+ emptyDir: {}
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: nuq-postgres
+spec:
+ selector:
+ app: nuq-postgres
+ ports:
+ - protocol: TCP
+ port: 5432
+ targetPort: 5432
+ type: ClusterIP
diff --git a/examples/kubernetes/cluster-install/nuq-worker.yaml b/examples/kubernetes/cluster-install/nuq-worker.yaml
new file mode 100644
index 000000000..06d4d8b1d
--- /dev/null
+++ b/examples/kubernetes/cluster-install/nuq-worker.yaml
@@ -0,0 +1,58 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: nuq-worker
+spec:
+ replicas: 5
+ selector:
+ matchLabels:
+ app: nuq-worker
+ template:
+ metadata:
+ labels:
+ app: nuq-worker
+ spec:
+ imagePullSecrets:
+ - name: docker-registry-secret
+ terminationGracePeriodSeconds: 60
+ containers:
+ - name: nuq-worker
+ image: ghcr.io/firecrawl/firecrawl:latest
+ imagePullPolicy: Always
+ command: [ "node" ]
+ args: [ "--max-old-space-size=3072", "dist/src/services/worker/nuq-worker.js" ]
+ env:
+ - name: FLY_PROCESS_GROUP
+ value: "nuq-worker"
+ - name: NUQ_WORKER_PORT
+ value: "3006"
+ envFrom:
+ - configMapRef:
+ name: firecrawl-config
+ - secretRef:
+ name: firecrawl-secret
+ resources:
+ requests:
+ memory: "3G"
+ cpu: "1000m"
+ limits:
+ memory: "4G"
+ cpu: "1000m"
+ livenessProbe:
+ httpGet:
+ path: /health
+ port: 3006
+ initialDelaySeconds: 5
+ periodSeconds: 5
+ timeoutSeconds: 5
+ successThreshold: 1
+ failureThreshold: 3
+ readinessProbe:
+ httpGet:
+ path: /health
+ port: 3006
+ initialDelaySeconds: 5
+ periodSeconds: 5
+ timeoutSeconds: 5
+ successThreshold: 1
+ failureThreshold: 3
diff --git a/examples/kubernetes/firecrawl-helm/templates/configmap.yaml b/examples/kubernetes/firecrawl-helm/templates/configmap.yaml
index ade1e1049..7d69a8691 100644
--- a/examples/kubernetes/firecrawl-helm/templates/configmap.yaml
+++ b/examples/kubernetes/firecrawl-helm/templates/configmap.yaml
@@ -11,3 +11,4 @@ data:
PLAYWRIGHT_MICROSERVICE_URL: {{ .Values.config.PLAYWRIGHT_MICROSERVICE_URL | quote }}
USE_DB_AUTHENTICATION: {{ .Values.config.USE_DB_AUTHENTICATION | quote }}
HDX_NODE_BETA_MODE: {{ .Values.config.HDX_NODE_BETA_MODE | quote }}
+ NUQ_DATABASE_URL: {{ .Values.config.NUQ_DATABASE_URL | quote }}
diff --git a/examples/kubernetes/firecrawl-helm/templates/nuq-postgres-deployment.yaml b/examples/kubernetes/firecrawl-helm/templates/nuq-postgres-deployment.yaml
new file mode 100644
index 000000000..4a5a01dcc
--- /dev/null
+++ b/examples/kubernetes/firecrawl-helm/templates/nuq-postgres-deployment.yaml
@@ -0,0 +1,71 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: {{ include "firecrawl.fullname" . }}-nuq-postgres
+ labels:
+ app: {{ include "firecrawl.name" . }}-nuq-postgres
+spec:
+ replicas: {{ .Values.nuqPostgres.replicaCount | default 1 }}
+ selector:
+ matchLabels:
+ app: {{ include "firecrawl.name" . }}-nuq-postgres
+ template:
+ metadata:
+ labels:
+ app: {{ include "firecrawl.name" . }}-nuq-postgres
+ spec:
+ {{- if .Values.image.dockerSecretEnabled }}
+ imagePullSecrets:
+ {{- toYaml .Values.imagePullSecrets | nindent 8 }}
+ {{- end }}
+ containers:
+ - name: nuq-postgres
+ image: "{{ .Values.nuqPostgres.image.repository }}:{{ .Values.nuqPostgres.image.tag | default "latest" }}"
+ imagePullPolicy: {{ .Values.nuqPostgres.image.pullPolicy | default "Always" }}
+ env:
+ - name: POSTGRES_USER
+ value: "{{ .Values.nuqPostgres.auth.username | default "postgres" }}"
+ - name: POSTGRES_PASSWORD
+ value: "password"
+ - name: POSTGRES_DB
+ value: "{{ .Values.nuqPostgres.auth.database | default "postgres" }}"
+ ports:
+ - containerPort: 5432
+ volumeMounts:
+ - name: postgres-storage
+ mountPath: /var/lib/postgresql/data
+ {{- if .Values.nuqPostgres.resources }}
+ resources:
+ {{- toYaml .Values.nuqPostgres.resources | nindent 12 }}
+ {{- else }}
+ resources:
+ requests:
+ memory: "512Mi"
+ cpu: "250m"
+ limits:
+ memory: "1Gi"
+ cpu: "500m"
+ {{- end }}
+ volumes:
+ - name: postgres-storage
+ {{- if .Values.nuqPostgres.persistence.enabled }}
+ persistentVolumeClaim:
+ claimName: {{ include "firecrawl.fullname" . }}-nuq-postgres-pvc
+ {{- else }}
+ emptyDir: {}
+ {{- end }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ include "firecrawl.fullname" . }}-nuq-postgres
+ labels:
+ app: {{ include "firecrawl.name" . }}-nuq-postgres
+spec:
+ selector:
+ app: {{ include "firecrawl.name" . }}-nuq-postgres
+ ports:
+ - protocol: TCP
+ port: 5432
+ targetPort: 5432
+ type: ClusterIP
diff --git a/examples/kubernetes/firecrawl-helm/templates/nuq-worker-deployment.yaml b/examples/kubernetes/firecrawl-helm/templates/nuq-worker-deployment.yaml
new file mode 100644
index 000000000..d36d83403
--- /dev/null
+++ b/examples/kubernetes/firecrawl-helm/templates/nuq-worker-deployment.yaml
@@ -0,0 +1,67 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: {{ include "firecrawl.fullname" . }}-nuq-worker
+ labels:
+ app: {{ include "firecrawl.name" . }}-nuq-worker
+spec:
+ replicas: {{ .Values.nuqWorker.replicaCount | default 5 }}
+ selector:
+ matchLabels:
+ app: {{ include "firecrawl.name" . }}-nuq-worker
+ template:
+ metadata:
+ labels:
+ app: {{ include "firecrawl.name" . }}-nuq-worker
+ spec:
+ {{- if .Values.image.dockerSecretEnabled }}
+ imagePullSecrets:
+ {{- toYaml .Values.imagePullSecrets | nindent 8 }}
+ {{- end }}
+ terminationGracePeriodSeconds: 60
+ containers:
+ - name: nuq-worker
+ image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+ imagePullPolicy: {{ .Values.image.pullPolicy }}
+ command: [ "node" ]
+ args: [ "--max-old-space-size=3072", "dist/src/services/worker/nuq-worker.js" ]
+ env:
+ - name: FLY_PROCESS_GROUP
+ value: "nuq-worker"
+ - name: NUQ_WORKER_PORT
+ value: "3006"
+ envFrom:
+ - configMapRef:
+ name: {{ include "firecrawl.fullname" . }}-config
+ - secretRef:
+ name: {{ include "firecrawl.fullname" . }}-secret
+ {{- if .Values.nuqWorker.resources }}
+ resources:
+ {{- toYaml .Values.nuqWorker.resources | nindent 12 }}
+ {{- else }}
+ resources:
+ requests:
+ memory: "3G"
+ cpu: "1000m"
+ limits:
+ memory: "4G"
+ cpu: "1000m"
+ {{- end }}
+ livenessProbe:
+ httpGet:
+ path: /health
+ port: 3006
+ initialDelaySeconds: 5
+ periodSeconds: 5
+ timeoutSeconds: 5
+ successThreshold: 1
+ failureThreshold: 3
+ readinessProbe:
+ httpGet:
+ path: /health
+ port: 3006
+ initialDelaySeconds: 5
+ periodSeconds: 5
+ timeoutSeconds: 5
+ successThreshold: 1
+ failureThreshold: 3
diff --git a/examples/kubernetes/firecrawl-helm/values.yaml b/examples/kubernetes/firecrawl-helm/values.yaml
index fb6ca2ba4..4f4f0ca68 100644
--- a/examples/kubernetes/firecrawl-helm/values.yaml
+++ b/examples/kubernetes/firecrawl-helm/values.yaml
@@ -1,5 +1,34 @@
replicaCount: 1
+nuqWorker:
+ replicaCount: 5
+ resources:
+ requests:
+ memory: "3G"
+ cpu: "1000m"
+ limits:
+ memory: "4G"
+ cpu: "1000m"
+
+nuqPostgres:
+ replicaCount: 1
+ image:
+ repository: ghcr.io/firecrawl/nuq-postgres
+ tag: latest
+ pullPolicy: Always
+ auth:
+ username: postgres
+ database: postgres
+ resources:
+ requests:
+ memory: "512Mi"
+ cpu: "250m"
+ limits:
+ memory: "1Gi"
+ cpu: "500m"
+ persistence:
+ enabled: false
+
image:
repository: ghcr.io/firecrawl/firecrawl
tag: "latest"
@@ -35,6 +64,7 @@ config:
PLAYWRIGHT_MICROSERVICE_URL: "http://playwright-service:3000"
USE_DB_AUTHENTICATION: "false"
HDX_NODE_BETA_MODE: "1"
+ NUQ_DATABASE_URL: "postgresql://postgres:password@nuq-postgres:5432/postgres"
playwrightConfig:
PORT: "3000"