Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat: add NuQ worker deployment to Kubernetes examples (#2163)
devin-ai-integration[bot]26f843802d2b
diff --git a/.github/workflows/deploy-nuq-postgres.yml b/.github/workflows/deploy-nuq-postgres.ymlnew file mode 100644index 000000000..200ea814d--- /dev/null+++ b/.github/workflows/deploy-nuq-postgres.yml@@ -0,0 +1,31 @@+name: Deploy NuQ Postgres to GHCR++on:+ push:+ branches:+ - main+ paths:+ - apps/nuq-postgres/**+ workflow_dispatch:++jobs:+ push-app-image:+ runs-on: ubuntu-latest+ defaults:+ run:+ working-directory: './apps/nuq-postgres'+ steps:+ - name: 'Checkout GitHub Action'+ uses: actions/checkout@main++ - name: 'Login to GitHub Container Registry'+ uses: docker/login-action@v3+ with:+ registry: ghcr.io+ username: ${{github.actor}}+ password: ${{secrets.GITHUB_TOKEN}}++ - name: 'Build NuQ Postgres Image'+ run: |+ docker build . --tag ghcr.io/firecrawl/nuq-postgres:latest+ docker push ghcr.io/firecrawl/nuq-postgres:latestdiff --git a/examples/kubernetes/cluster-install/README.md b/examples/kubernetes/cluster-install/README.mdindex d820d7c92..909dba6bb 100644--- a/examples/kubernetes/cluster-install/README.md+++ b/examples/kubernetes/cluster-install/README.md@@ -15,6 +15,8 @@ kubectl apply -f secret.yamlkubectl apply -f playwright-service.yamlkubectl apply -f api.yamlkubectl apply -f worker.yaml+kubectl apply -f nuq-worker.yaml+kubectl apply -f nuq-postgres.yamlkubectl apply -f redis.yaml```@@ -31,5 +33,7 @@ kubectl delete -f secret.yamlkubectl delete -f playwright-service.yamlkubectl delete -f api.yamlkubectl delete -f worker.yaml+kubectl delete -f nuq-worker.yaml+kubectl delete -f nuq-postgres.yamlkubectl delete -f redis.yaml```diff --git a/examples/kubernetes/cluster-install/configmap.yaml b/examples/kubernetes/cluster-install/configmap.yamlindex b60dbacfc..bb1072d46 100644--- a/examples/kubernetes/cluster-install/configmap.yaml+++ b/examples/kubernetes/cluster-install/configmap.yaml@@ -12,3 +12,4 @@ data:ENV: "production"LOGGING_LEVEL: "DEBUG"IS_KUBERNETES: "true"+ NUQ_DATABASE_URL: "postgresql://postgres:password@nuq-postgres:5432/postgres"diff --git a/examples/kubernetes/cluster-install/nuq-postgres.yaml b/examples/kubernetes/cluster-install/nuq-postgres.yamlnew file mode 100644index 000000000..16cf38002--- /dev/null+++ b/examples/kubernetes/cluster-install/nuq-postgres.yaml@@ -0,0 +1,53 @@+apiVersion: apps/v1+kind: Deployment+metadata:+ name: nuq-postgres+spec:+ replicas: 1+ selector:+ matchLabels:+ app: nuq-postgres+ template:+ metadata:+ labels:+ app: nuq-postgres+ spec:+ containers:+ - name: nuq-postgres+ image: ghcr.io/firecrawl/nuq-postgres:latest+ imagePullPolicy: Always+ env:+ - name: POSTGRES_USER+ value: "postgres"+ - name: POSTGRES_PASSWORD+ value: "password"+ - name: POSTGRES_DB+ value: "postgres"+ ports:+ - containerPort: 5432+ volumeMounts:+ - name: postgres-storage+ mountPath: /var/lib/postgresql/data+ resources:+ requests:+ memory: "512Mi"+ cpu: "250m"+ limits:+ memory: "1Gi"+ cpu: "500m"+ volumes:+ - name: postgres-storage+ emptyDir: {}+---+apiVersion: v1+kind: Service+metadata:+ name: nuq-postgres+spec:+ selector:+ app: nuq-postgres+ ports:+ - protocol: TCP+ port: 5432+ targetPort: 5432+ type: ClusterIPdiff --git a/examples/kubernetes/cluster-install/nuq-worker.yaml b/examples/kubernetes/cluster-install/nuq-worker.yamlnew file mode 100644index 000000000..06d4d8b1d--- /dev/null+++ b/examples/kubernetes/cluster-install/nuq-worker.yaml@@ -0,0 +1,58 @@+apiVersion: apps/v1+kind: Deployment+metadata:+ name: nuq-worker+spec:+ replicas: 5+ selector:+ matchLabels:+ app: nuq-worker+ template:+ metadata:+ labels:+ app: nuq-worker+ spec:+ imagePullSecrets:+ - name: docker-registry-secret+ terminationGracePeriodSeconds: 60+ containers:+ - name: nuq-worker+ image: ghcr.io/firecrawl/firecrawl:latest+ imagePullPolicy: Always+ command: [ "node" ]+ args: [ "--max-old-space-size=3072", "dist/src/services/worker/nuq-worker.js" ]+ env:+ - name: FLY_PROCESS_GROUP+ value: "nuq-worker"+ - name: NUQ_WORKER_PORT+ value: "3006"+ envFrom:+ - configMapRef:+ name: firecrawl-config+ - secretRef:+ name: firecrawl-secret+ resources:+ requests:+ memory: "3G"+ cpu: "1000m"+ limits:+ memory: "4G"+ cpu: "1000m"+ livenessProbe:+ httpGet:+ path: /health+ port: 3006+ initialDelaySeconds: 5+ periodSeconds: 5+ timeoutSeconds: 5+ successThreshold: 1+ failureThreshold: 3+ readinessProbe:+ httpGet:+ path: /health+ port: 3006+ initialDelaySeconds: 5+ periodSeconds: 5+ timeoutSeconds: 5+ successThreshold: 1+ failureThreshold: 3diff --git a/examples/kubernetes/firecrawl-helm/templates/configmap.yaml b/examples/kubernetes/firecrawl-helm/templates/configmap.yamlindex ade1e1049..7d69a8691 100644--- a/examples/kubernetes/firecrawl-helm/templates/configmap.yaml+++ b/examples/kubernetes/firecrawl-helm/templates/configmap.yaml@@ -11,3 +11,4 @@ data:PLAYWRIGHT_MICROSERVICE_URL: {{ .Values.config.PLAYWRIGHT_MICROSERVICE_URL | quote }}USE_DB_AUTHENTICATION: {{ .Values.config.USE_DB_AUTHENTICATION | quote }}HDX_NODE_BETA_MODE: {{ .Values.config.HDX_NODE_BETA_MODE | quote }}+ NUQ_DATABASE_URL: {{ .Values.config.NUQ_DATABASE_URL | quote }}diff --git a/examples/kubernetes/firecrawl-helm/templates/nuq-postgres-deployment.yaml b/examples/kubernetes/firecrawl-helm/templates/nuq-postgres-deployment.yamlnew file mode 100644index 000000000..4a5a01dcc--- /dev/null+++ b/examples/kubernetes/firecrawl-helm/templates/nuq-postgres-deployment.yaml@@ -0,0 +1,71 @@+apiVersion: apps/v1+kind: Deployment+metadata:+ name: {{ include "firecrawl.fullname" . }}-nuq-postgres+ labels:+ app: {{ include "firecrawl.name" . }}-nuq-postgres+spec:+ replicas: {{ .Values.nuqPostgres.replicaCount | default 1 }}+ selector:+ matchLabels:+ app: {{ include "firecrawl.name" . }}-nuq-postgres+ template:+ metadata:+ labels:+ app: {{ include "firecrawl.name" . }}-nuq-postgres+ spec:+ {{- if .Values.image.dockerSecretEnabled }}+ imagePullSecrets:+ {{- toYaml .Values.imagePullSecrets | nindent 8 }}+ {{- end }}+ containers:+ - name: nuq-postgres+ image: "{{ .Values.nuqPostgres.image.repository }}:{{ .Values.nuqPostgres.image.tag | default "latest" }}"+ imagePullPolicy: {{ .Values.nuqPostgres.image.pullPolicy | default "Always" }}+ env:+ - name: POSTGRES_USER+ value: "{{ .Values.nuqPostgres.auth.username | default "postgres" }}"+ - name: POSTGRES_PASSWORD+ value: "password"+ - name: POSTGRES_DB+ value: "{{ .Values.nuqPostgres.auth.database | default "postgres" }}"+ ports:+ - containerPort: 5432+ volumeMounts:+ - name: postgres-storage+ mountPath: /var/lib/postgresql/data+ {{- if .Values.nuqPostgres.resources }}+ resources:+ {{- toYaml .Values.nuqPostgres.resources | nindent 12 }}+ {{- else }}+ resources:+ requests:+ memory: "512Mi"+ cpu: "250m"+ limits:+ memory: "1Gi"+ cpu: "500m"+ {{- end }}+ volumes:+ - name: postgres-storage+ {{- if .Values.nuqPostgres.persistence.enabled }}+ persistentVolumeClaim:+ claimName: {{ include "firecrawl.fullname" . }}-nuq-postgres-pvc+ {{- else }}+ emptyDir: {}+ {{- end }}+---+apiVersion: v1+kind: Service+metadata:+ name: {{ include "firecrawl.fullname" . }}-nuq-postgres+ labels:+ app: {{ include "firecrawl.name" . }}-nuq-postgres+spec:+ selector:+ app: {{ include "firecrawl.name" . }}-nuq-postgres+ ports:+ - protocol: TCP+ port: 5432+ targetPort: 5432+ type: ClusterIPdiff --git a/examples/kubernetes/firecrawl-helm/templates/nuq-worker-deployment.yaml b/examples/kubernetes/firecrawl-helm/templates/nuq-worker-deployment.yamlnew file mode 100644index 000000000..d36d83403--- /dev/null+++ b/examples/kubernetes/firecrawl-helm/templates/nuq-worker-deployment.yaml@@ -0,0 +1,67 @@+apiVersion: apps/v1+kind: Deployment+metadata:+ name: {{ include "firecrawl.fullname" . }}-nuq-worker+ labels:+ app: {{ include "firecrawl.name" . }}-nuq-worker+spec:+ replicas: {{ .Values.nuqWorker.replicaCount | default 5 }}+ selector:+ matchLabels:+ app: {{ include "firecrawl.name" . }}-nuq-worker+ template:+ metadata:+ labels:+ app: {{ include "firecrawl.name" . }}-nuq-worker+ spec:+ {{- if .Values.image.dockerSecretEnabled }}+ imagePullSecrets:+ {{- toYaml .Values.imagePullSecrets | nindent 8 }}+ {{- end }}+ terminationGracePeriodSeconds: 60+ containers:+ - name: nuq-worker+ image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"+ imagePullPolicy: {{ .Values.image.pullPolicy }}+ command: [ "node" ]+ args: [ "--max-old-space-size=3072", "dist/src/services/worker/nuq-worker.js" ]+ env:+ - name: FLY_PROCESS_GROUP+ value: "nuq-worker"+ - name: NUQ_WORKER_PORT+ value: "3006"+ envFrom:+ - configMapRef:+ name: {{ include "firecrawl.fullname" . }}-config+ - secretRef:+ name: {{ include "firecrawl.fullname" . }}-secret+ {{- if .Values.nuqWorker.resources }}+ resources:+ {{- toYaml .Values.nuqWorker.resources | nindent 12 }}+ {{- else }}+ resources:+ requests:+ memory: "3G"+ cpu: "1000m"+ limits:+ memory: "4G"+ cpu: "1000m"+ {{- end }}+ livenessProbe:+ httpGet:+ path: /health+ port: 3006+ initialDelaySeconds: 5+ periodSeconds: 5+ timeoutSeconds: 5+ successThreshold: 1+ failureThreshold: 3+ readinessProbe:+ httpGet:+ path: /health+ port: 3006+ initialDelaySeconds: 5+ periodSeconds: 5+ timeoutSeconds: 5+ successThreshold: 1+ failureThreshold: 3diff --git a/examples/kubernetes/firecrawl-helm/values.yaml b/examples/kubernetes/firecrawl-helm/values.yamlindex fb6ca2ba4..4f4f0ca68 100644--- a/examples/kubernetes/firecrawl-helm/values.yaml+++ b/examples/kubernetes/firecrawl-helm/values.yaml@@ -1,5 +1,34 @@replicaCount: 1+nuqWorker:+ replicaCount: 5+ resources:+ requests:+ memory: "3G"+ cpu: "1000m"+ limits:+ memory: "4G"+ cpu: "1000m"++nuqPostgres:+ replicaCount: 1+ image:+ repository: ghcr.io/firecrawl/nuq-postgres+ tag: latest+ pullPolicy: Always+ auth:+ username: postgres+ database: postgres+ resources:+ requests:+ memory: "512Mi"+ cpu: "250m"+ limits:+ memory: "1Gi"+ cpu: "500m"+ persistence:+ enabled: false+image:repository: ghcr.io/firecrawl/firecrawltag: "latest"@@ -35,6 +64,7 @@ config:PLAYWRIGHT_MICROSERVICE_URL: "http://playwright-service:3000"USE_DB_AUTHENTICATION: "false"HDX_NODE_BETA_MODE: "1"+ NUQ_DATABASE_URL: "postgresql://postgres:password@nuq-postgres:5432/postgres"playwrightConfig:PORT: "3000"