Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Move cclog into a worker (#3884)

Gergő Móriczb4620ad6f43f
diff --git a/apps/api/package.json b/apps/api/package.json
index 2a192f1f4..ad1ee1236 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -31,6 +31,8 @@
"nuq-prefetch-worker:production": "node dist/src/services/worker/nuq-prefetch-worker.js",
"nuq-reconciler-worker": "tsc-watch --onSuccess \"node dist/src/services/worker/nuq-reconciler-worker.js\"",
"nuq-reconciler-worker:production": "node dist/src/services/worker/nuq-reconciler-worker.js",
+ "cclog-worker": "tsc-watch --onSuccess \"node dist/src/services/cclog-worker.js\"",
+ "cclog-worker:production": "node dist/src/services/cclog-worker.js",
"extract-worker": "tsc-watch --onSuccess \"node dist/src/services/extract-worker.js\"",
"extract-worker:production": "node dist/src/services/extract-worker.js",
"index-worker": "tsc-watch --onSuccess \"node dist/src/services/indexing/index-worker.js\"",
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index 077be98c5..3daa1ce54 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -177,6 +177,7 @@ const configSchema = z.object({
NUQ_WORKER_COUNT: z.coerce.number().default(5),
NUQ_PREFETCH_WORKER_PORT: z.coerce.number().default(3011).catch(3011), // todo: investigate why .catch is needed
NUQ_RECONCILER_WORKER_PORT: z.coerce.number().default(3012).catch(3012),
+ CCLOG_WORKER_PORT: z.coerce.number().default(3013).catch(3013),
EXTRACT_WORKER_PORT: z.coerce.number().default(3004),
NUQ_WAIT_MODE: z.string().optional(),
diff --git a/apps/api/src/controllers/v0/admin/cclog.ts b/apps/api/src/controllers/v0/admin/cclog.ts
deleted file mode 100644
index 50b8e6916..000000000
--- a/apps/api/src/controllers/v0/admin/cclog.ts
+++ /dev/null
@@ -1,95 +0,0 @@
-import { getRedisConnection } from "../../../services/queue-service";
-import { db } from "../../../db/connection";
-import * as schema from "../../../db/schema";
-import { logger as _logger } from "../../../lib/logger";
-import { Request, Response } from "express";
-import {
- nuqFdbHealthCheck,
- scrapeQueueFdb,
- withFdbTimeout,
-} from "../../../services/worker/nuq-fdb";
-import { fdbQueueEnabled } from "../../../services/worker/nuq-router";
-
-const FDB_OPTIONAL_COUNT_TIMEOUT_MS = 500;
-
-async function cclog() {
- const logger = _logger.child({
- module: "cclog",
- });
-
- const concurrencyByTeam = new Map<string, number>();
- const redis = getRedisConnection();
- let cursor = 0;
- do {
- const result = await redis.scan(
- cursor,
- "MATCH",
- "concurrency-limiter:*",
- "COUNT",
- 100000,
- );
- cursor = parseInt(result[0], 10);
- const usable = result[1].filter(x => !x.includes("preview_"));
-
- logger.info("Stepped", { cursor, usable: usable.length });
-
- if (usable.length > 0) {
- for (const x of usable) {
- const concurrency = await redis.zrangebyscore(x, Date.now(), Infinity);
- const teamId = x.split(":")[1];
- concurrencyByTeam.set(
- teamId,
- (concurrencyByTeam.get(teamId) ?? 0) + concurrency.length,
- );
- }
- }
- } while (cursor != 0);
-
- if (fdbQueueEnabled()) {
- try {
- if (await nuqFdbHealthCheck(FDB_OPTIONAL_COUNT_TIMEOUT_MS)) {
- const fdbCounts = await withFdbTimeout(
- scrapeQueueFdb.getTeamActiveCounts(),
- FDB_OPTIONAL_COUNT_TIMEOUT_MS,
- );
- for (const [teamId, concurrency] of fdbCounts) {
- concurrencyByTeam.set(
- teamId,
- (concurrencyByTeam.get(teamId) ?? 0) + concurrency,
- );
- }
- }
- } catch (e) {
- logger.warn("Error reading FDB concurrency", { error: e });
- }
- }
-
- const at = new Date();
- const entries = Array.from(concurrencyByTeam.entries()).map(
- ([team_id, concurrency]) => ({
- team_id,
- concurrency,
- created_at: at.toISOString(),
- }),
- );
-
- if (entries.length === 0) return;
-
- try {
- await db.insert(schema.concurrency_log).values(entries);
- } catch (e) {
- logger.error("Error inserting", { error: e });
- }
-}
-
-export async function cclogController(req: Request, res: Response) {
- try {
- await cclog();
- res.status(200).json({ ok: true });
- } catch (e) {
- _logger.error("Error", { module: "cclog", error: e });
- res.status(500).json({
- message: "Error",
- });
- }
-}
diff --git a/apps/api/src/lib/cclog.ts b/apps/api/src/lib/cclog.ts
new file mode 100644
index 000000000..bb191aacf
--- /dev/null
+++ b/apps/api/src/lib/cclog.ts
@@ -0,0 +1,227 @@
+import type IORedis from "ioredis";
+import { db } from "../db/connection";
+import * as schema from "../db/schema";
+import { logger as _logger } from "./logger";
+import {
+ nuqFdbHealthCheck,
+ scrapeQueueFdb,
+ withFdbTimeout,
+} from "../services/worker/nuq-fdb";
+import { fdbQueueEnabled } from "../services/worker/nuq-router";
+
+const FDB_OPTIONAL_COUNT_TIMEOUT_MS = 500;
+const CONCURRENCY_LIMITER_KEY_PATTERN = "concurrency-limiter:*";
+const PREVIEW_KEY_FRAGMENT = "preview_";
+const CCLOG_SAMPLE_KEY_PREFIX = "cclog:minute";
+const CCLOG_SAMPLE_TTL_SECONDS = 60 * 60;
+
+const CCLOG_MINUTE_MS = 60 * 1000;
+const CCLOG_AGGREGATE_INTERVAL_MINUTES = 10;
+
+type CclogSample = {
+ at: Date;
+ concurrencyByTeam: Map<string, number>;
+};
+
+type CclogAggregateEntry = {
+ team_id: string;
+ concurrency: number;
+ created_at: string;
+};
+
+export function floorToMinute(date: Date): Date {
+ const at = new Date(date);
+ at.setSeconds(0, 0);
+ return at;
+}
+
+export function getMsUntilNextMinute(now = new Date()): number {
+ return CCLOG_MINUTE_MS - (now.getSeconds() * 1000 + now.getMilliseconds());
+}
+
+function getSampleKey(at: Date): string {
+ return `${CCLOG_SAMPLE_KEY_PREFIX}:${Math.floor(
+ floorToMinute(at).getTime() / CCLOG_MINUTE_MS,
+ )}`;
+}
+
+function shouldAggregateAt(at: Date): boolean {
+ return (
+ floorToMinute(at).getMinutes() % CCLOG_AGGREGATE_INTERVAL_MINUTES === 0
+ );
+}
+
+function getAggregationWindow(at: Date): Date[] {
+ const end = floorToMinute(at).getTime();
+ return Array.from({ length: CCLOG_AGGREGATE_INTERVAL_MINUTES }, (_, i) => {
+ return new Date(
+ end - (CCLOG_AGGREGATE_INTERVAL_MINUTES - 1 - i) * CCLOG_MINUTE_MS,
+ );
+ });
+}
+
+async function collectCurrentConcurrency(
+ redis: IORedis,
+): Promise<Map<string, number>> {
+ const logger = _logger.child({ module: "cclog" });
+ const concurrencyByTeam = new Map<string, number>();
+
+ let cursor = "0";
+ do {
+ const [nextCursor, keys] = await redis.scan(
+ cursor,
+ "MATCH",
+ CONCURRENCY_LIMITER_KEY_PATTERN,
+ "COUNT",
+ 100000,
+ );
+ cursor = nextCursor;
+
+ const usable = keys.filter(x => !x.includes(PREVIEW_KEY_FRAGMENT));
+ logger.info("Scanned concurrency limiter keys", {
+ cursor,
+ usable: usable.length,
+ });
+
+ for (const key of usable) {
+ const concurrency = await redis.zrangebyscore(key, Date.now(), Infinity);
+ const teamId = key.split(":")[1];
+
+ if (!teamId) continue;
+
+ concurrencyByTeam.set(
+ teamId,
+ (concurrencyByTeam.get(teamId) ?? 0) + concurrency.length,
+ );
+ }
+ } while (cursor !== "0");
+
+ if (fdbQueueEnabled()) {
+ try {
+ if (await nuqFdbHealthCheck(FDB_OPTIONAL_COUNT_TIMEOUT_MS)) {
+ const fdbCounts = await withFdbTimeout(
+ scrapeQueueFdb.getTeamActiveCounts(),
+ FDB_OPTIONAL_COUNT_TIMEOUT_MS,
+ );
+
+ for (const [teamId, concurrency] of fdbCounts) {
+ concurrencyByTeam.set(
+ teamId,
+ (concurrencyByTeam.get(teamId) ?? 0) + concurrency,
+ );
+ }
+ }
+ } catch (error) {
+ logger.warn("Error reading FDB concurrency", { error });
+ }
+ }
+
+ return concurrencyByTeam;
+}
+
+async function saveCclogMinuteSample(
+ redis: IORedis,
+ sample: CclogSample,
+): Promise<void> {
+ const key = getSampleKey(sample.at);
+ const pipeline = redis.pipeline();
+
+ if (sample.concurrencyByTeam.size > 0) {
+ const values: Record<string, string> = {};
+
+ for (const [teamId, concurrency] of sample.concurrencyByTeam) {
+ values[teamId] = String(concurrency);
+ }
+
+ pipeline.hset(key, values);
+ } else {
+ pipeline.hset(key, "__empty__", "0");
+ }
+
+ pipeline.expire(key, CCLOG_SAMPLE_TTL_SECONDS);
+ await pipeline.exec();
+}
+
+async function buildCclogAggregateEntries(
+ redis: IORedis,
+ at: Date,
+): Promise<CclogAggregateEntry[]> {
+ const window = getAggregationWindow(at);
+ const samples = await Promise.all(
+ window.map(async minute => {
+ const values = await redis.hgetall(getSampleKey(minute));
+ delete values.__empty__;
+ return values;
+ }),
+ );
+
+ const teamIds = new Set<string>();
+ for (const sample of samples) {
+ for (const teamId of Object.keys(sample)) {
+ teamIds.add(teamId);
+ }
+ }
+
+ const created_at = floorToMinute(at).toISOString();
+ return Array.from(teamIds).map(team_id => {
+ let total = 0;
+
+ for (const sample of samples) {
+ total += Number.parseInt(sample[team_id] ?? "0", 10);
+ }
+
+ return {
+ team_id,
+ concurrency: Math.round(total / CCLOG_AGGREGATE_INTERVAL_MINUTES),
+ created_at,
+ };
+ });
+}
+
+async function insertCclogAggregate(entries: CclogAggregateEntry[]) {
+ if (entries.length === 0) return;
+
+ await db.insert(schema.concurrency_log).values(entries);
+}
+
+export async function runCclogTick(redis: IORedis, at = new Date()) {
+ const logger = _logger.child({ module: "cclog" });
+ const minute = floorToMinute(at);
+ const concurrencyByTeam = await collectCurrentConcurrency(redis);
+
+ await saveCclogMinuteSample(redis, {
+ at: minute,
+ concurrencyByTeam,
+ });
+
+ logger.info("Saved cclog minute sample", {
+ at: minute.toISOString(),
+ teams: concurrencyByTeam.size,
+ });
+
+ if (!shouldAggregateAt(minute)) {
+ return {
+ sampledTeams: concurrencyByTeam.size,
+ insertedRows: 0,
+ };
+ }
+
+ const entries = await buildCclogAggregateEntries(redis, minute);
+ let insertedRows = 0;
+
+ try {
+ await insertCclogAggregate(entries);
+ insertedRows = entries.length;
+ logger.info("Inserted cclog aggregate", {
+ at: minute.toISOString(),
+ rows: insertedRows,
+ });
+ } catch (error) {
+ logger.error("Error inserting cclog aggregate", { error });
+ }
+
+ return {
+ sampledTeams: concurrencyByTeam.size,
+ insertedRows,
+ };
+}
diff --git a/apps/api/src/routes/admin.ts b/apps/api/src/routes/admin.ts
index c3c198147..def358e82 100644
--- a/apps/api/src/routes/admin.ts
+++ b/apps/api/src/routes/admin.ts
@@ -5,7 +5,6 @@ import { autumnHealthController } from "../controllers/v0/admin/autumn-health";
import { authMiddleware, checkCreditsMiddleware, wrap } from "./shared";
import { acucCacheClearController } from "../controllers/v0/admin/acuc-cache-clear";
import { checkFireEngine } from "../controllers/v0/admin/check-fire-engine";
-import { cclogController } from "../controllers/v0/admin/cclog";
import { indexQueuePrometheus } from "../controllers/v0/admin/index-queue-prometheus";
import { triggerPrecrawl } from "../controllers/v0/admin/precrawl";
import {
@@ -46,11 +45,6 @@ if (config.BULL_AUTH_KEY) {
wrap(checkFireEngine),
);
- adminRouter.get(
- `/admin/${config.BULL_AUTH_KEY}/cclog`,
- wrap(cclogController),
- );
-
adminRouter.get(
`/admin/${config.BULL_AUTH_KEY}/index-queue-prometheus`,
wrap(indexQueuePrometheus),
diff --git a/apps/api/src/services/cclog-worker.ts b/apps/api/src/services/cclog-worker.ts
new file mode 100644
index 000000000..8631a6872
--- /dev/null
+++ b/apps/api/src/services/cclog-worker.ts
@@ -0,0 +1,106 @@
+import "dotenv/config";
+import { config } from "../config";
+import "./sentry";
+import { setSentryServiceTag } from "./sentry";
+import Express from "express";
+import { logger as _logger } from "../lib/logger";
+import {
+ floorToMinute,
+ getMsUntilNextMinute,
+ runCclogTick,
+} from "../lib/cclog";
+import { getRedisConnection } from "./queue-service";
+
+const CCLOG_WORKER_LOCK_KEY = "cclog:worker:tick-lock";
+const CCLOG_WORKER_LOCK_TTL_SECONDS = 55;
+
+const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
+
+(async () => {
+ setSentryServiceTag("cclog-worker");
+
+ let isShuttingDown = false;
+ let tickInFlight = false;
+
+ const app = Express();
+ app.get("/health", (_, res) => {
+ res.status(200).send("OK");
+ });
+
+ const server = app.listen(config.CCLOG_WORKER_PORT, (error?: Error) => {
+ if (error) {
+ _logger.error("Failed to start cclog worker", {
+ error,
+ port: config.CCLOG_WORKER_PORT,
+ });
+ throw error;
+ }
+
+ _logger.info("cclog worker started", {
+ port: config.CCLOG_WORKER_PORT,
+ });
+ });
+
+ async function shutdown() {
+ if (isShuttingDown) return;
+ isShuttingDown = true;
+ _logger.info("cclog worker shutting down");
+
+ while (tickInFlight) {
+ _logger.info("Waiting for in-flight cclog tick to complete...");
+ await sleep(1000);
+ }
+
+ server.close(() => {
+ _logger.info("cclog worker shut down");
+ process.exit(0);
+ });
+ }
+
+ if (require.main === module) {
+ process.on("SIGINT", shutdown);
+ process.on("SIGTERM", shutdown);
+ }
+
+ _logger.info("Waiting for next exact minute to start cclog worker", {
+ waitMs: getMsUntilNextMinute(),
+ });
+ await sleep(getMsUntilNextMinute());
+
+ const redis = getRedisConnection();
+
+ while (!isShuttingDown) {
+ const at = floorToMinute(new Date());
+ tickInFlight = true;
+
+ try {
+ const lock = await redis.set(
+ CCLOG_WORKER_LOCK_KEY,
+ `${process.pid}:${at.toISOString()}`,
+ "EX",
+ CCLOG_WORKER_LOCK_TTL_SECONDS,
+ "NX",
+ );
+
+ if (lock === "OK") {
+ const summary = await runCclogTick(redis, at);
+ _logger.info("cclog tick complete", {
+ at: at.toISOString(),
+ ...summary,
+ });
+ } else {
+ _logger.info("Skipping cclog tick because another worker holds lock", {
+ at: at.toISOString(),
+ });
+ }
+ } catch (error) {
+ _logger.error("cclog tick failed", { error });
+ } finally {
+ tickInFlight = false;
+ }
+
+ if (!isShuttingDown) {
+ await sleep(getMsUntilNextMinute());
+ }
+ }
+})();
diff --git a/examples/kubernetes/firecrawl-helm/README.md b/examples/kubernetes/firecrawl-helm/README.md
index 5cdf1f34b..9864fa18e 100644
--- a/examples/kubernetes/firecrawl-helm/README.md
+++ b/examples/kubernetes/firecrawl-helm/README.md
@@ -1,11 +1,13 @@
# Firecrawl Helm Chart
This chart deploys Firecrawl on Kubernetes with:
+
- `api`
- `worker` (queue-worker)
- `extract-worker`
- `nuq-worker`
- `nuq-prefetch-worker`
+- `cclog-worker`
- `playwright-service`
- `redis`
- `nuq-postgres`
@@ -23,12 +25,13 @@ Official Firecrawl images are fine for x86. Use winkk images only when ARM suppo
Use `values.yaml` plus one overlay.
Important fields:
+
- `secret.*` for API keys and sensitive values.
- `config.extra` / `secret.extra` for custom env vars.
- `image.dockerSecretEnabled` and `imagePullSecrets` for private registries.
- `resources.enabled` enables/disables all container resource requests/limits.
Default: `false`.
-- `rabbitmq.enabled`, `extractWorker.enabled`, `nuqPrefetchWorker.enabled` to toggle components.
+- `rabbitmq.enabled`, `extractWorker.enabled`, `nuqPrefetchWorker.enabled`, `cclogWorker.enabled` to toggle components.
## Deploy
diff --git a/examples/kubernetes/firecrawl-helm/templates/cclog-worker-deployment.yaml b/examples/kubernetes/firecrawl-helm/templates/cclog-worker-deployment.yaml
new file mode 100644
index 000000000..08be1f107
--- /dev/null
+++ b/examples/kubernetes/firecrawl-helm/templates/cclog-worker-deployment.yaml
@@ -0,0 +1,67 @@
+{{- if .Values.cclogWorker.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: {{ include "firecrawl.fullname" . }}-cclog-worker
+ labels:
+ app: {{ include "firecrawl.name" . }}-cclog-worker
+spec:
+ replicas: {{ .Values.cclogWorker.replicaCount | default 1 }}
+ selector:
+ matchLabels:
+ app: {{ include "firecrawl.name" . }}-cclog-worker
+ template:
+ metadata:
+ labels:
+ app: {{ include "firecrawl.name" . }}-cclog-worker
+ spec:
+ {{- if .Values.image.dockerSecretEnabled }}
+ imagePullSecrets:
+ {{- toYaml .Values.imagePullSecrets | nindent 8 }}
+ {{- end }}
+ terminationGracePeriodSeconds: 60
+ containers:
+ - name: cclog-worker
+ image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+ imagePullPolicy: {{ .Values.image.pullPolicy }}
+ command: [ "node" ]
+ args: [ "--max-old-space-size=1024", "dist/src/services/cclog-worker.js" ]
+ ports:
+ - containerPort: {{ .Values.cclogWorker.port | int }}
+ env:
+ - name: FLY_PROCESS_GROUP
+ value: "cclog-worker"
+ - name: CCLOG_WORKER_PORT
+ value: {{ .Values.cclogWorker.port | quote }}
+ - name: NUQ_POD_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.name
+ envFrom:
+ - configMapRef:
+ name: {{ include "firecrawl.fullname" . }}-config
+ - secretRef:
+ name: {{ include "firecrawl.fullname" . }}-secret
+ {{- if and .Values.resources.enabled .Values.cclogWorker.resources }}
+ resources:
+ {{- toYaml .Values.cclogWorker.resources | nindent 12 }}
+ {{- end }}
+ livenessProbe:
+ httpGet:
+ path: /health
+ port: {{ .Values.cclogWorker.port | int }}
+ initialDelaySeconds: 10
+ periodSeconds: 10
+ timeoutSeconds: 5
+ successThreshold: 1
+ failureThreshold: 3
+ readinessProbe:
+ httpGet:
+ path: /health
+ port: {{ .Values.cclogWorker.port | int }}
+ initialDelaySeconds: 10
+ periodSeconds: 10
+ timeoutSeconds: 5
+ successThreshold: 1
+ failureThreshold: 3
+{{- end }}
diff --git a/examples/kubernetes/firecrawl-helm/templates/configmap.yaml b/examples/kubernetes/firecrawl-helm/templates/configmap.yaml
index e36e1fc61..066ffc43f 100644
--- a/examples/kubernetes/firecrawl-helm/templates/configmap.yaml
+++ b/examples/kubernetes/firecrawl-helm/templates/configmap.yaml
@@ -21,6 +21,7 @@ data:
EXTRACT_WORKER_PORT: {{ default (printf "%v" .Values.extractWorker.port) .Values.config.EXTRACT_WORKER_PORT | quote }}
NUQ_WORKER_PORT: {{ default (printf "%v" .Values.nuqWorker.port) .Values.config.NUQ_WORKER_PORT | quote }}
NUQ_PREFETCH_WORKER_PORT: {{ default (printf "%v" .Values.nuqPrefetchWorker.port) .Values.config.NUQ_PREFETCH_WORKER_PORT | quote }}
+ CCLOG_WORKER_PORT: {{ default (printf "%v" .Values.cclogWorker.port) .Values.config.CCLOG_WORKER_PORT | quote }}
NUQ_WORKER_COUNT: {{ default (printf "%v" .Values.nuqWorker.replicaCount) .Values.config.NUQ_WORKER_COUNT | quote }}
REDIS_URL: {{ default $redisDefault .Values.config.REDIS_URL | quote }}
REDIS_RATE_LIMIT_URL: {{ default $redisDefault .Values.config.REDIS_RATE_LIMIT_URL | quote }}
diff --git a/examples/kubernetes/firecrawl-helm/values.yaml b/examples/kubernetes/firecrawl-helm/values.yaml
index b809ee188..50416db52 100644
--- a/examples/kubernetes/firecrawl-helm/values.yaml
+++ b/examples/kubernetes/firecrawl-helm/values.yaml
@@ -59,6 +59,18 @@ nuqPrefetchWorker:
memory: "2G"
cpu: "1000m"
+cclogWorker:
+ enabled: false
+ replicaCount: 1
+ port: 3013
+ resources:
+ requests:
+ memory: "512Mi"
+ cpu: "250m"
+ limits:
+ memory: "1Gi"
+ cpu: "500m"
+
nuqPostgres:
replicaCount: 1
image:
@@ -132,6 +144,7 @@ config:
EXTRACT_WORKER_PORT: "3004"
NUQ_WORKER_PORT: "3006"
NUQ_PREFETCH_WORKER_PORT: "3011"
+ CCLOG_WORKER_PORT: "3013"
NUQ_WORKER_COUNT: "5"
USE_DB_AUTHENTICATION: "false"
IS_KUBERNETES: "true"