Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Move cclog into a worker (#3884)
Gergő Móriczb4620ad6f43f
diff --git a/apps/api/package.json b/apps/api/package.jsonindex 2a192f1f4..ad1ee1236 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -31,6 +31,8 @@"nuq-prefetch-worker:production": "node dist/src/services/worker/nuq-prefetch-worker.js","nuq-reconciler-worker": "tsc-watch --onSuccess \"node dist/src/services/worker/nuq-reconciler-worker.js\"","nuq-reconciler-worker:production": "node dist/src/services/worker/nuq-reconciler-worker.js",+ "cclog-worker": "tsc-watch --onSuccess \"node dist/src/services/cclog-worker.js\"",+ "cclog-worker:production": "node dist/src/services/cclog-worker.js","extract-worker": "tsc-watch --onSuccess \"node dist/src/services/extract-worker.js\"","extract-worker:production": "node dist/src/services/extract-worker.js","index-worker": "tsc-watch --onSuccess \"node dist/src/services/indexing/index-worker.js\"",diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 077be98c5..3daa1ce54 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -177,6 +177,7 @@ const configSchema = z.object({NUQ_WORKER_COUNT: z.coerce.number().default(5),NUQ_PREFETCH_WORKER_PORT: z.coerce.number().default(3011).catch(3011), // todo: investigate why .catch is neededNUQ_RECONCILER_WORKER_PORT: z.coerce.number().default(3012).catch(3012),+ CCLOG_WORKER_PORT: z.coerce.number().default(3013).catch(3013),EXTRACT_WORKER_PORT: z.coerce.number().default(3004),NUQ_WAIT_MODE: z.string().optional(),diff --git a/apps/api/src/controllers/v0/admin/cclog.ts b/apps/api/src/controllers/v0/admin/cclog.tsdeleted file mode 100644index 50b8e6916..000000000--- a/apps/api/src/controllers/v0/admin/cclog.ts+++ /dev/null@@ -1,95 +0,0 @@-import { getRedisConnection } from "../../../services/queue-service";-import { db } from "../../../db/connection";-import * as schema from "../../../db/schema";-import { logger as _logger } from "../../../lib/logger";-import { Request, Response } from "express";-import {- nuqFdbHealthCheck,- scrapeQueueFdb,- withFdbTimeout,-} from "../../../services/worker/nuq-fdb";-import { fdbQueueEnabled } from "../../../services/worker/nuq-router";--const FDB_OPTIONAL_COUNT_TIMEOUT_MS = 500;--async function cclog() {- const logger = _logger.child({- module: "cclog",- });-- const concurrencyByTeam = new Map<string, number>();- const redis = getRedisConnection();- let cursor = 0;- do {- const result = await redis.scan(- cursor,- "MATCH",- "concurrency-limiter:*",- "COUNT",- 100000,- );- cursor = parseInt(result[0], 10);- const usable = result[1].filter(x => !x.includes("preview_"));-- logger.info("Stepped", { cursor, usable: usable.length });-- if (usable.length > 0) {- for (const x of usable) {- const concurrency = await redis.zrangebyscore(x, Date.now(), Infinity);- const teamId = x.split(":")[1];- concurrencyByTeam.set(- teamId,- (concurrencyByTeam.get(teamId) ?? 0) + concurrency.length,- );- }- }- } while (cursor != 0);-- if (fdbQueueEnabled()) {- try {- if (await nuqFdbHealthCheck(FDB_OPTIONAL_COUNT_TIMEOUT_MS)) {- const fdbCounts = await withFdbTimeout(- scrapeQueueFdb.getTeamActiveCounts(),- FDB_OPTIONAL_COUNT_TIMEOUT_MS,- );- for (const [teamId, concurrency] of fdbCounts) {- concurrencyByTeam.set(- teamId,- (concurrencyByTeam.get(teamId) ?? 0) + concurrency,- );- }- }- } catch (e) {- logger.warn("Error reading FDB concurrency", { error: e });- }- }-- const at = new Date();- const entries = Array.from(concurrencyByTeam.entries()).map(- ([team_id, concurrency]) => ({- team_id,- concurrency,- created_at: at.toISOString(),- }),- );-- if (entries.length === 0) return;-- try {- await db.insert(schema.concurrency_log).values(entries);- } catch (e) {- logger.error("Error inserting", { error: e });- }-}--export async function cclogController(req: Request, res: Response) {- try {- await cclog();- res.status(200).json({ ok: true });- } catch (e) {- _logger.error("Error", { module: "cclog", error: e });- res.status(500).json({- message: "Error",- });- }-}diff --git a/apps/api/src/lib/cclog.ts b/apps/api/src/lib/cclog.tsnew file mode 100644index 000000000..bb191aacf--- /dev/null+++ b/apps/api/src/lib/cclog.ts@@ -0,0 +1,227 @@+import type IORedis from "ioredis";+import { db } from "../db/connection";+import * as schema from "../db/schema";+import { logger as _logger } from "./logger";+import {+ nuqFdbHealthCheck,+ scrapeQueueFdb,+ withFdbTimeout,+} from "../services/worker/nuq-fdb";+import { fdbQueueEnabled } from "../services/worker/nuq-router";++const FDB_OPTIONAL_COUNT_TIMEOUT_MS = 500;+const CONCURRENCY_LIMITER_KEY_PATTERN = "concurrency-limiter:*";+const PREVIEW_KEY_FRAGMENT = "preview_";+const CCLOG_SAMPLE_KEY_PREFIX = "cclog:minute";+const CCLOG_SAMPLE_TTL_SECONDS = 60 * 60;++const CCLOG_MINUTE_MS = 60 * 1000;+const CCLOG_AGGREGATE_INTERVAL_MINUTES = 10;++type CclogSample = {+ at: Date;+ concurrencyByTeam: Map<string, number>;+};++type CclogAggregateEntry = {+ team_id: string;+ concurrency: number;+ created_at: string;+};++export function floorToMinute(date: Date): Date {+ const at = new Date(date);+ at.setSeconds(0, 0);+ return at;+}++export function getMsUntilNextMinute(now = new Date()): number {+ return CCLOG_MINUTE_MS - (now.getSeconds() * 1000 + now.getMilliseconds());+}++function getSampleKey(at: Date): string {+ return `${CCLOG_SAMPLE_KEY_PREFIX}:${Math.floor(+ floorToMinute(at).getTime() / CCLOG_MINUTE_MS,+ )}`;+}++function shouldAggregateAt(at: Date): boolean {+ return (+ floorToMinute(at).getMinutes() % CCLOG_AGGREGATE_INTERVAL_MINUTES === 0+ );+}++function getAggregationWindow(at: Date): Date[] {+ const end = floorToMinute(at).getTime();+ return Array.from({ length: CCLOG_AGGREGATE_INTERVAL_MINUTES }, (_, i) => {+ return new Date(+ end - (CCLOG_AGGREGATE_INTERVAL_MINUTES - 1 - i) * CCLOG_MINUTE_MS,+ );+ });+}++async function collectCurrentConcurrency(+ redis: IORedis,+): Promise<Map<string, number>> {+ const logger = _logger.child({ module: "cclog" });+ const concurrencyByTeam = new Map<string, number>();++ let cursor = "0";+ do {+ const [nextCursor, keys] = await redis.scan(+ cursor,+ "MATCH",+ CONCURRENCY_LIMITER_KEY_PATTERN,+ "COUNT",+ 100000,+ );+ cursor = nextCursor;++ const usable = keys.filter(x => !x.includes(PREVIEW_KEY_FRAGMENT));+ logger.info("Scanned concurrency limiter keys", {+ cursor,+ usable: usable.length,+ });++ for (const key of usable) {+ const concurrency = await redis.zrangebyscore(key, Date.now(), Infinity);+ const teamId = key.split(":")[1];++ if (!teamId) continue;++ concurrencyByTeam.set(+ teamId,+ (concurrencyByTeam.get(teamId) ?? 0) + concurrency.length,+ );+ }+ } while (cursor !== "0");++ if (fdbQueueEnabled()) {+ try {+ if (await nuqFdbHealthCheck(FDB_OPTIONAL_COUNT_TIMEOUT_MS)) {+ const fdbCounts = await withFdbTimeout(+ scrapeQueueFdb.getTeamActiveCounts(),+ FDB_OPTIONAL_COUNT_TIMEOUT_MS,+ );++ for (const [teamId, concurrency] of fdbCounts) {+ concurrencyByTeam.set(+ teamId,+ (concurrencyByTeam.get(teamId) ?? 0) + concurrency,+ );+ }+ }+ } catch (error) {+ logger.warn("Error reading FDB concurrency", { error });+ }+ }++ return concurrencyByTeam;+}++async function saveCclogMinuteSample(+ redis: IORedis,+ sample: CclogSample,+): Promise<void> {+ const key = getSampleKey(sample.at);+ const pipeline = redis.pipeline();++ if (sample.concurrencyByTeam.size > 0) {+ const values: Record<string, string> = {};++ for (const [teamId, concurrency] of sample.concurrencyByTeam) {+ values[teamId] = String(concurrency);+ }++ pipeline.hset(key, values);+ } else {+ pipeline.hset(key, "__empty__", "0");+ }++ pipeline.expire(key, CCLOG_SAMPLE_TTL_SECONDS);+ await pipeline.exec();+}++async function buildCclogAggregateEntries(+ redis: IORedis,+ at: Date,+): Promise<CclogAggregateEntry[]> {+ const window = getAggregationWindow(at);+ const samples = await Promise.all(+ window.map(async minute => {+ const values = await redis.hgetall(getSampleKey(minute));+ delete values.__empty__;+ return values;+ }),+ );++ const teamIds = new Set<string>();+ for (const sample of samples) {+ for (const teamId of Object.keys(sample)) {+ teamIds.add(teamId);+ }+ }++ const created_at = floorToMinute(at).toISOString();+ return Array.from(teamIds).map(team_id => {+ let total = 0;++ for (const sample of samples) {+ total += Number.parseInt(sample[team_id] ?? "0", 10);+ }++ return {+ team_id,+ concurrency: Math.round(total / CCLOG_AGGREGATE_INTERVAL_MINUTES),+ created_at,+ };+ });+}++async function insertCclogAggregate(entries: CclogAggregateEntry[]) {+ if (entries.length === 0) return;++ await db.insert(schema.concurrency_log).values(entries);+}++export async function runCclogTick(redis: IORedis, at = new Date()) {+ const logger = _logger.child({ module: "cclog" });+ const minute = floorToMinute(at);+ const concurrencyByTeam = await collectCurrentConcurrency(redis);++ await saveCclogMinuteSample(redis, {+ at: minute,+ concurrencyByTeam,+ });++ logger.info("Saved cclog minute sample", {+ at: minute.toISOString(),+ teams: concurrencyByTeam.size,+ });++ if (!shouldAggregateAt(minute)) {+ return {+ sampledTeams: concurrencyByTeam.size,+ insertedRows: 0,+ };+ }++ const entries = await buildCclogAggregateEntries(redis, minute);+ let insertedRows = 0;++ try {+ await insertCclogAggregate(entries);+ insertedRows = entries.length;+ logger.info("Inserted cclog aggregate", {+ at: minute.toISOString(),+ rows: insertedRows,+ });+ } catch (error) {+ logger.error("Error inserting cclog aggregate", { error });+ }++ return {+ sampledTeams: concurrencyByTeam.size,+ insertedRows,+ };+}diff --git a/apps/api/src/routes/admin.ts b/apps/api/src/routes/admin.tsindex c3c198147..def358e82 100644--- a/apps/api/src/routes/admin.ts+++ b/apps/api/src/routes/admin.ts@@ -5,7 +5,6 @@ import { autumnHealthController } from "../controllers/v0/admin/autumn-health";import { authMiddleware, checkCreditsMiddleware, wrap } from "./shared";import { acucCacheClearController } from "../controllers/v0/admin/acuc-cache-clear";import { checkFireEngine } from "../controllers/v0/admin/check-fire-engine";-import { cclogController } from "../controllers/v0/admin/cclog";import { indexQueuePrometheus } from "../controllers/v0/admin/index-queue-prometheus";import { triggerPrecrawl } from "../controllers/v0/admin/precrawl";import {@@ -46,11 +45,6 @@ if (config.BULL_AUTH_KEY) {wrap(checkFireEngine),);- adminRouter.get(- `/admin/${config.BULL_AUTH_KEY}/cclog`,- wrap(cclogController),- );-adminRouter.get(`/admin/${config.BULL_AUTH_KEY}/index-queue-prometheus`,wrap(indexQueuePrometheus),diff --git a/apps/api/src/services/cclog-worker.ts b/apps/api/src/services/cclog-worker.tsnew file mode 100644index 000000000..8631a6872--- /dev/null+++ b/apps/api/src/services/cclog-worker.ts@@ -0,0 +1,106 @@+import "dotenv/config";+import { config } from "../config";+import "./sentry";+import { setSentryServiceTag } from "./sentry";+import Express from "express";+import { logger as _logger } from "../lib/logger";+import {+ floorToMinute,+ getMsUntilNextMinute,+ runCclogTick,+} from "../lib/cclog";+import { getRedisConnection } from "./queue-service";++const CCLOG_WORKER_LOCK_KEY = "cclog:worker:tick-lock";+const CCLOG_WORKER_LOCK_TTL_SECONDS = 55;++const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));++(async () => {+ setSentryServiceTag("cclog-worker");++ let isShuttingDown = false;+ let tickInFlight = false;++ const app = Express();+ app.get("/health", (_, res) => {+ res.status(200).send("OK");+ });++ const server = app.listen(config.CCLOG_WORKER_PORT, (error?: Error) => {+ if (error) {+ _logger.error("Failed to start cclog worker", {+ error,+ port: config.CCLOG_WORKER_PORT,+ });+ throw error;+ }++ _logger.info("cclog worker started", {+ port: config.CCLOG_WORKER_PORT,+ });+ });++ async function shutdown() {+ if (isShuttingDown) return;+ isShuttingDown = true;+ _logger.info("cclog worker shutting down");++ while (tickInFlight) {+ _logger.info("Waiting for in-flight cclog tick to complete...");+ await sleep(1000);+ }++ server.close(() => {+ _logger.info("cclog worker shut down");+ process.exit(0);+ });+ }++ if (require.main === module) {+ process.on("SIGINT", shutdown);+ process.on("SIGTERM", shutdown);+ }++ _logger.info("Waiting for next exact minute to start cclog worker", {+ waitMs: getMsUntilNextMinute(),+ });+ await sleep(getMsUntilNextMinute());++ const redis = getRedisConnection();++ while (!isShuttingDown) {+ const at = floorToMinute(new Date());+ tickInFlight = true;++ try {+ const lock = await redis.set(+ CCLOG_WORKER_LOCK_KEY,+ `${process.pid}:${at.toISOString()}`,+ "EX",+ CCLOG_WORKER_LOCK_TTL_SECONDS,+ "NX",+ );++ if (lock === "OK") {+ const summary = await runCclogTick(redis, at);+ _logger.info("cclog tick complete", {+ at: at.toISOString(),+ ...summary,+ });+ } else {+ _logger.info("Skipping cclog tick because another worker holds lock", {+ at: at.toISOString(),+ });+ }+ } catch (error) {+ _logger.error("cclog tick failed", { error });+ } finally {+ tickInFlight = false;+ }++ if (!isShuttingDown) {+ await sleep(getMsUntilNextMinute());+ }+ }+})();diff --git a/examples/kubernetes/firecrawl-helm/README.md b/examples/kubernetes/firecrawl-helm/README.mdindex 5cdf1f34b..9864fa18e 100644--- a/examples/kubernetes/firecrawl-helm/README.md+++ b/examples/kubernetes/firecrawl-helm/README.md@@ -1,11 +1,13 @@# Firecrawl Helm ChartThis chart deploys Firecrawl on Kubernetes with:+- `api`- `worker` (queue-worker)- `extract-worker`- `nuq-worker`- `nuq-prefetch-worker`+- `cclog-worker`- `playwright-service`- `redis`- `nuq-postgres`@@ -23,12 +25,13 @@ Official Firecrawl images are fine for x86. Use winkk images only when ARM suppoUse `values.yaml` plus one overlay.Important fields:+- `secret.*` for API keys and sensitive values.- `config.extra` / `secret.extra` for custom env vars.- `image.dockerSecretEnabled` and `imagePullSecrets` for private registries.- `resources.enabled` enables/disables all container resource requests/limits.Default: `false`.-- `rabbitmq.enabled`, `extractWorker.enabled`, `nuqPrefetchWorker.enabled` to toggle components.+- `rabbitmq.enabled`, `extractWorker.enabled`, `nuqPrefetchWorker.enabled`, `cclogWorker.enabled` to toggle components.## Deploydiff --git a/examples/kubernetes/firecrawl-helm/templates/cclog-worker-deployment.yaml b/examples/kubernetes/firecrawl-helm/templates/cclog-worker-deployment.yamlnew file mode 100644index 000000000..08be1f107--- /dev/null+++ b/examples/kubernetes/firecrawl-helm/templates/cclog-worker-deployment.yaml@@ -0,0 +1,67 @@+{{- if .Values.cclogWorker.enabled }}+apiVersion: apps/v1+kind: Deployment+metadata:+ name: {{ include "firecrawl.fullname" . }}-cclog-worker+ labels:+ app: {{ include "firecrawl.name" . }}-cclog-worker+spec:+ replicas: {{ .Values.cclogWorker.replicaCount | default 1 }}+ selector:+ matchLabels:+ app: {{ include "firecrawl.name" . }}-cclog-worker+ template:+ metadata:+ labels:+ app: {{ include "firecrawl.name" . }}-cclog-worker+ spec:+ {{- if .Values.image.dockerSecretEnabled }}+ imagePullSecrets:+ {{- toYaml .Values.imagePullSecrets | nindent 8 }}+ {{- end }}+ terminationGracePeriodSeconds: 60+ containers:+ - name: cclog-worker+ image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"+ imagePullPolicy: {{ .Values.image.pullPolicy }}+ command: [ "node" ]+ args: [ "--max-old-space-size=1024", "dist/src/services/cclog-worker.js" ]+ ports:+ - containerPort: {{ .Values.cclogWorker.port | int }}+ env:+ - name: FLY_PROCESS_GROUP+ value: "cclog-worker"+ - name: CCLOG_WORKER_PORT+ value: {{ .Values.cclogWorker.port | quote }}+ - name: NUQ_POD_NAME+ valueFrom:+ fieldRef:+ fieldPath: metadata.name+ envFrom:+ - configMapRef:+ name: {{ include "firecrawl.fullname" . }}-config+ - secretRef:+ name: {{ include "firecrawl.fullname" . }}-secret+ {{- if and .Values.resources.enabled .Values.cclogWorker.resources }}+ resources:+ {{- toYaml .Values.cclogWorker.resources | nindent 12 }}+ {{- end }}+ livenessProbe:+ httpGet:+ path: /health+ port: {{ .Values.cclogWorker.port | int }}+ initialDelaySeconds: 10+ periodSeconds: 10+ timeoutSeconds: 5+ successThreshold: 1+ failureThreshold: 3+ readinessProbe:+ httpGet:+ path: /health+ port: {{ .Values.cclogWorker.port | int }}+ initialDelaySeconds: 10+ periodSeconds: 10+ timeoutSeconds: 5+ successThreshold: 1+ failureThreshold: 3+{{- end }}diff --git a/examples/kubernetes/firecrawl-helm/templates/configmap.yaml b/examples/kubernetes/firecrawl-helm/templates/configmap.yamlindex e36e1fc61..066ffc43f 100644--- a/examples/kubernetes/firecrawl-helm/templates/configmap.yaml+++ b/examples/kubernetes/firecrawl-helm/templates/configmap.yaml@@ -21,6 +21,7 @@ data:EXTRACT_WORKER_PORT: {{ default (printf "%v" .Values.extractWorker.port) .Values.config.EXTRACT_WORKER_PORT | quote }}NUQ_WORKER_PORT: {{ default (printf "%v" .Values.nuqWorker.port) .Values.config.NUQ_WORKER_PORT | quote }}NUQ_PREFETCH_WORKER_PORT: {{ default (printf "%v" .Values.nuqPrefetchWorker.port) .Values.config.NUQ_PREFETCH_WORKER_PORT | quote }}+ CCLOG_WORKER_PORT: {{ default (printf "%v" .Values.cclogWorker.port) .Values.config.CCLOG_WORKER_PORT | quote }}NUQ_WORKER_COUNT: {{ default (printf "%v" .Values.nuqWorker.replicaCount) .Values.config.NUQ_WORKER_COUNT | quote }}REDIS_URL: {{ default $redisDefault .Values.config.REDIS_URL | quote }}REDIS_RATE_LIMIT_URL: {{ default $redisDefault .Values.config.REDIS_RATE_LIMIT_URL | quote }}diff --git a/examples/kubernetes/firecrawl-helm/values.yaml b/examples/kubernetes/firecrawl-helm/values.yamlindex b809ee188..50416db52 100644--- a/examples/kubernetes/firecrawl-helm/values.yaml+++ b/examples/kubernetes/firecrawl-helm/values.yaml@@ -59,6 +59,18 @@ nuqPrefetchWorker:memory: "2G"cpu: "1000m"+cclogWorker:+ enabled: false+ replicaCount: 1+ port: 3013+ resources:+ requests:+ memory: "512Mi"+ cpu: "250m"+ limits:+ memory: "1Gi"+ cpu: "500m"+nuqPostgres:replicaCount: 1image:@@ -132,6 +144,7 @@ config:EXTRACT_WORKER_PORT: "3004"NUQ_WORKER_PORT: "3006"NUQ_PREFETCH_WORKER_PORT: "3011"+ CCLOG_WORKER_PORT: "3013"NUQ_WORKER_COUNT: "5"USE_DB_AUTHENTICATION: "false"IS_KUBERNETES: "true"