Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): move cclog to clickhouse (#3895)

Gergő Móriczc27746b2e594
diff --git a/apps/api/clickhouse/concurrency_logs.sql b/apps/api/clickhouse/concurrency_logs.sql
new file mode 100644
index 000000000..7f2b81498
--- /dev/null
+++ b/apps/api/clickhouse/concurrency_logs.sql
@@ -0,0 +1,11 @@
+CREATE TABLE IF NOT EXISTS concurrency_logs
+(
+ created_at DateTime,
+ team_id UUID,
+ avg_concurrency UInt32,
+ max_concurrency UInt32,
+ aggregate_minutes UInt8 DEFAULT 10
+)
+ENGINE = MergeTree
+PARTITION BY toYYYYMM(created_at)
+ORDER BY (team_id, created_at);
diff --git a/apps/api/src/db/schema/public.ts b/apps/api/src/db/schema/public.ts
index f03241d65..e5981cd93 100644
--- a/apps/api/src/db/schema/public.ts
+++ b/apps/api/src/db/schema/public.ts
@@ -129,13 +129,6 @@ export const browser_sessions = pgTable("browser_sessions", {
scrape_id: uuid("scrape_id"),
});
-export const concurrency_log = pgTable("concurrency_log", {
- id: bigintNum("id").notNull().generatedByDefaultAsIdentity(),
- created_at: ts("created_at").notNull().defaultNow(),
- team_id: uuid("team_id").notNull(),
- concurrency: integer("concurrency").notNull(),
-});
-
export const crawls = pgTable("crawls", {
id: uuid("id").notNull(),
request_id: uuid("request_id").notNull(),
diff --git a/apps/api/src/lib/cclog.test.ts b/apps/api/src/lib/cclog.test.ts
new file mode 100644
index 000000000..91c0c0ad4
--- /dev/null
+++ b/apps/api/src/lib/cclog.test.ts
@@ -0,0 +1,182 @@
+const { chInsertMock } = vi.hoisted(() => ({
+ chInsertMock: vi.fn(),
+}));
+
+vi.mock("./clickhouse-client", () => ({
+ chInsert: chInsertMock,
+}));
+
+vi.mock("../services/worker/nuq-router", () => ({
+ fdbQueueEnabled: () => false,
+}));
+
+vi.mock("../services/worker/nuq-fdb", () => ({
+ nuqFdbHealthCheck: vi.fn(),
+ scrapeQueueFdb: {
+ getTeamActiveCounts: vi.fn(),
+ },
+ withFdbTimeout: vi.fn(),
+}));
+
+import { runCclogTick } from "./cclog";
+
+const minuteMs = 60 * 1000;
+
+function sampleKey(at: Date): string {
+ const minute = new Date(at);
+ minute.setSeconds(0, 0);
+ return `cclog:minute:${Math.floor(minute.getTime() / minuteMs)}`;
+}
+
+class FakeRedis {
+ private hashes = new Map<string, Record<string, string>>();
+
+ constructor(
+ private readonly keys: string[],
+ private readonly activeCounts: Record<string, number>,
+ ) {}
+
+ seedHash(key: string, values: Record<string, string>) {
+ this.hashes.set(key, { ...values });
+ }
+
+ async scan() {
+ return ["0", this.keys] as [string, string[]];
+ }
+
+ async zrangebyscore(key: string) {
+ return Array.from({ length: this.activeCounts[key] ?? 0 }, (_, i) =>
+ String(i),
+ );
+ }
+
+ async hgetall(key: string) {
+ return { ...(this.hashes.get(key) ?? {}) };
+ }
+
+ pipeline() {
+ return {
+ hset: (
+ key: string,
+ valuesOrField: Record<string, string> | string,
+ value?: string,
+ ) => {
+ const values =
+ typeof valuesOrField === "string"
+ ? { [valuesOrField]: value ?? "" }
+ : valuesOrField;
+ this.hashes.set(key, { ...(this.hashes.get(key) ?? {}), ...values });
+ },
+ expire: () => {},
+ exec: async () => [],
+ };
+ }
+}
+
+describe("cclog", () => {
+ beforeEach(() => {
+ chInsertMock.mockReset();
+ chInsertMock.mockResolvedValue(true);
+ });
+
+ it("inserts avg and max aggregate concurrency rows into ClickHouse", async () => {
+ const teamA = "11111111-1111-1111-1111-111111111111";
+ const teamB = "22222222-2222-2222-2222-222222222222";
+ const at = new Date("2026-06-26T12:20:15.000Z");
+ const minute = new Date("2026-06-26T12:20:00.000Z");
+ const redis = new FakeRedis(
+ [`concurrency-limiter:${teamA}`, `concurrency-limiter:preview_${teamB}`],
+ {
+ [`concurrency-limiter:${teamA}`]: 7,
+ [`concurrency-limiter:preview_${teamB}`]: 50,
+ },
+ );
+
+ for (let i = 9; i >= 1; i--) {
+ const sampleAt = new Date(minute.getTime() - i * minuteMs);
+ redis.seedHash(sampleKey(sampleAt), {
+ [teamA]: String(10 - i),
+ ...(i === 9 ? { [teamB]: "10" } : {}),
+ });
+ }
+
+ const result = await runCclogTick(redis as any, at);
+
+ expect(result).toEqual({
+ sampledTeams: 1,
+ insertedRows: 2,
+ });
+ expect(chInsertMock).toHaveBeenCalledWith(
+ "concurrency_logs",
+ expect.arrayContaining([
+ {
+ team_id: teamA,
+ avg_concurrency: 5,
+ max_concurrency: 9,
+ created_at: "2026-06-26T12:20:00.000Z",
+ },
+ {
+ team_id: teamB,
+ avg_concurrency: 1,
+ max_concurrency: 10,
+ created_at: "2026-06-26T12:20:00.000Z",
+ },
+ ]),
+ { throwOnError: true },
+ );
+ });
+
+ it("does not report inserted rows when the ClickHouse insert fails", async () => {
+ const teamId = "11111111-1111-1111-1111-111111111111";
+ const at = new Date("2026-06-26T12:20:00.000Z");
+ const redis = new FakeRedis([], {});
+
+ redis.seedHash(sampleKey(new Date("2026-06-26T12:19:00.000Z")), {
+ [teamId]: "4",
+ });
+ chInsertMock.mockRejectedValueOnce(new Error("clickhouse unavailable"));
+
+ const result = await runCclogTick(redis as any, at);
+
+ expect(chInsertMock).toHaveBeenCalledWith(
+ "concurrency_logs",
+ [
+ {
+ team_id: teamId,
+ avg_concurrency: 0,
+ max_concurrency: 4,
+ created_at: "2026-06-26T12:20:00.000Z",
+ },
+ ],
+ { throwOnError: true },
+ );
+ expect(result.insertedRows).toBe(0);
+ });
+
+ it("does not report inserted rows when ClickHouse is not configured", async () => {
+ const teamId = "11111111-1111-1111-1111-111111111111";
+ const at = new Date("2026-06-26T12:20:00.000Z");
+ const redis = new FakeRedis([], {});
+
+ redis.seedHash(sampleKey(new Date("2026-06-26T12:19:00.000Z")), {
+ [teamId]: "4",
+ });
+ chInsertMock.mockResolvedValueOnce(false);
+
+ const result = await runCclogTick(redis as any, at);
+
+ expect(chInsertMock).toHaveBeenCalledWith(
+ "concurrency_logs",
+ [
+ {
+ team_id: teamId,
+ avg_concurrency: 0,
+ max_concurrency: 4,
+ created_at: "2026-06-26T12:20:00.000Z",
+ },
+ ],
+ { throwOnError: true },
+ );
+ expect(result.insertedRows).toBe(0);
+ });
+});
diff --git a/apps/api/src/lib/cclog.ts b/apps/api/src/lib/cclog.ts
index bb191aacf..596f3f22a 100644
--- a/apps/api/src/lib/cclog.ts
+++ b/apps/api/src/lib/cclog.ts
@@ -1,6 +1,4 @@
import type IORedis from "ioredis";
-import { db } from "../db/connection";
-import * as schema from "../db/schema";
import { logger as _logger } from "./logger";
import {
nuqFdbHealthCheck,
@@ -8,6 +6,7 @@ import {
withFdbTimeout,
} from "../services/worker/nuq-fdb";
import { fdbQueueEnabled } from "../services/worker/nuq-router";
+import { chInsert } from "./clickhouse-client";
const FDB_OPTIONAL_COUNT_TIMEOUT_MS = 500;
const CONCURRENCY_LIMITER_KEY_PATTERN = "concurrency-limiter:*";
@@ -25,7 +24,8 @@ type CclogSample = {
type CclogAggregateEntry = {
team_id: string;
- concurrency: number;
+ avg_concurrency: number;
+ max_concurrency: number;
created_at: string;
};
@@ -165,23 +165,29 @@ async function buildCclogAggregateEntries(
const created_at = floorToMinute(at).toISOString();
return Array.from(teamIds).map(team_id => {
let total = 0;
+ let max = 0;
for (const sample of samples) {
- total += Number.parseInt(sample[team_id] ?? "0", 10);
+ const concurrency = Number.parseInt(sample[team_id] ?? "0", 10);
+ total += concurrency;
+ max = Math.max(max, concurrency);
}
return {
team_id,
- concurrency: Math.round(total / CCLOG_AGGREGATE_INTERVAL_MINUTES),
+ avg_concurrency: Math.round(total / CCLOG_AGGREGATE_INTERVAL_MINUTES),
+ max_concurrency: max,
created_at,
};
});
}
-async function insertCclogAggregate(entries: CclogAggregateEntry[]) {
- if (entries.length === 0) return;
+async function insertCclogAggregate(
+ entries: CclogAggregateEntry[],
+): Promise<boolean> {
+ if (entries.length === 0) return true;
- await db.insert(schema.concurrency_log).values(entries);
+ return chInsert("concurrency_logs", entries, { throwOnError: true });
}
export async function runCclogTick(redis: IORedis, at = new Date()) {
@@ -210,12 +216,18 @@ export async function runCclogTick(redis: IORedis, at = new Date()) {
let insertedRows = 0;
try {
- await insertCclogAggregate(entries);
- insertedRows = entries.length;
- logger.info("Inserted cclog aggregate", {
- at: minute.toISOString(),
- rows: insertedRows,
- });
+ if (await insertCclogAggregate(entries)) {
+ insertedRows = entries.length;
+ logger.info("Inserted cclog aggregate", {
+ at: minute.toISOString(),
+ rows: insertedRows,
+ });
+ } else {
+ logger.warn("Skipped cclog aggregate insert", {
+ at: minute.toISOString(),
+ rows: entries.length,
+ });
+ }
} catch (error) {
logger.error("Error inserting cclog aggregate", { error });
}
diff --git a/apps/api/src/lib/clickhouse-client.ts b/apps/api/src/lib/clickhouse-client.ts
index 5d7b046fc..ed55d878a 100644
--- a/apps/api/src/lib/clickhouse-client.ts
+++ b/apps/api/src/lib/clickhouse-client.ts
@@ -16,8 +16,10 @@ const client = config.CLICKHOUSE_ANALYTICS_URL
export async function chInsert(
table: string,
rows: Record<string, unknown>[],
-): Promise<void> {
- if (!client || rows.length === 0) return;
+ opts?: { throwOnError?: boolean },
+): Promise<boolean> {
+ if (rows.length === 0) return true;
+ if (!client) return false;
try {
await client.insert({
@@ -25,11 +27,16 @@ export async function chInsert(
values: rows,
format: "JSONEachRow",
});
+ return true;
} catch (error: any) {
logger.error("ClickHouse insert failed", {
table,
rowCount: rows.length,
error: error?.message,
});
+ if (opts?.throwOnError) {
+ throw error;
+ }
+ return false;
}
}