Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): move cclog to clickhouse (#3895)
Gergő Móriczc27746b2e594
diff --git a/apps/api/clickhouse/concurrency_logs.sql b/apps/api/clickhouse/concurrency_logs.sqlnew file mode 100644index 000000000..7f2b81498--- /dev/null+++ b/apps/api/clickhouse/concurrency_logs.sql@@ -0,0 +1,11 @@+CREATE TABLE IF NOT EXISTS concurrency_logs+(+ created_at DateTime,+ team_id UUID,+ avg_concurrency UInt32,+ max_concurrency UInt32,+ aggregate_minutes UInt8 DEFAULT 10+)+ENGINE = MergeTree+PARTITION BY toYYYYMM(created_at)+ORDER BY (team_id, created_at);diff --git a/apps/api/src/db/schema/public.ts b/apps/api/src/db/schema/public.tsindex f03241d65..e5981cd93 100644--- a/apps/api/src/db/schema/public.ts+++ b/apps/api/src/db/schema/public.ts@@ -129,13 +129,6 @@ export const browser_sessions = pgTable("browser_sessions", {scrape_id: uuid("scrape_id"),});-export const concurrency_log = pgTable("concurrency_log", {- id: bigintNum("id").notNull().generatedByDefaultAsIdentity(),- created_at: ts("created_at").notNull().defaultNow(),- team_id: uuid("team_id").notNull(),- concurrency: integer("concurrency").notNull(),-});-export const crawls = pgTable("crawls", {id: uuid("id").notNull(),request_id: uuid("request_id").notNull(),diff --git a/apps/api/src/lib/cclog.test.ts b/apps/api/src/lib/cclog.test.tsnew file mode 100644index 000000000..91c0c0ad4--- /dev/null+++ b/apps/api/src/lib/cclog.test.ts@@ -0,0 +1,182 @@+const { chInsertMock } = vi.hoisted(() => ({+ chInsertMock: vi.fn(),+}));++vi.mock("./clickhouse-client", () => ({+ chInsert: chInsertMock,+}));++vi.mock("../services/worker/nuq-router", () => ({+ fdbQueueEnabled: () => false,+}));++vi.mock("../services/worker/nuq-fdb", () => ({+ nuqFdbHealthCheck: vi.fn(),+ scrapeQueueFdb: {+ getTeamActiveCounts: vi.fn(),+ },+ withFdbTimeout: vi.fn(),+}));++import { runCclogTick } from "./cclog";++const minuteMs = 60 * 1000;++function sampleKey(at: Date): string {+ const minute = new Date(at);+ minute.setSeconds(0, 0);+ return `cclog:minute:${Math.floor(minute.getTime() / minuteMs)}`;+}++class FakeRedis {+ private hashes = new Map<string, Record<string, string>>();++ constructor(+ private readonly keys: string[],+ private readonly activeCounts: Record<string, number>,+ ) {}++ seedHash(key: string, values: Record<string, string>) {+ this.hashes.set(key, { ...values });+ }++ async scan() {+ return ["0", this.keys] as [string, string[]];+ }++ async zrangebyscore(key: string) {+ return Array.from({ length: this.activeCounts[key] ?? 0 }, (_, i) =>+ String(i),+ );+ }++ async hgetall(key: string) {+ return { ...(this.hashes.get(key) ?? {}) };+ }++ pipeline() {+ return {+ hset: (+ key: string,+ valuesOrField: Record<string, string> | string,+ value?: string,+ ) => {+ const values =+ typeof valuesOrField === "string"+ ? { [valuesOrField]: value ?? "" }+ : valuesOrField;+ this.hashes.set(key, { ...(this.hashes.get(key) ?? {}), ...values });+ },+ expire: () => {},+ exec: async () => [],+ };+ }+}++describe("cclog", () => {+ beforeEach(() => {+ chInsertMock.mockReset();+ chInsertMock.mockResolvedValue(true);+ });++ it("inserts avg and max aggregate concurrency rows into ClickHouse", async () => {+ const teamA = "11111111-1111-1111-1111-111111111111";+ const teamB = "22222222-2222-2222-2222-222222222222";+ const at = new Date("2026-06-26T12:20:15.000Z");+ const minute = new Date("2026-06-26T12:20:00.000Z");+ const redis = new FakeRedis(+ [`concurrency-limiter:${teamA}`, `concurrency-limiter:preview_${teamB}`],+ {+ [`concurrency-limiter:${teamA}`]: 7,+ [`concurrency-limiter:preview_${teamB}`]: 50,+ },+ );++ for (let i = 9; i >= 1; i--) {+ const sampleAt = new Date(minute.getTime() - i * minuteMs);+ redis.seedHash(sampleKey(sampleAt), {+ [teamA]: String(10 - i),+ ...(i === 9 ? { [teamB]: "10" } : {}),+ });+ }++ const result = await runCclogTick(redis as any, at);++ expect(result).toEqual({+ sampledTeams: 1,+ insertedRows: 2,+ });+ expect(chInsertMock).toHaveBeenCalledWith(+ "concurrency_logs",+ expect.arrayContaining([+ {+ team_id: teamA,+ avg_concurrency: 5,+ max_concurrency: 9,+ created_at: "2026-06-26T12:20:00.000Z",+ },+ {+ team_id: teamB,+ avg_concurrency: 1,+ max_concurrency: 10,+ created_at: "2026-06-26T12:20:00.000Z",+ },+ ]),+ { throwOnError: true },+ );+ });++ it("does not report inserted rows when the ClickHouse insert fails", async () => {+ const teamId = "11111111-1111-1111-1111-111111111111";+ const at = new Date("2026-06-26T12:20:00.000Z");+ const redis = new FakeRedis([], {});++ redis.seedHash(sampleKey(new Date("2026-06-26T12:19:00.000Z")), {+ [teamId]: "4",+ });+ chInsertMock.mockRejectedValueOnce(new Error("clickhouse unavailable"));++ const result = await runCclogTick(redis as any, at);++ expect(chInsertMock).toHaveBeenCalledWith(+ "concurrency_logs",+ [+ {+ team_id: teamId,+ avg_concurrency: 0,+ max_concurrency: 4,+ created_at: "2026-06-26T12:20:00.000Z",+ },+ ],+ { throwOnError: true },+ );+ expect(result.insertedRows).toBe(0);+ });++ it("does not report inserted rows when ClickHouse is not configured", async () => {+ const teamId = "11111111-1111-1111-1111-111111111111";+ const at = new Date("2026-06-26T12:20:00.000Z");+ const redis = new FakeRedis([], {});++ redis.seedHash(sampleKey(new Date("2026-06-26T12:19:00.000Z")), {+ [teamId]: "4",+ });+ chInsertMock.mockResolvedValueOnce(false);++ const result = await runCclogTick(redis as any, at);++ expect(chInsertMock).toHaveBeenCalledWith(+ "concurrency_logs",+ [+ {+ team_id: teamId,+ avg_concurrency: 0,+ max_concurrency: 4,+ created_at: "2026-06-26T12:20:00.000Z",+ },+ ],+ { throwOnError: true },+ );+ expect(result.insertedRows).toBe(0);+ });+});diff --git a/apps/api/src/lib/cclog.ts b/apps/api/src/lib/cclog.tsindex bb191aacf..596f3f22a 100644--- a/apps/api/src/lib/cclog.ts+++ b/apps/api/src/lib/cclog.ts@@ -1,6 +1,4 @@import type IORedis from "ioredis";-import { db } from "../db/connection";-import * as schema from "../db/schema";import { logger as _logger } from "./logger";import {nuqFdbHealthCheck,@@ -8,6 +6,7 @@ import {withFdbTimeout,} from "../services/worker/nuq-fdb";import { fdbQueueEnabled } from "../services/worker/nuq-router";+import { chInsert } from "./clickhouse-client";const FDB_OPTIONAL_COUNT_TIMEOUT_MS = 500;const CONCURRENCY_LIMITER_KEY_PATTERN = "concurrency-limiter:*";@@ -25,7 +24,8 @@ type CclogSample = {type CclogAggregateEntry = {team_id: string;- concurrency: number;+ avg_concurrency: number;+ max_concurrency: number;created_at: string;};@@ -165,23 +165,29 @@ async function buildCclogAggregateEntries(const created_at = floorToMinute(at).toISOString();return Array.from(teamIds).map(team_id => {let total = 0;+ let max = 0;for (const sample of samples) {- total += Number.parseInt(sample[team_id] ?? "0", 10);+ const concurrency = Number.parseInt(sample[team_id] ?? "0", 10);+ total += concurrency;+ max = Math.max(max, concurrency);}return {team_id,- concurrency: Math.round(total / CCLOG_AGGREGATE_INTERVAL_MINUTES),+ avg_concurrency: Math.round(total / CCLOG_AGGREGATE_INTERVAL_MINUTES),+ max_concurrency: max,created_at,};});}-async function insertCclogAggregate(entries: CclogAggregateEntry[]) {- if (entries.length === 0) return;+async function insertCclogAggregate(+ entries: CclogAggregateEntry[],+): Promise<boolean> {+ if (entries.length === 0) return true;- await db.insert(schema.concurrency_log).values(entries);+ return chInsert("concurrency_logs", entries, { throwOnError: true });}export async function runCclogTick(redis: IORedis, at = new Date()) {@@ -210,12 +216,18 @@ export async function runCclogTick(redis: IORedis, at = new Date()) {let insertedRows = 0;try {- await insertCclogAggregate(entries);- insertedRows = entries.length;- logger.info("Inserted cclog aggregate", {- at: minute.toISOString(),- rows: insertedRows,- });+ if (await insertCclogAggregate(entries)) {+ insertedRows = entries.length;+ logger.info("Inserted cclog aggregate", {+ at: minute.toISOString(),+ rows: insertedRows,+ });+ } else {+ logger.warn("Skipped cclog aggregate insert", {+ at: minute.toISOString(),+ rows: entries.length,+ });+ }} catch (error) {logger.error("Error inserting cclog aggregate", { error });}diff --git a/apps/api/src/lib/clickhouse-client.ts b/apps/api/src/lib/clickhouse-client.tsindex 5d7b046fc..ed55d878a 100644--- a/apps/api/src/lib/clickhouse-client.ts+++ b/apps/api/src/lib/clickhouse-client.ts@@ -16,8 +16,10 @@ const client = config.CLICKHOUSE_ANALYTICS_URLexport async function chInsert(table: string,rows: Record<string, unknown>[],-): Promise<void> {- if (!client || rows.length === 0) return;+ opts?: { throwOnError?: boolean },+): Promise<boolean> {+ if (rows.length === 0) return true;+ if (!client) return false;try {await client.insert({@@ -25,11 +27,16 @@ export async function chInsert(values: rows,format: "JSONEachRow",});+ return true;} catch (error: any) {logger.error("ClickHouse insert failed", {table,rowCount: rows.length,error: error?.message,});+ if (opts?.throwOnError) {+ throw error;+ }+ return false;}}