Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api/cclog): work around dragonfly limitation of 2^16 command arguments
mogery48b11f3bd3c6
diff --git a/apps/api/src/lib/cclog.test.ts b/apps/api/src/lib/cclog.test.tsindex 91c0c0ad4..c4f2f2a3e 100644--- a/apps/api/src/lib/cclog.test.ts+++ b/apps/api/src/lib/cclog.test.ts@@ -30,6 +30,8 @@ function sampleKey(at: Date): string {class FakeRedis {private hashes = new Map<string, Record<string, string>>();+ hsetCalls: Array<{ key: string; fields: number }> = [];+ execError: Error | null = null;constructor(private readonly keys: string[],@@ -40,6 +42,10 @@ class FakeRedis {this.hashes.set(key, { ...values });}+ getHash(key: string) {+ return this.hashes.get(key) ?? {};+ }+async scan() {return ["0", this.keys] as [string, string[]];}@@ -55,6 +61,8 @@ class FakeRedis {}pipeline() {+ const self = this;+ let commands = 0;return {hset: (key: string,@@ -65,10 +73,18 @@ class FakeRedis {typeof valuesOrField === "string"? { [valuesOrField]: value ?? "" }: valuesOrField;- this.hashes.set(key, { ...(this.hashes.get(key) ?? {}), ...values });+ self.hsetCalls.push({ key, fields: Object.keys(values).length });+ self.hashes.set(key, { ...(self.hashes.get(key) ?? {}), ...values });+ commands++;+ },+ expire: () => {+ commands++;},- expire: () => {},- exec: async () => [],+ exec: async (): Promise<[Error | null, unknown][]> =>+ Array.from({ length: commands }, (_, i) => [+ i === 0 ? self.execError : null,+ 0,+ ]),};}}@@ -179,4 +195,52 @@ describe("cclog", () => {);expect(result.insertedRows).toBe(0);});++ it("saves samples with more teams than fit in a single HSET in chunks", async () => {+ // Dragonfly rejects commands with > 2^16 arguments, so a single HSET+ // breaks once a sample holds ~32k+ teams (production incident: aggregate+ // inserts silently logged rows=0 while samples never landed).+ const teamCount = 35491;+ const keys: string[] = [];+ const activeCounts: Record<string, number> = {};+ for (let i = 0; i < teamCount; i++) {+ const key = `concurrency-limiter:team-${i}`;+ keys.push(key);+ activeCounts[key] = 1;+ }++ const at = new Date("2026-06-26T12:20:15.000Z");+ const redis = new FakeRedis(keys, activeCounts);++ const result = await runCclogTick(redis as any, at);++ expect(result.sampledTeams).toBe(teamCount);++ const saved = redis.getHash(+ sampleKey(new Date("2026-06-26T12:20:00.000Z")),+ );+ expect(Object.keys(saved)).toHaveLength(teamCount);++ const sampleHsets = redis.hsetCalls.filter(+ x => x.key === sampleKey(new Date("2026-06-26T12:20:00.000Z")),+ );+ expect(sampleHsets.length).toBeGreaterThan(1);+ for (const call of sampleHsets) {+ // 2 args (HSET + key) + 2 per field must stay under 2^16+ expect(2 + call.fields * 2).toBeLessThanOrEqual(2 ** 16);+ }+ });++ it("fails loudly when the sample save pipeline errors", async () => {+ const at = new Date("2026-06-26T12:21:15.000Z");+ const redis = new FakeRedis(["concurrency-limiter:team-a"], {+ "concurrency-limiter:team-a": 3,+ });+ redis.execError = new Error("ERR Protocol error: invalid multibulk length");++ await expect(runCclogTick(redis as any, at)).rejects.toThrow(+ "Failed to save cclog minute sample",+ );+ expect(chInsertMock).not.toHaveBeenCalled();+ });});diff --git a/apps/api/src/lib/cclog.ts b/apps/api/src/lib/cclog.tsindex ab7ccaeb4..78087f063 100644--- a/apps/api/src/lib/cclog.ts+++ b/apps/api/src/lib/cclog.ts@@ -17,6 +17,11 @@ const CCLOG_SAMPLE_TTL_SECONDS = 60 * 60;const CCLOG_MINUTE_MS = 60 * 1000;const CCLOG_AGGREGATE_INTERVAL_MINUTES = 10;+// Dragonfly rejects commands with more than 2^16 arguments ("invalid+// multibulk length") and drops the connection, so a single HSET silently+// fails once a sample holds ~32k+ teams. Chunk well below that limit.+const CCLOG_HSET_FIELD_CHUNK_SIZE = 16000;+type CclogSample = {at: Date;concurrencyByTeam: Map<string, number>;@@ -127,19 +132,31 @@ async function saveCclogMinuteSample(const pipeline = redis.pipeline();if (sample.concurrencyByTeam.size > 0) {- const values: Record<string, string> = {};+ let values: Record<string, string> = {};for (const [teamId, concurrency] of sample.concurrencyByTeam) {values[teamId] = String(concurrency);++ if (Object.keys(values).length >= CCLOG_HSET_FIELD_CHUNK_SIZE) {+ pipeline.hset(key, values);+ values = {};+ }}- pipeline.hset(key, values);+ if (Object.keys(values).length > 0) {+ pipeline.hset(key, values);+ }} else {pipeline.hset(key, "__empty__", "0");}pipeline.expire(key, CCLOG_SAMPLE_TTL_SECONDS);- await pipeline.exec();++ const results = await pipeline.exec();+ const error = (results ?? []).find(([err]) => err)?.[0];+ if (error) {+ throw new Error(`Failed to save cclog minute sample: ${error.message}`);+ }}async function buildCclogAggregateEntries(