Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api/cclog): work around dragonfly limitation of 2^16 command arguments

mogery48b11f3bd3c6
diff --git a/apps/api/src/lib/cclog.test.ts b/apps/api/src/lib/cclog.test.ts
index 91c0c0ad4..c4f2f2a3e 100644
--- a/apps/api/src/lib/cclog.test.ts
+++ b/apps/api/src/lib/cclog.test.ts
@@ -30,6 +30,8 @@ function sampleKey(at: Date): string {
class FakeRedis {
private hashes = new Map<string, Record<string, string>>();
+ hsetCalls: Array<{ key: string; fields: number }> = [];
+ execError: Error | null = null;
constructor(
private readonly keys: string[],
@@ -40,6 +42,10 @@ class FakeRedis {
this.hashes.set(key, { ...values });
}
+ getHash(key: string) {
+ return this.hashes.get(key) ?? {};
+ }
+
async scan() {
return ["0", this.keys] as [string, string[]];
}
@@ -55,6 +61,8 @@ class FakeRedis {
}
pipeline() {
+ const self = this;
+ let commands = 0;
return {
hset: (
key: string,
@@ -65,10 +73,18 @@ class FakeRedis {
typeof valuesOrField === "string"
? { [valuesOrField]: value ?? "" }
: valuesOrField;
- this.hashes.set(key, { ...(this.hashes.get(key) ?? {}), ...values });
+ self.hsetCalls.push({ key, fields: Object.keys(values).length });
+ self.hashes.set(key, { ...(self.hashes.get(key) ?? {}), ...values });
+ commands++;
+ },
+ expire: () => {
+ commands++;
},
- expire: () => {},
- exec: async () => [],
+ exec: async (): Promise<[Error | null, unknown][]> =>
+ Array.from({ length: commands }, (_, i) => [
+ i === 0 ? self.execError : null,
+ 0,
+ ]),
};
}
}
@@ -179,4 +195,52 @@ describe("cclog", () => {
);
expect(result.insertedRows).toBe(0);
});
+
+ it("saves samples with more teams than fit in a single HSET in chunks", async () => {
+ // Dragonfly rejects commands with > 2^16 arguments, so a single HSET
+ // breaks once a sample holds ~32k+ teams (production incident: aggregate
+ // inserts silently logged rows=0 while samples never landed).
+ const teamCount = 35491;
+ const keys: string[] = [];
+ const activeCounts: Record<string, number> = {};
+ for (let i = 0; i < teamCount; i++) {
+ const key = `concurrency-limiter:team-${i}`;
+ keys.push(key);
+ activeCounts[key] = 1;
+ }
+
+ const at = new Date("2026-06-26T12:20:15.000Z");
+ const redis = new FakeRedis(keys, activeCounts);
+
+ const result = await runCclogTick(redis as any, at);
+
+ expect(result.sampledTeams).toBe(teamCount);
+
+ const saved = redis.getHash(
+ sampleKey(new Date("2026-06-26T12:20:00.000Z")),
+ );
+ expect(Object.keys(saved)).toHaveLength(teamCount);
+
+ const sampleHsets = redis.hsetCalls.filter(
+ x => x.key === sampleKey(new Date("2026-06-26T12:20:00.000Z")),
+ );
+ expect(sampleHsets.length).toBeGreaterThan(1);
+ for (const call of sampleHsets) {
+ // 2 args (HSET + key) + 2 per field must stay under 2^16
+ expect(2 + call.fields * 2).toBeLessThanOrEqual(2 ** 16);
+ }
+ });
+
+ it("fails loudly when the sample save pipeline errors", async () => {
+ const at = new Date("2026-06-26T12:21:15.000Z");
+ const redis = new FakeRedis(["concurrency-limiter:team-a"], {
+ "concurrency-limiter:team-a": 3,
+ });
+ redis.execError = new Error("ERR Protocol error: invalid multibulk length");
+
+ await expect(runCclogTick(redis as any, at)).rejects.toThrow(
+ "Failed to save cclog minute sample",
+ );
+ expect(chInsertMock).not.toHaveBeenCalled();
+ });
});
diff --git a/apps/api/src/lib/cclog.ts b/apps/api/src/lib/cclog.ts
index ab7ccaeb4..78087f063 100644
--- a/apps/api/src/lib/cclog.ts
+++ b/apps/api/src/lib/cclog.ts
@@ -17,6 +17,11 @@ const CCLOG_SAMPLE_TTL_SECONDS = 60 * 60;
const CCLOG_MINUTE_MS = 60 * 1000;
const CCLOG_AGGREGATE_INTERVAL_MINUTES = 10;
+// Dragonfly rejects commands with more than 2^16 arguments ("invalid
+// multibulk length") and drops the connection, so a single HSET silently
+// fails once a sample holds ~32k+ teams. Chunk well below that limit.
+const CCLOG_HSET_FIELD_CHUNK_SIZE = 16000;
+
type CclogSample = {
at: Date;
concurrencyByTeam: Map<string, number>;
@@ -127,19 +132,31 @@ async function saveCclogMinuteSample(
const pipeline = redis.pipeline();
if (sample.concurrencyByTeam.size > 0) {
- const values: Record<string, string> = {};
+ let values: Record<string, string> = {};
for (const [teamId, concurrency] of sample.concurrencyByTeam) {
values[teamId] = String(concurrency);
+
+ if (Object.keys(values).length >= CCLOG_HSET_FIELD_CHUNK_SIZE) {
+ pipeline.hset(key, values);
+ values = {};
+ }
}
- pipeline.hset(key, values);
+ if (Object.keys(values).length > 0) {
+ pipeline.hset(key, values);
+ }
} else {
pipeline.hset(key, "__empty__", "0");
}
pipeline.expire(key, CCLOG_SAMPLE_TTL_SECONDS);
- await pipeline.exec();
+
+ const results = await pipeline.exec();
+ const error = (results ?? []).find(([err]) => err)?.[0];
+ if (error) {
+ throw new Error(`Failed to save cclog minute sample: ${error.message}`);
+ }
}
async function buildCclogAggregateEntries(