Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api/cclog): remove quadratic HSET chunking and early minute wakes (#5189)
Gergő Móriczdc8a23c59ed2
diff --git a/apps/api/src/lib/cclog.ts b/apps/api/src/lib/cclog.tsindex 78087f063..7e84f046a 100644--- a/apps/api/src/lib/cclog.ts+++ b/apps/api/src/lib/cclog.ts@@ -133,17 +133,20 @@ async function saveCclogMinuteSample(if (sample.concurrencyByTeam.size > 0) {let values: Record<string, string> = {};+ let fields = 0;for (const [teamId, concurrency] of sample.concurrencyByTeam) {values[teamId] = String(concurrency);+ fields++;- if (Object.keys(values).length >= CCLOG_HSET_FIELD_CHUNK_SIZE) {+ if (fields >= CCLOG_HSET_FIELD_CHUNK_SIZE) {pipeline.hset(key, values);values = {};+ fields = 0;}}- if (Object.keys(values).length > 0) {+ if (fields > 0) {pipeline.hset(key, values);}} else {diff --git a/apps/api/src/services/cclog-worker.ts b/apps/api/src/services/cclog-worker.tsindex 84cf10e9c..3f6db7499 100644--- a/apps/api/src/services/cclog-worker.ts+++ b/apps/api/src/services/cclog-worker.ts@@ -15,6 +15,14 @@ const CCLOG_WORKER_LOCK_TTL_SECONDS = 55;const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));+async function sleepUntilNextMinute() {+ const now = new Date();+ const nextMinute = now.getTime() + getMsUntilNextMinute(now);+ while (Date.now() < nextMinute) {+ await sleep(nextMinute - Date.now());+ }+}+async function sendHeartbeat() {if (config.CCLOG_WORKER_HEARTBEAT_URL) {try {@@ -80,7 +88,7 @@ async function sendHeartbeat() {_logger.info("Waiting for next exact minute to start cclog worker", {waitMs: getMsUntilNextMinute(),});- await sleep(getMsUntilNextMinute());+ await sleepUntilNextMinute();const redis = getRedisConnection();@@ -116,7 +124,7 @@ async function sendHeartbeat() {}if (!isShuttingDown) {- await sleep(getMsUntilNextMinute());+ await sleepUntilNextMinute();}}})();