Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

fix(gcp-triage): say when the log read hit its limit so a 6h sweep stops passing off 4 minutes of logs as the whole window

nirholascde0ca536ca9
diff --git a/scripts/gcp-triage.mjs b/scripts/gcp-triage.mjs
index 296aba2a5..a8e0ea9e5 100644
--- a/scripts/gcp-triage.mjs
+++ b/scripts/gcp-triage.mjs
@@ -293,7 +293,7 @@ const DEEP_PROBE_IDS = [
];
function parseArgs(argv) {
- const opts = { since: '1h', json: false, limit: 1000, project: PROJECT, deep: false, skip: [] };
+ const opts = { since: '1h', json: false, limit: null, project: PROJECT, deep: false, skip: [] };
for (let i = 0; i < argv.length; i++) {
const a = argv[i];
const next = () => argv[++i];
@@ -305,7 +305,7 @@ function parseArgs(argv) {
case '-n': case '--limit': opts.limit = Number(next()); break;
case '--project': opts.project = next(); break;
case '-h': case '--help':
- console.log('Usage: node scripts/gcp-triage.mjs [--since 1h] [--json] [--deep] [--skip id,id] [--limit 1000] [--project <id>]');
+ console.log('Usage: node scripts/gcp-triage.mjs [--since 1h] [--json] [--deep] [--skip id,id] [--limit 1000 (10000 with --deep)] [--project <id>]');
console.log(`Deep probe ids: ${DEEP_PROBE_IDS.join(', ')}`);
process.exit(0);
break;
@@ -318,6 +318,9 @@ function parseArgs(argv) {
console.error(`--since must look like 30m, 2h, 1d (got "${opts.since}")`);
process.exit(2);
}
+ // A deep sweep answers "is anything wrong?" over its whole window, so it reads
+ // far more entries; the fast pulse check keeps the cheap 1000.
+ if (opts.limit == null) opts.limit = opts.deep ? DEEP_LOG_LIMIT : FAST_LOG_LIMIT;
const badSkips = opts.skip.filter((s) => !DEEP_PROBE_IDS.includes(s));
if (badSkips.length) {
console.error(`Unknown --skip probe id(s): ${badSkips.join(', ')} (valid: ${DEEP_PROBE_IDS.join(', ')})`);
@@ -348,6 +351,37 @@ async function fetchHealthz() {
}
}
+const FAST_LOG_LIMIT = 1000;
+const DEEP_LOG_LIMIT = 10000;
+const SINCE_UNIT_MS = { s: 1000, m: 60_000, h: 3_600_000, d: 86_400_000, w: 604_800_000 };
+
+// gcloud returns entries newest first and stops at --limit, so a noisy fleet can
+// fill the limit in minutes and every count below silently describes a fraction
+// of the requested window. Report how much of the window the read really covered.
+export function logCoverage(entries, { since, limit, now = Date.now() }) {
+ const windowMs = Number(since.slice(0, -1)) * SINCE_UNIT_MS[since.slice(-1)];
+ const truncated = entries.length >= limit;
+ let oldestMs = null;
+ for (const e of entries) {
+ const t = Date.parse(e.timestamp || '');
+ if (Number.isFinite(t) && (oldestMs == null || t < oldestMs)) oldestMs = t;
+ }
+ const coveredMs = truncated && oldestMs != null ? Math.min(windowMs, now - oldestMs) : windowMs;
+ return {
+ truncated,
+ windowMs,
+ coveredMs,
+ coveredFraction: windowMs ? Math.round((coveredMs / windowMs) * 1000) / 1000 : 1,
+ oldest: oldestMs == null ? null : new Date(oldestMs).toISOString(),
+ };
+}
+
+function formatSpan(ms) {
+ if (ms >= 3_600_000) return `${Math.round(ms / 360_000) / 10}h`;
+ if (ms >= 60_000) return `${Math.round(ms / 60_000)}m`;
+ return `${Math.round(ms / 1000)}s`;
+}
+
async function readLogs(opts) {
const query = 'resource.type="cloud_run_revision" severity>=WARNING';
// Keep this asynchronous. The deep sweep deliberately runs its HTTP/TLS
@@ -520,7 +554,7 @@ const CLASS_BADGE = {
const PROBE_BADGE = { ok: '✅', findings: '❗', skipped: '⏭️', error: '💥' };
-function renderReport({ opts, healthz, findings, scanned, deep }) {
+function renderReport({ opts, healthz, findings, scanned, coverage, deep }) {
const lines = [];
lines.push(`# gcp-triage: last ${opts.since}, project ${opts.project}${opts.deep ? ' (deep sweep)' : ''}`);
lines.push('');
@@ -542,6 +576,9 @@ function renderReport({ opts, healthz, findings, scanned, deep }) {
}
const actionable = findings.filter((f) => f.class !== 'self-healing');
lines.push(`## Log sweep: ${scanned} WARNING+ entries → ${findings.length} distinct signatures (${actionable.length} actionable)`);
+ if (coverage?.truncated) {
+ lines.push(`⚠️ Hit --limit ${opts.limit}: these entries cover only the last ${formatSpan(coverage.coveredMs)} of the ${opts.since} window (back to ${coverage.oldest}). Counts are a floor; re-run with a higher --limit for the full picture.`);
+ }
lines.push('');
if (!findings.length) lines.push('Nothing above WARNING in the window. Production is quiet.');
for (const f of findings) {
@@ -897,6 +934,7 @@ async function main() {
fetchHealthz(),
readLogs(opts),
]);
+ const coverage = logCoverage(entries, opts);
const findings = buildFindings(entries);
findings.push(...gpuCapacityFindings(findings));
const deep = await deepPromise;
@@ -915,6 +953,8 @@ async function main() {
window: opts.since,
project: opts.project,
scannedEntries: entries.length,
+ logLimit: opts.limit,
+ logCoverage: coverage,
healthz,
deep: deep ? deep.map(({ findings: _f, ...rest }) => rest) : null,
findings,
@@ -922,7 +962,7 @@ async function main() {
healthy: !unhealthy,
}, null, 2) + '\n');
} else {
- console.log(renderReport({ opts, healthz, findings, scanned: entries.length, deep }));
+ console.log(renderReport({ opts, healthz, findings, scanned: entries.length, coverage, deep }));
}
process.exit(unhealthy ? 1 : 0);
}
diff --git a/tests/gcp-triage.test.js b/tests/gcp-triage.test.js
index c6e41e3dd..05162712d 100644
--- a/tests/gcp-triage.test.js
+++ b/tests/gcp-triage.test.js
@@ -2,7 +2,7 @@
// added to KNOWN_SIGNATURES should get a case here: the monitor learning a
// benign pattern must never silently swallow a genuine fault that shares text.
import { describe, expect, it } from 'vitest';
-import { buildFindings, classify, summarizePageCheckFailure } from '../scripts/gcp-triage.mjs';
+import { buildFindings, classify, logCoverage, summarizePageCheckFailure } from '../scripts/gcp-triage.mjs';
describe('gcp-triage classify', () => {
it('classifies the Colyseus stale-seat refusal as self-healing', () => {
@@ -112,3 +112,28 @@ describe('gcp-triage page failure summary', () => {
expect(summary.sample).not.toContain('50/822');
});
});
+
+describe('logCoverage', () => {
+ const now = Date.parse('2026-10-10T12:00:00Z');
+ const at = (iso) => ({ timestamp: iso });
+
+ it('reports full coverage when the read stopped short of the limit', () => {
+ const cov = logCoverage([at('2026-10-10T11:58:00Z')], { since: '6h', limit: 1000, now });
+ expect(cov.truncated).toBe(false);
+ expect(cov.coveredFraction).toBe(1);
+ });
+
+ it('flags a read that hit the limit and measures how far back it reached', () => {
+ const entries = [at('2026-10-10T11:59:00Z'), at('2026-10-10T11:56:00Z')];
+ const cov = logCoverage(entries, { since: '6h', limit: 2, now });
+ expect(cov.truncated).toBe(true);
+ expect(cov.coveredMs).toBe(4 * 60_000);
+ expect(cov.coveredFraction).toBeCloseTo(4 / 360, 3);
+ expect(cov.oldest).toBe('2026-10-10T11:56:00.000Z');
+ });
+
+ it('never claims more than the requested window', () => {
+ const cov = logCoverage([at('2026-10-09T00:00:00Z')], { since: '1h', limit: 1, now });
+ expect(cov.coveredMs).toBe(3_600_000);
+ });
+});