Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

fix(ci): export telemetry on a timer that fits the session

shukieshahf13487fea468
diff --git a/cli/beacon/internal/ci/harness.go b/cli/beacon/internal/ci/harness.go
index f80b5954..7f269e64 100644
--- a/cli/beacon/internal/ci/harness.go
+++ b/cli/beacon/internal/ci/harness.go
@@ -40,6 +40,33 @@ func ClaudeEnv(base []string, endpoint string) []string {
delete(env, "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT")
env["OTEL_LOG_TOOL_DETAILS"] = "1"
env["OTEL_LOG_USER_PROMPTS"] = "1"
+
+ // Export on a timer that fits the session, instead of relying on the flush at shutdown.
+ //
+ // The OpenTelemetry default metric interval is 60 seconds, and `ci exec` wraps sessions that
+ // finish in five or ten. Nothing is exported on a timer in that window, so every event depends on
+ // the exporter flushing during shutdown, before the process exits and before `ci exec` stops the
+ // collector behind it. That is a race, and it was being lost about two runs in three: three
+ // dispatched runs of the same commit captured 21 events once and zero twice, with the agent doing
+ // identical work each time and the passing run being the longest of the three (#320).
+ //
+ // A shutdown flush is still the backstop and still has to work. This just stops it being the only
+ // thing between a captured session and an empty log.
+ //
+ // Set only when the caller has not. Someone who has deliberately tuned these has a reason, and
+ // this is a wrapper around their session rather than an owner of their telemetry configuration.
+ for key, value := range map[string]string{
+ "OTEL_METRIC_EXPORT_INTERVAL": "5000",
+ // The batch processors default to 1s and 5s respectively, which is already inside a short
+ // session -- pinned anyway so the whole pipeline's timing is stated in one place rather than
+ // inherited from three different SDK defaults.
+ "OTEL_BLRP_SCHEDULE_DELAY": "1000",
+ "OTEL_BSP_SCHEDULE_DELAY": "1000",
+ } {
+ if strings.TrimSpace(env[key]) == "" {
+ env[key] = value
+ }
+ }
return flattenEnv(env)
}
diff --git a/cli/beacon/internal/ci/harness_test.go b/cli/beacon/internal/ci/harness_test.go
index b7c5aa0f..f7550ab6 100644
--- a/cli/beacon/internal/ci/harness_test.go
+++ b/cli/beacon/internal/ci/harness_test.go
@@ -60,3 +60,47 @@ func TestNormalizeHarnessesSupportsClaudeAndCodexAliases(t *testing.T) {
t.Fatalf("NormalizeHarnesses = %q, want claude,codex", joined)
}
}
+
+// TestClaudeEnvExportsOnATimerShorterThanASession covers the intermittency in #320.
+//
+// The OpenTelemetry default metric interval is 60 seconds and `ci exec` wraps sessions that finish in
+// five or ten, so without these nothing is exported on a timer and every event rides on the flush at
+// shutdown -- a race that three dispatched runs of the same commit lost twice.
+func TestClaudeEnvExportsOnATimerShorterThanASession(t *testing.T) {
+ env := envMap(ClaudeEnv(nil, "http://127.0.0.1:4317"))
+
+ for key, want := range map[string]string{
+ "OTEL_METRIC_EXPORT_INTERVAL": "5000",
+ "OTEL_BLRP_SCHEDULE_DELAY": "1000",
+ "OTEL_BSP_SCHEDULE_DELAY": "1000",
+ } {
+ if env[key] != want {
+ t.Fatalf("%s = %q, want %q; a session shorter than the export interval captures nothing "+
+ "unless the shutdown flush wins a race", key, env[key], want)
+ }
+ }
+}
+
+// TestClaudeEnvDoesNotOverrideDeliberateExportTuning keeps this a wrapper rather than an owner.
+//
+// Someone who set these has a reason -- a long-running session, a slow collector, a cost concern --
+// and `ci exec` runs their session rather than administering their telemetry.
+func TestClaudeEnvDoesNotOverrideDeliberateExportTuning(t *testing.T) {
+ base := flattenEnv(map[string]string{
+ "OTEL_METRIC_EXPORT_INTERVAL": "30000",
+ "OTEL_BSP_SCHEDULE_DELAY": "250",
+ })
+ env := envMap(ClaudeEnv(base, "http://127.0.0.1:4317"))
+
+ if env["OTEL_METRIC_EXPORT_INTERVAL"] != "30000" {
+ t.Fatalf("OTEL_METRIC_EXPORT_INTERVAL = %q, want the caller's 30000", env["OTEL_METRIC_EXPORT_INTERVAL"])
+ }
+ if env["OTEL_BSP_SCHEDULE_DELAY"] != "250" {
+ t.Fatalf("OTEL_BSP_SCHEDULE_DELAY = %q, want the caller's 250", env["OTEL_BSP_SCHEDULE_DELAY"])
+ }
+ // The one it did not set still gets the default, so a partially-tuned environment is completed
+ // rather than left with a 60-second gap in it.
+ if env["OTEL_BLRP_SCHEDULE_DELAY"] != "1000" {
+ t.Fatalf("OTEL_BLRP_SCHEDULE_DELAY = %q, want it filled in", env["OTEL_BLRP_SCHEDULE_DELAY"])
+ }
+}