Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

OTLP forward: keep Codex token counts, keep headers through reinstall

Claudedabb3eddd7ff
diff --git a/cli/beacon/cmd/endpoint_install.go b/cli/beacon/cmd/endpoint_install.go
index 1e4b2a54..01c04344 100644
--- a/cli/beacon/cmd/endpoint_install.go
+++ b/cli/beacon/cmd/endpoint_install.go
@@ -10,6 +10,7 @@ import (
"strings"
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/auth"
+ endpointconfig "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/config"
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/dashboard"
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/harness"
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/lifecycle"
@@ -140,6 +141,24 @@ func endpointInstallOptions(selection endpointTargetSelection, serviceKind servi
}
}
+// warnMissingOTLPForwardHeaders says when the configured OTLP forward names headers whose values
+// are gone -- a package removal or cleanup took otlp-forward-headers.json -- and this run does not
+// set them again. The forward keeps running, so without this the backend would just start
+// refusing it.
+func warnMissingOTLPForwardHeaders(out io.Writer) {
+ if forward, _ := otlpForwardOptions(); forward != nil {
+ return
+ }
+ cfg, err := endpointconfig.Load(endpointUserMode())
+ if err != nil || cfg.Destinations == nil || cfg.Destinations.OTLPForward == nil {
+ return
+ }
+ if forward := cfg.Destinations.OTLPForward; forward.Enabled && forward.HeadersMissing {
+ fmt.Fprintf(out, "Warning: the OTLP forward to %s names headers (%s) whose values are missing (%s was removed); it will send without them until you run this again with --otlp-forward-header\n",
+ forward.Endpoint, strings.Join(forward.HeaderNames, ", "), endpointconfig.OTLPForwardSecretsPath(endpointUserMode()))
+ }
+}
+
// errSetupCancelled replaces the wizard's own "onboarding cancelled", which read as a failure
// and left the person to guess what state the machine was in. The wizard runs before anything is
// written, so the answer is always the same: nothing changed, and here is how to start again.
@@ -158,6 +177,7 @@ func runEndpointInstall(cmd *cobra.Command, args []string) error {
if _, err := otlpForwardOptions(); err != nil {
return err
}
+ warnMissingOTLPForwardHeaders(cmd.ErrOrStderr())
if endpointOpts.dryRun {
return printPlannedActions(plannedInstallActions(false, serviceKind, selection.OTLP, selection.Hooks))
}
@@ -460,6 +480,7 @@ func runEndpointRepair(cmd *cobra.Command, args []string) error {
if _, err := otlpForwardOptions(); err != nil {
return err
}
+ warnMissingOTLPForwardHeaders(cmd.ErrOrStderr())
if endpointOpts.dryRun {
return printPlannedActions(plannedInstallActions(true, serviceKind, selection.OTLP, selection.Hooks))
}
diff --git a/cli/beacon/internal/endpoint/config/config.go b/cli/beacon/internal/endpoint/config/config.go
index d3c9d1e3..2bea4120 100644
--- a/cli/beacon/internal/endpoint/config/config.go
+++ b/cli/beacon/internal/endpoint/config/config.go
@@ -207,11 +207,15 @@ type OTLPForward struct {
// Their values are credentials, so they are kept out of config.json (which ordinary accounts
// read to find the system log) and stored in the root-only OTLPForwardSecretsPath; config.json
// carries only HeaderNames.
- Headers map[string]string `json:"-"`
- HeaderNames []string `json:"header_names,omitempty"`
- Insecure bool `json:"insecure,omitempty"`
- InsecureSkipVerify bool `json:"insecure_skip_verify,omitempty"`
- CAFile string `json:"ca_file,omitempty"`
+ Headers map[string]string `json:"-"`
+ HeaderNames []string `json:"header_names,omitempty"`
+ // HeadersMissing is set by Load when config.json names headers but their file is gone (a
+ // package removal or a manual cleanup took it). The forward would then send no credentials,
+ // so status reports it and install and repair warn.
+ HeadersMissing bool `json:"-"`
+ Insecure bool `json:"insecure,omitempty"`
+ InsecureSkipVerify bool `json:"insecure_skip_verify,omitempty"`
+ CAFile string `json:"ca_file,omitempty"`
}
// DefaultOTLPForwardHarnesses are the runtimes a forward carries when none are named: the two
@@ -235,6 +239,10 @@ func loadOTLPForwardSecrets(cfg *Config) {
return
}
data, err := os.ReadFile(OTLPForwardSecretsPath(cfg.UserMode))
+ if os.IsNotExist(err) {
+ cfg.Destinations.OTLPForward.HeadersMissing = true
+ return
+ }
if err != nil {
return
}
diff --git a/cli/beacon/internal/endpoint/config/config_test.go b/cli/beacon/internal/endpoint/config/config_test.go
index 78e9070f..9be757b7 100644
--- a/cli/beacon/internal/endpoint/config/config_test.go
+++ b/cli/beacon/internal/endpoint/config/config_test.go
@@ -340,3 +340,28 @@ func TestValidateDestinationsRejectsBadOTLPForward(t *testing.T) {
})
}
}
+
+// A package removal or cleanup can take the headers file while config.json still names them. Load
+// says so instead of handing back a forward that silently sends without credentials.
+func TestLoadReportsMissingOTLPForwardHeaders(t *testing.T) {
+ home := t.TempDir()
+ testenv.SetHome(t, home)
+ cfg := Default(true, filepath.Join(home, "logs", "runtime.jsonl"))
+ cfg.Destinations = &Destinations{OTLPForward: &OTLPForward{Endpoint: "otel.example:4317", Headers: map[string]string{"authorization": "Bearer x"}}}
+ if _, err := Save(cfg); err != nil {
+ t.Fatal(err)
+ }
+ if loaded, err := Load(true); err != nil || loaded.Destinations.OTLPForward.HeadersMissing {
+ t.Fatalf("HeadersMissing with the file present: %v", err)
+ }
+ if err := os.Remove(OTLPForwardSecretsPath(true)); err != nil {
+ t.Fatal(err)
+ }
+ loaded, err := Load(true)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if f := loaded.Destinations.OTLPForward; !f.HeadersMissing || f.Headers != nil || strings.Join(f.HeaderNames, ",") != "authorization" {
+ t.Fatalf("forward without its headers file = %#v", f)
+ }
+}
diff --git a/cli/beacon/internal/endpoint/lifecycle/lifecycle.go b/cli/beacon/internal/endpoint/lifecycle/lifecycle.go
index 4d026b6b..5cedd18e 100644
--- a/cli/beacon/internal/endpoint/lifecycle/lifecycle.go
+++ b/cli/beacon/internal/endpoint/lifecycle/lifecycle.go
@@ -409,6 +409,9 @@ type OTLPForwardStatus struct {
Protocol string `json:"protocol,omitempty"`
Harnesses []string `json:"harnesses,omitempty"`
Headers []string `json:"headers,omitempty"`
+ // HeadersMissing means config.json names headers whose values file is gone, so the forward
+ // sends without them; re-run install or repair with --otlp-forward-header.
+ HeadersMissing bool `json:"headers_missing,omitempty"`
}
type ConfiguredStatus struct {
@@ -1268,7 +1271,7 @@ func destinationStatus(cfg endpointconfig.Config) DestinationStatus {
}
}
if forward := cfg.Destinations.OTLPForward; forward != nil && forward.Enabled {
- status.OTLPForward = OTLPForwardStatus{Configured: true, Endpoint: forward.Endpoint, Protocol: forward.Protocol, Harnesses: forward.Harnesses, Headers: forward.HeaderNames}
+ status.OTLPForward = OTLPForwardStatus{Configured: true, Endpoint: forward.Endpoint, Protocol: forward.Protocol, Harnesses: forward.Harnesses, Headers: forward.HeaderNames, HeadersMissing: forward.HeadersMissing}
}
return status
}
diff --git a/docs/guides/kandji-otlp-forward-mdm.mdx b/docs/guides/kandji-otlp-forward-mdm.mdx
index 5e4fca58..8838a2bc 100644
--- a/docs/guides/kandji-otlp-forward-mdm.mdx
+++ b/docs/guides/kandji-otlp-forward-mdm.mdx
@@ -215,7 +215,8 @@ Beacon's runtime log should contain the marker from both runtimes. In the existi
```
`Unauthenticated` or `401` means a wrong or missing header; `connection refused`, `deadline exceeded` or a TLS error means the endpoint, protocol, or `--otlp-forward-insecure`/`--otlp-forward-ca-file` is wrong. HTTP endpoints take the base URL; the exporter adds `/v1/logs`, `/v1/metrics` and `/v1/traces`.
-3. Check that the runtimes still point at the collector (validation step 2). A runtime update or an old Kandji item can rewrite the settings; the Step 4 audit repairs them on its next run.
+3. If status shows `"headers_missing": true`, the header values file (`otlp-forward-headers.json`) was removed, for example by a manual cleanup, and the forward is sending without credentials. Run the Step 3 script again.
+4. Check that the runtimes still point at the collector (validation step 2). A runtime update or an old Kandji item can rewrite the settings; the Step 4 audit repairs them on its next run.
### `repair` rejects the forward flags
diff --git a/packaging/linux/nfpm/preremove.sh b/packaging/linux/nfpm/preremove.sh
index 68f9d9e6..328a1ec7 100755
--- a/packaging/linux/nfpm/preremove.sh
+++ b/packaging/linux/nfpm/preremove.sh
@@ -40,7 +40,7 @@ esac
CONFIG_DIR=/etc/beacon/endpoint
STASH="$(mktemp -d)"
saved=0
-for f in config.json otelcol.yaml; do
+for f in config.json otelcol.yaml otlp-forward-headers.json; do
if [ -f "$CONFIG_DIR/$f" ]; then
cp -p "$CONFIG_DIR/$f" "$STASH/$f" && saved=1
fi
@@ -53,7 +53,7 @@ fi
if [ "$saved" = 1 ]; then
mkdir -p "$CONFIG_DIR"
- for f in config.json otelcol.yaml; do
+ for f in config.json otelcol.yaml otlp-forward-headers.json; do
[ -f "$STASH/$f" ] && cp -p "$STASH/$f" "$CONFIG_DIR/$f"
done
fi
diff --git a/packaging/windows/uninstall-endpoint.ps1 b/packaging/windows/uninstall-endpoint.ps1
index 62b9610a..73253a6c 100644
--- a/packaging/windows/uninstall-endpoint.ps1
+++ b/packaging/windows/uninstall-endpoint.ps1
@@ -37,7 +37,7 @@ if (-not (Test-Path -LiteralPath $beaconBin)) {
}
$configDir = Join-Path $env:ProgramData 'Beacon\Endpoint'
-$retained = @('config.json', 'otelcol.yaml')
+$retained = @('config.json', 'otelcol.yaml', 'otlp-forward-headers.json')
$stash = $null
if ($purge -ne '1') {
diff --git a/pkg/asymptoteobserve/privacy.go b/pkg/asymptoteobserve/privacy.go
index 3ac8536e..80caaf9a 100644
--- a/pkg/asymptoteobserve/privacy.go
+++ b/pkg/asymptoteobserve/privacy.go
@@ -982,15 +982,31 @@ func secretKeyName(key string) (secret, onlyTokens bool) {
var name strings.Builder
name.Grow(len(key))
start := 0
- // endWord reports whether the word that ends here names a credential other than "tokens", and
- // starts the next one.
+ // tokenPending is a "token" word not yet judged: followed by "count" or "usage" it names a
+ // count (Codex and Gemini log `input_token_count`, `total_token_count`), like "tokens", and a
+ // number under it is no secret; followed by anything else, or nothing, it names a credential.
+ tokenPending := false
+ // endWord reports whether the word that ends here names a credential other than a token count,
+ // and starts the next one.
endWord := func() bool {
word := name.String()[start:]
start = name.Len()
+ if tokenPending {
+ tokenPending = false
+ if slices.Contains(tokenCountWords, word) {
+ onlyTokens = true
+ return false
+ }
+ return true
+ }
if word == "tokens" {
onlyTokens = true
return false
}
+ if word == "token" {
+ tokenPending = true
+ return false
+ }
return slices.Contains(redactKeyWords, word)
}
for i := range len(key) {
@@ -1011,7 +1027,7 @@ func secretKeyName(key string) (secret, onlyTokens bool) {
}
}
}
- if endWord() {
+ if endWord() || tokenPending {
return true, false
}
whole := name.String()
@@ -1034,6 +1050,9 @@ var redactKeyWords = []string{
"accesskey", "secretkey", "privatekey", "auth", "authorization", "credential", "credentials", "cookie",
}
+// tokenCountWords, after "token", make a field name a count rather than a credential.
+var tokenCountWords = []string{"count", "counts", "usage"}
+
// redactKeySuffixes mark a field name that runs a credential's word into the words before it.
var redactKeySuffixes = []string{
"password", "passwd", "passphrase", "secret", "token", "apikey", "accesskey", "secretkey",
diff --git a/pkg/asymptoteobserve/redact_cost_test.go b/pkg/asymptoteobserve/redact_cost_test.go
index eeb99fda..5d5f6357 100644
--- a/pkg/asymptoteobserve/redact_cost_test.go
+++ b/pkg/asymptoteobserve/redact_cost_test.go
@@ -33,6 +33,11 @@ func TestIsSecretField(t *testing.T) {
{"X-Api-Key", "x", true}, {"SECRET_KEY_BASE", "x", true}, {"DB_PASS", "x", true}, {"SMTP_PASS", "x", true},
{"GITHUB_TOKENS", "abc", true}, {"pin_token", 1234, true}, {"password", "12345678", true},
{"max_tokens", 4096, false}, {"input_tokens", float64(812), false}, {"input_tokens", "1200", false},
+ // "token" followed by count or usage names a count (Codex's and Gemini's usage attributes,
+ // which Codex sends as strings); anything else after it, or nothing, is still a credential.
+ {"input_token_count", "1200", false}, {"total_token_count", int64(9), false}, {"inputTokenCount", "7", false},
+ {"cached_content_token_count", "0", false}, {"token_usage", "42", false}, {"input_token_count", "sk-abc", true},
+ {"GITHUB_TOKEN", "abc", true}, {"github_token", "12345678", true}, {"token_id", "1234", true}, {"refresh_token", "77", true},
{"token_type", "Bearer", false}, {"author", "Ada", false}, {"PWD", "/home/ada", false}, {"keyboard", "dvorak", false},
} {
if got := IsSecretField(tc.key, tc.value); got != tc.want {