The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
skill: fix jq examples that read missing fields; ask before ambiguous bulk writes
narrow-adventurec1778e443fa6
diff --git a/cli/pkg/mcpserver/knowledge/debug-flow.md b/cli/pkg/mcpserver/knowledge/debug-flow.mdindex 6b7a59a2..6547ad95 100644--- a/cli/pkg/mcpserver/knowledge/debug-flow.md+++ b/cli/pkg/mcpserver/knowledge/debug-flow.md@@ -53,7 +53,7 @@ traceway exceptions list --since 24h --order-by lastSeen # what is erroritraceway logs query --since 24h --min-severity 17 # errors and worsetraceway logs query --since 24h --search "payment declined" # search log bodiestraceway logs query --since 24h --service checkout-api --min-severity 13-traceway endpoints list --since 24h --search "checkout" # latency p50/p95/p99 and error counts, --order-by impact|count|p95|lastSeen+traceway endpoints list --since 24h --search "checkout" # latency p50/p95/p99, count and impact, --order-by impact|count|p95|lastSeen```Severity is an OTel number, not a name: 1 TRACE, 5 DEBUG, 9 INFO, 13 WARN, 17 ERROR, 21 FATAL. The flag is `--min-severity 17`, never `--severity error`.diff --git a/cli/pkg/mcpserver/knowledge/jq_examples_test.go b/cli/pkg/mcpserver/knowledge/jq_examples_test.gonew file mode 100644index 00000000..b9bc41c3--- /dev/null+++ b/cli/pkg/mcpserver/knowledge/jq_examples_test.go@@ -0,0 +1,179 @@+package knowledge_test++import (+ "io/fs"+ "reflect"+ "regexp"+ "strings"+ "testing"++ "github.com/tracewayapp/traceway/cli/pkg/client"+ "github.com/tracewayapp/traceway/cli/pkg/mcpserver/knowledge"+)++// jqOutputs is the type each command prints with --output json. An example+// piping a command that is missing here fails the test: add the command.+var jqOutputs = map[string]any{+ "endpoints list": client.ListEndpointsResponse{},+ "endpoints chart": client.EndpointChartResponse{},+ "endpoints show": client.EndpointDetailResponse{},+ "metrics query": client.QueryMetricsResponse{},+ "exceptions list": client.ListExceptionsResponse{},+ "exceptions show": client.GetExceptionResponse{},+ "logs query": client.QueryLogsResponse{},+}++// jqBuiltins are names a filter may use that are not output fields: the+// key/value pairs to_entries makes.+var jqBuiltins = map[string]bool{"key": true, "value": true}++var (+ exampleRe = regexp.MustCompile(`traceway(?:-write)? ([a-z-]+) ([a-z-]+)[^|'\n]*\|\s*jq(?: -[a-z]+)* '([^']*)'`)+ stringRe = regexp.MustCompile(`"(?:[^"\\]|\\.)*"`)+ fieldRe = regexp.MustCompile(`\.([A-Za-z_][A-Za-z0-9_]*)`)+ identRe = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)+ leadRe = regexp.MustCompile(`^\s*\.([A-Za-z_][A-Za-z0-9_]*)`)+)++// TestJQExamplesNameRealFields fails when a jq filter in a knowledge example+// reads a field the piped command's JSON does not have. Such a filter prints+// null, and an agent that copies it re-runs the command to find the real+// shape. A filter's first field must be a top-level field; the others are+// checked by name, at any depth of the output type.+func TestJQExamplesNameRealFields(t *testing.T) {+ examples := 0+ err := fs.WalkDir(knowledge.FS, ".", func(path string, d fs.DirEntry, err error) error {+ if err != nil || d.IsDir() || !strings.HasSuffix(path, ".md") {+ return err+ }+ raw, err := knowledge.FS.ReadFile(path)+ if err != nil {+ return err+ }+ text := strings.ReplaceAll(string(raw), "\\\n", " ")+ for _, m := range exampleRe.FindAllStringSubmatch(text, -1) {+ examples+++ cmd, filter := m[1]+" "+m[2], m[3]+ out, ok := jqOutputs[cmd]+ if !ok {+ t.Errorf("%s: no output type for %q; add it to jqOutputs", path, cmd)+ continue+ }+ if lead := leadRe.FindStringSubmatch(filter); lead != nil && !jsonNames(reflect.TypeOf(out), nil)[lead[1]] {+ t.Errorf("%s: `%s | jq '%s'` starts at %q, which is not a top-level field of %s output", path, cmd, filter, lead[1], cmd)+ }+ names := jsonNames(reflect.TypeOf(out), map[reflect.Type]bool{})+ for _, f := range filterFields(filter) {+ if !names[f] && !jqBuiltins[f] {+ t.Errorf("%s: `%s | jq '%s'` reads %q, which %s output does not have", path, cmd, filter, f, cmd)+ }+ }+ }+ return nil+ })+ if err != nil {+ t.Fatal(err)+ }+ if examples == 0 {+ t.Fatal("found no jq examples; the pattern no longer matches the knowledge files")+ }+}++// filterFields returns the field names a jq filter reads: every .name, and+// every shorthand key of an object construction ({endpoint, count: .x}).+func filterFields(filter string) []string {+ f := stringRe.ReplaceAllString(filter, `""`)+ var out []string+ for _, m := range fieldRe.FindAllStringSubmatch(f, -1) {+ out = append(out, m[1])+ }+ for _, obj := range objectBodies(f) {+ for _, part := range splitTop(obj) {+ if p := strings.TrimSpace(part); identRe.MatchString(p) {+ out = append(out, p)+ }+ }+ }+ return out+}++// objectBodies returns the text inside each {...}, outermost first.+func objectBodies(s string) []string {+ var out []string+ var stack []int+ for i, r := range s {+ switch r {+ case '{':+ stack = append(stack, i)+ case '}':+ if n := len(stack); n > 0 {+ out = append(out, s[stack[n-1]+1:i])+ stack = stack[:n-1]+ }+ }+ }+ return out+}++// splitTop splits an object body on commas outside brackets and parentheses.+func splitTop(s string) []string {+ var out []string+ depth, start := 0, 0+ for i, r := range s {+ switch r {+ case '(', '[', '{':+ depth+++ case ')', ']', '}':+ depth--+ case ',':+ if depth == 0 {+ out = append(out, s[start:i])+ start = i + 1+ }+ }+ }+ return append(out, s[start:])+}++// jsonNames collects the JSON field names reachable from t, or only its+// top-level ones when seen is nil.+func jsonNames(t reflect.Type, seen map[reflect.Type]bool) map[string]bool {+ names := map[string]bool{}+ for t.Kind() == reflect.Pointer || t.Kind() == reflect.Slice || t.Kind() == reflect.Array || t.Kind() == reflect.Map {+ t = t.Elem()+ }+ if t.Kind() != reflect.Struct || seen[t] {+ return names+ }+ if seen != nil {+ seen[t] = true+ }+ for i := range t.NumField() {+ f := t.Field(i)+ tag := f.Tag.Get("json")+ name, _, _ := strings.Cut(tag, ",")+ if name == "-" {+ continue+ }+ if f.Anonymous && name == "" {+ for n := range jsonNames(f.Type, seen) {+ names[n] = true+ }+ continue+ }+ if !f.IsExported() {+ continue+ }+ if name == "" {+ name = f.Name+ }+ names[name] = true+ if seen == nil {+ continue+ }+ for n := range jsonNames(f.Type, seen) {+ names[n] = true+ }+ }+ return names+}diff --git a/cli/pkg/mcpserver/knowledge/notifications.md b/cli/pkg/mcpserver/knowledge/notifications.mdindex 29fde2b3..b9209e49 100644--- a/cli/pkg/mcpserver/knowledge/notifications.md+++ b/cli/pkg/mcpserver/knowledge/notifications.md@@ -35,7 +35,7 @@ To resolve, hand straight to the **Performance flow** scoped to that endpoint, u```bash# 1. confirm the alert against current stats for that endpoint-traceway endpoints list --search "<endpoint>" --since 1h --output json | jq '.data[0] | {p50, p95, p99, count, impact, impactReason}'+traceway endpoints list --search "<endpoint>" --since 1h --output json | jq '.data[0] | {p50_ms: (.p50Duration / 1e6), p95_ms: (.p95Duration / 1e6), p99_ms: (.p99Duration / 1e6), count, impact, impactReason}'# 2. is this an accepted baseline? check whether an operator marked it slowtraceway endpoints slow "<endpoint>" --output json # {offsetMs, reason}; offsetMs 0 = not marked# 3. find when it crossed the threshold, then a slow trace (see Flow: Performance)diff --git a/cli/pkg/mcpserver/knowledge/operations.md b/cli/pkg/mcpserver/knowledge/operations.mdindex 150fe533..f5102bf3 100644--- a/cli/pkg/mcpserver/knowledge/operations.md+++ b/cli/pkg/mcpserver/knowledge/operations.md@@ -26,6 +26,7 @@ Everything else (creating rules and channels, editing schedules and policies, st### Rules for every write- **Only on request.** Investigating, debugging, summarizing or "taking a look" never implies a write. The user has to ask for the change.+- **Ask when the request could mean more than the user knows.** When a write would cover several records and they differ in a way the user may not know (one is still occurring, has a different cause, or belongs to another team), list them, say how they differ, and ask which ones to change before writing anything. Do not settle it yourself, even by leaving the odd one out.- **Say it before doing it.** One line naming the action, the record and the new value: "Snoozing rule 12 (Endpoint P95 on GET /api/search) for 60 minutes." The CLI asks for confirmation on a TTY and needs `--yes` otherwise. MCP clients run their own approval step.- **Prefer the reversible option.** Snooze rather than disable. Acknowledge rather than resolve while the problem is still live. Archive an issue rather than resolving its page with `--archive-issue` if you are unsure the page is done.- **Report what changed and how to undo it.**diff --git a/cli/pkg/mcpserver/knowledge/performance.md b/cli/pkg/mcpserver/knowledge/performance.mdindex 116515c7..0bdd1180 100644--- a/cli/pkg/mcpserver/knowledge/performance.md+++ b/cli/pkg/mcpserver/knowledge/performance.md@@ -9,8 +9,9 @@ All commands are read-only. Never archive or mutate anything during a performanc1. **Quantify and localize.** Find the slow endpoint(s) and read the shape of the latency:```bashtraceway endpoints list --since 24h --order-by p95 --page-size 10 --output json \- | jq '.data[]? | {endpoint, count, p50, p95, p99, errorCount}'+ | jq '.data[]? | {endpoint, count, p50_ms: (.p50Duration / 1e6), p95_ms: (.p95Duration / 1e6), p99_ms: (.p99Duration / 1e6), impact}'```+ The rows are under `data`. `p50Duration`, `p95Duration`, `p99Duration` and `avgDuration` are nanoseconds, hence the `/ 1e6`.Read p50 vs p99 before anything else (see "Reading the latency shape").2. **Get a representative slow trace.** Drill into one slow request to see where the time goes. Capture an id and its `recordedAt` together, then:@@ -73,10 +74,10 @@ traceway endpoints chart --metric-type p95 --since 7d --interval-minutes 60 --ou```Caveat: it always returns the *top 5* endpoints by that metric. If your suspect route is not in the top 5 it is folded into "Other"; fall back to method 3 for it.-**2. Infra and runtime over time: `metrics query --interval-minutes`.** For the *cause* behind a latency step (saturation, GC, throughput), bucket any metric the same way. The returned `series` is a list of `{timestamp, value}` points; read down them for the bucket where the value steps up.+**2. Infra and runtime over time: `metrics query --interval-minutes`.** For the *cause* behind a latency step (saturation, GC, throughput), bucket any metric the same way. The JSON is `{results: [{name, unit, series}]}`, one result per query, and `series` maps each group to its list of `{timestamp, value}` points (a single `__all__` group when ungrouped); read down the points for the bucket where the value steps up.```bashtraceway metrics query --name system.cpu.utilization --aggregation max --since 7d --interval-minutes 60 --output json \- | jq '.series | to_entries[] | .value[] | {t: .timestamp, v: .value}'+ | jq '.results[0].series | to_entries[] | .value[] | {t: .timestamp, v: .value}'```A flat line that steps up at the same bucket the latency jumped is the saturation signal; flat metrics under a latency step point at code.@@ -84,9 +85,9 @@ A flat line that steps up at the same bucket the latency jumped is the saturatio```bash# is the regression in the first or second half of the suspect day?traceway endpoints list --search "GET /api/checkout" --from 2026-06-25T00:00:00Z --to 2026-06-25T12:00:00Z \- --output json | jq '.data[0] | {p95, p99, count}'+ --output json | jq '.data[0] | {p95_ms: (.p95Duration / 1e6), p99_ms: (.p99Duration / 1e6), count}'traceway endpoints list --search "GET /api/checkout" --from 2026-06-25T12:00:00Z --to 2026-06-26T00:00:00Z \- --output json | jq '.data[0] | {p95, p99, count}'+ --output json | jq '.data[0] | {p95_ms: (.p95Duration / 1e6), p99_ms: (.p99Duration / 1e6), count}'```A few bisection steps localize the jump to an hour or less. Watch `count` alongside p95: if the slow half also carries far more traffic, suspect load, not a code change.diff --git a/cli/pkg/mcpserver/knowledge/write-instructions.md b/cli/pkg/mcpserver/knowledge/write-instructions.mdindex 18ef3685..bf5c943e 100644--- a/cli/pkg/mcpserver/knowledge/write-instructions.md+++ b/cli/pkg/mcpserver/knowledge/write-instructions.md@@ -3,6 +3,7 @@This server was started with write access, so it also has mutating tools: archive_exceptions, unarchive_exceptions, acknowledge_page, resolve_page, set_notification_rule_enabled, snooze_notification_rule, run_check, create_post_mortem, update_post_mortem and set_slow_endpoint. Every one of them changes what the whole team sees.- Only call a mutating tool when the user asked for that change. Investigating, debugging or summarizing never implies a write.+- When a write would cover several records and they differ in a way the user may not know (one is still occurring, has a different cause, or belongs to another team), list them, say how they differ, and ask which ones to change before calling the tool. Do not settle it yourself, even by leaving the odd one out.- Before the call, state the exact change in one line: the tool, the record and the new value. Then make it.- Prefer the reversible option: snooze a noisy rule rather than disabling it, acknowledge a page before resolving it, archive rather than resolve with archive_issue.- After the call, report what changed and how to undo it.diff --git a/plugins/tw-write/skills/traceway-write/SKILL.md b/plugins/tw-write/skills/traceway-write/SKILL.mdindex 91e47a59..2759bd9e 100644--- a/plugins/tw-write/skills/traceway-write/SKILL.md+++ b/plugins/tw-write/skills/traceway-write/SKILL.md@@ -87,6 +87,7 @@ Everything else (creating rules and channels, editing schedules and policies, st### Rules for every write- **Only on request.** Investigating, debugging, summarizing or "taking a look" never implies a write. The user has to ask for the change.+- **Ask when the request could mean more than the user knows.** When a write would cover several records and they differ in a way the user may not know (one is still occurring, has a different cause, or belongs to another team), list them, say how they differ, and ask which ones to change before writing anything. Do not settle it yourself, even by leaving the odd one out.- **Say it before doing it.** One line naming the action, the record and the new value: "Snoozing rule 12 (Endpoint P95 on GET /api/search) for 60 minutes." The CLI asks for confirmation on a TTY and needs `--yes` otherwise. MCP clients run their own approval step.- **Prefer the reversible option.** Snooze rather than disable. Acknowledge rather than resolve while the problem is still live. Archive an issue rather than resolving its page with `--archive-issue` if you are unsure the page is done.- **Report what changed and how to undo it.**diff --git a/skills/traceway/SKILL.md b/skills/traceway/SKILL.mdindex 2031ecba..20b233b4 100644--- a/skills/traceway/SKILL.md+++ b/skills/traceway/SKILL.md@@ -114,7 +114,7 @@ To resolve, hand straight to the **Performance flow** scoped to that endpoint, u```bash# 1. confirm the alert against current stats for that endpoint-traceway endpoints list --search "<endpoint>" --since 1h --output json | jq '.data[0] | {p50, p95, p99, count, impact, impactReason}'+traceway endpoints list --search "<endpoint>" --since 1h --output json | jq '.data[0] | {p50_ms: (.p50Duration / 1e6), p95_ms: (.p95Duration / 1e6), p99_ms: (.p99Duration / 1e6), count, impact, impactReason}'# 2. is this an accepted baseline? check whether an operator marked it slowtraceway endpoints slow "<endpoint>" --output json # {offsetMs, reason}; offsetMs 0 = not marked# 3. find when it crossed the threshold, then a slow trace (see Flow: Performance)@@ -243,7 +243,7 @@ traceway exceptions list --since 24h --order-by lastSeen # what is erroritraceway logs query --since 24h --min-severity 17 # errors and worsetraceway logs query --since 24h --search "payment declined" # search log bodiestraceway logs query --since 24h --service checkout-api --min-severity 13-traceway endpoints list --since 24h --search "checkout" # latency p50/p95/p99 and error counts, --order-by impact|count|p95|lastSeen+traceway endpoints list --since 24h --search "checkout" # latency p50/p95/p99, count and impact, --order-by impact|count|p95|lastSeen```Severity is an OTel number, not a name: 1 TRACE, 5 DEBUG, 9 INFO, 13 WARN, 17 ERROR, 21 FATAL. The flag is `--min-severity 17`, never `--severity error`.diff --git a/skills/traceway/performance.md b/skills/traceway/performance.mdindex 2bdbe5bc..300f9d58 100644--- a/skills/traceway/performance.md+++ b/skills/traceway/performance.md@@ -10,8 +10,9 @@ All commands are read-only. Never archive or mutate anything during a performanc1. **Quantify and localize.** Find the slow endpoint(s) and read the shape of the latency:```bashtraceway endpoints list --since 24h --order-by p95 --page-size 10 --output json \- | jq '.data[]? | {endpoint, count, p50, p95, p99, errorCount}'+ | jq '.data[]? | {endpoint, count, p50_ms: (.p50Duration / 1e6), p95_ms: (.p95Duration / 1e6), p99_ms: (.p99Duration / 1e6), impact}'```+ The rows are under `data`. `p50Duration`, `p95Duration`, `p99Duration` and `avgDuration` are nanoseconds, hence the `/ 1e6`.Read p50 vs p99 before anything else (see "Reading the latency shape").2. **Get a representative slow trace.** Drill into one slow request to see where the time goes. Capture an id and its `recordedAt` together, then:@@ -74,10 +75,10 @@ traceway endpoints chart --metric-type p95 --since 7d --interval-minutes 60 --ou```Caveat: it always returns the *top 5* endpoints by that metric. If your suspect route is not in the top 5 it is folded into "Other"; fall back to method 3 for it.-**2. Infra and runtime over time: `metrics query --interval-minutes`.** For the *cause* behind a latency step (saturation, GC, throughput), bucket any metric the same way. The returned `series` is a list of `{timestamp, value}` points; read down them for the bucket where the value steps up.+**2. Infra and runtime over time: `metrics query --interval-minutes`.** For the *cause* behind a latency step (saturation, GC, throughput), bucket any metric the same way. The JSON is `{results: [{name, unit, series}]}`, one result per query, and `series` maps each group to its list of `{timestamp, value}` points (a single `__all__` group when ungrouped); read down the points for the bucket where the value steps up.```bashtraceway metrics query --name system.cpu.utilization --aggregation max --since 7d --interval-minutes 60 --output json \- | jq '.series | to_entries[] | .value[] | {t: .timestamp, v: .value}'+ | jq '.results[0].series | to_entries[] | .value[] | {t: .timestamp, v: .value}'```A flat line that steps up at the same bucket the latency jumped is the saturation signal; flat metrics under a latency step point at code.@@ -85,9 +86,9 @@ A flat line that steps up at the same bucket the latency jumped is the saturatio```bash# is the regression in the first or second half of the suspect day?traceway endpoints list --search "GET /api/checkout" --from 2026-06-25T00:00:00Z --to 2026-06-25T12:00:00Z \- --output json | jq '.data[0] | {p95, p99, count}'+ --output json | jq '.data[0] | {p95_ms: (.p95Duration / 1e6), p99_ms: (.p99Duration / 1e6), count}'traceway endpoints list --search "GET /api/checkout" --from 2026-06-25T12:00:00Z --to 2026-06-26T00:00:00Z \- --output json | jq '.data[0] | {p95, p99, count}'+ --output json | jq '.data[0] | {p95_ms: (.p95Duration / 1e6), p99_ms: (.p99Duration / 1e6), count}'```A few bisection steps localize the jump to an hour or less. Watch `count` alongside p95: if the slow half also carries far more traffic, suspect load, not a code change.