The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
feat: added skills and api to figure out latency issues (#245)
narrow-adventure7b21bb322bd5
diff --git a/cli/cmd/traceway/endpoints.go b/cli/cmd/traceway/endpoints.goindex 4d2d33a9..e29bf100 100644--- a/cli/cmd/traceway/endpoints.go+++ b/cli/cmd/traceway/endpoints.go@@ -13,12 +13,16 @@ import (var endpointsOrderBy = []string{"impact", "count", "p95", "lastSeen"}+var endpointChartMetricTypes = []string{"total_time", "p50", "p95", "p99"}+func newEndpointsCmd() *cobra.Command {cmd := &cobra.Command{Use: "endpoints",Short: "Query HTTP endpoint performance",}cmd.AddCommand(newEndpointsListCmd())+ cmd.AddCommand(newEndpointsChartCmd())+ cmd.AddCommand(newEndpointsSlowCmd())cmd.AddCommand(newEndpointsShowCmd())return cmd}@@ -100,6 +104,164 @@ func runEndpointsList(cmd *cobra.Command, _ []string) error {}}+func newEndpointsChartCmd() *cobra.Command {+ cmd := &cobra.Command{+ Use: "chart",+ Short: "Endpoint latency bucketed over time (find when latency changed)",+ Long: `Show endpoint latency as a time series, the curve behind /endpoints in the+dashboard. The server returns the top 5 endpoints ranked by --metric-type plus+an "Other" bucket, each bucketed by --interval-minutes, so you can read down the+buckets to find when a latency jump started.++This always returns the top endpoints by the chosen metric; it does not filter+to one route. To track a specific endpoint that is not in the top 5, bisect+"endpoints list --search <name>" over narrowing --from/--to windows instead.++Values are in milliseconds. Use --output json for the full per-bucket series.`,+ RunE: runEndpointsChart,+ }+ addTimeRangeFlags(cmd)+ cmd.Flags().String("metric-type", "p95", "Metric to chart: total_time, p50, p95, p99")+ cmd.Flags().Int("interval-minutes", 0, "Time bucket size in minutes (0 = server default of 5)")+ return cmd+}++func runEndpointsChart(cmd *cobra.Command, _ []string) error {+ ctx := cmd.Context()+ mode := output.ResolveMode(flagOutput, output.StdoutIsTerminal())++ sess, err := loadSession()+ if err != nil {+ return renderSessionError(cmd.ErrOrStderr(), mode, err)+ }+ tr, err := resolveTimeRange(cmd)+ if err != nil {+ return renderTimeRangeError(cmd.ErrOrStderr(), mode, err)+ }+ metricType, _ := cmd.Flags().GetString("metric-type")+ if err := validateEnumFlag("--metric-type", metricType, endpointChartMetricTypes); err != nil {+ return renderUsageError(cmd.ErrOrStderr(), mode, err.Error(),+ enumFlagHint("traceway endpoints chart", "--metric-type", endpointChartMetricTypes))+ }+ intervalMin, _ := cmd.Flags().GetInt("interval-minutes")++ c := client.New(sess.URL, client.WithJWT(sess.JWT))+ resp, err := c.GetEndpointChart(ctx, sess.ProjectID, client.EndpointChartRequest{+ TimeRange: tr,+ MetricType: metricType,+ IntervalMinutes: intervalMin,+ })+ if err != nil {+ return renderAPIError(cmd.ErrOrStderr(), mode, err, false)+ }++ switch mode {+ case output.ModeJSON:+ return output.RenderJSON(cmd.OutOrStdout(), resp, output.ParseFieldsFlag(flagFields))+ case output.ModeYAML:+ return output.RenderYAML(cmd.OutOrStdout(), resp, output.ParseFieldsFlag(flagFields))+ default:+ return renderEndpointChartTable(cmd.OutOrStdout(), resp)+ }+}++// renderEndpointChartTable prints a per-endpoint summary of the time series+// (point count and min/max/latest value in ms), in the server's ranked order.+// The full per-bucket series is only in --output json.+func renderEndpointChartTable(out io.Writer, resp *client.EndpointChartResponse) error {+ type seriesAgg struct {+ points int+ min, max float64+ latest float64+ latestTime time.Time+ seen bool+ }+ byEndpoint := make(map[string]*seriesAgg, len(resp.Endpoints))+ for _, p := range resp.Series {+ a := byEndpoint[p.Endpoint]+ if a == nil {+ a = &seriesAgg{min: p.Value, max: p.Value}+ byEndpoint[p.Endpoint] = a+ }+ a.points+++ if p.Value < a.min {+ a.min = p.Value+ }+ if p.Value > a.max {+ a.max = p.Value+ }+ if !a.seen || p.Timestamp.After(a.latestTime) {+ a.latest = p.Value+ a.latestTime = p.Timestamp+ a.seen = true+ }+ }++ tw := output.NewTabWriter(out)+ _, _ = fmt.Fprintln(tw, "ENDPOINT\tPOINTS\tMIN(ms)\tMAX(ms)\tLATEST(ms)")+ for _, name := range resp.Endpoints {+ a := byEndpoint[name]+ if a == nil {+ _, _ = fmt.Fprintf(tw, "%s\t0\t-\t-\t-\n", name)+ continue+ }+ _, _ = fmt.Fprintf(tw, "%s\t%d\t%.1f\t%.1f\t%.1f\n", name, a.points, a.min, a.max, a.latest)+ }+ return tw.Flush()+}++func newEndpointsSlowCmd() *cobra.Command {+ cmd := &cobra.Command{+ Use: "slow <endpoint>",+ Short: "Show the user-configured slow allowance for one endpoint",+ Long: `Report whether an endpoint has been marked slow by an operator, and by how+much. The allowance (offsetMs) is added to the apdex and impact thresholds for+that endpoint server-side, so latency up to the offset is an accepted baseline+rather than a regression. The reason is the operator's note.++An endpoint that was never marked slow returns offsetMs 0 and an empty reason.++The offset shifts apdex/impact only; the raw p50/p95/p99 from "endpoints list"+and "endpoints chart" are NOT adjusted by it. Pass the full route name, e.g.+"GET /api/users/:id".`,+ Args: cobra.ExactArgs(1),+ RunE: runEndpointsSlow,+ }+ return cmd+}++func runEndpointsSlow(cmd *cobra.Command, args []string) error {+ ctx := cmd.Context()+ mode := output.ResolveMode(flagOutput, output.StdoutIsTerminal())++ sess, err := loadSession()+ if err != nil {+ return renderSessionError(cmd.ErrOrStderr(), mode, err)+ }++ c := client.New(sess.URL, client.WithJWT(sess.JWT))+ resp, err := c.GetSlowEndpoint(ctx, sess.ProjectID, args[0])+ if err != nil {+ return renderAPIError(cmd.ErrOrStderr(), mode, err, false)+ }++ switch mode {+ case output.ModeJSON:+ return output.RenderJSON(cmd.OutOrStdout(), resp, output.ParseFieldsFlag(flagFields))+ case output.ModeYAML:+ return output.RenderYAML(cmd.OutOrStdout(), resp, output.ParseFieldsFlag(flagFields))+ default:+ out := cmd.OutOrStdout()+ if resp.OffsetMs == 0 && resp.Reason == "" {+ _, _ = fmt.Fprintf(out, "ENDPOINT: %s\nMARKED: no (default thresholds)\n", args[0])+ return nil+ }+ _, _ = fmt.Fprintf(out, "ENDPOINT: %s\nMARKED: yes\nALLOWANCE: %dms\nREASON: %s\n",+ args[0], resp.OffsetMs, pickStr(resp.Reason, "-"))+ return nil+ }+}+func newEndpointsShowCmd() *cobra.Command {cmd := &cobra.Command{Use: "show <endpointId>",diff --git a/cli/cmd/traceway/endpoints_test.go b/cli/cmd/traceway/endpoints_test.goindex 34d0b9d2..cd30ae71 100644--- a/cli/cmd/traceway/endpoints_test.go+++ b/cli/cmd/traceway/endpoints_test.go@@ -1,6 +1,7 @@package mainimport (+ "encoding/json""net/http""net/http/httptest""strings"@@ -51,3 +52,208 @@ func TestEndpointsList_jsonShape(t *testing.T) {t.Errorf("missing endpoint in JSON: %s", stdout.String())}}++const chartResponseJSON = `{+ "endpoints":["GET /api/checkout","GET /api/projects","Other"],+ "series":[+ {"timestamp":"2026-06-25T12:00:00Z","endpoint":"GET /api/checkout","value":120.5},+ {"timestamp":"2026-06-25T13:00:00Z","endpoint":"GET /api/checkout","value":480.0},+ {"timestamp":"2026-06-25T12:00:00Z","endpoint":"GET /api/projects","value":30.0},+ {"timestamp":"2026-06-25T13:00:00Z","endpoint":"GET /api/projects","value":35.0}+ ]+}`++func TestEndpointsChart_jsonShape(t *testing.T) {+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ _, _ = w.Write([]byte(chartResponseJSON))+ }))+ defer srv.Close()+ seedSessionFor(t, srv.URL)++ stdout, _, err := runCmd(t, "", "endpoints", "chart", "--output", "json")+ if err != nil {+ t.Fatal(err)+ }+ out := stdout.String()+ if !strings.Contains(out, `"GET /api/checkout"`) || !strings.Contains(out, "480") {+ t.Errorf("json missing series data: %s", out)+ }+}++func TestEndpointsChart_table(t *testing.T) {+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ _, _ = w.Write([]byte(chartResponseJSON))+ }))+ defer srv.Close()+ seedSessionFor(t, srv.URL)++ stdout, _, err := runCmd(t, "", "endpoints", "chart", "--output", "table")+ if err != nil {+ t.Fatal(err)+ }+ out := stdout.String()+ if !strings.Contains(out, "ENDPOINT") || !strings.Contains(out, "MIN(ms)") || !strings.Contains(out, "MAX(ms)") || !strings.Contains(out, "LATEST(ms)") {+ t.Errorf("table missing headers: %s", out)+ }+ // The slow endpoint's peak should surface as its MAX and its later bucket as LATEST.+ if !strings.Contains(out, "GET /api/checkout") || !strings.Contains(out, "480.0") {+ t.Errorf("table missing endpoint summary: %s", out)+ }+ // Endpoints with no points (the "Other" bucket here) still render a row.+ if !strings.Contains(out, "Other") {+ t.Errorf("table missing Other row: %s", out)+ }+}++func TestEndpointsChart_sendsRequestBody(t *testing.T) {+ var gotPath, gotMethod string+ var gotBody map[string]any+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ gotPath = r.URL.Path+ gotMethod = r.Method+ _ = json.NewDecoder(r.Body).Decode(&gotBody)+ if r.URL.Query().Get("projectId") != "proj-1" {+ t.Errorf("projectId = %q", r.URL.Query().Get("projectId"))+ }+ _, _ = w.Write([]byte(chartResponseJSON))+ }))+ defer srv.Close()+ seedSessionFor(t, srv.URL)++ _, _, err := runCmd(t, "", "endpoints", "chart", "--metric-type", "p99", "--interval-minutes", "60", "--since", "24h", "--output", "json")+ if err != nil {+ t.Fatal(err)+ }+ if gotMethod != http.MethodPost {+ t.Errorf("method = %q, want POST", gotMethod)+ }+ if gotPath != "/api/endpoints/chart" {+ t.Errorf("path = %q, want /api/endpoints/chart", gotPath)+ }+ if gotBody["metricType"] != "p99" {+ t.Errorf("metricType = %v, want p99", gotBody["metricType"])+ }+ if gotBody["intervalMinutes"] != float64(60) {+ t.Errorf("intervalMinutes = %v, want 60", gotBody["intervalMinutes"])+ }+ // Endpoints routes use fromDate/toDate (not from/to like metrics).+ if gotBody["fromDate"] == nil || gotBody["toDate"] == nil {+ t.Errorf("body missing fromDate/toDate: %v", gotBody)+ }+ if gotBody["from"] != nil || gotBody["to"] != nil {+ t.Errorf("body should not carry from/to: %v", gotBody)+ }+}++func TestEndpointsChart_metricTypeValidation(t *testing.T) {+ // Bogus --metric-type is rejected client-side, before any request is sent.+ srv := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {+ t.Fatal("server should not be called for invalid --metric-type")+ }))+ defer srv.Close()+ seedSessionFor(t, srv.URL)++ _, stderr, err := runCmd(t, "", "endpoints", "chart", "--metric-type", "nope", "--output", "json")+ if err == nil {+ t.Fatal("expected error for bogus --metric-type")+ }+ if !strings.Contains(stderr.String(), "--metric-type") {+ t.Errorf("expected --metric-type in error, got: %s", stderr.String())+ }++ // Every documented metric type is accepted.+ for _, mt := range endpointChartMetricTypes {+ okSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ _, _ = w.Write([]byte(chartResponseJSON))+ }))+ seedSessionFor(t, okSrv.URL)+ if _, stderr, err := runCmd(t, "", "endpoints", "chart", "--metric-type", mt, "--output", "json"); err != nil {+ okSrv.Close()+ t.Fatalf("metric-type %q rejected: %v\nstderr: %s", mt, err, stderr.String())+ }+ okSrv.Close()+ }+}++func TestEndpointsSlow_marked(t *testing.T) {+ var gotPath, gotMethod, gotEndpoint string+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ gotPath = r.URL.Path+ gotMethod = r.Method+ gotEndpoint = r.URL.Query().Get("endpoint")+ _, _ = w.Write([]byte(`{"offsetMs":2000,"reason":"third-party export, accepted"}`))+ }))+ defer srv.Close()+ seedSessionFor(t, srv.URL)++ stdout, _, err := runCmd(t, "", "endpoints", "slow", "GET /api/checkout", "--output", "table")+ if err != nil {+ t.Fatal(err)+ }+ out := stdout.String()+ if !strings.Contains(out, "MARKED:") || !strings.Contains(out, "yes") || !strings.Contains(out, "2000ms") {+ t.Errorf("table missing marked summary: %s", out)+ }+ if !strings.Contains(out, "third-party export") {+ t.Errorf("table missing reason: %s", out)+ }+ if gotMethod != http.MethodGet {+ t.Errorf("method = %q, want GET", gotMethod)+ }+ if gotPath != "/api/endpoints/slow" {+ t.Errorf("path = %q, want /api/endpoints/slow", gotPath)+ }+ if gotEndpoint != "GET /api/checkout" {+ t.Errorf("endpoint query = %q, want %q", gotEndpoint, "GET /api/checkout")+ }+}++func TestEndpointsSlow_unmarked(t *testing.T) {+ // The server returns offsetMs 0 / empty reason for an endpoint never marked.+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ _, _ = w.Write([]byte(`{"offsetMs":0,"reason":""}`))+ }))+ defer srv.Close()+ seedSessionFor(t, srv.URL)++ stdout, _, err := runCmd(t, "", "endpoints", "slow", "GET /api/healthz", "--output", "table")+ if err != nil {+ t.Fatal(err)+ }+ out := stdout.String()+ if !strings.Contains(out, "no") || !strings.Contains(out, "default thresholds") {+ t.Errorf("unmarked endpoint should render 'no': %s", out)+ }+}++func TestEndpointsSlow_jsonShape(t *testing.T) {+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ _, _ = w.Write([]byte(`{"offsetMs":2000,"reason":"batch job"}`))+ }))+ defer srv.Close()+ seedSessionFor(t, srv.URL)++ stdout, _, err := runCmd(t, "", "endpoints", "slow", "GET /api/x", "--output", "json")+ if err != nil {+ t.Fatal(err)+ }+ var got map[string]any+ if err := json.Unmarshal(stdout.Bytes(), &got); err != nil {+ t.Fatalf("invalid json: %v\n%s", err, stdout.String())+ }+ if got["offsetMs"] != float64(2000) || got["reason"] != "batch job" {+ t.Errorf("json = %v", got)+ }+}++func TestEndpointsSlow_requiresEndpointArg(t *testing.T) {+ srv := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {+ t.Fatal("server should not be called without an endpoint arg")+ }))+ defer srv.Close()+ seedSessionFor(t, srv.URL)++ if _, _, err := runCmd(t, "", "endpoints", "slow", "--output", "json"); err == nil {+ t.Fatal("expected error when endpoint arg is missing")+ }+}diff --git a/cli/pkg/client/endpoints.go b/cli/pkg/client/endpoints.goindex d63629ec..59189539 100644--- a/cli/pkg/client/endpoints.go+++ b/cli/pkg/client/endpoints.go@@ -103,3 +103,71 @@ func (c *Client) GetEndpoint(ctx context.Context, projectID, id string, recorded}return &resp, nil}++// EndpointChartRequest is the body for POST /api/endpoints/chart. The server+// returns the top 5 endpoints ranked by MetricType (plus an "Other" bucket),+// each as a time series bucketed by IntervalMinutes, which is endpoint latency+// over time in a single call.+type EndpointChartRequest struct {+ TimeRange TimeRange `json:"-"`+ MetricType string `json:"metricType,omitempty"`+ IntervalMinutes int `json:"intervalMinutes,omitempty"`+}++// MarshalJSON expands TimeRange into top-level fromDate/toDate (the endpoints+// routes use fromDate/toDate, unlike metrics which uses from/to).+func (r EndpointChartRequest) MarshalJSON() ([]byte, error) {+ type alias EndpointChartRequest+ wire := struct {+ FromDate time.Time `json:"fromDate"`+ ToDate time.Time `json:"toDate"`+ alias+ }{r.TimeRange.From, r.TimeRange.To, alias(r)}+ return jsonMarshalNoHTMLEscape(wire)+}++// EndpointChartPoint is one bucket of one endpoint's series. Value is in+// milliseconds: total request time for total_time, the quantile for p50/p95/p99.+type EndpointChartPoint struct {+ Timestamp time.Time `json:"timestamp"`+ Endpoint string `json:"endpoint"`+ Value float64 `json:"value"`+}++// EndpointChartResponse mirrors models.EndpointStackedChartResponse.+type EndpointChartResponse struct {+ Endpoints []string `json:"endpoints"` // top 5 ranked by the metric, plus "Other"+ Series []EndpointChartPoint `json:"series"`+}++// GetEndpointChart returns endpoint latency bucketed over time for the top+// endpoints, which is the curve used to pinpoint when latency changed.+func (c *Client) GetEndpointChart(ctx context.Context, projectID string, req EndpointChartRequest) (*EndpointChartResponse, error) {+ path := "/api/endpoints/chart?projectId=" + url.QueryEscape(projectID)+ var resp EndpointChartResponse+ if err := c.do(ctx, http.MethodPost, path, req, &resp); err != nil {+ return nil, err+ }+ return &resp, nil+}++// SlowEndpointResponse is the body of GET /api/endpoints/slow. OffsetMs is the+// user-set allowance (in ms) added to the apdex and impact thresholds for this+// endpoint; Reason is the operator's note. An endpoint that was never marked+// slow returns {offsetMs:0, reason:""}, not an error.+type SlowEndpointResponse struct {+ OffsetMs uint32 `json:"offsetMs"`+ Reason string `json:"reason"`+}++// GetSlowEndpoint returns the user-configured slow allowance for one endpoint.+// The offset shifts the server-side apdex/impact thresholds only; the raw+// p50/p95/p99 from endpoints list/chart are not adjusted by it.+func (c *Client) GetSlowEndpoint(ctx context.Context, projectID, endpoint string) (*SlowEndpointResponse, error) {+ path := "/api/endpoints/slow?projectId=" + url.QueryEscape(projectID) + "&endpoint=" + url.QueryEscape(endpoint)+ var resp SlowEndpointResponse+ if err := c.do(ctx, http.MethodGet, path, nil, &resp); err != nil {+ return nil, err+ }+ return &resp, nil+}diff --git a/cli/pkg/client/endpoints_test.go b/cli/pkg/client/endpoints_test.goindex a8be64c6..d115d520 100644--- a/cli/pkg/client/endpoints_test.go+++ b/cli/pkg/client/endpoints_test.go@@ -56,3 +56,80 @@ func TestListEndpoints_callsGroupedRoute(t *testing.T) {t.Errorf("Impact = %v", resp.Data[0].Impact)}}++func TestGetEndpointChart_callsChartRoute(t *testing.T) {+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Path != "/api/endpoints/chart" {+ t.Errorf("path = %q (want chart)", r.URL.Path)+ }+ if r.URL.Query().Get("projectId") != "proj-1" {+ t.Errorf("projectId = %q", r.URL.Query().Get("projectId"))+ }+ var body map[string]any+ _ = json.NewDecoder(r.Body).Decode(&body)+ if body["fromDate"] == nil || body["toDate"] == nil {+ t.Errorf("body missing fromDate/toDate: %v", body)+ }+ if body["metricType"] != "p95" {+ t.Errorf("metricType = %v, want p95", body["metricType"])+ }+ if body["intervalMinutes"] != float64(60) {+ t.Errorf("intervalMinutes = %v, want 60", body["intervalMinutes"])+ }+ _, _ = w.Write([]byte(`{+ "endpoints":["GET /api/checkout","Other"],+ "series":[+ {"timestamp":"2026-06-25T12:00:00Z","endpoint":"GET /api/checkout","value":120.5},+ {"timestamp":"2026-06-25T13:00:00Z","endpoint":"GET /api/checkout","value":480}+ ]+ }`))+ }))+ defer srv.Close()++ c := New(srv.URL, WithJWT("tok"))+ resp, err := c.GetEndpointChart(context.Background(), "proj-1", EndpointChartRequest{+ TimeRange: TimeRange{From: time.Now().Add(-24 * time.Hour), To: time.Now()},+ MetricType: "p95",+ IntervalMinutes: 60,+ })+ if err != nil {+ t.Fatalf("GetEndpointChart: %v", err)+ }+ if len(resp.Endpoints) != 2 || resp.Endpoints[0] != "GET /api/checkout" {+ t.Errorf("Endpoints = %v", resp.Endpoints)+ }+ if len(resp.Series) != 2 {+ t.Fatalf("got %d series points", len(resp.Series))+ }+ if resp.Series[1].Value != 480 || resp.Series[1].Endpoint != "GET /api/checkout" {+ t.Errorf("series[1] = %+v", resp.Series[1])+ }+}++func TestGetSlowEndpoint_callsSlowRoute(t *testing.T) {+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.Method != http.MethodGet {+ t.Errorf("method = %q (want GET)", r.Method)+ }+ if r.URL.Path != "/api/endpoints/slow" {+ t.Errorf("path = %q (want slow)", r.URL.Path)+ }+ if r.URL.Query().Get("projectId") != "proj-1" {+ t.Errorf("projectId = %q", r.URL.Query().Get("projectId"))+ }+ if r.URL.Query().Get("endpoint") != "GET /api/users/:id" {+ t.Errorf("endpoint = %q", r.URL.Query().Get("endpoint"))+ }+ _, _ = w.Write([]byte(`{"offsetMs":1500,"reason":"known heavy report"}`))+ }))+ defer srv.Close()++ c := New(srv.URL, WithJWT("tok"))+ resp, err := c.GetSlowEndpoint(context.Background(), "proj-1", "GET /api/users/:id")+ if err != nil {+ t.Fatalf("GetSlowEndpoint: %v", err)+ }+ if resp.OffsetMs != 1500 || resp.Reason != "known heavy report" {+ t.Errorf("resp = %+v", resp)+ }+}diff --git a/cli/test/contract/contract_test.go b/cli/test/contract/contract_test.goindex f4646eeb..cd962c1a 100644--- a/cli/test/contract/contract_test.go+++ b/cli/test/contract/contract_test.go@@ -62,6 +62,32 @@ func TestContract_endpointDetail(t *testing.T) {goldenAssert(t, "endpoint-detail", rt.body)}+func TestContract_endpointsChart(t *testing.T) {+ c, rt := capturedClient()+ resp, err := c.GetEndpointChart(context.Background(), projectID, client.EndpointChartRequest{+ TimeRange: timeWindow(),+ MetricType: "p95",+ IntervalMinutes: 5,+ })+ if err != nil {+ t.Fatalf("endpoints chart: %v", err)+ }+ if len(resp.Series) == 0 {+ t.Fatal("expected seeded endpoint to produce a chart series point")+ }+ goldenAssert(t, "endpoints-chart", rt.body)+}++func TestContract_endpointsSlow(t *testing.T) {+ c, rt := capturedClient()+ // The seeded endpoint is not marked slow; the server still returns the+ // {offsetMs, reason} shape (offsetMs 0), which is what the golden locks.+ if _, err := c.GetSlowEndpoint(context.Background(), projectID, "GET /api/contract"); err != nil {+ t.Fatalf("endpoints slow: %v", err)+ }+ goldenAssert(t, "endpoints-slow", rt.body)+}+func TestContract_exceptionsGrouped(t *testing.T) {c, rt := capturedClient()resp, err := c.ListExceptions(context.Background(), projectID, client.ListExceptionsRequest{diff --git a/cli/test/contract/testdata/endpoints-chart.schema.txt b/cli/test/contract/testdata/endpoints-chart.schema.txtnew file mode 100644index 00000000..c4eab032--- /dev/null+++ b/cli/test/contract/testdata/endpoints-chart.schema.txt@@ -0,0 +1,4 @@+endpoints[]: string+series[].endpoint: string+series[].timestamp: string+series[].value: numberdiff --git a/cli/test/contract/testdata/endpoints-slow.schema.txt b/cli/test/contract/testdata/endpoints-slow.schema.txtnew file mode 100644index 00000000..278114ce--- /dev/null+++ b/cli/test/contract/testdata/endpoints-slow.schema.txt@@ -0,0 +1,2 @@+offsetMs: number+reason: stringdiff --git a/skills/traceway/SKILL.md b/skills/traceway/SKILL.mdindex a28e0c36..94fc0647 100644--- a/skills/traceway/SKILL.md+++ b/skills/traceway/SKILL.md@@ -11,6 +11,7 @@ Drive a Traceway instance from the terminal with the `traceway` CLI. The first w|---|---|| `/traceway login` | **Login**: install the CLI if missing, authenticate, select a project || `/traceway debug <issue ref or bug description>` | **Debug**: resolve the issue and investigate to root cause |+| `/traceway perf <endpoint or symptom>` | **Performance**: diagnose latency/slowness to root cause against a checklist of common bottlenecks || `/traceway <anything else>` | **Query**: answer the observability question with CLI reads || `/traceway` (no argument) | Ask what they want: log in, debug an issue, or run a query |@@ -71,9 +72,13 @@ The flag is required, so you must supply *something* — but it can be approxima2. **Estimate from context.** A notification's send time, the issue's `firstSeen`/`lastSeen`, or the URL's `preset`/`from` window all put you inside ±24h — good enough for a fast lookup.3. **Ask the user.** If nothing pins it down (e.g. a bare occurrence/endpoint id with no hash, no time, and no list to recover from), ask roughly when it happened — "around when did this fire? within a day is enough" — and pass that. Don't invent a placeholder like "now" when the issue is old; that defeats the pruning and can miss the ±24h window entirely.-## Resolving an issue notification+## Resolving a notification-Traceway issue notifications (email / Slack / webhook) embed everything for a direct, fast lookup. The body contains:+Traceway notifications (email / Slack / webhook) come in two shapes. Read the body to tell them apart: an **exception** notification carries a `Hash:` and `Exception ID:`; a **performance** notification (latency, apdex, impact) names an endpoint and a threshold and has no hash. Webhook payloads also carry a `ruleType` field that names the rule exactly. Resolve each differently.++### Exception notification++The body embeds everything for a direct, fast lookup. It contains:- `Hash: <16-hex>` — the exception group → `traceway exceptions show <hash>`.- `Exception ID: <uuid>` — the specific occurrence.@@ -87,6 +92,32 @@ traceway exceptions occurrence <Exception ID> --recorded-at <Occurred at → RFC# the result carries distributedTraceId and sessionId → traces show / sessions show below```+### Performance notification (an endpoint became slow or critical)++These fire from latency/throughput rules, not from an error. There is no hash, no exception id, no occurrence to fetch. The webhook `ruleType` (or the subject wording) tells you which:++| ruleType | Subject / body shape | Deep link |+|---|---|---|+| `endpoint_p95_threshold` / `endpoint_p99_threshold` | `P95 latency 1250ms on <endpoint>` · "reached 1250ms over the last N minutes (threshold: 1000ms)" | `/endpoints?from=<iso>&to=<iso>` |+| `apdex_drop` | `Apdex dropped to 0.62 (threshold: 0.80)` · "across N requests over the last N minutes" | `/endpoints?from=<iso>&to=<iso>` |+| `impact_score_critical\|high\|medium` | `Endpoint <endpoint> impact became critical` · "impact score: 0.82. Reason: <reason>" | `/endpoints?from=<iso>&to=<iso>` |+| `metric_threshold` | `Metric <name> is <value> (threshold: gt 100)` | `/metrics?preset=1h` |++What the body gives you: the **endpoint name** (parse it from the subject/body; it is not a separate field), the **metric/percentile**, the **current value**, the **threshold**, and the **lookback window**. The `/endpoints?from=&to=` link is only a ~3-minute window around the fire time, so treat the fire time as the onset hint, not the incident window.++To resolve, hand straight to the **Performance flow** scoped to that endpoint, using the fire time as the time anchor:++```bash+# 1. confirm the alert against current stats for that endpoint+traceway endpoints list --search "<endpoint>" --since 1h --output json | jq '.data[0] | {p50, p95, p99, count, impact, impactReason}'+# 2. is this an accepted baseline? check whether an operator marked it slow+traceway endpoints slow "<endpoint>" --output json # {offsetMs, reason}; offsetMs 0 = not marked+# 3. find when it crossed the threshold, then a slow trace (see Flow: Performance)+traceway endpoints chart --metric-type p95 --since 6h --interval-minutes 15 --output json+```++Step 2 matters: an `impact_score_*` or `apdex_drop` alert is already offset-aware, but `endpoint_p95_threshold` / `endpoint_p99_threshold` fire on **raw** latency regardless of any slow-marking, so a marked-slow endpoint can alert on latency its operators have already accepted. See "Account for user-marked slow endpoints" in `performance.md`.+## Flow: Login### 1. Check for an existing install@@ -234,9 +265,26 @@ Summarize: symptom, evidence (exception hashes, log excerpts, metric anomalies),traceway exceptions archive <hash> --yes```+## Flow: Performance++`/traceway perf <endpoint or symptom>`, or any request about slowness, high latency, p95/p99, or "why is X slow". Approach it as a senior performance engineer: quantify first, localize the cost to a span, confirm the cause against the checklist, separate code from saturation, then root cause. Reads only; never archive or mutate.++For the full methodology and the bottleneck checklist (Select N+1, missing index, chatty IPC, connection-pool exhaustion, GC pauses, and ~20 more, each tied to the Traceway signal that confirms it), read `performance.md` in this skill directory. It is the authoritative reference.++The loop, using the read commands documented in this skill:++1. **Quantify and localize.** `traceway endpoints list --since 24h --order-by p95` (or `impact`). Read the latency shape first: p50 already high means every request pays it (systemic: query, index, algorithm); p99 much greater than p50 means a tail (contention, pool exhaustion, GC, retries, a flaky dependency). Note that `impact`/`impactReason` are offset-adjusted for marked-slow endpoints but the raw p50/p95/p99 are not.+2. **Check the accepted baseline.** Before calling an endpoint slow, run `traceway endpoints slow "<endpoint>"`. A non-zero `offsetMs` means an operator accepted that much latency (with a `reason`), so only latency *beyond* the offset is a real regression; `offsetMs 0` means default thresholds apply. Skip this only for a clearly systemic, multi-endpoint slowdown.+3. **Pinpoint when it started: adjust the window down.** Do not investigate the default window blindly; narrow it until it brackets the onset. `traceway endpoints chart --metric-type p95 --interval-minutes <n>` returns latency over time for the top endpoints (`{timestamp, endpoint, value}` in ms): read down one endpoint's buckets for the step where p95 jumps. Confirm the cause with `traceway metrics query --name <metric> --interval-minutes <n>` (the infra/runtime curve). For a specific route not in the top 5, bisect `traceway endpoints list --search <name> --from <a> --to <b>` over adjacent windows. See `performance.md`, "Pinpointing when the slowness started".+4. **Get a representative slow trace.** `traceway endpoints show <endpointId> --recorded-at <t>` for one request's span waterfall, or `traceway traces show <distributedTraceId> --recorded-at <t>` for the cross-service timeline. Take the trace from inside the slow window you just found; find the long pole.+5. **Match the long pole to the checklist.** Is the dominant span a database call, an external call, in-process compute, or a *gap* before work starts (queueing / lock wait / pool exhaustion)? Look it up in `performance.md`.+6. **Separate code from saturation.** Pull infra/runtime metrics over the same window: `traceway metrics query --name system.cpu.utilization --aggregation max`, then `mem.used_pcnt`, `go.gc_pause`. A spike that lines up with the latency onset points at saturation, not a code bug.+7. **Correlate with code and deploys.** With the onset time from step 3, check what shipped then: `git log --since '<onset - 30m>' --until '<onset + 30m>'` or the deploy history. A jump at a release is a regression; a gradual ramp with no deploy is data growth (N+1, missing index, no pagination).+8. **Report.** The bottleneck, the evidence (endpoint percentiles, the onset time, the dominating span, any correlated metric anomaly), the root cause mapped to a checklist item, and a concrete fix, with the `endpoints show` / `traces show` references so the user can verify. If the endpoint is marked slow, state whether the latency exceeded the accepted offset.+## Flow: Query-For free-form requests ("what's broken in prod?", "is /api/checkout slow?", "show errors for service X"), use the read commands directly.+For free-form requests ("what's broken in prod?", "is /api/checkout slow?", "show errors for service X"), use the read commands directly. A request specifically about latency or slowness routes to the Performance flow above.### Command reference@@ -249,6 +297,8 @@ For free-form requests ("what's broken in prod?", "is /api/checkout slow?", "sho| `traceway exceptions archive/unarchive <hash>...` | Mutating; explicit user request + `--yes` only || `traceway logs query` | Logs; `--search` (`--search-type body\|attribute`), `--service`, `--min-severity <n>`, `--trace-id` || `traceway endpoints list` | Per-endpoint p50/p95/p99 and counts; `--search`, `--order-by impact\|count\|p95\|lastSeen` |+| `traceway endpoints chart` | Latency over time for the top endpoints; `--metric-type total_time\|p50\|p95\|p99`, `--interval-minutes`. Use to find when latency changed |+| `traceway endpoints slow <endpoint>` | Whether an operator marked this endpoint slow: `{offsetMs, reason}`. `offsetMs 0` = not marked. The offset is the accepted-latency baseline || `traceway endpoints show <id> --recorded-at <t>` | One request by id: span waterfall + linked errors || `traceway tasks show <id> --recorded-at <t>` | One background task run by id || `traceway ai-traces show <id> --recorded-at <t>` | One AI trace by id + its conversation |diff --git a/skills/traceway/performance.md b/skills/traceway/performance.mdnew file mode 100644index 00000000..9fd07a10--- /dev/null+++ b/skills/traceway/performance.md@@ -0,0 +1,154 @@+# Performance debugging++Authoritative reference for the **Performance** flow. Approach the investigation as a senior software engineer with deep performance-debugging experience: do not guess at fixes, quantify first, localize the cost to a specific span, confirm the cause against the checklist below, and only then propose a change. Every suspect here is paired with the Traceway signal that confirms it, so the checklist is a lookup table from "what the waterfall shows" to "what is actually wrong".++All commands are read-only. Never archive or mutate anything during a performance investigation. For exact CLI syntax, the mandatory `--recorded-at` / `--started-at` timestamp rules, and dashboard-URL resolution, the parent `SKILL.md` is authoritative.++## The investigation loop++1. **Quantify and localize.** Find the slow endpoint(s) and read the shape of the latency:+ ```bash+ traceway endpoints list --since 24h --order-by p95 --page-size 10 --output json \+ | jq '.data[]? | {endpoint, count, p50, p95, p99, errorCount}'+ ```+ Read p50 vs p99 before anything else (see "Reading the latency shape").++2. **Get a representative slow trace.** Drill into one slow request to see where the time goes. Capture an id and its `recordedAt` together, then:+ ```bash+ traceway endpoints show <endpointId> --recorded-at <t> # span waterfall for one request+ traceway traces show <distributedTraceId> --recorded-at <t> # cross-service timeline+ ```+ `traces show` is the highest-value call when more than one service is involved: it stitches the whole logical request together end to end.++3. **Find the long pole and match it to the checklist.** In the waterfall, identify the single span (or repeated group of spans) that dominates the duration. Classify it: is it a database call, an external/upstream call, in-process compute, or a *gap* before work starts (which means queueing, lock wait, or connection-pool exhaustion, not the work itself)? Then look it up in the checklist.++4. **Separate code from saturation.** A slow endpoint can be a code bug or an overloaded host. Pull infrastructure and runtime metrics over the same window:+ ```bash+ traceway metrics query --name system.cpu.utilization --aggregation max --since 24h+ traceway metrics query --name mem.used_pcnt --aggregation max --since 24h+ traceway metrics query --name go.gc_pause --aggregation max --since 24h+ ```+ A metric spike that lines up with the latency rise points at saturation; flat metrics under rising latency point at a code or query problem.++5. **Correlate with code and deploys.** Open the slow path in the source and read it. If p95 climbed at a specific time, check what shipped then (`git log --since ... --until ...` or deploy history). Latency that jumped at a release is a regression; latency that crept up without one is usually data growth.++6. **Report.** State the bottleneck, the evidence (endpoint percentiles, the dominating span, any correlated metric), the root cause mapped to a specific checklist item, and a concrete fix. Include the `endpoints show` / `traces show` references so the user can verify.++## Reading the latency shape++The relationship between percentiles, load, and time tells you which family of causes to look at before you open a single trace:++- **p50 already high** (the median request is slow): every request pays the cost, so it is systemic. Suspect an inefficient query, a missing index, an O(n^2) hot path, or synchronous work that should be backgrounded.+- **p99 much greater than p50** (median fine, tail slow): a minority of requests stall. Suspect contention, connection-pool exhaustion, GC pauses, retries, cache stampedes, or a flaky upstream dependency.+- **Latency that grew over weeks with no deploy**: data-volume driven. Suspect Select N+1, a missing index, or an unbounded result set, all of which scale with table size.+- **Latency that jumped at a deploy**: a regression. Correlate `firstSeen` / the climb time with `git log`.+- **Latency that tracks request volume**: saturation or contention. Confirm with infra metrics and pool-wait gaps in the waterfall.++## Account for user-marked slow endpoints++Some endpoints are *expected* to be slow: a CSV export, a heavy report, a third-party sync. An operator can mark such an endpoint slow with an allowance (`offsetMs`) and a `reason`. Check before you treat latency as a problem:++```bash+traceway endpoints slow "GET /api/reports/export" --output json # {offsetMs, reason}+```++- `offsetMs 0` (or "MARKED: no"): default thresholds apply; judge latency normally.+- `offsetMs > 0`: the operator accepted that much latency. The apdex and impact thresholds are shifted up by the offset server-side, so the endpoint's `impact`/`impactReason` already discount it. Only latency *beyond* `offsetMs`, or a fresh jump above the prior baseline, is a real regression.++Two things to keep straight:++- **Raw percentiles are not offset-adjusted.** The p50/p95/p99 from `endpoints list` and `endpoints chart` come from raw durations; the offset only moves the apdex/impact thresholds. A marked-slow endpoint can show a high p95 that is entirely expected, so compare p95 against `offsetMs`, not against zero.+- **p95/p99 alert rules are not offset-aware.** `endpoint_p95_threshold` / `endpoint_p99_threshold` notifications fire on raw latency regardless of marking, so a marked-slow endpoint can still page on latency its operators accepted. `apdex_drop` and `impact_score_*` rules are offset-aware. (See "Performance notification" in `SKILL.md`.)++When you report, state the p95 and the allowance together: "p95 is 2.1s but this endpoint is marked slow with a 2s allowance (reason: nightly export), so it is within baseline", or "p95 is 4s against a 2s allowance, so it regressed past the accepted offset".++## Pinpointing when the slowness started++Narrowing the window is how you turn "it is slow" into "it got slow at 14:10, right after deploy X". Do not investigate the default window blindly: adjust it down until it brackets the onset, then correlate that moment with deploys and metrics.++**1. Latency over time: `endpoints chart`.** This is the direct curve. It buckets latency over the window and returns the top 5 endpoints ranked by the chosen metric (plus an "Other" bucket), each as a `{timestamp, endpoint, value}` series in milliseconds. Read down one endpoint's buckets for the step where its value jumps: that bucket dates the onset. Use `--metric-type p95` (or `p99`/`p50`/`total_time`) and a coarse `--interval-minutes` over a wide window first, then re-run the suspect span with a finer interval to sharpen.+```bash+traceway endpoints chart --metric-type p95 --since 7d --interval-minutes 60 --output json \+ | jq '.series[] | select(.endpoint == "GET /api/checkout") | {t: .timestamp, p95_ms: .value}'+```+Caveat: it always returns the *top 5* endpoints by that metric. If your suspect route is not in the top 5 it is folded into "Other"; fall back to method 3 for it.++**2. Infra and runtime over time: `metrics query --interval-minutes`.** For the *cause* behind a latency step (saturation, GC, throughput), bucket any metric the same way. The returned `series` is a list of `{timestamp, value}` points; read down them for the bucket where the value steps up.+```bash+traceway metrics query --name system.cpu.utilization --aggregation max --since 7d --interval-minutes 60 --output json \+ | jq '.series | to_entries[] | .value[] | {t: .timestamp, v: .value}'+```+A flat line that steps up at the same bucket the latency jumped is the saturation signal; flat metrics under a latency step point at code.++**3. A specific non-top-5 endpoint: window bisection.** When the route you care about is in "Other", reconstruct its onset from `endpoints list` (the only source of true p50/p95/p99, computed from raw durations) by comparing the *same endpoint's* percentiles across adjacent windows with `--from`/`--to`. Start wide, split, keep the half where p95 is elevated, recurse until the bracket is tight:+```bash+# is the regression in the first or second half of the suspect day?+traceway endpoints list --search "GET /api/checkout" --from 2026-06-25T00:00:00Z --to 2026-06-25T12:00:00Z \+ --output json | jq '.data[0] | {p95, p99, count}'+traceway endpoints list --search "GET /api/checkout" --from 2026-06-25T12:00:00Z --to 2026-06-26T00:00:00Z \+ --output json | jq '.data[0] | {p95, p99, count}'+```+A few bisection steps localize the jump to an hour or less. Watch `count` alongside p95: if the slow half also carries far more traffic, suspect load, not a code change.++**4. Cross-check and correlate.** Once the methods agree on an onset time:+- A latency jump that coincides with a metric inflection (CPU or GC stepping up at the same bucket) points at saturation.+- A latency jump with flat infra metrics points at a code or query regression: `git log --since '<onset - 30m>' --until '<onset + 30m>'` or the deploy history for what shipped then.+- A latency curve that ramps gradually over days with no single step is data growth (N+1, missing index, unbounded result set), not a deploy.++Then pass the narrowed `--from`/`--to` back into the by-id detail lookups so `--recorded-at` lands inside the slow window and the trace you pull is actually from the slow period, not a fast one before the regression.++## Bottleneck checklist++Each row: the suspect, and how to confirm it from Traceway telemetry.++### Database and data access+| Suspect | How to confirm in Traceway |+|---|---|+| **Select N+1** | Many repeated, near-identical short query spans inside one request in the span waterfall; total DB time dominates while each call is tiny. |+| **Missing or unused index** | One query span is the long pole; its duration grows with table size; same query slow across many traces. |+| **Over-fetching (`SELECT *` / no projection)** | Large query-result spans, high memory; latency tracks row width/count rather than logic. |+| **No pagination / unbounded result set** | p50 has crept up over time with data growth; one query returns more rows every week. |+| **Expensive JOIN or aggregation** | A single query span dominates the trace consistently, independent of load. |+| **Connection-pool exhaustion** | A gap *before* the first DB span (time waiting to acquire a connection); p99 rises with concurrency while p50 stays flat. |+| **Lock / transaction contention** | DB spans queue behind each other; tail latency appears under write load; correlated WARN/ERROR locks in `logs query`. |+| **Missing query/result caching** | The same query runs on every request; high DB-call count per endpoint with low cardinality of inputs. |++### Application and compute+| Suspect | How to confirm in Traceway |+|---|---|+| **Synchronous work that belongs in a background task** | A long handler span runs before the response returns, for work the user does not wait on (emails, exports, image processing). It should be a `CaptureTask`. |+| **CPU-bound serialization** | A compute gap between IO spans (large JSON encode/decode); `system.cpu.utilization` up during the request. |+| **Inefficient algorithm / O(n^2) hot path** | Latency scales superlinearly with input size across traces; no IO to blame in the waterfall. |+| **Unbounded in-memory work / large allocations** | `mem.used_pcnt` and `go.num_gc` climb with p99; large compute spans. |+| **Excessive logging / telemetry in the hot path** | Many log lines per request in `logs query`; logging spans visible in the waterfall. |++### Network, IO, and external dependencies+| Suspect | How to confirm in Traceway |+|---|---|+| **Chatty IPC / microservice fan-out** | `traces show` reveals many sequential cross-service hops, each adding round-trip latency. |+| **Slow third-party / upstream API** | An external span dominates the trace; p99 spikes correlate with the provider; not your code. |+| **Sequential calls that could run in parallel** | A staircase waterfall: each span starts only when the previous ends, though they are independent. |+| **No timeouts / aggressive retries** | Long-tail p99; retry bursts in `logs query`; the same downstream span repeated within one trace. |+| **No connection reuse (DNS/TLS per call)** | Many short external spans each paying a handshake; per-request connection setup cost. |+| **Large request/response payloads** | Transfer-dominated spans; latency correlates with payload size. |++### Caching and delivery+| Suspect | How to confirm in Traceway |+|---|---|+| **No cache / low hit ratio** | DB or upstream is hit on every request; cache-hit metric (if reported) is low. |+| **Cache stampede / thundering herd** | Latency spikes at cache-expiry boundaries; a correlated burst of identical traces. |+| **Cold start / JIT / connection warmup** | First-request-after-deploy outliers; latency normalizes after warmup; `firstSeen`-correlated. |++### Infrastructure and saturation+| Suspect | How to confirm in Traceway |+|---|---|+| **CPU saturation** | `system.cpu.utilization` (max) spike lines up with the latency rise across many endpoints at once. |+| **Memory pressure / GC pauses** | `mem.used_pcnt`, `go.gc_pause`, `go.num_gc` correlate with p99; pauses visible as gaps in the waterfall. |+| **Disk / IO bottleneck** | `system.disk.*` or IO-wait metrics up; slow IO spans with no CPU cost. |+| **Undersized / noisy-neighbor instance** | Broad latency rise with no single span culprit; infra metrics elevated with no code change. |++## Notes on the signals++- Host and runtime metrics live under `system.*` (Traceway OTel Agent) and the SDK runtime names (`go.gc_pause`, `mem.used_pcnt`, `go.go_routines`). The server has no quantile aggregation for metric points: `p50/p95/p99` on `metrics query` silently return `avg`, so never present those as percentiles. Real latency percentiles come only from `endpoints list`, computed from raw request durations.+- OTLP histogram metrics are stored as two series, `<name>.avg` and `<name>.count`. A bogus metric name returns an empty `series: {}` cleanly, so probing names is safe.+- When the waterfall shows a gap with no span, that gap is the finding: it is time the request spent waiting (for a connection, a lock, the scheduler), not time spent doing work.