Skip to content

The only tool you need to know what is happening and how to fix it.

feat: added skills and api to figure out latency issues (#245)

narrow-adventure7b21bb322bd5
diff --git a/cli/cmd/traceway/endpoints.go b/cli/cmd/traceway/endpoints.go
index 4d2d33a9..e29bf100 100644
--- a/cli/cmd/traceway/endpoints.go
+++ b/cli/cmd/traceway/endpoints.go
@@ -13,12 +13,16 @@ import (
var endpointsOrderBy = []string{"impact", "count", "p95", "lastSeen"}
+var endpointChartMetricTypes = []string{"total_time", "p50", "p95", "p99"}
+
func newEndpointsCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "endpoints",
Short: "Query HTTP endpoint performance",
}
cmd.AddCommand(newEndpointsListCmd())
+ cmd.AddCommand(newEndpointsChartCmd())
+ cmd.AddCommand(newEndpointsSlowCmd())
cmd.AddCommand(newEndpointsShowCmd())
return cmd
}
@@ -100,6 +104,164 @@ func runEndpointsList(cmd *cobra.Command, _ []string) error {
}
}
+func newEndpointsChartCmd() *cobra.Command {
+ cmd := &cobra.Command{
+ Use: "chart",
+ Short: "Endpoint latency bucketed over time (find when latency changed)",
+ Long: `Show endpoint latency as a time series, the curve behind /endpoints in the
+dashboard. The server returns the top 5 endpoints ranked by --metric-type plus
+an "Other" bucket, each bucketed by --interval-minutes, so you can read down the
+buckets to find when a latency jump started.
+
+This always returns the top endpoints by the chosen metric; it does not filter
+to one route. To track a specific endpoint that is not in the top 5, bisect
+"endpoints list --search <name>" over narrowing --from/--to windows instead.
+
+Values are in milliseconds. Use --output json for the full per-bucket series.`,
+ RunE: runEndpointsChart,
+ }
+ addTimeRangeFlags(cmd)
+ cmd.Flags().String("metric-type", "p95", "Metric to chart: total_time, p50, p95, p99")
+ cmd.Flags().Int("interval-minutes", 0, "Time bucket size in minutes (0 = server default of 5)")
+ return cmd
+}
+
+func runEndpointsChart(cmd *cobra.Command, _ []string) error {
+ ctx := cmd.Context()
+ mode := output.ResolveMode(flagOutput, output.StdoutIsTerminal())
+
+ sess, err := loadSession()
+ if err != nil {
+ return renderSessionError(cmd.ErrOrStderr(), mode, err)
+ }
+ tr, err := resolveTimeRange(cmd)
+ if err != nil {
+ return renderTimeRangeError(cmd.ErrOrStderr(), mode, err)
+ }
+ metricType, _ := cmd.Flags().GetString("metric-type")
+ if err := validateEnumFlag("--metric-type", metricType, endpointChartMetricTypes); err != nil {
+ return renderUsageError(cmd.ErrOrStderr(), mode, err.Error(),
+ enumFlagHint("traceway endpoints chart", "--metric-type", endpointChartMetricTypes))
+ }
+ intervalMin, _ := cmd.Flags().GetInt("interval-minutes")
+
+ c := client.New(sess.URL, client.WithJWT(sess.JWT))
+ resp, err := c.GetEndpointChart(ctx, sess.ProjectID, client.EndpointChartRequest{
+ TimeRange: tr,
+ MetricType: metricType,
+ IntervalMinutes: intervalMin,
+ })
+ if err != nil {
+ return renderAPIError(cmd.ErrOrStderr(), mode, err, false)
+ }
+
+ switch mode {
+ case output.ModeJSON:
+ return output.RenderJSON(cmd.OutOrStdout(), resp, output.ParseFieldsFlag(flagFields))
+ case output.ModeYAML:
+ return output.RenderYAML(cmd.OutOrStdout(), resp, output.ParseFieldsFlag(flagFields))
+ default:
+ return renderEndpointChartTable(cmd.OutOrStdout(), resp)
+ }
+}
+
+// renderEndpointChartTable prints a per-endpoint summary of the time series
+// (point count and min/max/latest value in ms), in the server's ranked order.
+// The full per-bucket series is only in --output json.
+func renderEndpointChartTable(out io.Writer, resp *client.EndpointChartResponse) error {
+ type seriesAgg struct {
+ points int
+ min, max float64
+ latest float64
+ latestTime time.Time
+ seen bool
+ }
+ byEndpoint := make(map[string]*seriesAgg, len(resp.Endpoints))
+ for _, p := range resp.Series {
+ a := byEndpoint[p.Endpoint]
+ if a == nil {
+ a = &seriesAgg{min: p.Value, max: p.Value}
+ byEndpoint[p.Endpoint] = a
+ }
+ a.points++
+ if p.Value < a.min {
+ a.min = p.Value
+ }
+ if p.Value > a.max {
+ a.max = p.Value
+ }
+ if !a.seen || p.Timestamp.After(a.latestTime) {
+ a.latest = p.Value
+ a.latestTime = p.Timestamp
+ a.seen = true
+ }
+ }
+
+ tw := output.NewTabWriter(out)
+ _, _ = fmt.Fprintln(tw, "ENDPOINT\tPOINTS\tMIN(ms)\tMAX(ms)\tLATEST(ms)")
+ for _, name := range resp.Endpoints {
+ a := byEndpoint[name]
+ if a == nil {
+ _, _ = fmt.Fprintf(tw, "%s\t0\t-\t-\t-\n", name)
+ continue
+ }
+ _, _ = fmt.Fprintf(tw, "%s\t%d\t%.1f\t%.1f\t%.1f\n", name, a.points, a.min, a.max, a.latest)
+ }
+ return tw.Flush()
+}
+
+func newEndpointsSlowCmd() *cobra.Command {
+ cmd := &cobra.Command{
+ Use: "slow <endpoint>",
+ Short: "Show the user-configured slow allowance for one endpoint",
+ Long: `Report whether an endpoint has been marked slow by an operator, and by how
+much. The allowance (offsetMs) is added to the apdex and impact thresholds for
+that endpoint server-side, so latency up to the offset is an accepted baseline
+rather than a regression. The reason is the operator's note.
+
+An endpoint that was never marked slow returns offsetMs 0 and an empty reason.
+
+The offset shifts apdex/impact only; the raw p50/p95/p99 from "endpoints list"
+and "endpoints chart" are NOT adjusted by it. Pass the full route name, e.g.
+"GET /api/users/:id".`,
+ Args: cobra.ExactArgs(1),
+ RunE: runEndpointsSlow,
+ }
+ return cmd
+}
+
+func runEndpointsSlow(cmd *cobra.Command, args []string) error {
+ ctx := cmd.Context()
+ mode := output.ResolveMode(flagOutput, output.StdoutIsTerminal())
+
+ sess, err := loadSession()
+ if err != nil {
+ return renderSessionError(cmd.ErrOrStderr(), mode, err)
+ }
+
+ c := client.New(sess.URL, client.WithJWT(sess.JWT))
+ resp, err := c.GetSlowEndpoint(ctx, sess.ProjectID, args[0])
+ if err != nil {
+ return renderAPIError(cmd.ErrOrStderr(), mode, err, false)
+ }
+
+ switch mode {
+ case output.ModeJSON:
+ return output.RenderJSON(cmd.OutOrStdout(), resp, output.ParseFieldsFlag(flagFields))
+ case output.ModeYAML:
+ return output.RenderYAML(cmd.OutOrStdout(), resp, output.ParseFieldsFlag(flagFields))
+ default:
+ out := cmd.OutOrStdout()
+ if resp.OffsetMs == 0 && resp.Reason == "" {
+ _, _ = fmt.Fprintf(out, "ENDPOINT: %s\nMARKED: no (default thresholds)\n", args[0])
+ return nil
+ }
+ _, _ = fmt.Fprintf(out, "ENDPOINT: %s\nMARKED: yes\nALLOWANCE: %dms\nREASON: %s\n",
+ args[0], resp.OffsetMs, pickStr(resp.Reason, "-"))
+ return nil
+ }
+}
+
func newEndpointsShowCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "show <endpointId>",
diff --git a/cli/cmd/traceway/endpoints_test.go b/cli/cmd/traceway/endpoints_test.go
index 34d0b9d2..cd30ae71 100644
--- a/cli/cmd/traceway/endpoints_test.go
+++ b/cli/cmd/traceway/endpoints_test.go
@@ -1,6 +1,7 @@
package main
import (
+ "encoding/json"
"net/http"
"net/http/httptest"
"strings"
@@ -51,3 +52,208 @@ func TestEndpointsList_jsonShape(t *testing.T) {
t.Errorf("missing endpoint in JSON: %s", stdout.String())
}
}
+
+const chartResponseJSON = `{
+ "endpoints":["GET /api/checkout","GET /api/projects","Other"],
+ "series":[
+ {"timestamp":"2026-06-25T12:00:00Z","endpoint":"GET /api/checkout","value":120.5},
+ {"timestamp":"2026-06-25T13:00:00Z","endpoint":"GET /api/checkout","value":480.0},
+ {"timestamp":"2026-06-25T12:00:00Z","endpoint":"GET /api/projects","value":30.0},
+ {"timestamp":"2026-06-25T13:00:00Z","endpoint":"GET /api/projects","value":35.0}
+ ]
+}`
+
+func TestEndpointsChart_jsonShape(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ _, _ = w.Write([]byte(chartResponseJSON))
+ }))
+ defer srv.Close()
+ seedSessionFor(t, srv.URL)
+
+ stdout, _, err := runCmd(t, "", "endpoints", "chart", "--output", "json")
+ if err != nil {
+ t.Fatal(err)
+ }
+ out := stdout.String()
+ if !strings.Contains(out, `"GET /api/checkout"`) || !strings.Contains(out, "480") {
+ t.Errorf("json missing series data: %s", out)
+ }
+}
+
+func TestEndpointsChart_table(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ _, _ = w.Write([]byte(chartResponseJSON))
+ }))
+ defer srv.Close()
+ seedSessionFor(t, srv.URL)
+
+ stdout, _, err := runCmd(t, "", "endpoints", "chart", "--output", "table")
+ if err != nil {
+ t.Fatal(err)
+ }
+ out := stdout.String()
+ if !strings.Contains(out, "ENDPOINT") || !strings.Contains(out, "MIN(ms)") || !strings.Contains(out, "MAX(ms)") || !strings.Contains(out, "LATEST(ms)") {
+ t.Errorf("table missing headers: %s", out)
+ }
+ // The slow endpoint's peak should surface as its MAX and its later bucket as LATEST.
+ if !strings.Contains(out, "GET /api/checkout") || !strings.Contains(out, "480.0") {
+ t.Errorf("table missing endpoint summary: %s", out)
+ }
+ // Endpoints with no points (the "Other" bucket here) still render a row.
+ if !strings.Contains(out, "Other") {
+ t.Errorf("table missing Other row: %s", out)
+ }
+}
+
+func TestEndpointsChart_sendsRequestBody(t *testing.T) {
+ var gotPath, gotMethod string
+ var gotBody map[string]any
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ gotPath = r.URL.Path
+ gotMethod = r.Method
+ _ = json.NewDecoder(r.Body).Decode(&gotBody)
+ if r.URL.Query().Get("projectId") != "proj-1" {
+ t.Errorf("projectId = %q", r.URL.Query().Get("projectId"))
+ }
+ _, _ = w.Write([]byte(chartResponseJSON))
+ }))
+ defer srv.Close()
+ seedSessionFor(t, srv.URL)
+
+ _, _, err := runCmd(t, "", "endpoints", "chart", "--metric-type", "p99", "--interval-minutes", "60", "--since", "24h", "--output", "json")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if gotMethod != http.MethodPost {
+ t.Errorf("method = %q, want POST", gotMethod)
+ }
+ if gotPath != "/api/endpoints/chart" {
+ t.Errorf("path = %q, want /api/endpoints/chart", gotPath)
+ }
+ if gotBody["metricType"] != "p99" {
+ t.Errorf("metricType = %v, want p99", gotBody["metricType"])
+ }
+ if gotBody["intervalMinutes"] != float64(60) {
+ t.Errorf("intervalMinutes = %v, want 60", gotBody["intervalMinutes"])
+ }
+ // Endpoints routes use fromDate/toDate (not from/to like metrics).
+ if gotBody["fromDate"] == nil || gotBody["toDate"] == nil {
+ t.Errorf("body missing fromDate/toDate: %v", gotBody)
+ }
+ if gotBody["from"] != nil || gotBody["to"] != nil {
+ t.Errorf("body should not carry from/to: %v", gotBody)
+ }
+}
+
+func TestEndpointsChart_metricTypeValidation(t *testing.T) {
+ // Bogus --metric-type is rejected client-side, before any request is sent.
+ srv := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {
+ t.Fatal("server should not be called for invalid --metric-type")
+ }))
+ defer srv.Close()
+ seedSessionFor(t, srv.URL)
+
+ _, stderr, err := runCmd(t, "", "endpoints", "chart", "--metric-type", "nope", "--output", "json")
+ if err == nil {
+ t.Fatal("expected error for bogus --metric-type")
+ }
+ if !strings.Contains(stderr.String(), "--metric-type") {
+ t.Errorf("expected --metric-type in error, got: %s", stderr.String())
+ }
+
+ // Every documented metric type is accepted.
+ for _, mt := range endpointChartMetricTypes {
+ okSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ _, _ = w.Write([]byte(chartResponseJSON))
+ }))
+ seedSessionFor(t, okSrv.URL)
+ if _, stderr, err := runCmd(t, "", "endpoints", "chart", "--metric-type", mt, "--output", "json"); err != nil {
+ okSrv.Close()
+ t.Fatalf("metric-type %q rejected: %v\nstderr: %s", mt, err, stderr.String())
+ }
+ okSrv.Close()
+ }
+}
+
+func TestEndpointsSlow_marked(t *testing.T) {
+ var gotPath, gotMethod, gotEndpoint string
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ gotPath = r.URL.Path
+ gotMethod = r.Method
+ gotEndpoint = r.URL.Query().Get("endpoint")
+ _, _ = w.Write([]byte(`{"offsetMs":2000,"reason":"third-party export, accepted"}`))
+ }))
+ defer srv.Close()
+ seedSessionFor(t, srv.URL)
+
+ stdout, _, err := runCmd(t, "", "endpoints", "slow", "GET /api/checkout", "--output", "table")
+ if err != nil {
+ t.Fatal(err)
+ }
+ out := stdout.String()
+ if !strings.Contains(out, "MARKED:") || !strings.Contains(out, "yes") || !strings.Contains(out, "2000ms") {
+ t.Errorf("table missing marked summary: %s", out)
+ }
+ if !strings.Contains(out, "third-party export") {
+ t.Errorf("table missing reason: %s", out)
+ }
+ if gotMethod != http.MethodGet {
+ t.Errorf("method = %q, want GET", gotMethod)
+ }
+ if gotPath != "/api/endpoints/slow" {
+ t.Errorf("path = %q, want /api/endpoints/slow", gotPath)
+ }
+ if gotEndpoint != "GET /api/checkout" {
+ t.Errorf("endpoint query = %q, want %q", gotEndpoint, "GET /api/checkout")
+ }
+}
+
+func TestEndpointsSlow_unmarked(t *testing.T) {
+ // The server returns offsetMs 0 / empty reason for an endpoint never marked.
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ _, _ = w.Write([]byte(`{"offsetMs":0,"reason":""}`))
+ }))
+ defer srv.Close()
+ seedSessionFor(t, srv.URL)
+
+ stdout, _, err := runCmd(t, "", "endpoints", "slow", "GET /api/healthz", "--output", "table")
+ if err != nil {
+ t.Fatal(err)
+ }
+ out := stdout.String()
+ if !strings.Contains(out, "no") || !strings.Contains(out, "default thresholds") {
+ t.Errorf("unmarked endpoint should render 'no': %s", out)
+ }
+}
+
+func TestEndpointsSlow_jsonShape(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ _, _ = w.Write([]byte(`{"offsetMs":2000,"reason":"batch job"}`))
+ }))
+ defer srv.Close()
+ seedSessionFor(t, srv.URL)
+
+ stdout, _, err := runCmd(t, "", "endpoints", "slow", "GET /api/x", "--output", "json")
+ if err != nil {
+ t.Fatal(err)
+ }
+ var got map[string]any
+ if err := json.Unmarshal(stdout.Bytes(), &got); err != nil {
+ t.Fatalf("invalid json: %v\n%s", err, stdout.String())
+ }
+ if got["offsetMs"] != float64(2000) || got["reason"] != "batch job" {
+ t.Errorf("json = %v", got)
+ }
+}
+
+func TestEndpointsSlow_requiresEndpointArg(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {
+ t.Fatal("server should not be called without an endpoint arg")
+ }))
+ defer srv.Close()
+ seedSessionFor(t, srv.URL)
+
+ if _, _, err := runCmd(t, "", "endpoints", "slow", "--output", "json"); err == nil {
+ t.Fatal("expected error when endpoint arg is missing")
+ }
+}
diff --git a/cli/pkg/client/endpoints.go b/cli/pkg/client/endpoints.go
index d63629ec..59189539 100644
--- a/cli/pkg/client/endpoints.go
+++ b/cli/pkg/client/endpoints.go
@@ -103,3 +103,71 @@ func (c *Client) GetEndpoint(ctx context.Context, projectID, id string, recorded
}
return &resp, nil
}
+
+// EndpointChartRequest is the body for POST /api/endpoints/chart. The server
+// returns the top 5 endpoints ranked by MetricType (plus an "Other" bucket),
+// each as a time series bucketed by IntervalMinutes, which is endpoint latency
+// over time in a single call.
+type EndpointChartRequest struct {
+ TimeRange TimeRange `json:"-"`
+ MetricType string `json:"metricType,omitempty"`
+ IntervalMinutes int `json:"intervalMinutes,omitempty"`
+}
+
+// MarshalJSON expands TimeRange into top-level fromDate/toDate (the endpoints
+// routes use fromDate/toDate, unlike metrics which uses from/to).
+func (r EndpointChartRequest) MarshalJSON() ([]byte, error) {
+ type alias EndpointChartRequest
+ wire := struct {
+ FromDate time.Time `json:"fromDate"`
+ ToDate time.Time `json:"toDate"`
+ alias
+ }{r.TimeRange.From, r.TimeRange.To, alias(r)}
+ return jsonMarshalNoHTMLEscape(wire)
+}
+
+// EndpointChartPoint is one bucket of one endpoint's series. Value is in
+// milliseconds: total request time for total_time, the quantile for p50/p95/p99.
+type EndpointChartPoint struct {
+ Timestamp time.Time `json:"timestamp"`
+ Endpoint string `json:"endpoint"`
+ Value float64 `json:"value"`
+}
+
+// EndpointChartResponse mirrors models.EndpointStackedChartResponse.
+type EndpointChartResponse struct {
+ Endpoints []string `json:"endpoints"` // top 5 ranked by the metric, plus "Other"
+ Series []EndpointChartPoint `json:"series"`
+}
+
+// GetEndpointChart returns endpoint latency bucketed over time for the top
+// endpoints, which is the curve used to pinpoint when latency changed.
+func (c *Client) GetEndpointChart(ctx context.Context, projectID string, req EndpointChartRequest) (*EndpointChartResponse, error) {
+ path := "/api/endpoints/chart?projectId=" + url.QueryEscape(projectID)
+ var resp EndpointChartResponse
+ if err := c.do(ctx, http.MethodPost, path, req, &resp); err != nil {
+ return nil, err
+ }
+ return &resp, nil
+}
+
+// SlowEndpointResponse is the body of GET /api/endpoints/slow. OffsetMs is the
+// user-set allowance (in ms) added to the apdex and impact thresholds for this
+// endpoint; Reason is the operator's note. An endpoint that was never marked
+// slow returns {offsetMs:0, reason:""}, not an error.
+type SlowEndpointResponse struct {
+ OffsetMs uint32 `json:"offsetMs"`
+ Reason string `json:"reason"`
+}
+
+// GetSlowEndpoint returns the user-configured slow allowance for one endpoint.
+// The offset shifts the server-side apdex/impact thresholds only; the raw
+// p50/p95/p99 from endpoints list/chart are not adjusted by it.
+func (c *Client) GetSlowEndpoint(ctx context.Context, projectID, endpoint string) (*SlowEndpointResponse, error) {
+ path := "/api/endpoints/slow?projectId=" + url.QueryEscape(projectID) + "&endpoint=" + url.QueryEscape(endpoint)
+ var resp SlowEndpointResponse
+ if err := c.do(ctx, http.MethodGet, path, nil, &resp); err != nil {
+ return nil, err
+ }
+ return &resp, nil
+}
diff --git a/cli/pkg/client/endpoints_test.go b/cli/pkg/client/endpoints_test.go
index a8be64c6..d115d520 100644
--- a/cli/pkg/client/endpoints_test.go
+++ b/cli/pkg/client/endpoints_test.go
@@ -56,3 +56,80 @@ func TestListEndpoints_callsGroupedRoute(t *testing.T) {
t.Errorf("Impact = %v", resp.Data[0].Impact)
}
}
+
+func TestGetEndpointChart_callsChartRoute(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/api/endpoints/chart" {
+ t.Errorf("path = %q (want chart)", r.URL.Path)
+ }
+ if r.URL.Query().Get("projectId") != "proj-1" {
+ t.Errorf("projectId = %q", r.URL.Query().Get("projectId"))
+ }
+ var body map[string]any
+ _ = json.NewDecoder(r.Body).Decode(&body)
+ if body["fromDate"] == nil || body["toDate"] == nil {
+ t.Errorf("body missing fromDate/toDate: %v", body)
+ }
+ if body["metricType"] != "p95" {
+ t.Errorf("metricType = %v, want p95", body["metricType"])
+ }
+ if body["intervalMinutes"] != float64(60) {
+ t.Errorf("intervalMinutes = %v, want 60", body["intervalMinutes"])
+ }
+ _, _ = w.Write([]byte(`{
+ "endpoints":["GET /api/checkout","Other"],
+ "series":[
+ {"timestamp":"2026-06-25T12:00:00Z","endpoint":"GET /api/checkout","value":120.5},
+ {"timestamp":"2026-06-25T13:00:00Z","endpoint":"GET /api/checkout","value":480}
+ ]
+ }`))
+ }))
+ defer srv.Close()
+
+ c := New(srv.URL, WithJWT("tok"))
+ resp, err := c.GetEndpointChart(context.Background(), "proj-1", EndpointChartRequest{
+ TimeRange: TimeRange{From: time.Now().Add(-24 * time.Hour), To: time.Now()},
+ MetricType: "p95",
+ IntervalMinutes: 60,
+ })
+ if err != nil {
+ t.Fatalf("GetEndpointChart: %v", err)
+ }
+ if len(resp.Endpoints) != 2 || resp.Endpoints[0] != "GET /api/checkout" {
+ t.Errorf("Endpoints = %v", resp.Endpoints)
+ }
+ if len(resp.Series) != 2 {
+ t.Fatalf("got %d series points", len(resp.Series))
+ }
+ if resp.Series[1].Value != 480 || resp.Series[1].Endpoint != "GET /api/checkout" {
+ t.Errorf("series[1] = %+v", resp.Series[1])
+ }
+}
+
+func TestGetSlowEndpoint_callsSlowRoute(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Method != http.MethodGet {
+ t.Errorf("method = %q (want GET)", r.Method)
+ }
+ if r.URL.Path != "/api/endpoints/slow" {
+ t.Errorf("path = %q (want slow)", r.URL.Path)
+ }
+ if r.URL.Query().Get("projectId") != "proj-1" {
+ t.Errorf("projectId = %q", r.URL.Query().Get("projectId"))
+ }
+ if r.URL.Query().Get("endpoint") != "GET /api/users/:id" {
+ t.Errorf("endpoint = %q", r.URL.Query().Get("endpoint"))
+ }
+ _, _ = w.Write([]byte(`{"offsetMs":1500,"reason":"known heavy report"}`))
+ }))
+ defer srv.Close()
+
+ c := New(srv.URL, WithJWT("tok"))
+ resp, err := c.GetSlowEndpoint(context.Background(), "proj-1", "GET /api/users/:id")
+ if err != nil {
+ t.Fatalf("GetSlowEndpoint: %v", err)
+ }
+ if resp.OffsetMs != 1500 || resp.Reason != "known heavy report" {
+ t.Errorf("resp = %+v", resp)
+ }
+}
diff --git a/cli/test/contract/contract_test.go b/cli/test/contract/contract_test.go
index f4646eeb..cd962c1a 100644
--- a/cli/test/contract/contract_test.go
+++ b/cli/test/contract/contract_test.go
@@ -62,6 +62,32 @@ func TestContract_endpointDetail(t *testing.T) {
goldenAssert(t, "endpoint-detail", rt.body)
}
+func TestContract_endpointsChart(t *testing.T) {
+ c, rt := capturedClient()
+ resp, err := c.GetEndpointChart(context.Background(), projectID, client.EndpointChartRequest{
+ TimeRange: timeWindow(),
+ MetricType: "p95",
+ IntervalMinutes: 5,
+ })
+ if err != nil {
+ t.Fatalf("endpoints chart: %v", err)
+ }
+ if len(resp.Series) == 0 {
+ t.Fatal("expected seeded endpoint to produce a chart series point")
+ }
+ goldenAssert(t, "endpoints-chart", rt.body)
+}
+
+func TestContract_endpointsSlow(t *testing.T) {
+ c, rt := capturedClient()
+ // The seeded endpoint is not marked slow; the server still returns the
+ // {offsetMs, reason} shape (offsetMs 0), which is what the golden locks.
+ if _, err := c.GetSlowEndpoint(context.Background(), projectID, "GET /api/contract"); err != nil {
+ t.Fatalf("endpoints slow: %v", err)
+ }
+ goldenAssert(t, "endpoints-slow", rt.body)
+}
+
func TestContract_exceptionsGrouped(t *testing.T) {
c, rt := capturedClient()
resp, err := c.ListExceptions(context.Background(), projectID, client.ListExceptionsRequest{
diff --git a/cli/test/contract/testdata/endpoints-chart.schema.txt b/cli/test/contract/testdata/endpoints-chart.schema.txt
new file mode 100644
index 00000000..c4eab032
--- /dev/null
+++ b/cli/test/contract/testdata/endpoints-chart.schema.txt
@@ -0,0 +1,4 @@
+endpoints[]: string
+series[].endpoint: string
+series[].timestamp: string
+series[].value: number
diff --git a/cli/test/contract/testdata/endpoints-slow.schema.txt b/cli/test/contract/testdata/endpoints-slow.schema.txt
new file mode 100644
index 00000000..278114ce
--- /dev/null
+++ b/cli/test/contract/testdata/endpoints-slow.schema.txt
@@ -0,0 +1,2 @@
+offsetMs: number
+reason: string
diff --git a/skills/traceway/SKILL.md b/skills/traceway/SKILL.md
index a28e0c36..94fc0647 100644
--- a/skills/traceway/SKILL.md
+++ b/skills/traceway/SKILL.md
@@ -11,6 +11,7 @@ Drive a Traceway instance from the terminal with the `traceway` CLI. The first w
|---|---|
| `/traceway login` | **Login**: install the CLI if missing, authenticate, select a project |
| `/traceway debug <issue ref or bug description>` | **Debug**: resolve the issue and investigate to root cause |
+| `/traceway perf <endpoint or symptom>` | **Performance**: diagnose latency/slowness to root cause against a checklist of common bottlenecks |
| `/traceway <anything else>` | **Query**: answer the observability question with CLI reads |
| `/traceway` (no argument) | Ask what they want: log in, debug an issue, or run a query |
@@ -71,9 +72,13 @@ The flag is required, so you must supply *something* — but it can be approxima
2. **Estimate from context.** A notification's send time, the issue's `firstSeen`/`lastSeen`, or the URL's `preset`/`from` window all put you inside ±24h — good enough for a fast lookup.
3. **Ask the user.** If nothing pins it down (e.g. a bare occurrence/endpoint id with no hash, no time, and no list to recover from), ask roughly when it happened — "around when did this fire? within a day is enough" — and pass that. Don't invent a placeholder like "now" when the issue is old; that defeats the pruning and can miss the ±24h window entirely.
-## Resolving an issue notification
+## Resolving a notification
-Traceway issue notifications (email / Slack / webhook) embed everything for a direct, fast lookup. The body contains:
+Traceway notifications (email / Slack / webhook) come in two shapes. Read the body to tell them apart: an **exception** notification carries a `Hash:` and `Exception ID:`; a **performance** notification (latency, apdex, impact) names an endpoint and a threshold and has no hash. Webhook payloads also carry a `ruleType` field that names the rule exactly. Resolve each differently.
+
+### Exception notification
+
+The body embeds everything for a direct, fast lookup. It contains:
- `Hash: <16-hex>` — the exception group → `traceway exceptions show <hash>`.
- `Exception ID: <uuid>` — the specific occurrence.
@@ -87,6 +92,32 @@ traceway exceptions occurrence <Exception ID> --recorded-at <Occurred at → RFC
# the result carries distributedTraceId and sessionId → traces show / sessions show below
```
+### Performance notification (an endpoint became slow or critical)
+
+These fire from latency/throughput rules, not from an error. There is no hash, no exception id, no occurrence to fetch. The webhook `ruleType` (or the subject wording) tells you which:
+
+| ruleType | Subject / body shape | Deep link |
+|---|---|---|
+| `endpoint_p95_threshold` / `endpoint_p99_threshold` | `P95 latency 1250ms on <endpoint>` · "reached 1250ms over the last N minutes (threshold: 1000ms)" | `/endpoints?from=<iso>&to=<iso>` |
+| `apdex_drop` | `Apdex dropped to 0.62 (threshold: 0.80)` · "across N requests over the last N minutes" | `/endpoints?from=<iso>&to=<iso>` |
+| `impact_score_critical\|high\|medium` | `Endpoint <endpoint> impact became critical` · "impact score: 0.82. Reason: <reason>" | `/endpoints?from=<iso>&to=<iso>` |
+| `metric_threshold` | `Metric <name> is <value> (threshold: gt 100)` | `/metrics?preset=1h` |
+
+What the body gives you: the **endpoint name** (parse it from the subject/body; it is not a separate field), the **metric/percentile**, the **current value**, the **threshold**, and the **lookback window**. The `/endpoints?from=&to=` link is only a ~3-minute window around the fire time, so treat the fire time as the onset hint, not the incident window.
+
+To resolve, hand straight to the **Performance flow** scoped to that endpoint, using the fire time as the time anchor:
+
+```bash
+# 1. confirm the alert against current stats for that endpoint
+traceway endpoints list --search "<endpoint>" --since 1h --output json | jq '.data[0] | {p50, p95, p99, count, impact, impactReason}'
+# 2. is this an accepted baseline? check whether an operator marked it slow
+traceway endpoints slow "<endpoint>" --output json # {offsetMs, reason}; offsetMs 0 = not marked
+# 3. find when it crossed the threshold, then a slow trace (see Flow: Performance)
+traceway endpoints chart --metric-type p95 --since 6h --interval-minutes 15 --output json
+```
+
+Step 2 matters: an `impact_score_*` or `apdex_drop` alert is already offset-aware, but `endpoint_p95_threshold` / `endpoint_p99_threshold` fire on **raw** latency regardless of any slow-marking, so a marked-slow endpoint can alert on latency its operators have already accepted. See "Account for user-marked slow endpoints" in `performance.md`.
+
## Flow: Login
### 1. Check for an existing install
@@ -234,9 +265,26 @@ Summarize: symptom, evidence (exception hashes, log excerpts, metric anomalies),
traceway exceptions archive <hash> --yes
```
+## Flow: Performance
+
+`/traceway perf <endpoint or symptom>`, or any request about slowness, high latency, p95/p99, or "why is X slow". Approach it as a senior performance engineer: quantify first, localize the cost to a span, confirm the cause against the checklist, separate code from saturation, then root cause. Reads only; never archive or mutate.
+
+For the full methodology and the bottleneck checklist (Select N+1, missing index, chatty IPC, connection-pool exhaustion, GC pauses, and ~20 more, each tied to the Traceway signal that confirms it), read `performance.md` in this skill directory. It is the authoritative reference.
+
+The loop, using the read commands documented in this skill:
+
+1. **Quantify and localize.** `traceway endpoints list --since 24h --order-by p95` (or `impact`). Read the latency shape first: p50 already high means every request pays it (systemic: query, index, algorithm); p99 much greater than p50 means a tail (contention, pool exhaustion, GC, retries, a flaky dependency). Note that `impact`/`impactReason` are offset-adjusted for marked-slow endpoints but the raw p50/p95/p99 are not.
+2. **Check the accepted baseline.** Before calling an endpoint slow, run `traceway endpoints slow "<endpoint>"`. A non-zero `offsetMs` means an operator accepted that much latency (with a `reason`), so only latency *beyond* the offset is a real regression; `offsetMs 0` means default thresholds apply. Skip this only for a clearly systemic, multi-endpoint slowdown.
+3. **Pinpoint when it started: adjust the window down.** Do not investigate the default window blindly; narrow it until it brackets the onset. `traceway endpoints chart --metric-type p95 --interval-minutes <n>` returns latency over time for the top endpoints (`{timestamp, endpoint, value}` in ms): read down one endpoint's buckets for the step where p95 jumps. Confirm the cause with `traceway metrics query --name <metric> --interval-minutes <n>` (the infra/runtime curve). For a specific route not in the top 5, bisect `traceway endpoints list --search <name> --from <a> --to <b>` over adjacent windows. See `performance.md`, "Pinpointing when the slowness started".
+4. **Get a representative slow trace.** `traceway endpoints show <endpointId> --recorded-at <t>` for one request's span waterfall, or `traceway traces show <distributedTraceId> --recorded-at <t>` for the cross-service timeline. Take the trace from inside the slow window you just found; find the long pole.
+5. **Match the long pole to the checklist.** Is the dominant span a database call, an external call, in-process compute, or a *gap* before work starts (queueing / lock wait / pool exhaustion)? Look it up in `performance.md`.
+6. **Separate code from saturation.** Pull infra/runtime metrics over the same window: `traceway metrics query --name system.cpu.utilization --aggregation max`, then `mem.used_pcnt`, `go.gc_pause`. A spike that lines up with the latency onset points at saturation, not a code bug.
+7. **Correlate with code and deploys.** With the onset time from step 3, check what shipped then: `git log --since '<onset - 30m>' --until '<onset + 30m>'` or the deploy history. A jump at a release is a regression; a gradual ramp with no deploy is data growth (N+1, missing index, no pagination).
+8. **Report.** The bottleneck, the evidence (endpoint percentiles, the onset time, the dominating span, any correlated metric anomaly), the root cause mapped to a checklist item, and a concrete fix, with the `endpoints show` / `traces show` references so the user can verify. If the endpoint is marked slow, state whether the latency exceeded the accepted offset.
+
## Flow: Query
-For free-form requests ("what's broken in prod?", "is /api/checkout slow?", "show errors for service X"), use the read commands directly.
+For free-form requests ("what's broken in prod?", "is /api/checkout slow?", "show errors for service X"), use the read commands directly. A request specifically about latency or slowness routes to the Performance flow above.
### Command reference
@@ -249,6 +297,8 @@ For free-form requests ("what's broken in prod?", "is /api/checkout slow?", "sho
| `traceway exceptions archive/unarchive <hash>...` | Mutating; explicit user request + `--yes` only |
| `traceway logs query` | Logs; `--search` (`--search-type body\|attribute`), `--service`, `--min-severity <n>`, `--trace-id` |
| `traceway endpoints list` | Per-endpoint p50/p95/p99 and counts; `--search`, `--order-by impact\|count\|p95\|lastSeen` |
+| `traceway endpoints chart` | Latency over time for the top endpoints; `--metric-type total_time\|p50\|p95\|p99`, `--interval-minutes`. Use to find when latency changed |
+| `traceway endpoints slow <endpoint>` | Whether an operator marked this endpoint slow: `{offsetMs, reason}`. `offsetMs 0` = not marked. The offset is the accepted-latency baseline |
| `traceway endpoints show <id> --recorded-at <t>` | One request by id: span waterfall + linked errors |
| `traceway tasks show <id> --recorded-at <t>` | One background task run by id |
| `traceway ai-traces show <id> --recorded-at <t>` | One AI trace by id + its conversation |
diff --git a/skills/traceway/performance.md b/skills/traceway/performance.md
new file mode 100644
index 00000000..9fd07a10
--- /dev/null
+++ b/skills/traceway/performance.md
@@ -0,0 +1,154 @@
+# Performance debugging
+
+Authoritative reference for the **Performance** flow. Approach the investigation as a senior software engineer with deep performance-debugging experience: do not guess at fixes, quantify first, localize the cost to a specific span, confirm the cause against the checklist below, and only then propose a change. Every suspect here is paired with the Traceway signal that confirms it, so the checklist is a lookup table from "what the waterfall shows" to "what is actually wrong".
+
+All commands are read-only. Never archive or mutate anything during a performance investigation. For exact CLI syntax, the mandatory `--recorded-at` / `--started-at` timestamp rules, and dashboard-URL resolution, the parent `SKILL.md` is authoritative.
+
+## The investigation loop
+
+1. **Quantify and localize.** Find the slow endpoint(s) and read the shape of the latency:
+ ```bash
+ traceway endpoints list --since 24h --order-by p95 --page-size 10 --output json \
+ | jq '.data[]? | {endpoint, count, p50, p95, p99, errorCount}'
+ ```
+ Read p50 vs p99 before anything else (see "Reading the latency shape").
+
+2. **Get a representative slow trace.** Drill into one slow request to see where the time goes. Capture an id and its `recordedAt` together, then:
+ ```bash
+ traceway endpoints show <endpointId> --recorded-at <t> # span waterfall for one request
+ traceway traces show <distributedTraceId> --recorded-at <t> # cross-service timeline
+ ```
+ `traces show` is the highest-value call when more than one service is involved: it stitches the whole logical request together end to end.
+
+3. **Find the long pole and match it to the checklist.** In the waterfall, identify the single span (or repeated group of spans) that dominates the duration. Classify it: is it a database call, an external/upstream call, in-process compute, or a *gap* before work starts (which means queueing, lock wait, or connection-pool exhaustion, not the work itself)? Then look it up in the checklist.
+
+4. **Separate code from saturation.** A slow endpoint can be a code bug or an overloaded host. Pull infrastructure and runtime metrics over the same window:
+ ```bash
+ traceway metrics query --name system.cpu.utilization --aggregation max --since 24h
+ traceway metrics query --name mem.used_pcnt --aggregation max --since 24h
+ traceway metrics query --name go.gc_pause --aggregation max --since 24h
+ ```
+ A metric spike that lines up with the latency rise points at saturation; flat metrics under rising latency point at a code or query problem.
+
+5. **Correlate with code and deploys.** Open the slow path in the source and read it. If p95 climbed at a specific time, check what shipped then (`git log --since ... --until ...` or deploy history). Latency that jumped at a release is a regression; latency that crept up without one is usually data growth.
+
+6. **Report.** State the bottleneck, the evidence (endpoint percentiles, the dominating span, any correlated metric), the root cause mapped to a specific checklist item, and a concrete fix. Include the `endpoints show` / `traces show` references so the user can verify.
+
+## Reading the latency shape
+
+The relationship between percentiles, load, and time tells you which family of causes to look at before you open a single trace:
+
+- **p50 already high** (the median request is slow): every request pays the cost, so it is systemic. Suspect an inefficient query, a missing index, an O(n^2) hot path, or synchronous work that should be backgrounded.
+- **p99 much greater than p50** (median fine, tail slow): a minority of requests stall. Suspect contention, connection-pool exhaustion, GC pauses, retries, cache stampedes, or a flaky upstream dependency.
+- **Latency that grew over weeks with no deploy**: data-volume driven. Suspect Select N+1, a missing index, or an unbounded result set, all of which scale with table size.
+- **Latency that jumped at a deploy**: a regression. Correlate `firstSeen` / the climb time with `git log`.
+- **Latency that tracks request volume**: saturation or contention. Confirm with infra metrics and pool-wait gaps in the waterfall.
+
+## Account for user-marked slow endpoints
+
+Some endpoints are *expected* to be slow: a CSV export, a heavy report, a third-party sync. An operator can mark such an endpoint slow with an allowance (`offsetMs`) and a `reason`. Check before you treat latency as a problem:
+
+```bash
+traceway endpoints slow "GET /api/reports/export" --output json # {offsetMs, reason}
+```
+
+- `offsetMs 0` (or "MARKED: no"): default thresholds apply; judge latency normally.
+- `offsetMs > 0`: the operator accepted that much latency. The apdex and impact thresholds are shifted up by the offset server-side, so the endpoint's `impact`/`impactReason` already discount it. Only latency *beyond* `offsetMs`, or a fresh jump above the prior baseline, is a real regression.
+
+Two things to keep straight:
+
+- **Raw percentiles are not offset-adjusted.** The p50/p95/p99 from `endpoints list` and `endpoints chart` come from raw durations; the offset only moves the apdex/impact thresholds. A marked-slow endpoint can show a high p95 that is entirely expected, so compare p95 against `offsetMs`, not against zero.
+- **p95/p99 alert rules are not offset-aware.** `endpoint_p95_threshold` / `endpoint_p99_threshold` notifications fire on raw latency regardless of marking, so a marked-slow endpoint can still page on latency its operators accepted. `apdex_drop` and `impact_score_*` rules are offset-aware. (See "Performance notification" in `SKILL.md`.)
+
+When you report, state the p95 and the allowance together: "p95 is 2.1s but this endpoint is marked slow with a 2s allowance (reason: nightly export), so it is within baseline", or "p95 is 4s against a 2s allowance, so it regressed past the accepted offset".
+
+## Pinpointing when the slowness started
+
+Narrowing the window is how you turn "it is slow" into "it got slow at 14:10, right after deploy X". Do not investigate the default window blindly: adjust it down until it brackets the onset, then correlate that moment with deploys and metrics.
+
+**1. Latency over time: `endpoints chart`.** This is the direct curve. It buckets latency over the window and returns the top 5 endpoints ranked by the chosen metric (plus an "Other" bucket), each as a `{timestamp, endpoint, value}` series in milliseconds. Read down one endpoint's buckets for the step where its value jumps: that bucket dates the onset. Use `--metric-type p95` (or `p99`/`p50`/`total_time`) and a coarse `--interval-minutes` over a wide window first, then re-run the suspect span with a finer interval to sharpen.
+```bash
+traceway endpoints chart --metric-type p95 --since 7d --interval-minutes 60 --output json \
+ | jq '.series[] | select(.endpoint == "GET /api/checkout") | {t: .timestamp, p95_ms: .value}'
+```
+Caveat: it always returns the *top 5* endpoints by that metric. If your suspect route is not in the top 5 it is folded into "Other"; fall back to method 3 for it.
+
+**2. Infra and runtime over time: `metrics query --interval-minutes`.** For the *cause* behind a latency step (saturation, GC, throughput), bucket any metric the same way. The returned `series` is a list of `{timestamp, value}` points; read down them for the bucket where the value steps up.
+```bash
+traceway metrics query --name system.cpu.utilization --aggregation max --since 7d --interval-minutes 60 --output json \
+ | jq '.series | to_entries[] | .value[] | {t: .timestamp, v: .value}'
+```
+A flat line that steps up at the same bucket the latency jumped is the saturation signal; flat metrics under a latency step point at code.
+
+**3. A specific non-top-5 endpoint: window bisection.** When the route you care about is in "Other", reconstruct its onset from `endpoints list` (the only source of true p50/p95/p99, computed from raw durations) by comparing the *same endpoint's* percentiles across adjacent windows with `--from`/`--to`. Start wide, split, keep the half where p95 is elevated, recurse until the bracket is tight:
+```bash
+# is the regression in the first or second half of the suspect day?
+traceway endpoints list --search "GET /api/checkout" --from 2026-06-25T00:00:00Z --to 2026-06-25T12:00:00Z \
+ --output json | jq '.data[0] | {p95, p99, count}'
+traceway endpoints list --search "GET /api/checkout" --from 2026-06-25T12:00:00Z --to 2026-06-26T00:00:00Z \
+ --output json | jq '.data[0] | {p95, p99, count}'
+```
+A few bisection steps localize the jump to an hour or less. Watch `count` alongside p95: if the slow half also carries far more traffic, suspect load, not a code change.
+
+**4. Cross-check and correlate.** Once the methods agree on an onset time:
+- A latency jump that coincides with a metric inflection (CPU or GC stepping up at the same bucket) points at saturation.
+- A latency jump with flat infra metrics points at a code or query regression: `git log --since '<onset - 30m>' --until '<onset + 30m>'` or the deploy history for what shipped then.
+- A latency curve that ramps gradually over days with no single step is data growth (N+1, missing index, unbounded result set), not a deploy.
+
+Then pass the narrowed `--from`/`--to` back into the by-id detail lookups so `--recorded-at` lands inside the slow window and the trace you pull is actually from the slow period, not a fast one before the regression.
+
+## Bottleneck checklist
+
+Each row: the suspect, and how to confirm it from Traceway telemetry.
+
+### Database and data access
+| Suspect | How to confirm in Traceway |
+|---|---|
+| **Select N+1** | Many repeated, near-identical short query spans inside one request in the span waterfall; total DB time dominates while each call is tiny. |
+| **Missing or unused index** | One query span is the long pole; its duration grows with table size; same query slow across many traces. |
+| **Over-fetching (`SELECT *` / no projection)** | Large query-result spans, high memory; latency tracks row width/count rather than logic. |
+| **No pagination / unbounded result set** | p50 has crept up over time with data growth; one query returns more rows every week. |
+| **Expensive JOIN or aggregation** | A single query span dominates the trace consistently, independent of load. |
+| **Connection-pool exhaustion** | A gap *before* the first DB span (time waiting to acquire a connection); p99 rises with concurrency while p50 stays flat. |
+| **Lock / transaction contention** | DB spans queue behind each other; tail latency appears under write load; correlated WARN/ERROR locks in `logs query`. |
+| **Missing query/result caching** | The same query runs on every request; high DB-call count per endpoint with low cardinality of inputs. |
+
+### Application and compute
+| Suspect | How to confirm in Traceway |
+|---|---|
+| **Synchronous work that belongs in a background task** | A long handler span runs before the response returns, for work the user does not wait on (emails, exports, image processing). It should be a `CaptureTask`. |
+| **CPU-bound serialization** | A compute gap between IO spans (large JSON encode/decode); `system.cpu.utilization` up during the request. |
+| **Inefficient algorithm / O(n^2) hot path** | Latency scales superlinearly with input size across traces; no IO to blame in the waterfall. |
+| **Unbounded in-memory work / large allocations** | `mem.used_pcnt` and `go.num_gc` climb with p99; large compute spans. |
+| **Excessive logging / telemetry in the hot path** | Many log lines per request in `logs query`; logging spans visible in the waterfall. |
+
+### Network, IO, and external dependencies
+| Suspect | How to confirm in Traceway |
+|---|---|
+| **Chatty IPC / microservice fan-out** | `traces show` reveals many sequential cross-service hops, each adding round-trip latency. |
+| **Slow third-party / upstream API** | An external span dominates the trace; p99 spikes correlate with the provider; not your code. |
+| **Sequential calls that could run in parallel** | A staircase waterfall: each span starts only when the previous ends, though they are independent. |
+| **No timeouts / aggressive retries** | Long-tail p99; retry bursts in `logs query`; the same downstream span repeated within one trace. |
+| **No connection reuse (DNS/TLS per call)** | Many short external spans each paying a handshake; per-request connection setup cost. |
+| **Large request/response payloads** | Transfer-dominated spans; latency correlates with payload size. |
+
+### Caching and delivery
+| Suspect | How to confirm in Traceway |
+|---|---|
+| **No cache / low hit ratio** | DB or upstream is hit on every request; cache-hit metric (if reported) is low. |
+| **Cache stampede / thundering herd** | Latency spikes at cache-expiry boundaries; a correlated burst of identical traces. |
+| **Cold start / JIT / connection warmup** | First-request-after-deploy outliers; latency normalizes after warmup; `firstSeen`-correlated. |
+
+### Infrastructure and saturation
+| Suspect | How to confirm in Traceway |
+|---|---|
+| **CPU saturation** | `system.cpu.utilization` (max) spike lines up with the latency rise across many endpoints at once. |
+| **Memory pressure / GC pauses** | `mem.used_pcnt`, `go.gc_pause`, `go.num_gc` correlate with p99; pauses visible as gaps in the waterfall. |
+| **Disk / IO bottleneck** | `system.disk.*` or IO-wait metrics up; slow IO spans with no CPU cost. |
+| **Undersized / noisy-neighbor instance** | Broad latency rise with no single span culprit; infra metrics elevated with no code change. |
+
+## Notes on the signals
+
+- Host and runtime metrics live under `system.*` (Traceway OTel Agent) and the SDK runtime names (`go.gc_pause`, `mem.used_pcnt`, `go.go_routines`). The server has no quantile aggregation for metric points: `p50/p95/p99` on `metrics query` silently return `avg`, so never present those as percentiles. Real latency percentiles come only from `endpoints list`, computed from raw request durations.
+- OTLP histogram metrics are stored as two series, `<name>.avg` and `<name>.count`. A bogus metric name returns an empty `series: {}` cleanly, so probing names is safe.
+- When the waterfall shows a gap with no span, that gap is the finding: it is time the request spent waiting (for a connection, a lock, the scheduler), not time spent doing work.