Skip to content

The only tool you need to know what is happening and how to fix it.

cli, skill: list an endpoint's requests; fix CPU and memory recipes; explain each 403

narrow-adventure5b9a665bd16a
diff --git a/cli/cmd/traceway/endpoints.go b/cli/cmd/traceway/endpoints.go
index 229e670d..da9c17ee 100644
--- a/cli/cmd/traceway/endpoints.go
+++ b/cli/cmd/traceway/endpoints.go
@@ -22,6 +22,7 @@ func newEndpointsCmd() *cobra.Command {
Short: "Query HTTP endpoint performance",
}
cmd.AddCommand(newEndpointsListCmd())
+ cmd.AddCommand(newEndpointsRequestsCmd())
cmd.AddCommand(newEndpointsChartCmd())
cmd.AddCommand(newEndpointsSlowCmd())
cmd.AddCommand(newEndpointsShowCmd())
@@ -105,6 +106,78 @@ func runEndpointsList(cmd *cobra.Command, _ []string) error {
}
}
+func newEndpointsRequestsCmd() *cobra.Command {
+ cmd := &cobra.Command{
+ Use: "requests <endpoint>",
+ Short: "List one endpoint's requests, slowest first, with the id and recordedAt endpoints show takes",
+ Args: cobra.ExactArgs(1),
+ RunE: runEndpointsRequests,
+ }
+ addTimeRangeFlags(cmd)
+ addPaginationFlags(cmd)
+ cmd.Flags().String("search", "", "Match request attribute values, such as a user id")
+ cmd.Flags().String("order-by", "duration", "Sort field (duration, recordedAt, statusCode)")
+ cmd.Flags().String("sort-direction", "desc", "Sort direction: asc or desc")
+ return cmd
+}
+
+func runEndpointsRequests(cmd *cobra.Command, args []string) error {
+ ctx := cmd.Context()
+ mode := output.ResolveMode(flagOutput, output.StdoutIsTerminal())
+
+ sess, err := loadSession()
+ if err != nil {
+ return renderSessionError(cmd.ErrOrStderr(), mode, err)
+ }
+ tr, err := resolveTimeRange(cmd)
+ if err != nil {
+ return renderTimeRangeError(cmd.ErrOrStderr(), mode, err)
+ }
+ if err := validatePaginationFlags(cmd); err != nil {
+ return renderUsageError(cmd.ErrOrStderr(), mode, err.Error(),
+ paginationHint(appdir.Binary+" endpoints requests"))
+ }
+ page := resolvePagination(cmd)
+ search, _ := cmd.Flags().GetString("search")
+ orderBy, _ := cmd.Flags().GetString("order-by")
+ if err := validateEnumFlag("--order-by", orderBy, client.EndpointRequestsOrderByValues); err != nil {
+ return renderUsageError(cmd.ErrOrStderr(), mode, err.Error(),
+ enumFlagHint(appdir.Binary+" endpoints requests", "--order-by", client.EndpointRequestsOrderByValues))
+ }
+ sortDir, _ := cmd.Flags().GetString("sort-direction")
+ if err := validateEnumFlag("--sort-direction", sortDir, sortDirections); err != nil {
+ return renderUsageError(cmd.ErrOrStderr(), mode, err.Error(),
+ enumFlagHint(appdir.Binary+" endpoints requests", "--sort-direction", sortDirections))
+ }
+
+ c := sess.Client()
+ resp, err := c.ListEndpointRequests(ctx, sess.ProjectID, args[0], client.ListEndpointRequestsRequest{
+ TimeRange: tr,
+ Pagination: page,
+ Search: search,
+ OrderBy: orderBy,
+ SortDirection: sortDir,
+ })
+ if err != nil {
+ return renderAPIError(cmd.ErrOrStderr(), mode, err, false)
+ }
+
+ switch mode {
+ case output.ModeJSON:
+ return output.RenderJSON(cmd.OutOrStdout(), resp, output.ParseFieldsFlag(flagFields))
+ case output.ModeYAML:
+ return output.RenderYAML(cmd.OutOrStdout(), resp, output.ParseFieldsFlag(flagFields))
+ default:
+ tw := output.NewTabWriter(cmd.OutOrStdout())
+ _, _ = fmt.Fprintln(tw, "ID\tRECORDED AT\tDURATION\tSTATUS\tTRACE ID")
+ for _, e := range resp.Data {
+ _, _ = fmt.Fprintf(tw, "%s\t%s\t%s\t%d\t%s\n",
+ e.Id, e.RecordedAt.Format(time.RFC3339Nano), formatDuration(e.Duration), e.StatusCode, e.TraceId)
+ }
+ return tw.Flush()
+ }
+}
+
func newEndpointsChartCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "chart",
@@ -275,9 +348,9 @@ p50/p95/p99 stats by route, use "endpoints list --search <name>" instead.
--recorded-at is REQUIRED. The endpoints table is daily-partitioned; the
timestamp bounds the lookup to a window around it so ClickHouse prunes
-partitions instead of scanning all of them. Source it from the dashboard URL's
-t= param (/endpoints/<name>/<id>?t=...) or a distributed-trace node's
-recordedAt.`,
+partitions instead of scanning all of them. Take the id and recordedAt
+together from "endpoints requests <endpoint>", the dashboard URL's t= param
+(/endpoints/<name>/<id>?t=...) or a distributed-trace node's recordedAt.`,
Args: cobra.ExactArgs(1),
RunE: runEndpointsShow,
}
diff --git a/cli/cmd/traceway/projects.go b/cli/cmd/traceway/projects.go
index c6ae714c..84ba8ca5 100644
--- a/cli/cmd/traceway/projects.go
+++ b/cli/cmd/traceway/projects.go
@@ -80,9 +80,9 @@ func runProjectsList(cmd *cobra.Command, _ []string) error {
return output.RenderYAML(cmd.OutOrStdout(), projects, output.ParseFieldsFlag(flagFields))
default:
tw := output.NewTabWriter(cmd.OutOrStdout())
- _, _ = fmt.Fprintln(tw, "ID\tNAME")
+ _, _ = fmt.Fprintln(tw, "ID\tNAME\tROLE")
for _, p := range projects {
- _, _ = fmt.Fprintf(tw, "%s\t%s\n", p.ID, p.Name)
+ _, _ = fmt.Fprintf(tw, "%s\t%s\t%s\n", p.ID, p.Name, p.Role)
}
return tw.Flush()
}
diff --git a/cli/pkg/client/endpoints.go b/cli/pkg/client/endpoints.go
index 5e2128a8..58616f23 100644
--- a/cli/pkg/client/endpoints.go
+++ b/cli/pkg/client/endpoints.go
@@ -85,6 +85,62 @@ type Endpoint struct {
IsRoot bool `json:"isRoot"`
}
+// ListEndpointRequestsRequest is the body for POST /api/endpoints/endpoint:
+// the individual requests of one endpoint, the ids endpoints show takes.
+type ListEndpointRequestsRequest struct {
+ TimeRange TimeRange `json:"-"`
+ Pagination PaginationParams `json:"pagination"`
+ OrderBy string `json:"orderBy,omitempty"`
+ SortDirection string `json:"sortDirection,omitempty"`
+ // Search matches request attribute values, such as a user id.
+ Search string `json:"search,omitempty"`
+}
+
+// MarshalJSON expands TimeRange into top-level fromDate/toDate and sends
+// OrderBy as the column name the backend sorts on.
+func (r ListEndpointRequestsRequest) MarshalJSON() ([]byte, error) {
+ type alias ListEndpointRequestsRequest
+ wire := struct {
+ FromDate time.Time `json:"fromDate"`
+ ToDate time.Time `json:"toDate"`
+ alias
+ }{r.TimeRange.From, r.TimeRange.To, alias(r)}
+ wire.OrderBy = orderByColumn(r.OrderBy)
+ return jsonMarshalNoHTMLEscape(wire)
+}
+
+// EndpointRequestStats mirrors models.EndpointDetailStats. Unlike the
+// requests themselves, its durations are milliseconds.
+type EndpointRequestStats struct {
+ Count int64 `json:"count"`
+ AvgDuration float64 `json:"avgDuration"`
+ MedianDuration float64 `json:"medianDuration"`
+ P95Duration float64 `json:"p95Duration"`
+ P99Duration float64 `json:"p99Duration"`
+ Apdex float64 `json:"apdex"`
+ ErrorRate float64 `json:"errorRate"`
+ Throughput float64 `json:"throughput"`
+ IsStream bool `json:"isStream"`
+}
+
+// ListEndpointRequestsResponse mirrors EndpointInstancesResponse.
+type ListEndpointRequestsResponse struct {
+ Data []Endpoint `json:"data"`
+ Stats *EndpointRequestStats `json:"stats"`
+ Pagination Pagination `json:"pagination"`
+}
+
+// ListEndpointRequests returns one page of an endpoint's requests. endpoint is
+// the name exactly as ListEndpoints returns it ("GET /api/orders/:id").
+func (c *Client) ListEndpointRequests(ctx context.Context, projectID, endpoint string, req ListEndpointRequestsRequest) (*ListEndpointRequestsResponse, error) {
+ path := "/api/endpoints/endpoint?projectId=" + url.QueryEscape(projectID) + "&endpoint=" + url.QueryEscape(endpoint)
+ var resp ListEndpointRequestsResponse
+ if err := c.do(ctx, http.MethodPost, path, req, &resp); err != nil {
+ return nil, err
+ }
+ return &resp, nil
+}
+
// EndpointDetailResponse is the body of POST /api/endpoints/:endpointId.
type EndpointDetailResponse struct {
Endpoint *Endpoint `json:"endpoint"`
diff --git a/cli/pkg/client/endpoints_test.go b/cli/pkg/client/endpoints_test.go
index c28ce33f..9548c802 100644
--- a/cli/pkg/client/endpoints_test.go
+++ b/cli/pkg/client/endpoints_test.go
@@ -83,6 +83,37 @@ func TestListRequests_sendOrderByColumns(t *testing.T) {
}
}
+func TestListEndpointRequests_callsEndpointRoute(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/api/endpoints/endpoint" {
+ t.Errorf("path = %q", r.URL.Path)
+ }
+ if got := r.URL.Query().Get("endpoint"); got != "GET /api/orders/:id" {
+ t.Errorf("endpoint = %q", got)
+ }
+ var body map[string]any
+ _ = json.NewDecoder(r.Body).Decode(&body)
+ if body["orderBy"] != "recorded_at" || body["fromDate"] == nil {
+ t.Errorf("body = %v", body)
+ }
+ _, _ = w.Write([]byte(`{"data":[{"id":"6f9619ff-8b86-d011-b42d-00c04fc964ff","endpoint":"GET /api/orders/:id","duration":45000000,"recordedAt":"2026-10-20T10:00:00Z","statusCode":200}],"stats":{"count":1,"p95Duration":45},"pagination":{"page":1,"pageSize":20,"total":1,"totalPages":1}}`))
+ }))
+ defer srv.Close()
+
+ c := New(srv.URL, WithJWT("tok"))
+ resp, err := c.ListEndpointRequests(context.Background(), "proj-1", "GET /api/orders/:id", ListEndpointRequestsRequest{
+ TimeRange: TimeRange{From: time.Now().Add(-time.Hour), To: time.Now()},
+ Pagination: PaginationParams{Page: 1, PageSize: 20},
+ OrderBy: "recordedAt",
+ })
+ if err != nil {
+ t.Fatalf("ListEndpointRequests: %v", err)
+ }
+ if len(resp.Data) != 1 || resp.Data[0].Duration != 45*time.Millisecond || resp.Stats == nil || resp.Stats.P95Duration != 45 {
+ t.Errorf("resp = %+v", resp)
+ }
+}
+
func TestGetEndpointChart_callsChartRoute(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/api/endpoints/chart" {
diff --git a/cli/pkg/client/enums.go b/cli/pkg/client/enums.go
index f32ad8ff..e85ca37d 100644
--- a/cli/pkg/client/enums.go
+++ b/cli/pkg/client/enums.go
@@ -4,12 +4,14 @@ package client
// contract and are shared by the CLI flag validation and the MCP server's
// input validation, so the two surfaces cannot drift.
var (
- ExceptionsOrderByValues = []string{"lastSeen", "firstSeen", "count"}
- ExceptionsSearchTypes = []string{"text", "regex"}
- LogsSearchTypes = []string{"body", "attribute"}
- SortDirections = []string{"asc", "desc"}
- EndpointsOrderByValues = []string{"impact", "count", "p95", "lastSeen"}
- EndpointChartMetricTypes = []string{"total_time", "p50", "p95", "p99"}
+ ExceptionsOrderByValues = []string{"lastSeen", "firstSeen", "count"}
+ ExceptionsSearchTypes = []string{"text", "regex"}
+ LogsSearchTypes = []string{"body", "attribute"}
+ SortDirections = []string{"asc", "desc"}
+ EndpointsOrderByValues = []string{"impact", "count", "p95", "lastSeen"}
+ // EndpointRequestsOrderByValues sort one endpoint's requests.
+ EndpointRequestsOrderByValues = []string{"duration", "recordedAt", "statusCode"}
+ EndpointChartMetricTypes = []string{"total_time", "p50", "p95", "p99"}
// MetricAggregations are the values the server accepts on metrics/query.
// The server has no quantile aggregation for metric points: p50/p95/p99
@@ -28,9 +30,11 @@ var (
// name it does not know, so sending "p95" or "lastSeen" as is silently did
// not sort.
var orderByColumns = map[string]string{
- "p95": "p95_duration",
- "lastSeen": "last_seen",
- "firstSeen": "first_seen",
+ "p95": "p95_duration",
+ "lastSeen": "last_seen",
+ "firstSeen": "first_seen",
+ "recordedAt": "recorded_at",
+ "statusCode": "status_code",
}
func orderByColumn(v string) string {
diff --git a/cli/pkg/client/projects.go b/cli/pkg/client/projects.go
index ed808dbc..de3f06b5 100644
--- a/cli/pkg/client/projects.go
+++ b/cli/pkg/client/projects.go
@@ -10,6 +10,9 @@ import (
type Project struct {
ID string `json:"id"`
Name string `json:"name"`
+ // Role is the caller's effective role in the project (owner, admin,
+ // user, readonly); a readonly role refuses writes whatever the credential.
+ Role string `json:"role,omitempty"`
}
// ListProjects returns all projects visible to the authenticated user.
diff --git a/cli/pkg/mcpserver/knowledge/debug-flow.md b/cli/pkg/mcpserver/knowledge/debug-flow.md
index 6547ad95..019a4f59 100644
--- a/cli/pkg/mcpserver/knowledge/debug-flow.md
+++ b/cli/pkg/mcpserver/knowledge/debug-flow.md
@@ -81,7 +81,7 @@ SID=$(jq -r '.sessionId // empty' <<<"$OCC")
**Check metrics for systemic causes** (spikes lining up with `firstSeen` suggest saturation rather than a code bug):
```bash
-traceway metrics query --name system.cpu.utilization --aggregation max --since 24h
+traceway metrics query --name system.cpu.utilization --tag state=idle --aggregation min --since 24h # busy = 1 - idle
traceway metrics query --name <name> --aggregation avg|sum|count|min|max [--tag key=value] [--group-by <tag>]
```
diff --git a/cli/pkg/mcpserver/knowledge/jq_examples_test.go b/cli/pkg/mcpserver/knowledge/jq_examples_test.go
index b9bc41c3..7c14ef46 100644
--- a/cli/pkg/mcpserver/knowledge/jq_examples_test.go
+++ b/cli/pkg/mcpserver/knowledge/jq_examples_test.go
@@ -14,13 +14,14 @@ import (
// jqOutputs is the type each command prints with --output json. An example
// piping a command that is missing here fails the test: add the command.
var jqOutputs = map[string]any{
- "endpoints list": client.ListEndpointsResponse{},
- "endpoints chart": client.EndpointChartResponse{},
- "endpoints show": client.EndpointDetailResponse{},
- "metrics query": client.QueryMetricsResponse{},
- "exceptions list": client.ListExceptionsResponse{},
- "exceptions show": client.GetExceptionResponse{},
- "logs query": client.QueryLogsResponse{},
+ "endpoints list": client.ListEndpointsResponse{},
+ "endpoints requests": client.ListEndpointRequestsResponse{},
+ "endpoints chart": client.EndpointChartResponse{},
+ "endpoints show": client.EndpointDetailResponse{},
+ "metrics query": client.QueryMetricsResponse{},
+ "exceptions list": client.ListExceptionsResponse{},
+ "exceptions show": client.GetExceptionResponse{},
+ "logs query": client.QueryLogsResponse{},
}
// jqBuiltins are names a filter may use that are not output fields: the
diff --git a/cli/pkg/mcpserver/knowledge/learn/endpoints-and-tasks.md b/cli/pkg/mcpserver/knowledge/learn/endpoints-and-tasks.md
index 630a7bf0..7feb9ce5 100644
--- a/cli/pkg/mcpserver/knowledge/learn/endpoints-and-tasks.md
+++ b/cli/pkg/mcpserver/knowledge/learn/endpoints-and-tasks.md
@@ -72,6 +72,7 @@ A queue worker whose span is parented to the request that enqueued it shares tha
|---|---|---|
| Endpoints with percentiles and impact | `traceway endpoints list --since 24h [--order-by impact\|count\|p95\|lastSeen] [--search S]` | `list_endpoints` |
| Latency over time (find when it changed) | `traceway endpoints chart [--metric-type total_time\|p50\|p95\|p99] [--interval-minutes N]` | `endpoints_chart` |
+| One endpoint's requests, slowest first (the ids `endpoints show` takes) | `traceway endpoints requests <endpoint> --since 24h [--order-by duration\|recordedAt\|statusCode] [--search S]` | `list_endpoint_requests` |
| One request with its spans and errors | `traceway endpoints show <id> --recorded-at <t>` | `get_endpoint_request` |
| An endpoint's expected-performance mark | `traceway endpoints slow <endpoint>` | `get_slow_endpoint_config` |
| One task run | `traceway tasks show <id> --recorded-at <t>` | `get_task` |
diff --git a/cli/pkg/mcpserver/knowledge/operations.md b/cli/pkg/mcpserver/knowledge/operations.md
index f5102bf3..b962e62f 100644
--- a/cli/pkg/mcpserver/knowledge/operations.md
+++ b/cli/pkg/mcpserver/knowledge/operations.md
@@ -4,7 +4,7 @@ Write access lets an agent take the routine actions an on-call engineer takes du
1. **The credential.** Only write credentials can write: a `tww_` personal access token, a `traceway-write login`, or an MCP client connected to the `/mcp/write` server. A `twp_` token, a `traceway login` and the `/mcp` server are read only, whatever the person's role.
2. **The organization.** An admin has to turn on Settings > Agent Access. Until then every agent write is refused with a 403 that says so.
-3. **The person.** The write runs with the role of the person who created the credential, under the same rules as the dashboard. A `readonly` member's write token cannot archive issues, change alerts, run monitors or edit post-mortems. Acknowledging and resolving pages needs no write role, so any member's write token can do that.
+3. **The person.** The write runs with the role of the person who created the credential, under the same rules as the dashboard. A `readonly` member's write token cannot archive issues, change alerts, run monitors or edit post-mortems: the write is refused with a 403 that starts "Your role in this project is read-only". `projects list` (`list_projects`) shows the person's role in each project. Acknowledging and resolving pages needs no write role, so any member's write token can do that.
Every write a write credential makes in the organization, accepted or refused because Agent Access is off, is recorded in the organization's agent activity log with the person, the credential, the project and the result.
@@ -30,7 +30,10 @@ Everything else (creating rules and channels, editing schedules and policies, st
- **Say it before doing it.** One line naming the action, the record and the new value: "Snoozing rule 12 (Endpoint P95 on GET /api/search) for 60 minutes." The CLI asks for confirmation on a TTY and needs `--yes` otherwise. MCP clients run their own approval step.
- **Prefer the reversible option.** Snooze rather than disable. Acknowledge rather than resolve while the problem is still live. Archive an issue rather than resolving its page with `--archive-issue` if you are unsure the page is done.
- **Report what changed and how to undo it.**
-- **Treat a 403 as an answer.** "This credential is read-only" or "Agent write access is turned off" is a policy decision. Do not retry, switch credentials or look for another route. Tell the user what it said and who can change it.
+- **Treat a 403 as an answer.** Each one names its level, and each is a policy decision. Do not retry, switch credentials or look for another route. Tell the user what it said and who can change it:
+ - "This credential is read-only": the credential is a read one. The user needs a write credential (a `tww_` token, `traceway-write login`, or `/mcp/write`).
+ - "Agent write access is turned off": an organization admin turns it on in Settings > Agent Access.
+ - "Your role in this project is read-only": the credential can write, but the person's role in the project is `readonly`. Another credential will not help. An organization owner or admin can change the role in Settings > Team Members, or make the change themselves.
### Playbooks
diff --git a/cli/pkg/mcpserver/knowledge/performance-flow.md b/cli/pkg/mcpserver/knowledge/performance-flow.md
index 0585323e..2c5b0c7f 100644
--- a/cli/pkg/mcpserver/knowledge/performance-flow.md
+++ b/cli/pkg/mcpserver/knowledge/performance-flow.md
@@ -11,7 +11,7 @@ The loop, using the read commands documented in this skill:
3. **Pinpoint when it started: adjust the window down.** Do not investigate the default window blindly; narrow it until it brackets the onset. `traceway endpoints chart --metric-type p95 --interval-minutes <n>` returns latency over time for the top endpoints (`{timestamp, endpoint, value}` in ms): read down one endpoint's buckets for the step where p95 jumps. Confirm the cause with `traceway metrics query --name <metric> --interval-minutes <n>` (the infra/runtime curve). For a specific route not in the top 5, bisect `traceway endpoints list --search <name> --from <a> --to <b>` over adjacent windows. See `performance.md`, "Pinpointing when the slowness started".
4. **Get a representative slow trace.** `traceway endpoints show <endpointId> --recorded-at <t>` for one request's span waterfall, or `traceway traces show <traceId> --recorded-at <t>` (the `traceId` that `endpoints show` prints) for the cross-service timeline. Take the trace from inside the slow window you just found; find the long pole.
5. **Match the long pole to the checklist.** Is the dominant span a database call, an external call, in-process compute, or a *gap* before work starts (queueing / lock wait / pool exhaustion)? Look it up in `performance.md`.
-6. **Separate code from saturation.** Pull infra/runtime metrics over the same window: `traceway metrics query --name system.cpu.utilization --aggregation max`, then `mem.used`, `go.gc_pause`. A spike that lines up with the latency onset points at saturation, not a code bug.
+6. **Separate code from saturation.** Pull infra/runtime metrics over the same window: `traceway metrics query --name system.cpu.utilization --tag state=idle --aggregation min` (busy is 1 minus idle; the metric is split by `state`, so never read it unfiltered), then `system.memory.utilization` with `state=used` and the runtime's heap and GC metrics (`jvm.*`, `go.*`, `v8js.*`). A spike that lines up with the latency onset points at saturation, not a code bug.
7. **Correlate with code and deploys.** With the onset time from step 3, check what shipped then: `git log --since '<onset - 30m>' --until '<onset + 30m>'` or the deploy history. A jump at a release is a regression; a gradual ramp with no deploy is data growth (N+1, missing index, no pagination).
8. **Report.** The bottleneck, the evidence (endpoint percentiles, the onset time, the dominating span, any correlated metric anomaly), the root cause mapped to a checklist item, and a concrete fix, with the `endpoints show` / `traces show` references so the user can verify. If the endpoint is marked slow, state whether the latency exceeded the accepted offset.
diff --git a/cli/pkg/mcpserver/knowledge/performance.md b/cli/pkg/mcpserver/knowledge/performance.md
index 0bdd1180..d7012bf4 100644
--- a/cli/pkg/mcpserver/knowledge/performance.md
+++ b/cli/pkg/mcpserver/knowledge/performance.md
@@ -14,21 +14,25 @@ All commands are read-only. Never archive or mutate anything during a performanc
The rows are under `data`. `p50Duration`, `p95Duration`, `p99Duration` and `avgDuration` are nanoseconds, hence the `/ 1e6`.
Read p50 vs p99 before anything else (see "Reading the latency shape").
-2. **Get a representative slow trace.** Drill into one slow request to see where the time goes. Capture an id and its `recordedAt` together, then:
+2. **Get a representative slow trace.** Drill into one slow request to see where the time goes. `endpoints requests` lists the endpoint's requests, slowest first; take an `id` and its `recordedAt` together, then open it:
```bash
- traceway endpoints show <endpointId> --recorded-at <t> # span waterfall for one request
- traceway traces show <traceId> --recorded-at <t> # cross-service timeline (traceId from endpoints show)
+ traceway endpoints requests "GET /api/checkout" --since 24h --page-size 5 --output json \
+ | jq '.data[] | {id, recordedAt, duration_ms: (.duration / 1e6), statusCode, traceId}'
+ traceway endpoints show <id> --recorded-at <recordedAt> # span waterfall for one request
+ traceway traces show <traceId> --recorded-at <recordedAt> # cross-service timeline
```
+ Open a typical slow request and the slowest one: if they differ, the tail has its own cause.
`traces show` is the highest-value call when more than one service is involved: it stitches the whole logical request together end to end.
3. **Find the long pole and match it to the checklist.** In the waterfall, identify the single span (or repeated group of spans) that dominates the duration. Classify it: is it a database call, an external/upstream call, in-process compute, or a *gap* before work starts (which means queueing, lock wait, or connection-pool exhaustion, not the work itself)? Then look it up in the checklist.
4. **Separate code from saturation.** A slow endpoint can be a code bug or an overloaded host. Pull infrastructure and runtime metrics over the same window:
```bash
- traceway metrics query --name system.cpu.utilization --aggregation max --since 24h
- traceway metrics query --name mem.used --aggregation max --since 24h
- traceway metrics query --name go.gc_pause --aggregation max --since 24h
+ traceway metrics query --name system.cpu.utilization --tag state=idle --aggregation min --group-by host.name --since 24h
+ traceway metrics query --name system.memory.utilization --tag state=used --aggregation max --group-by host.name --since 24h
+ traceway metrics query --name jvm.gc.duration.avg --aggregation max --since 24h
```
+ CPU and memory host metrics are split by a `state` tag, so always filter or group by it. `system.cpu.utilization` has one series per state (user, system, idle, wait, steal, ...) whose fractions add up to 1: an unfiltered query averages the states and reads as a flat fraction whatever the load. The busy share is 1 minus `state=idle`, and the minimum idle in a bucket is its peak load. Runtime metrics use the OpenTelemetry name for the language: `jvm.memory.used` and `jvm.gc.duration.avg` (JVM), `go.memory.used` and `go.memory.gc.goal` (Go), `v8js.memory.heap.used` (Node.js), `process.runtime.cpython.memory` (Python). Projects on the native protocol report `cpu.used_pcnt` and `mem.used`/`mem.total` instead.
A metric spike that lines up with the latency rise points at saturation; flat metrics under rising latency point at a code or query problem.
5. **Correlate with code and deploys.** Open the slow path in the source and read it. If p95 climbed at a specific time, check what shipped then (`git log --since ... --until ...` or deploy history). Latency that jumped at a release is a regression; latency that crept up without one is usually data growth.
@@ -76,8 +80,8 @@ Caveat: it always returns the *top 5* endpoints by that metric. If your suspect
**2. Infra and runtime over time: `metrics query --interval-minutes`.** For the *cause* behind a latency step (saturation, GC, throughput), bucket any metric the same way. The JSON is `{results: [{name, unit, series}]}`, one result per query, and `series` maps each group to its list of `{timestamp, value}` points (a single `__all__` group when ungrouped); read down the points for the bucket where the value steps up.
```bash
-traceway metrics query --name system.cpu.utilization --aggregation max --since 7d --interval-minutes 60 --output json \
- | jq '.results[0].series | to_entries[] | .value[] | {t: .timestamp, v: .value}'
+traceway metrics query --name system.cpu.utilization --tag state=idle --aggregation min --since 7d --interval-minutes 60 --output json \
+ | jq '.results[0].series | to_entries[] | .value[] | {t: .timestamp, busy: (1 - .value)}'
```
A flat line that steps up at the same bucket the latency jumped is the saturation signal; flat metrics under a latency step point at code.
@@ -118,9 +122,9 @@ Each row: the suspect, and how to confirm it from Traceway telemetry.
| Suspect | How to confirm in Traceway |
|---|---|
| **Synchronous work that belongs in a background task** | A long handler span runs before the response returns, for work the user does not wait on (emails, exports, image processing). It should be a `CaptureTask`. |
-| **CPU-bound serialization** | A compute gap between IO spans (large JSON encode/decode); `system.cpu.utilization` up during the request. |
+| **CPU-bound serialization** | A compute gap between IO spans (large JSON encode/decode); busy CPU (1 minus `system.cpu.utilization` with `state=idle`) up during the request. |
| **Inefficient algorithm / O(n^2) hot path** | Latency scales superlinearly with input size across traces; no IO to blame in the waterfall. |
-| **Unbounded in-memory work / large allocations** | `mem.used` and `go.num_gc` climb with p99; large compute spans. |
+| **Unbounded in-memory work / large allocations** | The runtime heap (`jvm.memory.used`, `go.memory.used`, `v8js.memory.heap.used`) and GC activity climb with p99; large compute spans. |
| **Excessive logging / telemetry in the hot path** | Many log lines per request in `logs query`; logging spans visible in the waterfall. |
### Network, IO, and external dependencies
@@ -143,13 +147,13 @@ Each row: the suspect, and how to confirm it from Traceway telemetry.
### Infrastructure and saturation
| Suspect | How to confirm in Traceway |
|---|---|
-| **CPU saturation** | `system.cpu.utilization` (max) spike lines up with the latency rise across many endpoints at once. |
-| **Memory pressure / GC pauses** | `mem.used`, `go.gc_pause`, `go.num_gc` correlate with p99; pauses visible as gaps in the waterfall. |
+| **CPU saturation** | Busy CPU (1 minus `system.cpu.utilization` with `state=idle`) spikes in line with the latency rise across many endpoints at once. |
+| **Memory pressure / GC pauses** | `system.memory.utilization` (`state=used`), the runtime heap and GC time (`jvm.gc.duration.avg`, `go.memory.gc.goal`) correlate with p99; pauses visible as gaps in the waterfall. |
| **Disk / IO bottleneck** | `system.disk.*` or IO-wait metrics up; slow IO spans with no CPU cost. |
| **Undersized / noisy-neighbor instance** | Broad latency rise with no single span culprit; infra metrics elevated with no code change. |
## Notes on the signals
-- Host and runtime metrics live under `system.*` (Traceway OTel Agent) and the SDK runtime names (`go.gc_pause`, `mem.used`, `go.go_routines`). The server has no quantile aggregation for metric points: `p50/p95/p99` on `metrics query` silently return `avg`, so never present those as percentiles. Real latency percentiles come only from `endpoints list`, computed from raw request durations.
+- Host metrics live under `system.*` (OpenTelemetry hostmetrics, sent by the Traceway OTel Agent) and are split by tags such as `state`; runtime metrics use the OpenTelemetry names of the language (`jvm.*`, `go.*`, `v8js.*`, `process.runtime.*`). The server has no quantile aggregation for metric points: `p50/p95/p99` on `metrics query` silently return `avg`, so never present those as percentiles. Real latency percentiles come only from `endpoints list`, computed from raw request durations.
- OTLP histogram metrics are stored as two series, `<name>.avg` and `<name>.count`. A bogus metric name returns an empty `series: {}` cleanly, so probing names is safe.
- When the waterfall shows a gap with no span, that gap is the finding: it is time the request spent waiting (for a connection, a lock, the scheduler), not time spent doing work.
diff --git a/cli/pkg/mcpserver/knowledge/tool-map.md b/cli/pkg/mcpserver/knowledge/tool-map.md
index 7506b6b9..9fbcb7d5 100644
--- a/cli/pkg/mcpserver/knowledge/tool-map.md
+++ b/cli/pkg/mcpserver/knowledge/tool-map.md
@@ -10,6 +10,7 @@ The knowledge resources show `traceway` CLI commands in their examples. Every co
| `traceway exceptions occurrence <id> --recorded-at <t>` | `get_exception_occurrence` |
| `traceway logs query` | `query_logs` |
| `traceway endpoints list` | `list_endpoints` |
+| `traceway endpoints requests <endpoint>` | `list_endpoint_requests` |
| `traceway endpoints show <id> --recorded-at <t>` | `get_endpoint_request` |
| `traceway endpoints chart` | `endpoints_chart` |
| `traceway endpoints slow <endpoint>` | `get_slow_endpoint_config` |
diff --git a/cli/pkg/mcpserver/knowledge/write-instructions.md b/cli/pkg/mcpserver/knowledge/write-instructions.md
index bf5c943e..08b3ada3 100644
--- a/cli/pkg/mcpserver/knowledge/write-instructions.md
+++ b/cli/pkg/mcpserver/knowledge/write-instructions.md
@@ -7,7 +7,7 @@ This server was started with write access, so it also has mutating tools: archiv
- Before the call, state the exact change in one line: the tool, the record and the new value. Then make it.
- Prefer the reversible option: snooze a noisy rule rather than disabling it, acknowledge a page before resolving it, archive rather than resolve with archive_issue.
- After the call, report what changed and how to undo it.
-- A 403 that says the credential is read-only, or that agent write access is off for the organization, is a policy decision. Do not retry or look for another route. Tell the user what it said; an organization admin turns agent writes on in Settings > Agent Access.
+- A 403 is a policy decision; do not retry or look for another route. Tell the user what it said and who can change it. "This credential is read-only" means the client is not connected with a write credential. "Agent write access is turned off" means an organization admin has to turn it on in Settings > Agent Access. "Your role in this project is read-only" means the credential can write but the person's project role is readonly: another credential will not help, and an organization owner or admin can change the role in Settings > Team Members.
- Every agent write is recorded in the organization's agent activity log with the credential that made it (the token, or the OAuth client).
Read traceway://knowledge/operations before your first write: it covers the rules, what each write does, and playbooks for pages, alerts, monitors and post-mortems. For how each area of the product works, read traceway://knowledge/learn.
diff --git a/cli/pkg/mcpserver/server_test.go b/cli/pkg/mcpserver/server_test.go
index de15d8d0..3a4d77c4 100644
--- a/cli/pkg/mcpserver/server_test.go
+++ b/cli/pkg/mcpserver/server_test.go
@@ -5,6 +5,7 @@ import (
"io/fs"
"net/http"
"net/http/httptest"
+ "net/url"
"strings"
"testing"
@@ -88,7 +89,7 @@ func TestListTools_surfaceAndAnnotations(t *testing.T) {
wantReadOnly := map[string]bool{
"list_projects": true, "list_exceptions": true, "get_exception": true,
"get_exception_occurrence": true, "query_logs": true, "list_endpoints": true,
- "get_endpoint_request": true, "endpoints_chart": true, "get_slow_endpoint_config": true,
+ "list_endpoint_requests": true, "get_endpoint_request": true, "endpoints_chart": true, "get_slow_endpoint_config": true,
"query_metrics": true, "get_task": true, "get_ai_trace": true, "get_session": true,
"get_trace": true, "list_pages": true, "get_page": true,
"list_notification_rules": true, "list_checks": true, "get_check": true,
@@ -106,7 +107,7 @@ func TestListTools_writeServerAddsMutatingTools(t *testing.T) {
want := map[string]bool{
"list_projects": true, "list_exceptions": true, "get_exception": true,
"get_exception_occurrence": true, "query_logs": true, "list_endpoints": true,
- "get_endpoint_request": true, "endpoints_chart": true, "get_slow_endpoint_config": true,
+ "list_endpoint_requests": true, "get_endpoint_request": true, "endpoints_chart": true, "get_slow_endpoint_config": true,
"query_metrics": true, "get_task": true, "get_ai_trace": true, "get_session": true,
"get_trace": true, "list_pages": true, "get_page": true,
"list_notification_rules": true, "list_checks": true, "get_check": true,
@@ -162,6 +163,43 @@ func TestInitialize_carriesInstructions(t *testing.T) {
}
}
+func TestListEndpointRequests_wireShapeAndDefaults(t *testing.T) {
+ var gotPath string
+ var gotQuery url.Values
+ var gotBody map[string]any
+ backend := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ gotPath = r.URL.Path
+ gotQuery = r.URL.Query()
+ _ = json.NewDecoder(r.Body).Decode(&gotBody)
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = w.Write([]byte(`{"data":[{"id":"6f9619ff-8b86-d011-b42d-00c04fc964ff","endpoint":"GET /api/orders","duration":597000000,"recordedAt":"2026-10-20T10:00:00Z","statusCode":200,"traceId":"ab69e041ab69e041ab69e041ab69e041"}],"stats":{"count":1},"pagination":{"page":1,"pageSize":20,"total":1,"totalPages":1}}`))
+ })
+ cs := connect(t, backend, "p-1")
+
+ res := callTool(t, cs, "list_endpoint_requests", map[string]any{"endpoint": "GET /api/orders", "since": "24h"})
+ if res.IsError {
+ t.Fatalf("unexpected tool error: %s", resultText(t, res))
+ }
+ if gotPath != "/api/endpoints/endpoint" || gotQuery.Get("projectId") != "p-1" || gotQuery.Get("endpoint") != "GET /api/orders" {
+ t.Errorf("path = %q, query = %v", gotPath, gotQuery)
+ }
+ if gotBody["orderBy"] != "duration" || gotBody["sortDirection"] != "desc" || gotBody["fromDate"] == nil {
+ t.Errorf("defaults not applied: %v", gotBody)
+ }
+ if !strings.Contains(resultText(t, res), "6f9619ff-8b86-d011-b42d-00c04fc964ff") {
+ t.Errorf("result text missing the request id: %s", resultText(t, res))
+ }
+
+ res = callTool(t, cs, "list_endpoint_requests", map[string]any{"since": "24h"})
+ if !res.IsError || !strings.Contains(resultText(t, res), "endpoint") {
+ t.Errorf("a missing endpoint must be a usage error: %s", resultText(t, res))
+ }
+ res = callTool(t, cs, "list_endpoint_requests", map[string]any{"endpoint": "GET /api/orders", "order_by": "recordedAt"})
+ if res.IsError || gotBody["orderBy"] != "recorded_at" {
+ t.Errorf("order_by recordedAt sent %v: %s", gotBody["orderBy"], resultText(t, res))
+ }
+}
+
func TestListExceptions_wireShapeAndDefaults(t *testing.T) {
var gotPath, gotQuery string
var gotBody map[string]any
diff --git a/cli/pkg/mcpserver/tools.go b/cli/pkg/mcpserver/tools.go
index f0845d4a..2eb3ff2f 100644
--- a/cli/pkg/mcpserver/tools.go
+++ b/cli/pkg/mcpserver/tools.go
@@ -52,10 +52,17 @@ func (s *server) addTools(srv *mcp.Server) {
Annotations: readOnly(),
}, s.listEndpoints)
+ mcp.AddTool(srv, &mcp.Tool{
+ Name: "list_endpoint_requests",
+ OutputSchema: outSchema[client.ListEndpointRequestsResponse](),
+ Description: "One endpoint's individual requests in a window, slowest first by default: each with its id, recordedAt, duration (nanoseconds), status code and traceId, plus the endpoint's stats (milliseconds). This is where a request id comes from: pass id and recordedAt to get_endpoint_request, or traceId and recordedAt to get_trace.",
+ Annotations: readOnly(),
+ }, s.listEndpointRequests)
+
mcp.AddTool(srv, &mcp.Tool{
Name: "get_endpoint_request",
OutputSchema: outSchema[client.EndpointDetailResponse](),
- Description: "Get one request (transaction) by id: the request, its span waterfall, and any linked exception/messages. In the waterfall, a gap before work starts is itself the finding: time spent queueing, waiting on a lock, or acquiring a pool connection.",
+ Description: "Get one request (transaction) by id (from list_endpoint_requests or a distributed trace node): the request, its span waterfall, and any linked exception/messages. In the waterfall, a gap before work starts is itself the finding: time spent queueing, waiting on a lock, or acquiring a pool connection.",
Annotations: readOnly(),
}, s.getEndpointRequest)
@@ -76,7 +83,7 @@ func (s *server) addTools(srv *mcp.Server) {
mcp.AddTool(srv, &mcp.Tool{
Name: "query_metrics",
OutputSchema: outSchema[client.QueryMetricsResponse](),
- Description: "Query time-bucketed metric series (aggregations: avg, sum, count, min, max). There is NO quantile aggregation: latency percentiles come from list_endpoints only. Host metrics from the Traceway OTel Agent live under system.* names; runtime names include go.gc_pause, mem.used_pcnt, go.go_routines. OTLP histogram metrics are stored as two series, <name>.avg and <name>.count. An unknown name returns an empty series cleanly, so probing names is safe. A grouped query is cut to the first 200 groups (alphabetical) and that result carries truncatedGroups: true, while the top-level intervalMinutes is the bucket size actually used after the server widens it so no series exceeds 2000 points.",
+ Description: "Query time-bucketed metric series (aggregations: avg, sum, count, min, max). There is NO quantile aggregation: latency percentiles come from list_endpoints only. Host metrics from the Traceway OTel Agent live under system.* names and are split by tags: system.cpu.utilization and system.memory.utilization have one series per state, so filter with tag_filters {state: idle} (busy CPU is 1 minus idle) or {state: used}, or group by state; never read them unfiltered. Runtime metrics use the OpenTelemetry names of the language (jvm.*, go.*, v8js.*, process.runtime.*). OTLP histogram metrics are stored as two series, <name>.avg and <name>.count. An unknown name returns an empty series cleanly, so probing names is safe. A grouped query is cut to the first 200 groups (alphabetical) and that result carries truncatedGroups: true, while the top-level intervalMinutes is the bucket size actually used after the server widens it so no series exceeds 2000 points.",
Annotations: readOnly(),
}, s.queryMetrics)
diff --git a/cli/pkg/mcpserver/tools_endpoints.go b/cli/pkg/mcpserver/tools_endpoints.go
index 07057f35..00714e32 100644
--- a/cli/pkg/mcpserver/tools_endpoints.go
+++ b/cli/pkg/mcpserver/tools_endpoints.go
@@ -50,9 +50,54 @@ func (s *server) listEndpoints(ctx context.Context, req *mcp.CallToolRequest, in
return nil, resp, nil
}
+type listEndpointRequestsIn struct {
+ projectIn
+ timeRangeIn
+ pageIn
+ Endpoint string `json:"endpoint" jsonschema:"The endpoint name exactly as list_endpoints returns it, like GET /api/users/:id."`
+ Search string `json:"search,omitempty" jsonschema:"Match request attribute values, such as a user id."`
+ OrderBy string `json:"order_by,omitempty" jsonschema:"Sort field: duration (default), recordedAt, or statusCode."`
+ SortDirection string `json:"sort_direction,omitempty" jsonschema:"asc or desc (default)."`
+}
+
+func (s *server) listEndpointRequests(ctx context.Context, req *mcp.CallToolRequest, in listEndpointRequestsIn) (*mcp.CallToolResult, any, error) {
+ projectID, err := s.project(in.ProjectID)
+ if err != nil {
+ return nil, nil, err
+ }
+ if in.Endpoint == "" {
+ return nil, nil, usageErrf("endpoint is required: the endpoint name exactly as returned by list_endpoints")
+ }
+ tr, err := in.timeRangeIn.resolve()
+ if err != nil {
+ return nil, nil, err
+ }
+ page, err := in.pageIn.resolve()
+ if err != nil {
+ return nil, nil, err
+ }
+ if err := validateEnum("order_by", in.OrderBy, client.EndpointRequestsOrderByValues); err != nil {
+ return nil, nil, err
+ }
+ if err := validateEnum("sort_direction", in.SortDirection, client.SortDirections); err != nil {
+ return nil, nil, err
+ }
+ resp, err := s.client(req).ListEndpointRequests(ctx, projectID, in.Endpoint, client.ListEndpointRequestsRequest{
+ TimeRange: tr,
+ Pagination: page,
+ Search: in.Search,
+ OrderBy: cmp.Or(in.OrderBy, "duration"),
+ SortDirection: cmp.Or(in.SortDirection, "desc"),
+ })
+ if err != nil {
+ return nil, nil, s.apiErr(err)
+ }
+ return nil, resp, nil
+}
+
type getEndpointRequestIn struct {
projectIn
- ID string `json:"id" jsonschema:"The request's UUID, from a dashboard /endpoints/<endpoint>/<endpointId> URL or a distributed trace node."`
+ ID string `json:"id" jsonschema:"The request's UUID, from list_endpoint_requests, a dashboard /endpoints/<endpoint>/<endpointId> URL or a distributed trace node."`
RecordedAt string `json:"recorded_at" jsonschema:"REQUIRED for a fast lookup: the record's timestamp, RFC3339. Approximate is fine (within 24h). Recover it from the dashboard URL's ?t= param or the trace node; see traceway://knowledge/timestamps. Never pass the current time for an old record."`
}
diff --git a/cli/pkg/mcpserver/tools_metrics.go b/cli/pkg/mcpserver/tools_metrics.go
index 87877da1..c55f675c 100644
--- a/cli/pkg/mcpserver/tools_metrics.go
+++ b/cli/pkg/mcpserver/tools_metrics.go
@@ -12,10 +12,10 @@ import (
)
type metricQueryIn struct {
- Name string `json:"name" jsonschema:"Metric name, e.g. system.cpu.utilization, mem.used_pcnt, go.gc_pause. Histogram metrics are stored as <name>.avg and <name>.count."`
+ Name string `json:"name" jsonschema:"Metric name, e.g. system.cpu.utilization, system.memory.utilization, jvm.memory.used, go.memory.used. Histogram metrics are stored as <name>.avg and <name>.count."`
Aggregation string `json:"aggregation,omitempty" jsonschema:"avg (default), sum, count, min, or max. p50/p95/p99 are rejected: the server has no quantile aggregation for metric points; latency percentiles come from list_endpoints."`
TagFilters map[string]string `json:"tag_filters,omitempty" jsonschema:"Exact-match tag filters, e.g. {host: web-1}."`
- GroupBy string `json:"group_by,omitempty" jsonschema:"Tag key to split the series by; without it the series key is all."`
+ GroupBy string `json:"group_by,omitempty" jsonschema:"Tag key to split the series by; without it the series key is __all__."`
}
type queryMetricsIn struct {
diff --git a/cli/test/contract/contract_test.go b/cli/test/contract/contract_test.go
index 17554136..d884eacd 100644
--- a/cli/test/contract/contract_test.go
+++ b/cli/test/contract/contract_test.go
@@ -54,6 +54,22 @@ func TestContract_endpointsGrouped(t *testing.T) {
goldenAssert(t, "endpoints-grouped", rt.body)
}
+func TestContract_endpointRequests(t *testing.T) {
+ c, rt := capturedClient()
+ resp, err := c.ListEndpointRequests(context.Background(), projectID, "GET /api/contract", client.ListEndpointRequestsRequest{
+ TimeRange: timeWindow(),
+ Pagination: client.PaginationParams{Page: 1, PageSize: 20},
+ OrderBy: "duration",
+ })
+ if err != nil {
+ t.Fatalf("endpoint requests: %v", err)
+ }
+ if len(resp.Data) == 0 || resp.Data[0].Id != seedEndpointID || resp.Data[0].RecordedAt.IsZero() {
+ t.Fatalf("the seeded request is missing its id or recordedAt: %s", rt.body)
+ }
+ goldenAssert(t, "endpoint-requests", rt.body)
+}
+
func TestContract_endpointDetail(t *testing.T) {
c, rt := capturedClient()
resp, err := c.GetEndpoint(context.Background(), projectID, seedEndpointID.String(), seedAt)
diff --git a/cli/test/contract/mcp_test.go b/cli/test/contract/mcp_test.go
index 266a72ac..409a30d9 100644
--- a/cli/test/contract/mcp_test.go
+++ b/cli/test/contract/mcp_test.go
@@ -33,6 +33,7 @@ func TestContract_mcpToolSurface(t *testing.T) {
call("get_exception_occurrence", map[string]any{"id": seedExceptionID.String(), "recorded_at": at}, seedHash)
call("query_logs", map[string]any{"from": from, "to": to, "service_name": "contract-svc"}, "contract-svc")
call("list_endpoints", map[string]any{"from": from, "to": to}, "GET /api/contract")
+ call("list_endpoint_requests", map[string]any{"endpoint": "GET /api/contract", "from": from, "to": to}, seedEndpointID.String())
call("get_endpoint_request", map[string]any{"id": seedEndpointID.String(), "recorded_at": at}, seedEndpointID.String())
call("endpoints_chart", map[string]any{"from": from, "to": to}, "endpoints")
call("get_slow_endpoint_config", map[string]any{"endpoint": "GET /api/contract"}, "offsetMs")
diff --git a/cli/test/contract/oauth_mcp_test.go b/cli/test/contract/oauth_mcp_test.go
index fdee60e7..8be37fe5 100644
--- a/cli/test/contract/oauth_mcp_test.go
+++ b/cli/test/contract/oauth_mcp_test.go
@@ -394,8 +394,8 @@ func TestContract_mcpOverStreamableHTTP(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- if len(tools.Tools) != 21 {
- t.Errorf("got %d tools, want 21", len(tools.Tools))
+ if len(tools.Tools) != 22 {
+ t.Errorf("got %d tools, want 22", len(tools.Tools))
}
for _, tool := range tools.Tools {
if !tool.Annotations.ReadOnlyHint {
@@ -536,8 +536,8 @@ func TestContract_mcpWriteMount(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- if len(tools.Tools) != 31 {
- t.Errorf("/mcp/write tools = %d, want 31", len(tools.Tools))
+ if len(tools.Tools) != 32 {
+ t.Errorf("/mcp/write tools = %d, want 32", len(tools.Tools))
}
archive := func() (*mcp.CallToolResult, string) {
diff --git a/cli/test/contract/testdata/endpoint-requests.schema.txt b/cli/test/contract/testdata/endpoint-requests.schema.txt
new file mode 100644
index 00000000..19145d12
--- /dev/null
+++ b/cli/test/contract/testdata/endpoint-requests.schema.txt
@@ -0,0 +1,28 @@
+data[].appVersion: string
+data[].attributes: null
+data[].bodySize: number
+data[].clientIP: string
+data[].duration: number
+data[].endpoint: string
+data[].id: string
+data[].isRoot: bool
+data[].isStream: bool
+data[].projectId: string
+data[].recordedAt: string
+data[].serverName: string
+data[].spanId: string
+data[].statusCode: number
+data[].traceId: string
+pagination.page: number
+pagination.pageSize: number
+pagination.total: number
+pagination.totalPages: number
+stats.apdex: number
+stats.avgDuration: number
+stats.count: number
+stats.errorRate: number
+stats.isStream: bool
+stats.medianDuration: number
+stats.p95Duration: number
+stats.p99Duration: number
+stats.throughput: number
diff --git a/cli/test/evals/internal/ops/ops.go b/cli/test/evals/internal/ops/ops.go
index 9be178f5..b4c42a6d 100644
--- a/cli/test/evals/internal/ops/ops.go
+++ b/cli/test/evals/internal/ops/ops.go
@@ -43,6 +43,7 @@ var specs = []Spec{
{Op: "exceptions.occurrence", CLI: []string{"exceptions", "occurrence"}, MCP: "get_exception_occurrence", Positional: []string{"id"}},
{Op: "logs.query", CLI: []string{"logs", "query"}, MCP: "query_logs", Rename: map[string]string{"service": "service_name"}},
{Op: "endpoints.list", CLI: []string{"endpoints", "list"}, MCP: "list_endpoints"},
+ {Op: "endpoints.requests", CLI: []string{"endpoints", "requests"}, MCP: "list_endpoint_requests", Positional: []string{"endpoint"}},
{Op: "endpoints.show", CLI: []string{"endpoints", "show"}, MCP: "get_endpoint_request", Positional: []string{"id"}},
{Op: "endpoints.chart", CLI: []string{"endpoints", "chart"}, MCP: "endpoints_chart"},
{Op: "endpoints.slow", CLI: []string{"endpoints", "slow"}, MCP: "get_slow_endpoint_config", Positional: []string{"endpoint"}},
diff --git a/docs/pages/learn/mcp.mdx b/docs/pages/learn/mcp.mdx
index 7c8b0571..051ab8a4 100644
--- a/docs/pages/learn/mcp.mdx
+++ b/docs/pages/learn/mcp.mdx
@@ -76,6 +76,7 @@ Every tool wraps one API call and declares an output schema, so results come bac
| `get_exception_occurrence` | One occurrence by id |
| `query_logs` | Log search (severity, service, text, trace id) |
| `list_endpoints` | Per-endpoint p50/p95/p99, counts, impact |
+| `list_endpoint_requests` | One endpoint's requests, slowest first |
| `get_endpoint_request` | One request: span waterfall + linked errors |
| `endpoints_chart` | Latency over time (find when it changed) |
| `get_slow_endpoint_config` | Operator-accepted latency baseline |
diff --git a/plugins/tw-write/skills/traceway-write/SKILL.md b/plugins/tw-write/skills/traceway-write/SKILL.md
index 2759bd9e..0b9dd94d 100644
--- a/plugins/tw-write/skills/traceway-write/SKILL.md
+++ b/plugins/tw-write/skills/traceway-write/SKILL.md
@@ -65,7 +65,7 @@ Write access lets an agent take the routine actions an on-call engineer takes du
1. **The credential.** Only write credentials can write: a `tww_` personal access token, a `traceway-write login`, or an MCP client connected to the `/mcp/write` server. A `twp_` token, a `traceway login` and the `/mcp` server are read only, whatever the person's role.
2. **The organization.** An admin has to turn on Settings > Agent Access. Until then every agent write is refused with a 403 that says so.
-3. **The person.** The write runs with the role of the person who created the credential, under the same rules as the dashboard. A `readonly` member's write token cannot archive issues, change alerts, run monitors or edit post-mortems. Acknowledging and resolving pages needs no write role, so any member's write token can do that.
+3. **The person.** The write runs with the role of the person who created the credential, under the same rules as the dashboard. A `readonly` member's write token cannot archive issues, change alerts, run monitors or edit post-mortems: the write is refused with a 403 that starts "Your role in this project is read-only". `projects list` (`list_projects`) shows the person's role in each project. Acknowledging and resolving pages needs no write role, so any member's write token can do that.
Every write a write credential makes in the organization, accepted or refused because Agent Access is off, is recorded in the organization's agent activity log with the person, the credential, the project and the result.
@@ -91,7 +91,10 @@ Everything else (creating rules and channels, editing schedules and policies, st
- **Say it before doing it.** One line naming the action, the record and the new value: "Snoozing rule 12 (Endpoint P95 on GET /api/search) for 60 minutes." The CLI asks for confirmation on a TTY and needs `--yes` otherwise. MCP clients run their own approval step.
- **Prefer the reversible option.** Snooze rather than disable. Acknowledge rather than resolve while the problem is still live. Archive an issue rather than resolving its page with `--archive-issue` if you are unsure the page is done.
- **Report what changed and how to undo it.**
-- **Treat a 403 as an answer.** "This credential is read-only" or "Agent write access is turned off" is a policy decision. Do not retry, switch credentials or look for another route. Tell the user what it said and who can change it.
+- **Treat a 403 as an answer.** Each one names its level, and each is a policy decision. Do not retry, switch credentials or look for another route. Tell the user what it said and who can change it:
+ - "This credential is read-only": the credential is a read one. The user needs a write credential (a `tww_` token, `traceway-write login`, or `/mcp/write`).
+ - "Agent write access is turned off": an organization admin turns it on in Settings > Agent Access.
+ - "Your role in this project is read-only": the credential can write, but the person's role in the project is `readonly`. Another credential will not help. An organization owner or admin can change the role in Settings > Team Members, or make the change themselves.
### Playbooks
diff --git a/skills/traceway-learn/endpoints-and-tasks.md b/skills/traceway-learn/endpoints-and-tasks.md
index 1d130d77..9db4236d 100644
--- a/skills/traceway-learn/endpoints-and-tasks.md
+++ b/skills/traceway-learn/endpoints-and-tasks.md
@@ -73,6 +73,7 @@ A queue worker whose span is parented to the request that enqueued it shares tha
|---|---|---|
| Endpoints with percentiles and impact | `traceway endpoints list --since 24h [--order-by impact\|count\|p95\|lastSeen] [--search S]` | `list_endpoints` |
| Latency over time (find when it changed) | `traceway endpoints chart [--metric-type total_time\|p50\|p95\|p99] [--interval-minutes N]` | `endpoints_chart` |
+| One endpoint's requests, slowest first (the ids `endpoints show` takes) | `traceway endpoints requests <endpoint> --since 24h [--order-by duration\|recordedAt\|statusCode] [--search S]` | `list_endpoint_requests` |
| One request with its spans and errors | `traceway endpoints show <id> --recorded-at <t>` | `get_endpoint_request` |
| An endpoint's expected-performance mark | `traceway endpoints slow <endpoint>` | `get_slow_endpoint_config` |
| One task run | `traceway tasks show <id> --recorded-at <t>` | `get_task` |
diff --git a/skills/traceway/SKILL.md b/skills/traceway/SKILL.md
index 20b233b4..df481ec3 100644
--- a/skills/traceway/SKILL.md
+++ b/skills/traceway/SKILL.md
@@ -271,7 +271,7 @@ SID=$(jq -r '.sessionId // empty' <<<"$OCC")
**Check metrics for systemic causes** (spikes lining up with `firstSeen` suggest saturation rather than a code bug):
```bash
-traceway metrics query --name system.cpu.utilization --aggregation max --since 24h
+traceway metrics query --name system.cpu.utilization --tag state=idle --aggregation min --since 24h # busy = 1 - idle
traceway metrics query --name <name> --aggregation avg|sum|count|min|max [--tag key=value] [--group-by <tag>]
```
@@ -340,7 +340,7 @@ The loop, using the read commands documented in this skill:
3. **Pinpoint when it started: adjust the window down.** Do not investigate the default window blindly; narrow it until it brackets the onset. `traceway endpoints chart --metric-type p95 --interval-minutes <n>` returns latency over time for the top endpoints (`{timestamp, endpoint, value}` in ms): read down one endpoint's buckets for the step where p95 jumps. Confirm the cause with `traceway metrics query --name <metric> --interval-minutes <n>` (the infra/runtime curve). For a specific route not in the top 5, bisect `traceway endpoints list --search <name> --from <a> --to <b>` over adjacent windows. See `performance.md`, "Pinpointing when the slowness started".
4. **Get a representative slow trace.** `traceway endpoints show <endpointId> --recorded-at <t>` for one request's span waterfall, or `traceway traces show <traceId> --recorded-at <t>` (the `traceId` that `endpoints show` prints) for the cross-service timeline. Take the trace from inside the slow window you just found; find the long pole.
5. **Match the long pole to the checklist.** Is the dominant span a database call, an external call, in-process compute, or a *gap* before work starts (queueing / lock wait / pool exhaustion)? Look it up in `performance.md`.
-6. **Separate code from saturation.** Pull infra/runtime metrics over the same window: `traceway metrics query --name system.cpu.utilization --aggregation max`, then `mem.used`, `go.gc_pause`. A spike that lines up with the latency onset points at saturation, not a code bug.
+6. **Separate code from saturation.** Pull infra/runtime metrics over the same window: `traceway metrics query --name system.cpu.utilization --tag state=idle --aggregation min` (busy is 1 minus idle; the metric is split by `state`, so never read it unfiltered), then `system.memory.utilization` with `state=used` and the runtime's heap and GC metrics (`jvm.*`, `go.*`, `v8js.*`). A spike that lines up with the latency onset points at saturation, not a code bug.
7. **Correlate with code and deploys.** With the onset time from step 3, check what shipped then: `git log --since '<onset - 30m>' --until '<onset + 30m>'` or the deploy history. A jump at a release is a regression; a gradual ramp with no deploy is data growth (N+1, missing index, no pagination).
8. **Report.** The bottleneck, the evidence (endpoint percentiles, the onset time, the dominating span, any correlated metric anomaly), the root cause mapped to a checklist item, and a concrete fix, with the `endpoints show` / `traces show` references so the user can verify. If the endpoint is marked slow, state whether the latency exceeded the accepted offset.
diff --git a/skills/traceway/performance.md b/skills/traceway/performance.md
index 300f9d58..fe89e34b 100644
--- a/skills/traceway/performance.md
+++ b/skills/traceway/performance.md
@@ -15,21 +15,25 @@ All commands are read-only. Never archive or mutate anything during a performanc
The rows are under `data`. `p50Duration`, `p95Duration`, `p99Duration` and `avgDuration` are nanoseconds, hence the `/ 1e6`.
Read p50 vs p99 before anything else (see "Reading the latency shape").
-2. **Get a representative slow trace.** Drill into one slow request to see where the time goes. Capture an id and its `recordedAt` together, then:
+2. **Get a representative slow trace.** Drill into one slow request to see where the time goes. `endpoints requests` lists the endpoint's requests, slowest first; take an `id` and its `recordedAt` together, then open it:
```bash
- traceway endpoints show <endpointId> --recorded-at <t> # span waterfall for one request
- traceway traces show <traceId> --recorded-at <t> # cross-service timeline (traceId from endpoints show)
+ traceway endpoints requests "GET /api/checkout" --since 24h --page-size 5 --output json \
+ | jq '.data[] | {id, recordedAt, duration_ms: (.duration / 1e6), statusCode, traceId}'
+ traceway endpoints show <id> --recorded-at <recordedAt> # span waterfall for one request
+ traceway traces show <traceId> --recorded-at <recordedAt> # cross-service timeline
```
+ Open a typical slow request and the slowest one: if they differ, the tail has its own cause.
`traces show` is the highest-value call when more than one service is involved: it stitches the whole logical request together end to end.
3. **Find the long pole and match it to the checklist.** In the waterfall, identify the single span (or repeated group of spans) that dominates the duration. Classify it: is it a database call, an external/upstream call, in-process compute, or a *gap* before work starts (which means queueing, lock wait, or connection-pool exhaustion, not the work itself)? Then look it up in the checklist.
4. **Separate code from saturation.** A slow endpoint can be a code bug or an overloaded host. Pull infrastructure and runtime metrics over the same window:
```bash
- traceway metrics query --name system.cpu.utilization --aggregation max --since 24h
- traceway metrics query --name mem.used --aggregation max --since 24h
- traceway metrics query --name go.gc_pause --aggregation max --since 24h
+ traceway metrics query --name system.cpu.utilization --tag state=idle --aggregation min --group-by host.name --since 24h
+ traceway metrics query --name system.memory.utilization --tag state=used --aggregation max --group-by host.name --since 24h
+ traceway metrics query --name jvm.gc.duration.avg --aggregation max --since 24h
```
+ CPU and memory host metrics are split by a `state` tag, so always filter or group by it. `system.cpu.utilization` has one series per state (user, system, idle, wait, steal, ...) whose fractions add up to 1: an unfiltered query averages the states and reads as a flat fraction whatever the load. The busy share is 1 minus `state=idle`, and the minimum idle in a bucket is its peak load. Runtime metrics use the OpenTelemetry name for the language: `jvm.memory.used` and `jvm.gc.duration.avg` (JVM), `go.memory.used` and `go.memory.gc.goal` (Go), `v8js.memory.heap.used` (Node.js), `process.runtime.cpython.memory` (Python). Projects on the native protocol report `cpu.used_pcnt` and `mem.used`/`mem.total` instead.
A metric spike that lines up with the latency rise points at saturation; flat metrics under rising latency point at a code or query problem.
5. **Correlate with code and deploys.** Open the slow path in the source and read it. If p95 climbed at a specific time, check what shipped then (`git log --since ... --until ...` or deploy history). Latency that jumped at a release is a regression; latency that crept up without one is usually data growth.
@@ -77,8 +81,8 @@ Caveat: it always returns the *top 5* endpoints by that metric. If your suspect
**2. Infra and runtime over time: `metrics query --interval-minutes`.** For the *cause* behind a latency step (saturation, GC, throughput), bucket any metric the same way. The JSON is `{results: [{name, unit, series}]}`, one result per query, and `series` maps each group to its list of `{timestamp, value}` points (a single `__all__` group when ungrouped); read down the points for the bucket where the value steps up.
```bash
-traceway metrics query --name system.cpu.utilization --aggregation max --since 7d --interval-minutes 60 --output json \
- | jq '.results[0].series | to_entries[] | .value[] | {t: .timestamp, v: .value}'
+traceway metrics query --name system.cpu.utilization --tag state=idle --aggregation min --since 7d --interval-minutes 60 --output json \
+ | jq '.results[0].series | to_entries[] | .value[] | {t: .timestamp, busy: (1 - .value)}'
```
A flat line that steps up at the same bucket the latency jumped is the saturation signal; flat metrics under a latency step point at code.
@@ -119,9 +123,9 @@ Each row: the suspect, and how to confirm it from Traceway telemetry.
| Suspect | How to confirm in Traceway |
|---|---|
| **Synchronous work that belongs in a background task** | A long handler span runs before the response returns, for work the user does not wait on (emails, exports, image processing). It should be a `CaptureTask`. |
-| **CPU-bound serialization** | A compute gap between IO spans (large JSON encode/decode); `system.cpu.utilization` up during the request. |
+| **CPU-bound serialization** | A compute gap between IO spans (large JSON encode/decode); busy CPU (1 minus `system.cpu.utilization` with `state=idle`) up during the request. |
| **Inefficient algorithm / O(n^2) hot path** | Latency scales superlinearly with input size across traces; no IO to blame in the waterfall. |
-| **Unbounded in-memory work / large allocations** | `mem.used` and `go.num_gc` climb with p99; large compute spans. |
+| **Unbounded in-memory work / large allocations** | The runtime heap (`jvm.memory.used`, `go.memory.used`, `v8js.memory.heap.used`) and GC activity climb with p99; large compute spans. |
| **Excessive logging / telemetry in the hot path** | Many log lines per request in `logs query`; logging spans visible in the waterfall. |
### Network, IO, and external dependencies
@@ -144,13 +148,13 @@ Each row: the suspect, and how to confirm it from Traceway telemetry.
### Infrastructure and saturation
| Suspect | How to confirm in Traceway |
|---|---|
-| **CPU saturation** | `system.cpu.utilization` (max) spike lines up with the latency rise across many endpoints at once. |
-| **Memory pressure / GC pauses** | `mem.used`, `go.gc_pause`, `go.num_gc` correlate with p99; pauses visible as gaps in the waterfall. |
+| **CPU saturation** | Busy CPU (1 minus `system.cpu.utilization` with `state=idle`) spikes in line with the latency rise across many endpoints at once. |
+| **Memory pressure / GC pauses** | `system.memory.utilization` (`state=used`), the runtime heap and GC time (`jvm.gc.duration.avg`, `go.memory.gc.goal`) correlate with p99; pauses visible as gaps in the waterfall. |
| **Disk / IO bottleneck** | `system.disk.*` or IO-wait metrics up; slow IO spans with no CPU cost. |
| **Undersized / noisy-neighbor instance** | Broad latency rise with no single span culprit; infra metrics elevated with no code change. |
## Notes on the signals
-- Host and runtime metrics live under `system.*` (Traceway OTel Agent) and the SDK runtime names (`go.gc_pause`, `mem.used`, `go.go_routines`). The server has no quantile aggregation for metric points: `p50/p95/p99` on `metrics query` silently return `avg`, so never present those as percentiles. Real latency percentiles come only from `endpoints list`, computed from raw request durations.
+- Host metrics live under `system.*` (OpenTelemetry hostmetrics, sent by the Traceway OTel Agent) and are split by tags such as `state`; runtime metrics use the OpenTelemetry names of the language (`jvm.*`, `go.*`, `v8js.*`, `process.runtime.*`). The server has no quantile aggregation for metric points: `p50/p95/p99` on `metrics query` silently return `avg`, so never present those as percentiles. Real latency percentiles come only from `endpoints list`, computed from raw request durations.
- OTLP histogram metrics are stored as two series, `<name>.avg` and `<name>.count`. A bogus metric name returns an empty `series: {}` cleanly, so probing names is safe.
- When the waterfall shows a gap with no span, that gap is the finding: it is time the request spent waiting (for a connection, a lock, the scheduler), not time spent doing work.
diff --git a/website/app/product/mcp/page.tsx b/website/app/product/mcp/page.tsx
index 2315171c..830bf993 100644
--- a/website/app/product/mcp/page.tsx
+++ b/website/app/product/mcp/page.tsx
@@ -34,6 +34,7 @@ const TOOL_GROUPS = [
label: "Performance",
tools: [
"list_endpoints",
+ "list_endpoint_requests",
"endpoints_chart",
"get_endpoint_request",
"get_slow_endpoint_config",