The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
// Package seed builds realistic Traceway telemetry for one eval scenario and
// sends it through the real ingest endpoints (OTLP/HTTP and native
// /api/report). The DSL owns consistency: a request's duration is the sum of
// its children plus explicit gaps, percentiles come from the generated
// requests, and deploys are service.version changes on the resource.
package seed
import (
"crypto/sha256"
"encoding/binary"
"fmt"
"math"
"math/rand/v2"
"sort"
"time"
)
// ProjectKind selects how a project's telemetry arrives.
type ProjectKind int
const (
// OTel is a backend project fed over OTLP (framework opentelemetry).
OTel ProjectKind = iota
// Browser is a frontend project fed by a Traceway browser SDK.
Browser
// Mobile is a mobile project fed by a Traceway mobile SDK.
Mobile
)
// Scenario is the state of one seed run.
type Scenario struct {
CaseID string
Now time.Time
RNG *rand.Rand
Sink Sink
projects []*Project
services []*Service
exposed map[string]any
lazy map[string]func() any
strings []string
flushed bool
stats Stats
// SpanBudget caps the spans sent with full detail (default 20000).
SpanBudget int
afterFlush []func() error
// rows maps each endpoint row's id to the request that holds its span.
rows map[string]*Request
}
// Stats summarizes what a seed sent.
type Stats struct {
Requests int `json:"requests"`
Spans int `json:"spans"`
DetailedReqs int `json:"detailedRequests"`
Logs int `json:"logs"`
MetricPoints int `json:"metricPoints"`
Exceptions int `json:"exceptions"`
NativeFrames int `json:"nativeFrames"`
IngestElapsed time.Duration `json:"ingestElapsed"`
}
// NewScenario starts a scenario anchored at now (truncated to the minute)
// with an RNG seeded from the case id, so a case always draws the same data.
func NewScenario(caseID string, now time.Time, sink Sink) *Scenario {
sum := sha256.Sum256([]byte(caseID))
seed1 := binary.BigEndian.Uint64(sum[:8])
seed2 := binary.BigEndian.Uint64(sum[8:16])
return &Scenario{
CaseID: caseID,
Now: now.UTC().Truncate(time.Minute),
RNG: rand.New(rand.NewPCG(seed1, seed2)),
Sink: sink,
exposed: map[string]any{},
lazy: map[string]func() any{},
SpanBudget: 20000,
}
}
// Stats returns what was sent.
func (s *Scenario) Stats() Stats { return s.stats }
// Strings returns every human-readable string the seed emitted (span names,
// attribute values, log bodies, stack traces, object names), for the leakage
// lint.
func (s *Scenario) Strings() []string { return s.strings }
func (s *Scenario) record(v ...string) {
for _, x := range v {
if x != "" {
s.strings = append(s.strings, x)
}
}
}
// Project declares a Traceway project. framework defaults per kind.
func (s *Scenario) Project(name string, kind ProjectKind, framework ...string) *Project {
fw := "opentelemetry"
switch kind {
case Browser:
fw = "react"
case Mobile:
fw = "android"
}
if len(framework) > 0 {
fw = framework[0]
}
p := &Project{s: s, Name: name, Kind: kind, Framework: fw}
id, token, err := s.Sink.CreateProject(name, fw)
if err != nil {
panic(fmt.Errorf("create project %s: %w", name, err))
}
p.ID, p.token = id, token
s.projects = append(s.projects, p)
s.record(name)
return p
}
// Projects returns the declared projects in order.
func (s *Scenario) Projects() []*Project { return s.projects }
// Expose records a value for seed.json (graders and the oracle read it; the
// agent never sees it). A func() any value is resolved at Flush.
func (s *Scenario) Expose(key string, v any) {
if f, ok := v.(func() any); ok {
s.lazy[key] = f
return
}
s.exposed[key] = v
}
// Exposed returns the resolved exposed values.
func (s *Scenario) Exposed() map[string]any {
out := map[string]any{}
for k, v := range s.exposed {
out[k] = v
}
return out
}
// AfterFlush schedules work that needs the telemetry in place (for example
// objects that reference an ingested exception).
func (s *Scenario) AfterFlush(f func() error) { s.afterFlush = append(s.afterFlush, f) }
// Ago returns Now minus d.
func (s *Scenario) Ago(d time.Duration) time.Time { return s.Now.Add(-d) }
// Project is one Traceway project.
type Project struct {
s *Scenario
Name string
Kind ProjectKind
Framework string
ID string
token string
native []nativeFrame
}
// Service declares an OTel-instrumented service reporting to the project.
func (p *Project) Service(name string, stack Stack) *Service {
svc := &Service{s: p.s, p: p, Name: name, Stack: stack, Env: "production", Hosts: []string{name + "-" + hostSuffix(p.s, name)}}
p.s.services = append(p.s.services, svc)
p.s.record(name)
return svc
}
func hostSuffix(s *Scenario, name string) string {
h := sha256.Sum256([]byte(s.CaseID + name))
return fmt.Sprintf("%x-%x", h[0:3], h[3:5])
}
// Service is one deployable component.
type Service struct {
s *Scenario
p *Project
Name string
Stack Stack
Env string
Hosts []string
deploys []Deploy
// HostWeight biases request placement per host (default uniform).
HostWeight map[string]float64
endpoints []*Endpoint
tasks []*Endpoint
metrics []*metricSeries
logs []logRecord
defaultHost bool
}
// Deploy is a release: service.version changes at At.
type Deploy struct {
At time.Time
Version string
}
// Project returns the owning project.
func (svc *Service) Project() *Project { return svc.p }
// Instances replaces the host list (multi-instance service).
func (svc *Service) Instances(hosts ...string) *Service {
svc.Hosts = hosts
svc.s.record(hosts...)
return svc
}
// Versions sets the release history: the first version applies from the
// start, each later one from its deploy time.
func (svc *Service) Versions(initial string, deploys ...Deploy) *Service {
svc.deploys = append([]Deploy{{At: time.Time{}, Version: initial}}, deploys...)
sort.SliceStable(svc.deploys, func(i, j int) bool { return svc.deploys[i].At.Before(svc.deploys[j].At) })
return svc
}
// VersionAt returns the service.version live at t.
func (svc *Service) VersionAt(t time.Time) string {
if len(svc.deploys) == 0 {
return "1.0.0"
}
v := svc.deploys[0].Version
for _, d := range svc.deploys[1:] {
if !t.Before(d.At) {
v = d.Version
}
}
return v
}
// Endpoint declares an HTTP route served by the service.
func (svc *Service) Endpoint(method, route string) *Endpoint {
e := &Endpoint{svc: svc, Method: method, Route: route, policy: DetailPolicy{Recent: 6 * time.Hour, Rate: 0.03, Max: 60}}
svc.endpoints = append(svc.endpoints, e)
svc.s.record(method + " " + route)
return e
}
// Task declares a background task (a CONSUMER span root).
func (svc *Service) Task(name string) *Endpoint {
e := &Endpoint{svc: svc, Task: true, Route: name, policy: DetailPolicy{Recent: 6 * time.Hour, Rate: 0.05, Max: 40}}
svc.tasks = append(svc.tasks, e)
svc.s.record(name)
return e
}
// Endpoint is an HTTP route (or a task, when Task is set).
type Endpoint struct {
svc *Service
Method string
Route string
Task bool
shapes []Shape
window time.Duration
handler func(r *Req)
policy DetailPolicy
reqs []*Request
gen bool
// Status picks the response status; nil means 200.
status func(r *Req) int
// SkipAccessLog disables the per-request access log line.
SkipAccessLog bool
}
// Name returns "METHOD /route" (or the task name).
func (e *Endpoint) Name() string {
if e.Task {
return e.Route
}
return e.Method + " " + e.Route
}
// Service returns the serving service.
func (e *Endpoint) Service() *Service { return e.svc }
// Traffic sets the arrival process: rate shapes multiplied together over a
// window (Over, default 24h).
func (e *Endpoint) Traffic(shapes ...Shape) *Endpoint {
for _, sh := range shapes {
if o, ok := sh.(over); ok {
e.window = time.Duration(o)
continue
}
e.shapes = append(e.shapes, sh)
}
return e
}
// Handler sets the per-request span builder.
func (e *Endpoint) Handler(f func(r *Req)) *Endpoint { e.handler = f; return e }
// Detail tunes which requests are sent with their full span tree. Requests
// that carry logs or exceptions, or are exposed, always are.
func (e *Endpoint) Detail(p DetailPolicy) *Endpoint { e.policy = p; return e }
// DetailPolicy selects requests to send with full span trees: every request
// in the last Recent window with probability Rate, at most Max of them.
type DetailPolicy struct {
Recent time.Duration
Rate float64
Max int
}
// Shape is a factor on the arrival rate.
type Shape interface {
factor(s *Scenario, t time.Time, start, end time.Time) float64
}
type daily float64
func (d daily) factor(*Scenario, time.Time, time.Time, time.Time) float64 { return float64(d) }
// Daily sets the base rate in requests per day.
func Daily(n int) Shape { return daily(n) }
type diurnal struct{}
func (diurnal) factor(_ *Scenario, t, _, _ time.Time) float64 {
h := float64(t.Hour()) + float64(t.Minute())/60
// Peak around 14:00 UTC, trough around 02:00; mean ~1.
return 1 + 0.6*math.Cos((h-14)/24*2*math.Pi)
}
// Diurnal applies a day/night cycle (peak mid-afternoon UTC).
func Diurnal() Shape { return diurnal{} }
type growth struct{ from, to float64 }
func (g growth) factor(_ *Scenario, t, start, end time.Time) float64 {
return g.from + (g.to-g.from)*progress(t, start, end)
}
// GrowRate multiplies the rate from `from` at the window start to `to` at its end.
func GrowRate(from, to float64) Shape { return growth{from, to} }
type window struct {
from, to time.Time
mult float64
}
func (w window) factor(_ *Scenario, t, _, _ time.Time) float64 {
if !t.Before(w.from) && t.Before(w.to) {
return w.mult
}
return 1
}
// Burst multiplies the rate inside [from, to).
func Burst(from, to time.Time, mult float64) Shape { return window{from, to, mult} }
type funcShape func(t time.Time) float64
func (f funcShape) factor(_ *Scenario, t, _, _ time.Time) float64 { return f(t) }
// RateFunc is an arbitrary rate factor.
func RateFunc(f func(t time.Time) float64) Shape { return funcShape(f) }
type over time.Duration
func (over) factor(*Scenario, time.Time, time.Time, time.Time) float64 { return 1 }
// Over sets the traffic window ending now.
func Over(d time.Duration) Shape { return over(d) }
func progress(t, start, end time.Time) float64 {
if !end.After(start) {
return 1
}
p := float64(t.Sub(start)) / float64(end.Sub(start))
return math.Max(0, math.Min(1, p))
}
// Request is one generated request (or task run).
type Request struct {
e *Endpoint
Index int
Start time.Time
Host string
Version string
TraceID [16]byte
SpanID [8]byte
Root *Span
Status int
Detail bool
forced bool
listed bool // detail only because a listing shows it (listedRequests): no access log line
Progress float64
ParentRef *Span // for downstream SERVER spans: the CLIENT span calling it
}
// Duration is the request's total duration.
func (r *Request) Duration() time.Duration { return r.Root.Dur }
// TraceHex is the 32-hex trace id.
func (r *Request) TraceHex() string { return fmt.Sprintf("%x", r.TraceID) }
// SpanHex is the 16-hex root span id.
func (r *Request) SpanHex() string { return fmt.Sprintf("%x", r.SpanID) }
// Endpoint returns the request's endpoint.
func (r *Request) Endpoint() *Endpoint { return r.e }
// KeepDetail forces the full span tree to be sent.
func (r *Request) KeepDetail() { r.forced = true; r.Detail = true }
// Ref is what seed.json carries for a request.
func (r *Request) Ref() map[string]any {
return map[string]any{
"id": EntityID(r.e.svc.p.ID, r.TraceID, r.SpanID),
"recordedAt": r.Start.UTC().Format(time.RFC3339Nano),
"traceId": r.TraceHex(),
"spanId": r.SpanHex(),
"durationMs": float64(r.Duration()) / float64(time.Millisecond),
"endpoint": r.e.Name(),
"service": r.e.svc.Name,
"host": r.Host,
"version": r.Version,
"project": r.e.svc.p.Name,
}
}
// Span is one span in a request tree. Start is the offset from the
// request's start.
type Span struct {
Name string
Kind int
Start time.Duration
Dur time.Duration
Attrs map[string]any
Status int
StatusMsg string
Events []Event
Children []*Span
Scope string
SpanID [8]byte
// Service is set when the span belongs to another service (a
// downstream SERVER span and its subtree).
Service *Service
Host string
logs []logRecord
}
// Event is a span event.
type Event struct {
Name string
At time.Duration
Attrs map[string]any
}
// Requests generates (once) and returns the endpoint's requests in time order.
func (e *Endpoint) Requests() []*Request {
if e.gen {
return e.reqs
}
e.gen = true
if e.handler == nil {
panic(fmt.Errorf("endpoint %s has no handler", e.Name()))
}
s := e.svc.s
win := e.window
if win == 0 {
win = 24 * time.Hour
}
end := s.Now.Add(-time.Minute)
start := end.Add(-win)
base := 0.0
var mods []Shape
for _, sh := range e.shapes {
if d, ok := sh.(daily); ok {
base += float64(d)
continue
}
mods = append(mods, sh)
}
if base == 0 {
panic(fmt.Errorf("endpoint %s: traffic needs a Daily(n) base rate", e.Name()))
}
// Walk the window minute by minute; expected arrivals per minute is the
// base rate times every shape factor. Fractional arrivals carry over so
// low rates still produce requests (deterministic dithering).
carry := 0.0
idx := 0
for t := start; t.Before(end); t = t.Add(time.Minute) {
lambda := base / 1440
for _, sh := range mods {
lambda *= sh.factor(s, t, start, end)
}
carry += lambda
n := int(carry)
carry -= float64(n)
for k := 0; k < n; k++ {
at := t.Add(time.Duration(s.RNG.Int64N(int64(time.Minute))))
r := e.newRequest(idx, at, progress(at, start, end))
idx++
e.reqs = append(e.reqs, r)
}
}
sort.SliceStable(e.reqs, func(i, j int) bool { return e.reqs[i].Start.Before(e.reqs[j].Start) })
e.selectDetail()
return e.reqs
}
func (e *Endpoint) newRequest(idx int, at time.Time, prog float64) *Request {
s := e.svc.s
r := &Request{e: e, Index: idx, Start: at, Progress: prog}
r.Host = e.svc.pickHost()
r.Version = e.svc.VersionAt(at)
fillRandom(s, r.TraceID[:])
fillRandom(s, r.SpanID[:])
kind := KindServer
if e.Task {
kind = KindConsumer
}
root := &Span{Name: e.Name(), Kind: kind, Scope: e.svc.Stack.ServerScope, SpanID: r.SpanID, Attrs: map[string]any{}, Host: r.Host}
r.Root = root
b := &Req{s: s, req: r, cur: root}
if !e.Task {
root.Attrs["http.request.method"] = e.Method
root.Attrs["http.route"] = e.Route
root.Attrs["url.path"] = concretePath(s, e.Route)
root.Attrs["url.scheme"] = "https"
root.Attrs["network.protocol.version"] = "1.1"
root.Attrs["user_agent.original"] = pick(s, userAgents)
root.Attrs["client.address"] = fmt.Sprintf("10.%d.%d.%d", 1+s.RNG.IntN(30), s.RNG.IntN(255), 1+s.RNG.IntN(250))
} else {
root.Attrs["messaging.operation.type"] = "process"
}
e.handler(b)
b.finish()
r.Status = 200
if e.status != nil {
r.Status = e.status(b)
}
if b.status != 0 {
r.Status = b.status
}
if !e.Task {
root.Attrs["http.response.status_code"] = r.Status
if r.Status >= 500 {
root.Status = StatusError
}
}
if b.forceDetail {
r.forced, r.Detail = true, true
}
return r
}
// Statuses sets a per-request status picker.
func (e *Endpoint) Statuses(f func(r *Req) int) *Endpoint { e.status = f; return e }
func (e *Endpoint) selectDetail() {
s := e.svc.s
cut := s.Now.Add(-e.policy.Recent)
// Forced detail (logs, errors) never uses up the sample's cap.
n := 0
// Prefer the newest requests so the detail sample sits where the agent
// looks first, but keep it random within the window.
for i := len(e.reqs) - 1; i >= 0 && n < e.policy.Max; i-- {
r := e.reqs[i]
if r.Detail || r.Start.Before(cut) {
continue
}
if s.RNG.Float64() < e.policy.Rate {
r.Detail = true
n++
}
}
}
func (svc *Service) pickHost() string {
if len(svc.Hosts) == 1 {
return svc.Hosts[0]
}
total := 0.0
for _, h := range svc.Hosts {
total += svc.weight(h)
}
x := svc.s.RNG.Float64() * total
for _, h := range svc.Hosts {
x -= svc.weight(h)
if x <= 0 {
return h
}
}
return svc.Hosts[len(svc.Hosts)-1]
}
func (svc *Service) weight(h string) float64 {
if w, ok := svc.HostWeight[h]; ok {
return w
}
return 1
}
func fillRandom(s *Scenario, b []byte) {
for i := 0; i < len(b); i += 8 {
v := s.RNG.Uint64()
for j := 0; j < 8 && i+j < len(b); j++ {
b[i+j] = byte(v >> (8 * j))
}
}
if allZero(b) {
b[0] = 1
}
}
func allZero(b []byte) bool {
for _, x := range b {
if x != 0 {
return false
}
}
return true
}
// Slowest returns a lazy reference to the slowest request in the trailing
// window, for Expose.
func (e *Endpoint) Slowest(within time.Duration) func() any {
return func() any {
r := e.SlowestRequest(within)
if r == nil {
return nil
}
return r.Ref()
}
}
// SlowestRequest returns the slowest request in the trailing window and
// forces it to be sent in detail.
func (e *Endpoint) SlowestRequest(within time.Duration) *Request {
var best *Request
cut := e.svc.s.Now.Add(-within)
for _, r := range e.Requests() {
if r.Start.Before(cut) {
continue
}
if best == nil || r.Duration() > best.Duration() {
best = r
}
}
if best != nil {
best.KeepDetail()
}
return best
}
// Percentile returns the p-quantile (0..1) of durations in the trailing window.
func (e *Endpoint) Percentile(p float64, within time.Duration) time.Duration {
var ds []time.Duration
cut := e.svc.s.Now.Add(-within)
for _, r := range e.Requests() {
if !r.Start.Before(cut) {
ds = append(ds, r.Duration())
}
}
return quantile(ds, p)
}
// PercentileBetween returns the p-quantile of durations started in [from, to).
func (e *Endpoint) PercentileBetween(p float64, from, to time.Time) time.Duration {
var ds []time.Duration
for _, r := range e.Requests() {
if !r.Start.Before(from) && r.Start.Before(to) {
ds = append(ds, r.Duration())
}
}
return quantile(ds, p)
}
func quantile(ds []time.Duration, p float64) time.Duration {
if len(ds) == 0 {
return 0
}
sort.Slice(ds, func(i, j int) bool { return ds[i] < ds[j] })
pos := p * float64(len(ds)-1)
lo := int(math.Floor(pos))
hi := int(math.Ceil(pos))
frac := pos - float64(lo)
return time.Duration(float64(ds[lo])*(1-frac) + float64(ds[hi])*frac)
}
// Pick returns a lazy reference to the first request matching f (newest
// first), forcing it to detail.
func (e *Endpoint) Pick(f func(r *Request) bool) func() any {
return func() any {
r := e.PickRequest(f)
if r == nil {
return nil
}
return r.Ref()
}
}
// PickRequest returns the newest request matching f and forces its detail.
func (e *Endpoint) PickRequest(f func(r *Request) bool) *Request {
reqs := e.Requests()
for i := len(reqs) - 1; i >= 0; i-- {
if f(reqs[i]) {
reqs[i].KeepDetail()
return reqs[i]
}
}
return nil
}
// Count returns how many requests started in the trailing window.
func (e *Endpoint) Count(within time.Duration) int {
cut := e.svc.s.Now.Add(-within)
n := 0
for _, r := range e.Requests() {
if !r.Start.Before(cut) {
n++
}
}
return n
}
var userAgents = []string{
"Mozilla/5.0 (Macintosh; Intel Mac OS X 14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.1 Safari/605.1.15",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36",
"Mozilla/5.0 (iPhone; CPU iPhone OS 18_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148",
"okhttp/4.12.0",
}
func pick[T any](s *Scenario, xs []T) T { return xs[s.RNG.IntN(len(xs))] }
// concretePath turns a route template into a plausible concrete path.
func concretePath(s *Scenario, route string) string {
out := []byte{}
for i := 0; i < len(route); i++ {
if route[i] == ':' || route[i] == '{' {
j := i + 1
for j < len(route) && route[j] != '/' && route[j] != '}' {
j++
}
if j < len(route) && route[j] == '}' {
j++
}
out = append(out, []byte(fmt.Sprint(1000+s.RNG.IntN(90000)))...)
i = j - 1
continue
}
out = append(out, route[i])
}
return string(out)
}