The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
package controllers
import (
"context"
"database/sql"
"errors"
"fmt"
"net/http"
"strconv"
"time"
"github.com/tracewayapp/traceway/backend/app/db"
"github.com/tracewayapp/traceway/backend/app/middleware"
"github.com/tracewayapp/traceway/backend/app/models"
"github.com/tracewayapp/traceway/backend/app/repositories/telemetry"
"github.com/tracewayapp/traceway/backend/app/repositories/telemetry/shared"
"github.com/tracewayapp/traceway/backend/app/repositories/transactional"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
traceway "go.tracewayapp.com"
)
type metricQueryController struct{}
const (
metricQueryMaxQueries = 50
metricQueryMaxBuckets = shared.MaxTimeSeriesBuckets
metricQueryMaxGroups = 200
metricQueryMaxRange = 731 * 24 * time.Hour
metricQueryMaxTagFilters = 20
metricQueryTimeout = 30 * time.Second
metricQueryMaxIntervalMinutes = 366 * 24 * 60
)
func minIntervalMinutes(rangeSeconds, maxBuckets int) int {
if maxBuckets < 2 {
maxBuckets = 2
}
per := 60 * (maxBuckets - 1)
iv := (rangeSeconds + per - 1) / per
if iv < 1 {
iv = 1
}
return iv
}
func validateMetricQuery(req *MetricQueryRequest) string {
if len(req.Queries) > metricQueryMaxQueries {
return fmt.Sprintf("a maximum of %d queries can be requested at once", metricQueryMaxQueries)
}
if msg := validateMetricRange(req.From, req.To); msg != "" {
return msg
}
for _, q := range req.Queries {
if len(q.TagFilters) > metricQueryMaxTagFilters {
return fmt.Sprintf("a maximum of %d tag filters can be applied per query", metricQueryMaxTagFilters)
}
}
if req.IntervalMinutes <= 0 {
req.IntervalMinutes = calculateIntervalMinutes(req.To.Sub(req.From))
}
if req.IntervalMinutes > metricQueryMaxIntervalMinutes {
return fmt.Sprintf("intervalMinutes must be at most %d", metricQueryMaxIntervalMinutes)
}
if minInterval := minIntervalMinutes(int(req.To.Sub(req.From).Seconds()), metricQueryMaxBuckets); req.IntervalMinutes < minInterval {
req.IntervalMinutes = minInterval
}
return ""
}
func validateMetricRange(from, to time.Time) string {
if to.Before(from) {
return "to must not be before from"
}
if to.Sub(from) > metricQueryMaxRange {
return fmt.Sprintf("the time range must be at most %d days", int(metricQueryMaxRange.Hours()/24))
}
return ""
}
func trimGroupSeries(series map[string][]models.TimeSeriesPoint) bool {
truncated := false
for k, points := range series {
if points == nil {
delete(series, k)
truncated = true
}
}
return truncated
}
type MetricQueryRequest struct {
Queries []MetricQueryItem `json:"queries" binding:"required,min=1"`
From time.Time `json:"from" binding:"required"`
To time.Time `json:"to" binding:"required"`
IntervalMinutes int `json:"intervalMinutes"`
}
type MetricQueryItem struct {
Name string `json:"name" binding:"required"`
Aggregation string `json:"aggregation"`
TagFilters map[string]string `json:"tagFilters"`
GroupBy string `json:"groupBy"`
}
type MetricQueryResponse struct {
Results []MetricQueryResult `json:"results"`
IntervalMinutes int `json:"intervalMinutes"`
}
type MetricQueryResult struct {
Name string `json:"name"`
Unit string `json:"unit"`
Series map[string][]models.TimeSeriesPoint `json:"series"`
TruncatedGroups bool `json:"truncatedGroups,omitempty"`
}
func (c *metricQueryController) Query(ctx *gin.Context) {
projectId, err := middleware.GetProjectId(ctx)
if err != nil {
ctx.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("RequireProjectAccess middleware must be applied: %w", err))
return
}
limitJSONBody(ctx)
var req MetricQueryRequest
if err := ctx.ShouldBindJSON(&req); err != nil {
middleware.RejectBindError(ctx, err, "Invalid request body")
return
}
if msg := validateMetricQuery(&req); msg != "" {
ctx.JSON(http.StatusUnprocessableEntity, gin.H{"error": msg})
return
}
queryCtx, cancel := context.WithTimeout(ctx.Request.Context(), metricQueryTimeout)
defer cancel()
unitMap := make(map[string]string)
registry, regErr := db.ExecuteTransaction(func(tx *sql.Tx) ([]*models.MetricRegistry, error) {
return transactional.MetricRegistryRepository.FindByProject(tx, projectId)
})
if regErr != nil {
traceway.CaptureException(fmt.Errorf("failed to load metric registry for query: %w", regErr))
}
if regErr == nil {
for _, r := range registry {
unitMap[r.Name] = r.Unit
}
}
var results []MetricQueryResult
for _, q := range req.Queries {
agg := q.Aggregation
if agg == "" {
agg = "avg"
}
spanName := fmt.Sprintf("query metric: %s agg=%s", q.Name, agg)
if q.GroupBy != "" {
spanName += " group=" + q.GroupBy
}
if len(q.TagFilters) > 0 {
spanName += fmt.Sprintf(" filters=%d", len(q.TagFilters))
}
span := traceway.StartSpan(ctx, spanName)
series, err := telemetry.MetricPointRepository.QueryTimeSeries(
queryCtx, projectId, q.Name, req.From, req.To,
req.IntervalMinutes, agg, q.TagFilters, q.GroupBy, metricQueryMaxGroups,
)
span.End()
if err != nil {
if errors.Is(err, context.DeadlineExceeded) || errors.Is(queryCtx.Err(), context.DeadlineExceeded) {
ctx.JSON(http.StatusGatewayTimeout, gin.H{"error": fmt.Sprintf("metric query timed out after %s; narrow the time range or send fewer queries", metricQueryTimeout)})
return
}
ctx.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("failed to query metric %s: %w", q.Name, err))
return
}
truncated := trimGroupSeries(series)
resultUnit := rateUnit(agg, unitMap[q.Name])
if agg == "count" {
resultUnit = "count"
}
results = append(results, MetricQueryResult{
Name: q.Name,
Unit: resultUnit,
Series: series,
TruncatedGroups: truncated,
})
}
ctx.JSON(http.StatusOK, MetricQueryResponse{Results: results, IntervalMinutes: req.IntervalMinutes})
}
type DiscoverResponse struct {
Metrics []models.DiscoveredMetric `json:"metrics"`
}
type DiscoverInstancesResponse struct {
Instances []string `json:"instances"`
}
func parseMetricDiscoveryRange(ctx *gin.Context) (time.Time, time.Time, string) {
from := time.Now().AddDate(0, 0, -7)
to := time.Now()
if fromStr := ctx.Query("from"); fromStr != "" {
if parsed, err := time.Parse(time.RFC3339, fromStr); err == nil {
from = parsed
}
}
if toStr := ctx.Query("to"); toStr != "" {
if parsed, err := time.Parse(time.RFC3339, toStr); err == nil {
to = parsed
}
}
return from, to, validateMetricRange(from, to)
}
func (c *metricQueryController) Discover(ctx *gin.Context) {
projectId, err := middleware.GetProjectId(ctx)
if err != nil {
ctx.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("RequireProjectAccess middleware must be applied: %w", err))
return
}
from, to, rangeMsg := parseMetricDiscoveryRange(ctx)
if rangeMsg != "" {
ctx.JSON(http.StatusUnprocessableEntity, gin.H{"error": rangeMsg})
return
}
span := traceway.StartSpan(ctx, "discover metrics")
discovered, err := telemetry.MetricPointRepository.DiscoverMetrics(ctx, projectId, from, to)
span.End()
if err != nil {
ctx.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("failed to discover metrics: %w", err))
return
}
registry, regErr := db.ExecuteTransaction(func(tx *sql.Tx) ([]*models.MetricRegistry, error) {
return transactional.MetricRegistryRepository.FindByProject(tx, projectId)
})
if regErr != nil {
traceway.CaptureException(fmt.Errorf("failed to load metric registry for discover: %w", regErr))
}
if regErr == nil {
regMap := make(map[string]*models.MetricRegistry, len(registry))
for _, r := range registry {
regMap[r.Name] = r
}
for i := range discovered {
if reg, ok := regMap[discovered[i].Name]; ok {
discovered[i].MetricType = reg.MetricType
discovered[i].Unit = reg.Unit
}
}
}
ctx.JSON(http.StatusOK, DiscoverResponse{Metrics: discovered})
}
func (c *metricQueryController) DiscoverInstances(ctx *gin.Context) {
projectId, err := middleware.GetProjectId(ctx)
if err != nil {
ctx.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("RequireProjectAccess middleware must be applied: %w", err))
return
}
from, to, rangeMsg := parseMetricDiscoveryRange(ctx)
if rangeMsg != "" {
ctx.JSON(http.StatusUnprocessableEntity, gin.H{"error": rangeMsg})
return
}
span := traceway.StartSpan(ctx, "discover instances")
instances, err := telemetry.MetricPointRepository.GetDistinctServers(ctx, projectId, from, to)
span.End()
if err != nil {
ctx.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("failed to discover instances: %w", err))
return
}
if instances == nil {
instances = []string{}
}
ctx.JSON(http.StatusOK, DiscoverInstancesResponse{Instances: instances})
}
const discoverOrgMaxProjects = 25
type OrgDiscoveredMetric struct {
Name string `json:"name"`
TagKeys []string `json:"tagKeys"`
MetricType string `json:"metricType,omitempty"`
Unit string `json:"unit,omitempty"`
ProjectIds []uuid.UUID `json:"projectIds"`
}
func (c *metricQueryController) DiscoverOrg(ctx *gin.Context) {
organizationId, err := strconv.Atoi(ctx.Query("organizationId"))
if err != nil {
ctx.JSON(http.StatusBadRequest, gin.H{"error": "organizationId query param is required"})
return
}
userId := middleware.GetUserId(ctx)
projects, err := db.ExecuteTransaction(func(tx *sql.Tx) ([]*models.Project, error) {
role, err := transactional.OrganizationRepository.GetUserRole(tx, organizationId, userId)
if err != nil {
return nil, err
}
if role == "" {
return nil, nil
}
return transactional.ProjectRepository.FindByOrganizationId(tx, organizationId)
})
if err != nil {
ctx.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("failed to list organization projects: %w", err))
return
}
if projects == nil {
ctx.JSON(http.StatusNotFound, gin.H{"error": "Organization not found"})
return
}
truncated := false
if len(projects) > discoverOrgMaxProjects {
projects = projects[:discoverOrgMaxProjects]
truncated = true
}
from := time.Now().AddDate(0, 0, -7)
to := time.Now()
merged := map[string]*OrgDiscoveredMetric{}
order := []string{}
for _, project := range projects {
span := traceway.StartSpan(ctx, fmt.Sprintf("discover metrics: project %s", project.Id))
discovered, err := telemetry.MetricPointRepository.DiscoverMetrics(ctx, project.Id, from, to)
span.End()
if err != nil {
traceway.CaptureException(fmt.Errorf("failed to discover metrics for project %s: %w", project.Id, err))
continue
}
for _, m := range discovered {
entry := merged[m.Name]
if entry == nil {
entry = &OrgDiscoveredMetric{Name: m.Name, TagKeys: m.TagKeys, MetricType: m.MetricType, Unit: m.Unit, ProjectIds: []uuid.UUID{}}
merged[m.Name] = entry
order = append(order, m.Name)
}
entry.ProjectIds = append(entry.ProjectIds, project.Id)
if entry.Unit == "" {
entry.Unit = m.Unit
}
if entry.MetricType == "" {
entry.MetricType = m.MetricType
}
}
}
metrics := make([]OrgDiscoveredMetric, 0, len(order))
for _, name := range order {
metrics = append(metrics, *merged[name])
}
ctx.JSON(http.StatusOK, gin.H{"metrics": metrics, "truncated": truncated})
}
func (c *metricQueryController) DiscoverTags(ctx *gin.Context) {
projectId, err := middleware.GetProjectId(ctx)
if err != nil {
ctx.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("RequireProjectAccess middleware must be applied: %w", err))
return
}
name := ctx.Query("name")
key := ctx.Query("key")
if name == "" || key == "" {
ctx.JSON(http.StatusBadRequest, gin.H{"error": "name and key query params are required"})
return
}
from := time.Now().AddDate(0, 0, -7)
to := time.Now()
span := traceway.StartSpan(ctx, fmt.Sprintf("discover tag values: %s.%s", name, key))
values, err := telemetry.MetricPointRepository.DiscoverTagValues(ctx, projectId, name, key, from, to)
span.End()
if err != nil {
ctx.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("failed to discover tag values: %w", err))
return
}
ctx.JSON(http.StatusOK, gin.H{"values": values})
}
func (c *metricQueryController) UpdateRegistry(ctx *gin.Context) {
projectId, err := middleware.GetProjectId(ctx)
if err != nil {
ctx.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("RequireProjectAccess middleware must be applied: %w", err))
return
}
var req struct {
Name string `json:"name" binding:"required"`
MetricType string `json:"metricType"`
Unit string `json:"unit"`
Description string `json:"description"`
}
if err := ctx.ShouldBindJSON(&req); err != nil {
middleware.RejectBindError(ctx, err, "Invalid request body")
return
}
if req.MetricType != "" && req.MetricType != "gauge" && req.MetricType != "counter" {
ctx.JSON(http.StatusBadRequest, gin.H{"error": "metricType must be 'gauge' or 'counter'"})
return
}
updated, err := db.ExecuteTransaction(func(tx *sql.Tx) (*models.MetricRegistry, error) {
entry, err := transactional.MetricRegistryRepository.FindByProjectAndName(tx, projectId, req.Name)
if err != nil {
return nil, err
}
if entry == nil {
return nil, nil
}
if req.MetricType != "" {
entry.MetricType = req.MetricType
}
if req.Unit != "" {
entry.Unit = req.Unit
}
entry.Description = req.Description
if err := transactional.MetricRegistryRepository.Update(tx, entry); err != nil {
return nil, err
}
return entry, nil
})
if err != nil {
ctx.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("failed to update metric registry: %w", err))
return
}
if updated == nil {
ctx.JSON(http.StatusNotFound, gin.H{"error": "metric not found in registry"})
return
}
ctx.JSON(http.StatusOK, updated)
}
var MetricQueryController = metricQueryController{}
func rateUnit(aggregation string, unit string) string {
if aggregation != "rate" {
return unit
}
switch unit {
case "s":
return "1"
case "":
return "count/s"
}
return unit + "/s"
}