The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
package controllers
import (
"bytes"
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"net/http"
"strings"
"time"
"github.com/tracewayapp/traceway/backend/app/db"
"github.com/tracewayapp/traceway/backend/app/middleware"
"github.com/tracewayapp/traceway/backend/app/models"
"github.com/tracewayapp/traceway/backend/app/oncall"
"github.com/tracewayapp/traceway/backend/app/repositories/telemetry"
"github.com/tracewayapp/traceway/backend/app/repositories/transactional"
"github.com/tracewayapp/traceway/backend/app/storage"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
traceway "go.tracewayapp.com"
)
type exceptionStackTraceController struct{}
type ExceptionSearchRequest struct {
FromDate time.Time `json:"fromDate"`
ToDate time.Time `json:"toDate"`
OrderBy string `json:"orderBy"`
Pagination PaginationParams `json:"pagination"`
Search string `json:"search"`
SearchType string `json:"searchType"`
IncludeArchived bool `json:"includeArchived"`
}
const maxBulkIssueHashes = 100
type ArchiveRequest struct {
Hashes []string `json:"hashes"`
// ResolvePages also resolves any unresolved on-call pages that were opened
// for these issues (archive path only, confirmed by the user in the dialog).
ResolvePages bool `json:"resolvePages"`
}
type ExceptionDetailRequest struct {
Pagination PaginationParams `json:"pagination"`
}
type ExceptionDetailResponse struct {
Group *models.ExceptionGroup `json:"group"`
Occurrences []models.ExceptionStackTrace `json:"occurrences"`
Pagination Pagination `json:"pagination"`
SessionRecording json.RawMessage `json:"sessionRecording,omitempty"`
// RelatedEntity is the endpoint, task or AI trace the newest occurrence happened in.
RelatedEntity *telemetry.ExceptionOwner `json:"relatedEntity,omitempty"`
// SessionId is set when the exception is linked to a parent session row
// (always-on recording). The frontend swaps SessionRecording for a fetch
// against `/sessions/:sessionId/recording`, which spans every segment.
SessionId *uuid.UUID `json:"sessionId,omitempty"`
}
// wrapLegacyRecording handles the case where a recording in blob storage was
// written before the logs/actions wrapper landed. Older files contain just the
// events array as their top-level JSON; newer files contain a wrapper object
// with `events` / `logs` / `actions` / `startedAt` / `endedAt`. Returns the
// new shape either way so the frontend never branches on file format.
func wrapLegacyRecording(raw []byte) json.RawMessage {
trimmed := bytes.TrimLeft(raw, " \t\r\n")
if len(trimmed) > 0 && trimmed[0] == '[' {
wrapped := make([]byte, 0, len(raw)+len(`{"events":}`))
wrapped = append(wrapped, []byte(`{"events":`)...)
wrapped = append(wrapped, raw...)
wrapped = append(wrapped, '}')
return json.RawMessage(wrapped)
}
return json.RawMessage(raw)
}
func loadSessionRecording(ctx context.Context, filePath string) (json.RawMessage, error) {
raw, err := storage.Store.Read(ctx, filePath)
if err != nil {
return nil, err
}
return wrapLegacyRecording(raw), nil
}
func (e exceptionStackTraceController) FindGrouppedExceptionStackTraces(c *gin.Context) {
projectId, err := middleware.GetProjectId(c)
if err != nil {
c.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("RequireProjectAccess middleware must be applied: %w", err))
return
}
var request ExceptionSearchRequest
if err := c.ShouldBindJSON(&request); err != nil {
middleware.RejectBindError(c, err, err.Error())
return
}
span := traceway.StartSpan(c, "loading grouped exceptions")
exceptions, total, err := telemetry.ExceptionStackTraceRepository.FindGrouped(c, projectId, request.FromDate, request.ToDate, request.Pagination.Page, request.Pagination.PageSize, request.OrderBy, request.Search, request.SearchType, request.IncludeArchived)
span.End()
if err != nil {
c.AbortWithError(500, traceway.NewStackTraceErrorf("error loading exceptions: %w", err))
return
}
// Get hourly trends for all returned exception hashes (last 24 hours)
if len(exceptions) > 0 {
hashes := make([]string, len(exceptions))
for i, ex := range exceptions {
hashes[i] = ex.ExceptionHash
}
now := time.Now()
start24h := now.Add(-24 * time.Hour)
span = traceway.StartSpan(c, "loading hourly trends")
trends, err := telemetry.ExceptionStackTraceRepository.GetHourlyTrendForHashes(c, projectId, hashes, start24h, now)
span.End()
if err != nil {
c.AbortWithError(500, traceway.NewStackTraceErrorf("error loading trends: %w", err))
return
}
// Attach trends to each exception group
for i := range exceptions {
if trend, ok := trends[exceptions[i].ExceptionHash]; ok {
exceptions[i].HourlyTrend = trend
} else {
exceptions[i].HourlyTrend = []models.ExceptionTrendPoint{}
}
}
}
c.JSON(http.StatusOK, PaginatedResponse[models.ExceptionGroup]{
Data: exceptions,
Pagination: Pagination{
Page: request.Pagination.Page,
PageSize: request.Pagination.PageSize,
Total: total,
TotalPages: (total + int64(request.Pagination.PageSize) - 1) / int64(request.Pagination.PageSize),
},
})
}
func (e exceptionStackTraceController) FindByHash(c *gin.Context) {
projectId, err := middleware.GetProjectId(c)
if err != nil {
c.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("RequireProjectAccess middleware must be applied: %w", err))
return
}
exceptionHash := c.Param("hash")
if exceptionHash == "" {
c.JSON(http.StatusBadRequest, gin.H{"error": "exception hash is required"})
return
}
var request ExceptionDetailRequest
if err := c.ShouldBindJSON(&request); err != nil {
// Default pagination if not provided
request.Pagination = PaginationParams{Page: 1, PageSize: 20}
}
span := traceway.StartSpan(c, "loading exception by hash")
group, occurrences, total, err := telemetry.ExceptionStackTraceRepository.FindByHash(c, projectId, exceptionHash, request.Pagination.Page, request.Pagination.PageSize)
span.End()
if err != nil {
c.AbortWithError(500, traceway.NewStackTraceErrorf("error loading the group: %w", err))
return
}
if group == nil {
c.JSON(http.StatusNotFound, gin.H{"error": "Exception not found"})
return
}
response := ExceptionDetailResponse{
Group: group,
Occurrences: occurrences,
Pagination: Pagination{
Page: request.Pagination.Page,
PageSize: request.Pagination.PageSize,
Total: total,
TotalPages: (total + int64(request.Pagination.PageSize) - 1) / int64(request.Pagination.PageSize),
},
}
if len(occurrences) > 0 {
response.RelatedEntity = findRelatedEntity(c, occurrences[0])
// The per-exception 10 s clip and the parent session are independent
// attachments — surface both whenever they exist. The dashboard plays
// the clip inline and renders a link to the full session.
filePath, err := telemetry.SessionRecordingRepository.FindByExceptionId(c, projectId, occurrences[0].Id)
if err == nil && filePath != "" {
recording, err := loadSessionRecording(c, filePath)
if err == nil {
response.SessionRecording = recording
} else {
traceway.CaptureException(fmt.Errorf("failed to read session recording (key=%s): %w", filePath, err))
}
} else if err != nil && !errors.Is(err, sql.ErrNoRows) {
traceway.CaptureException(fmt.Errorf("failed to load session recording ref for exception %s: %w", occurrences[0].Id, err))
}
sessionId, err := telemetry.ExceptionStackTraceRepository.GetSessionIdForException(c, projectId, occurrences[0].Id)
if err == nil && sessionId != nil {
response.SessionId = sessionId
}
}
c.JSON(http.StatusOK, response)
}
func (e exceptionStackTraceController) ArchiveExceptions(c *gin.Context) {
projectId, err := middleware.GetProjectId(c)
if err != nil {
c.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("RequireProjectAccess middleware must be applied: %w", err))
return
}
limitJSONBody(c)
var request ArchiveRequest
if err := c.ShouldBindJSON(&request); err != nil {
middleware.RejectBindError(c, err, err.Error())
return
}
if len(request.Hashes) == 0 {
c.JSON(http.StatusBadRequest, gin.H{"error": "hashes array is required"})
return
}
if len(request.Hashes) > maxBulkIssueHashes {
c.JSON(http.StatusUnprocessableEntity, gin.H{"error": fmt.Sprintf("a maximum of %d issues can be archived at once", maxBulkIssueHashes)})
return
}
err = telemetry.ExceptionStackTraceRepository.ArchiveByHashes(c, projectId, request.Hashes)
if err != nil {
c.AbortWithError(500, traceway.NewStackTraceErrorf("error archiving %s: %w", strings.Join(request.Hashes, ","), err))
return
}
resolvedPages := 0
if request.ResolvePages {
userId := middleware.GetUserId(c)
now := time.Now().UTC()
resolvedPages, err = db.ExecuteTransaction(func(tx *sql.Tx) (int, error) {
resolved := 0
for _, hash := range request.Hashes {
pages, err := transactional.PageRepository.FindUnresolvedByIssueHash(tx, projectId, hash)
if err != nil {
return 0, err
}
for _, page := range pages {
ok, err := oncall.ResolvePage(tx, page.Id, userId, now)
if err != nil {
return 0, err
}
if ok {
resolved++
}
}
}
return resolved, nil
})
if err != nil {
c.AbortWithError(500, traceway.NewStackTraceErrorf("error resolving pages for archived issues %s: %w", strings.Join(request.Hashes, ","), err))
return
}
}
c.JSON(http.StatusOK, gin.H{"archived": len(request.Hashes), "resolvedPages": resolvedPages})
}
func (e exceptionStackTraceController) UnarchiveExceptions(c *gin.Context) {
projectId, err := middleware.GetProjectId(c)
if err != nil {
c.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("RequireProjectAccess middleware must be applied: %w", err))
return
}
limitJSONBody(c)
var request ArchiveRequest
if err := c.ShouldBindJSON(&request); err != nil {
middleware.RejectBindError(c, err, err.Error())
return
}
if len(request.Hashes) == 0 {
c.JSON(http.StatusBadRequest, gin.H{"error": "hashes array is required"})
return
}
if len(request.Hashes) > maxBulkIssueHashes {
c.JSON(http.StatusUnprocessableEntity, gin.H{"error": fmt.Sprintf("a maximum of %d issues can be unarchived at once", maxBulkIssueHashes)})
return
}
err = telemetry.ExceptionStackTraceRepository.UnarchiveByHashes(c, projectId, request.Hashes)
if err != nil {
c.AbortWithError(500, traceway.NewStackTraceErrorf("error unarchiving %s: %w", strings.Join(request.Hashes, ","), err))
return
}
c.JSON(http.StatusOK, gin.H{"unarchived": len(request.Hashes)})
}
// The related entity is a convenience on the page, so a failed lookup is reported and the page still loads.
func findRelatedEntity(ctx context.Context, exception models.ExceptionStackTrace) *telemetry.ExceptionOwner {
owner, err := telemetry.FindExceptionOwner(ctx, exception)
if err != nil {
traceway.CaptureException(fmt.Errorf("failed to find the related entity of exception %s: %w", exception.Id, err))
return nil
}
return owner
}
type exceptionByIdRequest struct {
RecordedAt *time.Time `json:"recordedAt"`
}
func (e exceptionStackTraceController) FindById(c *gin.Context) {
projectId, err := middleware.GetProjectId(c)
if err != nil {
c.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("RequireProjectAccess middleware must be applied: %w", err))
return
}
exceptionId, err := uuid.Parse(c.Param("exceptionId"))
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid exception id"})
return
}
var request exceptionByIdRequest
_ = c.ShouldBindJSON(&request)
span := traceway.StartSpan(c, "loading exception by id")
exception, err := telemetry.ExceptionStackTraceRepository.FindById(c, projectId, exceptionId, request.RecordedAt)
if exception == nil && err == nil && request.RecordedAt != nil {
exception, err = telemetry.ExceptionStackTraceRepository.FindById(c, projectId, exceptionId, nil)
}
span.End()
if err != nil {
c.AbortWithError(500, traceway.NewStackTraceErrorf("error loading the exception: %w", err))
return
}
if exception == nil {
c.JSON(http.StatusNotFound, gin.H{"error": "Exception not found"})
return
}
response := gin.H{"exception": exception}
if related := findRelatedEntity(c, *exception); related != nil {
response["relatedEntity"] = related
}
filePath, err := telemetry.SessionRecordingRepository.FindByExceptionId(c, projectId, exceptionId)
if err == nil && filePath != "" {
recording, err := loadSessionRecording(c, filePath)
if err == nil {
response["sessionRecording"] = recording
} else {
traceway.CaptureException(fmt.Errorf("failed to read session recording (key=%s): %w", filePath, err))
}
} else if err != nil && !errors.Is(err, sql.ErrNoRows) {
traceway.CaptureException(fmt.Errorf("failed to load session recording ref for exception %s: %w", exceptionId, err))
}
sessionId, err := telemetry.ExceptionStackTraceRepository.GetSessionIdForException(c, projectId, exceptionId)
if err == nil && sessionId != nil {
response["sessionId"] = sessionId
}
c.JSON(http.StatusOK, response)
}
var ExceptionStackTraceController = exceptionStackTraceController{}