The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
perf: invert the group-index guard and index the session, log-trace and notification-history reads (#354)
FrameAutomata3bf49cda7001
diff --git a/backend/app/migrations/sqlite_telemetry/0021_index_session_and_log_trace_time.up.sql b/backend/app/migrations/sqlite_telemetry/0021_index_session_and_log_trace_time.up.sqlnew file mode 100644index 00000000..e4f9256d--- /dev/null+++ b/backend/app/migrations/sqlite_telemetry/0021_index_session_and_log_trace_time.up.sql@@ -0,0 +1,4 @@+CREATE INDEX IF NOT EXISTS idx_exceptions_session_recorded ON exception_stack_traces(project_id, session_id, recorded_at) WHERE session_id IS NOT NULL;+DROP INDEX IF EXISTS idx_exceptions_session;+CREATE INDEX IF NOT EXISTS idx_log_records_project_trace_timestamp ON log_records(project_id, trace_id, timestamp);+DROP INDEX IF EXISTS idx_log_records_project_trace;diff --git a/backend/app/migrations/sqlite_telemetry/0022_index_fired_notifications_project_fired.up.sql b/backend/app/migrations/sqlite_telemetry/0022_index_fired_notifications_project_fired.up.sqlnew file mode 100644index 00000000..fea7033e--- /dev/null+++ b/backend/app/migrations/sqlite_telemetry/0022_index_fired_notifications_project_fired.up.sql@@ -0,0 +1 @@+CREATE INDEX IF NOT EXISTS idx_fired_notifications_project_fired ON fired_notifications(project_id, fired_at);diff --git a/backend/app/migrations/telemetry_group_index_test.go b/backend/app/migrations/telemetry_group_index_test.goindex 343c2acb..ba76bb77 100644--- a/backend/app/migrations/telemetry_group_index_test.go+++ b/backend/app/migrations/telemetry_group_index_test.go@@ -4,18 +4,27 @@ package migrationsimport ("database/sql"+ "slices"+ "strings""testing"+ "github.com/tracewayapp/traceway/backend/app/retention"_ "modernc.org/sqlite")// Every per-group telemetry read pairs a group column with the table's time-// column -- as a second filter on the grouped lists, as the ORDER BY on the-// per-group detail reads. An index that stops at the group column forces SQLite-// to either re-filter the group across the whole window once per group, which is-// quadratic in the number of groups, or seek the group and sort its entire-// history. Both are unbounded and both are invisible to functional tests. Guard-// the covering indexes so a later migration cannot narrow or drop one unnoticed.+// column: as a second filter on the grouped lists, as the ORDER BY on the+// per-group detail reads. An index that stops at the group column leaves SQLite+// to either re-filter the group across the whole window once per group, which+// is quadratic in the number of groups, or seek the group and sort its entire+// history. Both are unbounded and invisible to functional tests.+//+// The first loop guards the indexes behind known reads so a later migration+// cannot narrow or drop one unnoticed. The second inverts the question: every+// index that leads with project_id must reach its table's time column, so a+// short index fails here the day it lands rather than when somebody remembers+// to list it. Indexes that stop short on purpose are named with the reason,+// and each entry is itself checked so it cannot outlive the index it excuses.func TestTelemetryGroupIndexesCoverTimeColumn(t *testing.T) {telemetryDB, err := sql.Open("sqlite", ":memory:")if err != nil {@@ -28,28 +37,94 @@ func TestTelemetryGroupIndexesCoverTimeColumn(t *testing.T) {t.Fatalf("telemetry migrations failed: %v", err)}- for _, tc := range []struct{ table, group, timeCol string }{- {"endpoints", "endpoint", "recorded_at"},- {"tasks", "task_name", "recorded_at"},- {"ai_traces", "trace_name", "recorded_at"},- {"ai_traces", "conversation_id", "recorded_at"},- {"exception_stack_traces", "exception_hash", "recorded_at"},- {"check_results", "check_id", "recorded_at"},- {"log_records", "service_name", "timestamp"},- {"metric_points", "name", "recorded_at"},+ timeColumn := retention.TelemetryTimeColumns()+ indexes := createdIndexes(t, telemetryDB)++ for _, tc := range []struct{ table, group string }{+ {"endpoints", "endpoint"},+ {"tasks", "task_name"},+ {"ai_traces", "trace_name"},+ {"ai_traces", "conversation_id"},+ {"exception_stack_traces", "exception_hash"},+ {"exception_stack_traces", "session_id"},+ {"check_results", "check_id"},+ {"log_records", "trace_id"},+ {"log_records", "service_name"},+ {"metric_points", "name"},} {- var covered bool- if err := telemetryDB.QueryRow(`- SELECT EXISTS (- SELECT 1 FROM pragma_index_list(?1) AS il- WHERE (SELECT name FROM pragma_index_info(il.name) WHERE seqno = 0) = 'project_id'- AND (SELECT name FROM pragma_index_info(il.name) WHERE seqno = 1) = ?2- AND (SELECT name FROM pragma_index_info(il.name) WHERE seqno = 2) = ?3- )`, tc.table, tc.group, tc.timeCol).Scan(&covered); err != nil {- t.Fatalf("%s: failed to inspect indexes: %v", tc.table, err)+ timeCol, pruned := timeColumn[tc.table]+ if !pruned {+ t.Fatalf("%s is not a retention target, so it has no time column to guard", tc.table)}+ want := []string{"project_id", tc.group, timeCol}+ covered := slices.ContainsFunc(indexes, func(ix telemetryIndex) bool {+ return ix.table == tc.table && len(ix.cols) >= len(want) && slices.Equal(ix.cols[:len(want)], want)+ })if !covered {- t.Errorf("%s: no index leading with (project_id, %s, %s)", tc.table, tc.group, tc.timeCol)+ t.Errorf("%s: no index leading with (%s)", tc.table, strings.Join(want, ", "))+ }+ }++ intentionallyShort := map[string]string{+ "idx_spans_project_trace": "a trace has tens of spans; after the trace seek, the recorded_at window and the start_time sort touch only those rows",+ "idx_session_recordings_project_exception": "an exception has at most a handful of recordings, so ordering them costs nothing",+ }+ for _, ix := range indexes {+ timeCol, pruned := timeColumn[ix.table]+ if ix.cols[0] != "project_id" || !pruned {+ continue+ }+ _, listed := intentionallyShort[ix.name]+ delete(intentionallyShort, ix.name)+ reaches := slices.Contains(ix.cols[1:], timeCol)+ shape := ix.table + "(" + strings.Join(ix.cols, ", ") + ")"+ switch {+ case reaches && listed:+ t.Errorf("%s on %s reaches %s now; remove it from intentionallyShort", ix.name, shape, timeCol)+ case !reaches && !listed:+ t.Errorf("%s on %s leads with project_id but never reaches %s; widen it, or add it to intentionallyShort with the reason", ix.name, shape, timeCol)+ }+ }+ for index := range intentionallyShort {+ t.Errorf("intentionallyShort names %s, which is no longer an index leading with project_id on a pruned table; remove the entry", index)+ }+}++type telemetryIndex struct {+ table, name string+ cols []string+}++// createdIndexes lists every CREATE INDEX index with its columns in order. An+// expression column has no name and is listed as "". UNIQUE and PRIMARY KEY+// autoindexes are left out: their column set is dictated by the constraint,+// not by a read pattern.+func createdIndexes(t *testing.T, telemetryDB *sql.DB) []telemetryIndex {+ t.Helper()+ rows, err := telemetryDB.Query(`+ SELECT m.name, il.name, COALESCE(ii.name, '') FROM sqlite_master AS m+ JOIN pragma_index_list(m.name) AS il+ JOIN pragma_index_info(il.name) AS ii+ WHERE m.type = 'table' AND il.origin = 'c'+ ORDER BY m.name, il.name, ii.seqno`)+ if err != nil {+ t.Fatalf("failed to list indexes: %v", err)+ }+ defer rows.Close()++ var indexes []telemetryIndex+ for rows.Next() {+ var table, index, col string+ if err := rows.Scan(&table, &index, &col); err != nil {+ t.Fatalf("failed to scan index column: %v", err)}+ if n := len(indexes); n == 0 || indexes[n-1].name != index {+ indexes = append(indexes, telemetryIndex{table: table, name: index})+ }+ indexes[len(indexes)-1].cols = append(indexes[len(indexes)-1].cols, col)+ }+ if err := rows.Err(); err != nil {+ t.Fatalf("failed to iterate indexes: %v", err)}+ return indexes}diff --git a/backend/app/repositories/telemetry/sqlite/exception_stack_trace.repository.go b/backend/app/repositories/telemetry/sqlite/exception_stack_trace.repository.goindex ab58b51a..baf066ff 100644--- a/backend/app/repositories/telemetry/sqlite/exception_stack_trace.repository.go+++ b/backend/app/repositories/telemetry/sqlite/exception_stack_trace.repository.go@@ -492,6 +492,9 @@ func (e *exceptionStackTraceRepository) FindByDistributedTraceId(ctx context.Con}// FindAllBySessionId returns all exceptions/messages stamped with the given session_id, ordered by time.+// The plain `session_id = ?` is what lets SQLite prove the partial idx_exceptions_session_recorded+// (WHERE session_id IS NOT NULL) applies; an IS, IN or COALESCE rewrite would fall back to the+// (project_id, recorded_at) index and re-filter the whole project.func (e *exceptionStackTraceRepository) FindAllBySessionId(ctx context.Context, projectId, sessionId uuid.UUID) ([]models.ExceptionStackTrace, error) {rows, err := lit.SelectNamed[exceptionRow](db.TelemetryDB,`SELECT id, project_id, trace_id, trace_type, exception_hash, stack_trace, recorded_at, attributes, app_version, server_name, is_message, distributed_trace_id, session_iddiff --git a/backend/app/retention/sqlite.go b/backend/app/retention/sqlite.goindex 1028ecb7..2935fbee 100644--- a/backend/app/retention/sqlite.go+++ b/backend/app/retention/sqlite.go@@ -31,6 +31,17 @@ var telemetryRetentionTargets = []struct {{"profiling_stacks", "last_seen"},}+// TelemetryTimeColumns maps each pruned telemetry table to its retention time+// column, which is also the column the migrations package's index guard+// expects project-leading indexes to reach.+func TelemetryTimeColumns() map[string]string {+ columns := make(map[string]string, len(telemetryRetentionTargets))+ for _, tgt := range telemetryRetentionTargets {+ columns[tgt.table] = tgt.column+ }+ return columns+}+func startSQLiteRetention(ctx context.Context, days int, source string) {if !db.IsSQLite() {return