Skip to content

The only tool you need to know what is happening and how to fix it.

perf: invert the group-index guard and index the session, log-trace and notification-history reads (#354)

FrameAutomata3bf49cda7001
diff --git a/backend/app/migrations/sqlite_telemetry/0021_index_session_and_log_trace_time.up.sql b/backend/app/migrations/sqlite_telemetry/0021_index_session_and_log_trace_time.up.sql
new file mode 100644
index 00000000..e4f9256d
--- /dev/null
+++ b/backend/app/migrations/sqlite_telemetry/0021_index_session_and_log_trace_time.up.sql
@@ -0,0 +1,4 @@
+CREATE INDEX IF NOT EXISTS idx_exceptions_session_recorded ON exception_stack_traces(project_id, session_id, recorded_at) WHERE session_id IS NOT NULL;
+DROP INDEX IF EXISTS idx_exceptions_session;
+CREATE INDEX IF NOT EXISTS idx_log_records_project_trace_timestamp ON log_records(project_id, trace_id, timestamp);
+DROP INDEX IF EXISTS idx_log_records_project_trace;
diff --git a/backend/app/migrations/sqlite_telemetry/0022_index_fired_notifications_project_fired.up.sql b/backend/app/migrations/sqlite_telemetry/0022_index_fired_notifications_project_fired.up.sql
new file mode 100644
index 00000000..fea7033e
--- /dev/null
+++ b/backend/app/migrations/sqlite_telemetry/0022_index_fired_notifications_project_fired.up.sql
@@ -0,0 +1 @@
+CREATE INDEX IF NOT EXISTS idx_fired_notifications_project_fired ON fired_notifications(project_id, fired_at);
diff --git a/backend/app/migrations/telemetry_group_index_test.go b/backend/app/migrations/telemetry_group_index_test.go
index 343c2acb..ba76bb77 100644
--- a/backend/app/migrations/telemetry_group_index_test.go
+++ b/backend/app/migrations/telemetry_group_index_test.go
@@ -4,18 +4,27 @@ package migrations
import (
"database/sql"
+ "slices"
+ "strings"
"testing"
+ "github.com/tracewayapp/traceway/backend/app/retention"
_ "modernc.org/sqlite"
)
// Every per-group telemetry read pairs a group column with the table's time
-// column -- as a second filter on the grouped lists, as the ORDER BY on the
-// per-group detail reads. An index that stops at the group column forces SQLite
-// to either re-filter the group across the whole window once per group, which is
-// quadratic in the number of groups, or seek the group and sort its entire
-// history. Both are unbounded and both are invisible to functional tests. Guard
-// the covering indexes so a later migration cannot narrow or drop one unnoticed.
+// column: as a second filter on the grouped lists, as the ORDER BY on the
+// per-group detail reads. An index that stops at the group column leaves SQLite
+// to either re-filter the group across the whole window once per group, which
+// is quadratic in the number of groups, or seek the group and sort its entire
+// history. Both are unbounded and invisible to functional tests.
+//
+// The first loop guards the indexes behind known reads so a later migration
+// cannot narrow or drop one unnoticed. The second inverts the question: every
+// index that leads with project_id must reach its table's time column, so a
+// short index fails here the day it lands rather than when somebody remembers
+// to list it. Indexes that stop short on purpose are named with the reason,
+// and each entry is itself checked so it cannot outlive the index it excuses.
func TestTelemetryGroupIndexesCoverTimeColumn(t *testing.T) {
telemetryDB, err := sql.Open("sqlite", ":memory:")
if err != nil {
@@ -28,28 +37,94 @@ func TestTelemetryGroupIndexesCoverTimeColumn(t *testing.T) {
t.Fatalf("telemetry migrations failed: %v", err)
}
- for _, tc := range []struct{ table, group, timeCol string }{
- {"endpoints", "endpoint", "recorded_at"},
- {"tasks", "task_name", "recorded_at"},
- {"ai_traces", "trace_name", "recorded_at"},
- {"ai_traces", "conversation_id", "recorded_at"},
- {"exception_stack_traces", "exception_hash", "recorded_at"},
- {"check_results", "check_id", "recorded_at"},
- {"log_records", "service_name", "timestamp"},
- {"metric_points", "name", "recorded_at"},
+ timeColumn := retention.TelemetryTimeColumns()
+ indexes := createdIndexes(t, telemetryDB)
+
+ for _, tc := range []struct{ table, group string }{
+ {"endpoints", "endpoint"},
+ {"tasks", "task_name"},
+ {"ai_traces", "trace_name"},
+ {"ai_traces", "conversation_id"},
+ {"exception_stack_traces", "exception_hash"},
+ {"exception_stack_traces", "session_id"},
+ {"check_results", "check_id"},
+ {"log_records", "trace_id"},
+ {"log_records", "service_name"},
+ {"metric_points", "name"},
} {
- var covered bool
- if err := telemetryDB.QueryRow(`
- SELECT EXISTS (
- SELECT 1 FROM pragma_index_list(?1) AS il
- WHERE (SELECT name FROM pragma_index_info(il.name) WHERE seqno = 0) = 'project_id'
- AND (SELECT name FROM pragma_index_info(il.name) WHERE seqno = 1) = ?2
- AND (SELECT name FROM pragma_index_info(il.name) WHERE seqno = 2) = ?3
- )`, tc.table, tc.group, tc.timeCol).Scan(&covered); err != nil {
- t.Fatalf("%s: failed to inspect indexes: %v", tc.table, err)
+ timeCol, pruned := timeColumn[tc.table]
+ if !pruned {
+ t.Fatalf("%s is not a retention target, so it has no time column to guard", tc.table)
}
+ want := []string{"project_id", tc.group, timeCol}
+ covered := slices.ContainsFunc(indexes, func(ix telemetryIndex) bool {
+ return ix.table == tc.table && len(ix.cols) >= len(want) && slices.Equal(ix.cols[:len(want)], want)
+ })
if !covered {
- t.Errorf("%s: no index leading with (project_id, %s, %s)", tc.table, tc.group, tc.timeCol)
+ t.Errorf("%s: no index leading with (%s)", tc.table, strings.Join(want, ", "))
+ }
+ }
+
+ intentionallyShort := map[string]string{
+ "idx_spans_project_trace": "a trace has tens of spans; after the trace seek, the recorded_at window and the start_time sort touch only those rows",
+ "idx_session_recordings_project_exception": "an exception has at most a handful of recordings, so ordering them costs nothing",
+ }
+ for _, ix := range indexes {
+ timeCol, pruned := timeColumn[ix.table]
+ if ix.cols[0] != "project_id" || !pruned {
+ continue
+ }
+ _, listed := intentionallyShort[ix.name]
+ delete(intentionallyShort, ix.name)
+ reaches := slices.Contains(ix.cols[1:], timeCol)
+ shape := ix.table + "(" + strings.Join(ix.cols, ", ") + ")"
+ switch {
+ case reaches && listed:
+ t.Errorf("%s on %s reaches %s now; remove it from intentionallyShort", ix.name, shape, timeCol)
+ case !reaches && !listed:
+ t.Errorf("%s on %s leads with project_id but never reaches %s; widen it, or add it to intentionallyShort with the reason", ix.name, shape, timeCol)
+ }
+ }
+ for index := range intentionallyShort {
+ t.Errorf("intentionallyShort names %s, which is no longer an index leading with project_id on a pruned table; remove the entry", index)
+ }
+}
+
+type telemetryIndex struct {
+ table, name string
+ cols []string
+}
+
+// createdIndexes lists every CREATE INDEX index with its columns in order. An
+// expression column has no name and is listed as "". UNIQUE and PRIMARY KEY
+// autoindexes are left out: their column set is dictated by the constraint,
+// not by a read pattern.
+func createdIndexes(t *testing.T, telemetryDB *sql.DB) []telemetryIndex {
+ t.Helper()
+ rows, err := telemetryDB.Query(`
+ SELECT m.name, il.name, COALESCE(ii.name, '') FROM sqlite_master AS m
+ JOIN pragma_index_list(m.name) AS il
+ JOIN pragma_index_info(il.name) AS ii
+ WHERE m.type = 'table' AND il.origin = 'c'
+ ORDER BY m.name, il.name, ii.seqno`)
+ if err != nil {
+ t.Fatalf("failed to list indexes: %v", err)
+ }
+ defer rows.Close()
+
+ var indexes []telemetryIndex
+ for rows.Next() {
+ var table, index, col string
+ if err := rows.Scan(&table, &index, &col); err != nil {
+ t.Fatalf("failed to scan index column: %v", err)
}
+ if n := len(indexes); n == 0 || indexes[n-1].name != index {
+ indexes = append(indexes, telemetryIndex{table: table, name: index})
+ }
+ indexes[len(indexes)-1].cols = append(indexes[len(indexes)-1].cols, col)
+ }
+ if err := rows.Err(); err != nil {
+ t.Fatalf("failed to iterate indexes: %v", err)
}
+ return indexes
}
diff --git a/backend/app/repositories/telemetry/sqlite/exception_stack_trace.repository.go b/backend/app/repositories/telemetry/sqlite/exception_stack_trace.repository.go
index ab58b51a..baf066ff 100644
--- a/backend/app/repositories/telemetry/sqlite/exception_stack_trace.repository.go
+++ b/backend/app/repositories/telemetry/sqlite/exception_stack_trace.repository.go
@@ -492,6 +492,9 @@ func (e *exceptionStackTraceRepository) FindByDistributedTraceId(ctx context.Con
}
// FindAllBySessionId returns all exceptions/messages stamped with the given session_id, ordered by time.
+// The plain `session_id = ?` is what lets SQLite prove the partial idx_exceptions_session_recorded
+// (WHERE session_id IS NOT NULL) applies; an IS, IN or COALESCE rewrite would fall back to the
+// (project_id, recorded_at) index and re-filter the whole project.
func (e *exceptionStackTraceRepository) FindAllBySessionId(ctx context.Context, projectId, sessionId uuid.UUID) ([]models.ExceptionStackTrace, error) {
rows, err := lit.SelectNamed[exceptionRow](db.TelemetryDB,
`SELECT id, project_id, trace_id, trace_type, exception_hash, stack_trace, recorded_at, attributes, app_version, server_name, is_message, distributed_trace_id, session_id
diff --git a/backend/app/retention/sqlite.go b/backend/app/retention/sqlite.go
index 1028ecb7..2935fbee 100644
--- a/backend/app/retention/sqlite.go
+++ b/backend/app/retention/sqlite.go
@@ -31,6 +31,17 @@ var telemetryRetentionTargets = []struct {
{"profiling_stacks", "last_seen"},
}
+// TelemetryTimeColumns maps each pruned telemetry table to its retention time
+// column, which is also the column the migrations package's index guard
+// expects project-leading indexes to reach.
+func TelemetryTimeColumns() map[string]string {
+ columns := make(map[string]string, len(telemetryRetentionTargets))
+ for _, tgt := range telemetryRetentionTargets {
+ columns[tgt.table] = tgt.column
+ }
+ return columns
+}
+
func startSQLiteRetention(ctx context.Context, days int, source string) {
if !db.IsSQLite() {
return