Skip to content

The only tool you need to know what is happening and how to fix it.

perf: cover recorded_at in the per-group telemetry indexes (#334)

FrameAutomata0e7a3b2bf00a
diff --git a/backend/app/migrations/sqlite_telemetry/0020_index_telemetry_group_time.up.sql b/backend/app/migrations/sqlite_telemetry/0020_index_telemetry_group_time.up.sql
new file mode 100644
index 00000000..4d294a1a
--- /dev/null
+++ b/backend/app/migrations/sqlite_telemetry/0020_index_telemetry_group_time.up.sql
@@ -0,0 +1,5 @@
+DROP INDEX IF EXISTS idx_endpoints_project_endpoint;
+CREATE INDEX IF NOT EXISTS idx_endpoints_project_endpoint_recorded ON endpoints(project_id, endpoint, recorded_at);
+DROP INDEX IF EXISTS idx_ai_traces_project_trace_name;
+CREATE INDEX IF NOT EXISTS idx_ai_traces_project_trace_name_recorded ON ai_traces(project_id, trace_name, recorded_at);
+CREATE INDEX IF NOT EXISTS idx_tasks_project_task_name_recorded ON tasks(project_id, task_name, recorded_at);
diff --git a/backend/app/migrations/telemetry_group_index_test.go b/backend/app/migrations/telemetry_group_index_test.go
new file mode 100644
index 00000000..61ffc74b
--- /dev/null
+++ b/backend/app/migrations/telemetry_group_index_test.go
@@ -0,0 +1,49 @@
+//go:build !telemetry_ch && !telemetry_duckdb
+
+package migrations
+
+import (
+ "database/sql"
+ "testing"
+
+ _ "modernc.org/sqlite"
+)
+
+// The grouped-list reads for these three tables filter
+// (project_id, <group>, recorded_at). An index that stops at the group column
+// leaves SQLite to seek the (project_id, recorded_at) index and re-filter the
+// group column across the whole window once per group, which is quadratic in
+// the number of groups and invisible to functional tests. Guard the covering
+// indexes so a later migration cannot narrow or drop them unnoticed.
+func TestTelemetryGroupIndexesCoverRecordedAt(t *testing.T) {
+ telemetryDB, err := sql.Open("sqlite", ":memory:")
+ if err != nil {
+ t.Fatalf("failed to open in-memory sqlite: %v", err)
+ }
+ defer telemetryDB.Close()
+ telemetryDB.SetMaxOpenConns(1)
+
+ if err := runMigrationsOn(telemetryDB, migrationsSqliteTelemetryFS, "sqlite_telemetry", "schema_migrations", sqliteTrackingDDL); err != nil {
+ t.Fatalf("telemetry migrations failed: %v", err)
+ }
+
+ for _, tc := range []struct{ table, group string }{
+ {"endpoints", "endpoint"},
+ {"tasks", "task_name"},
+ {"ai_traces", "trace_name"},
+ } {
+ var covered bool
+ if err := telemetryDB.QueryRow(`
+ SELECT EXISTS (
+ SELECT 1 FROM pragma_index_list(?1) AS il
+ WHERE (SELECT name FROM pragma_index_info(il.name) WHERE seqno = 0) = 'project_id'
+ AND (SELECT name FROM pragma_index_info(il.name) WHERE seqno = 1) = ?2
+ AND (SELECT name FROM pragma_index_info(il.name) WHERE seqno = 2) = 'recorded_at'
+ )`, tc.table, tc.group).Scan(&covered); err != nil {
+ t.Fatalf("%s: failed to inspect indexes: %v", tc.table, err)
+ }
+ if !covered {
+ t.Errorf("%s: no index leading with (project_id, %s, recorded_at); per-group queries will scan the whole window once per group", tc.table, tc.group)
+ }
+ }
+}
diff --git a/backend/app/repositories/telemetry/testhelper_sqlite_test.go b/backend/app/repositories/telemetry/testhelper_sqlite_test.go
index 01f0421a..eafd9005 100644
--- a/backend/app/repositories/telemetry/testhelper_sqlite_test.go
+++ b/backend/app/repositories/telemetry/testhelper_sqlite_test.go
@@ -3,349 +3,12 @@
package telemetry
import (
- "database/sql"
"testing"
- "github.com/tracewayapp/lit/v2"
- "github.com/tracewayapp/traceway/backend/app/db"
- "github.com/tracewayapp/traceway/backend/app/models"
- _ "modernc.org/sqlite"
+ "github.com/tracewayapp/traceway/backend/app/dbtest"
)
func setupTestDB(t *testing.T) {
t.Helper()
-
- mainDB, err := sql.Open("sqlite", ":memory:")
- if err != nil {
- t.Fatalf("failed to open in-memory sqlite (main): %v", err)
- }
- mainDB.SetMaxOpenConns(1)
- if _, err := mainDB.Exec("PRAGMA foreign_keys = ON"); err != nil {
- t.Fatalf("failed to enable foreign keys: %v", err)
- }
- if _, err := mainDB.Exec("PRAGMA journal_mode = WAL"); err != nil {
- t.Fatalf("failed to set WAL mode: %v", err)
- }
-
- telemetryDB, err := sql.Open("sqlite", ":memory:")
- if err != nil {
- t.Fatalf("failed to open in-memory sqlite (telemetry): %v", err)
- }
- telemetryDB.SetMaxOpenConns(1)
- if _, err := telemetryDB.Exec("PRAGMA journal_mode = WAL"); err != nil {
- t.Fatalf("failed to set WAL mode (telemetry): %v", err)
- }
-
- db.DB = mainDB
- db.TelemetryDB = telemetryDB
- db.Driver = lit.SQLite
-
- models.Init(db.Driver)
-
- if err := runTelemetryMigrations(telemetryDB); err != nil {
- t.Fatalf("failed to run telemetry migrations: %v", err)
- }
-
- t.Cleanup(func() {
- mainDB.Close()
- telemetryDB.Close()
- })
-}
-
-func runTelemetryMigrations(telemetryDB *sql.DB) error {
- ddl := `
-CREATE TABLE IF NOT EXISTS endpoints (
- id TEXT NOT NULL,
- project_id TEXT NOT NULL,
- endpoint TEXT NOT NULL DEFAULT '',
- duration INTEGER NOT NULL DEFAULT 0,
- recorded_at DATETIME NOT NULL,
- status_code INTEGER NOT NULL DEFAULT 0,
- body_size INTEGER NOT NULL DEFAULT 0,
- client_ip TEXT NOT NULL DEFAULT '',
- attributes TEXT NOT NULL DEFAULT '{}',
- app_version TEXT NOT NULL DEFAULT '',
- server_name TEXT NOT NULL DEFAULT '',
- distributed_trace_id TEXT DEFAULT NULL,
- span_id TEXT DEFAULT NULL,
- is_stream INTEGER NOT NULL DEFAULT 0,
- is_root INTEGER NOT NULL DEFAULT 1
-);
-CREATE INDEX IF NOT EXISTS idx_endpoints_project_recorded ON endpoints(project_id, recorded_at);
-CREATE INDEX IF NOT EXISTS idx_endpoints_project_endpoint ON endpoints(project_id, endpoint);
-
-CREATE TABLE IF NOT EXISTS tasks (
- id TEXT NOT NULL,
- project_id TEXT NOT NULL,
- task_name TEXT NOT NULL DEFAULT '',
- duration INTEGER NOT NULL DEFAULT 0,
- recorded_at DATETIME NOT NULL,
- client_ip TEXT NOT NULL DEFAULT '',
- attributes TEXT NOT NULL DEFAULT '{}',
- app_version TEXT NOT NULL DEFAULT '',
- server_name TEXT NOT NULL DEFAULT '',
- distributed_trace_id TEXT DEFAULT NULL,
- span_id TEXT DEFAULT NULL,
- is_root INTEGER NOT NULL DEFAULT 1
-);
-CREATE INDEX IF NOT EXISTS idx_tasks_project_recorded ON tasks(project_id, recorded_at);
-
-CREATE TABLE IF NOT EXISTS ai_traces (
- id TEXT NOT NULL,
- project_id TEXT NOT NULL,
- recorded_at DATETIME NOT NULL,
- duration INTEGER NOT NULL DEFAULT 0,
- status_code INTEGER NOT NULL DEFAULT 0,
- model TEXT NOT NULL DEFAULT '',
- response_model TEXT NOT NULL DEFAULT '',
- provider TEXT NOT NULL DEFAULT '',
- operation TEXT NOT NULL DEFAULT '',
- input_tokens INTEGER NOT NULL DEFAULT 0,
- output_tokens INTEGER NOT NULL DEFAULT 0,
- total_tokens INTEGER NOT NULL DEFAULT 0,
- cached_tokens INTEGER NOT NULL DEFAULT 0,
- reasoning_tokens INTEGER NOT NULL DEFAULT 0,
- input_cost REAL NOT NULL DEFAULT 0,
- output_cost REAL NOT NULL DEFAULT 0,
- total_cost REAL NOT NULL DEFAULT 0,
- trace_name TEXT NOT NULL DEFAULT '',
- user_id TEXT NOT NULL DEFAULT '',
- finish_reason TEXT NOT NULL DEFAULT '',
- server_name TEXT NOT NULL DEFAULT '',
- app_version TEXT NOT NULL DEFAULT '',
- storage_key TEXT NOT NULL DEFAULT '',
- attributes TEXT NOT NULL DEFAULT '{}',
- distributed_trace_id TEXT DEFAULT NULL,
- is_root INTEGER NOT NULL DEFAULT 1,
- conversation_id TEXT NOT NULL DEFAULT '',
- tool_call_count INTEGER NOT NULL DEFAULT 0,
- tool_names TEXT NOT NULL DEFAULT '',
- flagged INTEGER NOT NULL DEFAULT 0,
- flagged_terms TEXT NOT NULL DEFAULT ''
-);
-CREATE INDEX IF NOT EXISTS idx_ai_traces_project_recorded ON ai_traces(project_id, recorded_at);
-CREATE INDEX IF NOT EXISTS idx_ai_traces_project_conversation ON ai_traces(project_id, conversation_id);
-
-CREATE TABLE IF NOT EXISTS exception_stack_traces (
- id TEXT NOT NULL,
- project_id TEXT NOT NULL,
- trace_id TEXT,
- trace_type TEXT NOT NULL DEFAULT 'endpoint',
- exception_hash TEXT NOT NULL DEFAULT '',
- stack_trace TEXT NOT NULL DEFAULT '',
- recorded_at DATETIME NOT NULL,
- attributes TEXT NOT NULL DEFAULT '{}',
- app_version TEXT NOT NULL DEFAULT '',
- server_name TEXT NOT NULL DEFAULT '',
- is_message INTEGER NOT NULL DEFAULT 0,
- distributed_trace_id TEXT DEFAULT NULL,
- session_id TEXT DEFAULT NULL
-);
-CREATE INDEX IF NOT EXISTS idx_exceptions_project_recorded ON exception_stack_traces(project_id, recorded_at);
-CREATE INDEX IF NOT EXISTS idx_exceptions_project_hash ON exception_stack_traces(project_id, exception_hash);
-
-CREATE TABLE IF NOT EXISTS spans (
- id TEXT NOT NULL,
- trace_id TEXT NOT NULL,
- project_id TEXT NOT NULL,
- name TEXT NOT NULL DEFAULT '',
- start_time DATETIME NOT NULL,
- duration INTEGER NOT NULL DEFAULT 0,
- recorded_at DATETIME NOT NULL,
- parent_span_id TEXT DEFAULT NULL,
- attributes TEXT NOT NULL DEFAULT '{}'
-);
-CREATE INDEX IF NOT EXISTS idx_spans_project_trace ON spans(project_id, trace_id);
-
-CREATE TABLE IF NOT EXISTS metric_points (
- project_id TEXT NOT NULL,
- name TEXT NOT NULL DEFAULT '',
- value REAL NOT NULL DEFAULT 0,
- tags TEXT NOT NULL DEFAULT '{}',
- recorded_at DATETIME NOT NULL
-);
-CREATE INDEX IF NOT EXISTS idx_metric_points_project_name ON metric_points(project_id, name, recorded_at);
-
-CREATE TABLE IF NOT EXISTS session_recordings (
- id TEXT NOT NULL,
- project_id TEXT NOT NULL,
- exception_id TEXT NOT NULL,
- file_path TEXT NOT NULL DEFAULT '',
- recorded_at DATETIME NOT NULL,
- session_id TEXT DEFAULT NULL,
- segment_index INTEGER NOT NULL DEFAULT 0
-);
-CREATE INDEX IF NOT EXISTS idx_session_recordings_project_exception ON session_recordings(project_id, exception_id);
-CREATE INDEX IF NOT EXISTS idx_session_recordings_session_id ON session_recordings(session_id);
-
-CREATE TABLE IF NOT EXISTS archived_exceptions (
- project_id TEXT NOT NULL,
- exception_hash TEXT NOT NULL,
- archived_at DATETIME NOT NULL DEFAULT (datetime('now')),
- UNIQUE(project_id, exception_hash)
-);
-
-CREATE TABLE IF NOT EXISTS slow_endpoints (
- project_id TEXT NOT NULL,
- endpoint TEXT NOT NULL,
- offset_ms INTEGER NOT NULL DEFAULT 0,
- reason TEXT NOT NULL DEFAULT '',
- UNIQUE(project_id, endpoint)
-);
-
-CREATE TABLE IF NOT EXISTS fired_notifications (
- project_id TEXT NOT NULL,
- rule_id INTEGER NOT NULL DEFAULT 0,
- rule_type TEXT NOT NULL DEFAULT '',
- rule_name TEXT NOT NULL DEFAULT '',
- channel_type TEXT NOT NULL DEFAULT '',
- channel_name TEXT NOT NULL DEFAULT '',
- severity TEXT NOT NULL DEFAULT '',
- subject TEXT NOT NULL DEFAULT '',
- body TEXT NOT NULL DEFAULT '',
- status TEXT NOT NULL DEFAULT '',
- error_message TEXT NOT NULL DEFAULT '',
- endpoint TEXT NOT NULL DEFAULT '',
- url TEXT NOT NULL DEFAULT '',
- fired_at DATETIME NOT NULL
-);
-
-CREATE TABLE IF NOT EXISTS profiling_stacks (
- project_id TEXT NOT NULL,
- service_name TEXT NOT NULL DEFAULT '',
- stack_hash INTEGER NOT NULL,
- stack TEXT NOT NULL DEFAULT '[]',
- last_seen DATETIME NOT NULL,
- UNIQUE(project_id, service_name, stack_hash)
-);
-
-CREATE TABLE IF NOT EXISTS profiling_samples (
- project_id TEXT NOT NULL,
- profile_id TEXT NOT NULL,
- service_name TEXT NOT NULL DEFAULT '',
- type TEXT NOT NULL DEFAULT '',
- unit TEXT NOT NULL DEFAULT '',
- is_gauge INTEGER NOT NULL DEFAULT 0,
- start_time DATETIME NOT NULL,
- end_time DATETIME NOT NULL,
- stack_hash INTEGER NOT NULL,
- value INTEGER NOT NULL DEFAULT 0,
- labels TEXT NOT NULL DEFAULT '{}',
- server_name TEXT NOT NULL DEFAULT '',
- app_version TEXT NOT NULL DEFAULT '',
- trace_id TEXT NOT NULL DEFAULT '',
- span_id TEXT NOT NULL DEFAULT ''
-);
-CREATE INDEX IF NOT EXISTS idx_profiling_samples_query ON profiling_samples(project_id, type, service_name, start_time);
-
-CREATE TABLE IF NOT EXISTS profiles (
- id TEXT NOT NULL,
- project_id TEXT NOT NULL,
- recorded_at DATETIME NOT NULL,
- duration INTEGER NOT NULL DEFAULT 0,
- service_name TEXT NOT NULL DEFAULT '',
- profile_type TEXT NOT NULL DEFAULT '',
- unit TEXT NOT NULL DEFAULT '',
- is_gauge INTEGER NOT NULL DEFAULT 0,
- sample_count INTEGER NOT NULL DEFAULT 0,
- total_value INTEGER NOT NULL DEFAULT 0,
- server_name TEXT NOT NULL DEFAULT '',
- app_version TEXT NOT NULL DEFAULT '',
- attributes TEXT NOT NULL DEFAULT '{}',
- storage_key TEXT NOT NULL DEFAULT '',
- trace_id TEXT NOT NULL DEFAULT '',
- span_id TEXT NOT NULL DEFAULT '',
- distributed_trace_id TEXT DEFAULT NULL
-);
-CREATE INDEX IF NOT EXISTS idx_profiles_project_recorded ON profiles(project_id, recorded_at);
-
-CREATE TABLE IF NOT EXISTS log_records (
- id TEXT NOT NULL,
- project_id TEXT NOT NULL,
- timestamp DATETIME NOT NULL,
- trace_id TEXT NOT NULL DEFAULT '',
- span_id TEXT NOT NULL DEFAULT '',
- trace_flags INTEGER NOT NULL DEFAULT 0,
- severity_text TEXT NOT NULL DEFAULT '',
- severity_number INTEGER NOT NULL DEFAULT 0,
- service_name TEXT NOT NULL DEFAULT '',
- body TEXT NOT NULL DEFAULT '',
- resource_schema_url TEXT NOT NULL DEFAULT '',
- resource_attributes TEXT NOT NULL DEFAULT '{}',
- scope_schema_url TEXT NOT NULL DEFAULT '',
- scope_name TEXT NOT NULL DEFAULT '',
- scope_version TEXT NOT NULL DEFAULT '',
- scope_attributes TEXT NOT NULL DEFAULT '{}',
- log_attributes TEXT NOT NULL DEFAULT '{}'
-);
-CREATE INDEX IF NOT EXISTS idx_log_records_project_timestamp ON log_records(project_id, timestamp);
-
-CREATE TABLE IF NOT EXISTS sessions (
- id TEXT NOT NULL PRIMARY KEY,
- project_id TEXT NOT NULL,
- started_at DATETIME NOT NULL,
- ended_at DATETIME,
- duration INTEGER NOT NULL DEFAULT 0,
- client_ip TEXT NOT NULL DEFAULT '',
- attributes TEXT NOT NULL DEFAULT '{}',
- app_version TEXT NOT NULL DEFAULT '',
- server_name TEXT NOT NULL DEFAULT '',
- distributed_trace_id TEXT DEFAULT NULL
-);
-CREATE INDEX IF NOT EXISTS idx_sessions_project_started ON sessions(project_id, started_at);
-`
-
- statements := splitStatements(ddl)
- for _, stmt := range statements {
- if _, err := telemetryDB.Exec(stmt); err != nil {
- return err
- }
- }
- return nil
-}
-
-func splitStatements(sql string) []string {
- var stmts []string
- current := ""
- for _, line := range splitLines(sql) {
- trimmed := trimSpace(line)
- if trimmed == "" {
- continue
- }
- current += line + "\n"
- if len(trimmed) > 0 && trimmed[len(trimmed)-1] == ';' {
- stmts = append(stmts, current)
- current = ""
- }
- }
- if trimSpace(current) != "" {
- stmts = append(stmts, current)
- }
- return stmts
-}
-
-func splitLines(s string) []string {
- var lines []string
- start := 0
- for i := 0; i < len(s); i++ {
- if s[i] == '\n' {
- lines = append(lines, s[start:i])
- start = i + 1
- }
- }
- if start < len(s) {
- lines = append(lines, s[start:])
- }
- return lines
-}
-
-func trimSpace(s string) string {
- start, end := 0, len(s)
- for start < end && (s[start] == ' ' || s[start] == '\t' || s[start] == '\r' || s[start] == '\n') {
- start++
- }
- for end > start && (s[end-1] == ' ' || s[end-1] == '\t' || s[end-1] == '\r' || s[end-1] == '\n') {
- end--
- }
- return s[start:end]
+ dbtest.SetupSQLite(t)
}