The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
perf: cover recorded_at in the per-group telemetry indexes (#334)
FrameAutomata0e7a3b2bf00a
diff --git a/backend/app/migrations/sqlite_telemetry/0020_index_telemetry_group_time.up.sql b/backend/app/migrations/sqlite_telemetry/0020_index_telemetry_group_time.up.sqlnew file mode 100644index 00000000..4d294a1a--- /dev/null+++ b/backend/app/migrations/sqlite_telemetry/0020_index_telemetry_group_time.up.sql@@ -0,0 +1,5 @@+DROP INDEX IF EXISTS idx_endpoints_project_endpoint;+CREATE INDEX IF NOT EXISTS idx_endpoints_project_endpoint_recorded ON endpoints(project_id, endpoint, recorded_at);+DROP INDEX IF EXISTS idx_ai_traces_project_trace_name;+CREATE INDEX IF NOT EXISTS idx_ai_traces_project_trace_name_recorded ON ai_traces(project_id, trace_name, recorded_at);+CREATE INDEX IF NOT EXISTS idx_tasks_project_task_name_recorded ON tasks(project_id, task_name, recorded_at);diff --git a/backend/app/migrations/telemetry_group_index_test.go b/backend/app/migrations/telemetry_group_index_test.gonew file mode 100644index 00000000..61ffc74b--- /dev/null+++ b/backend/app/migrations/telemetry_group_index_test.go@@ -0,0 +1,49 @@+//go:build !telemetry_ch && !telemetry_duckdb++package migrations++import (+ "database/sql"+ "testing"++ _ "modernc.org/sqlite"+)++// The grouped-list reads for these three tables filter+// (project_id, <group>, recorded_at). An index that stops at the group column+// leaves SQLite to seek the (project_id, recorded_at) index and re-filter the+// group column across the whole window once per group, which is quadratic in+// the number of groups and invisible to functional tests. Guard the covering+// indexes so a later migration cannot narrow or drop them unnoticed.+func TestTelemetryGroupIndexesCoverRecordedAt(t *testing.T) {+ telemetryDB, err := sql.Open("sqlite", ":memory:")+ if err != nil {+ t.Fatalf("failed to open in-memory sqlite: %v", err)+ }+ defer telemetryDB.Close()+ telemetryDB.SetMaxOpenConns(1)++ if err := runMigrationsOn(telemetryDB, migrationsSqliteTelemetryFS, "sqlite_telemetry", "schema_migrations", sqliteTrackingDDL); err != nil {+ t.Fatalf("telemetry migrations failed: %v", err)+ }++ for _, tc := range []struct{ table, group string }{+ {"endpoints", "endpoint"},+ {"tasks", "task_name"},+ {"ai_traces", "trace_name"},+ } {+ var covered bool+ if err := telemetryDB.QueryRow(`+ SELECT EXISTS (+ SELECT 1 FROM pragma_index_list(?1) AS il+ WHERE (SELECT name FROM pragma_index_info(il.name) WHERE seqno = 0) = 'project_id'+ AND (SELECT name FROM pragma_index_info(il.name) WHERE seqno = 1) = ?2+ AND (SELECT name FROM pragma_index_info(il.name) WHERE seqno = 2) = 'recorded_at'+ )`, tc.table, tc.group).Scan(&covered); err != nil {+ t.Fatalf("%s: failed to inspect indexes: %v", tc.table, err)+ }+ if !covered {+ t.Errorf("%s: no index leading with (project_id, %s, recorded_at); per-group queries will scan the whole window once per group", tc.table, tc.group)+ }+ }+}diff --git a/backend/app/repositories/telemetry/testhelper_sqlite_test.go b/backend/app/repositories/telemetry/testhelper_sqlite_test.goindex 01f0421a..eafd9005 100644--- a/backend/app/repositories/telemetry/testhelper_sqlite_test.go+++ b/backend/app/repositories/telemetry/testhelper_sqlite_test.go@@ -3,349 +3,12 @@package telemetryimport (- "database/sql""testing"- "github.com/tracewayapp/lit/v2"- "github.com/tracewayapp/traceway/backend/app/db"- "github.com/tracewayapp/traceway/backend/app/models"- _ "modernc.org/sqlite"+ "github.com/tracewayapp/traceway/backend/app/dbtest")func setupTestDB(t *testing.T) {t.Helper()-- mainDB, err := sql.Open("sqlite", ":memory:")- if err != nil {- t.Fatalf("failed to open in-memory sqlite (main): %v", err)- }- mainDB.SetMaxOpenConns(1)- if _, err := mainDB.Exec("PRAGMA foreign_keys = ON"); err != nil {- t.Fatalf("failed to enable foreign keys: %v", err)- }- if _, err := mainDB.Exec("PRAGMA journal_mode = WAL"); err != nil {- t.Fatalf("failed to set WAL mode: %v", err)- }-- telemetryDB, err := sql.Open("sqlite", ":memory:")- if err != nil {- t.Fatalf("failed to open in-memory sqlite (telemetry): %v", err)- }- telemetryDB.SetMaxOpenConns(1)- if _, err := telemetryDB.Exec("PRAGMA journal_mode = WAL"); err != nil {- t.Fatalf("failed to set WAL mode (telemetry): %v", err)- }-- db.DB = mainDB- db.TelemetryDB = telemetryDB- db.Driver = lit.SQLite-- models.Init(db.Driver)-- if err := runTelemetryMigrations(telemetryDB); err != nil {- t.Fatalf("failed to run telemetry migrations: %v", err)- }-- t.Cleanup(func() {- mainDB.Close()- telemetryDB.Close()- })-}--func runTelemetryMigrations(telemetryDB *sql.DB) error {- ddl := `-CREATE TABLE IF NOT EXISTS endpoints (- id TEXT NOT NULL,- project_id TEXT NOT NULL,- endpoint TEXT NOT NULL DEFAULT '',- duration INTEGER NOT NULL DEFAULT 0,- recorded_at DATETIME NOT NULL,- status_code INTEGER NOT NULL DEFAULT 0,- body_size INTEGER NOT NULL DEFAULT 0,- client_ip TEXT NOT NULL DEFAULT '',- attributes TEXT NOT NULL DEFAULT '{}',- app_version TEXT NOT NULL DEFAULT '',- server_name TEXT NOT NULL DEFAULT '',- distributed_trace_id TEXT DEFAULT NULL,- span_id TEXT DEFAULT NULL,- is_stream INTEGER NOT NULL DEFAULT 0,- is_root INTEGER NOT NULL DEFAULT 1-);-CREATE INDEX IF NOT EXISTS idx_endpoints_project_recorded ON endpoints(project_id, recorded_at);-CREATE INDEX IF NOT EXISTS idx_endpoints_project_endpoint ON endpoints(project_id, endpoint);--CREATE TABLE IF NOT EXISTS tasks (- id TEXT NOT NULL,- project_id TEXT NOT NULL,- task_name TEXT NOT NULL DEFAULT '',- duration INTEGER NOT NULL DEFAULT 0,- recorded_at DATETIME NOT NULL,- client_ip TEXT NOT NULL DEFAULT '',- attributes TEXT NOT NULL DEFAULT '{}',- app_version TEXT NOT NULL DEFAULT '',- server_name TEXT NOT NULL DEFAULT '',- distributed_trace_id TEXT DEFAULT NULL,- span_id TEXT DEFAULT NULL,- is_root INTEGER NOT NULL DEFAULT 1-);-CREATE INDEX IF NOT EXISTS idx_tasks_project_recorded ON tasks(project_id, recorded_at);--CREATE TABLE IF NOT EXISTS ai_traces (- id TEXT NOT NULL,- project_id TEXT NOT NULL,- recorded_at DATETIME NOT NULL,- duration INTEGER NOT NULL DEFAULT 0,- status_code INTEGER NOT NULL DEFAULT 0,- model TEXT NOT NULL DEFAULT '',- response_model TEXT NOT NULL DEFAULT '',- provider TEXT NOT NULL DEFAULT '',- operation TEXT NOT NULL DEFAULT '',- input_tokens INTEGER NOT NULL DEFAULT 0,- output_tokens INTEGER NOT NULL DEFAULT 0,- total_tokens INTEGER NOT NULL DEFAULT 0,- cached_tokens INTEGER NOT NULL DEFAULT 0,- reasoning_tokens INTEGER NOT NULL DEFAULT 0,- input_cost REAL NOT NULL DEFAULT 0,- output_cost REAL NOT NULL DEFAULT 0,- total_cost REAL NOT NULL DEFAULT 0,- trace_name TEXT NOT NULL DEFAULT '',- user_id TEXT NOT NULL DEFAULT '',- finish_reason TEXT NOT NULL DEFAULT '',- server_name TEXT NOT NULL DEFAULT '',- app_version TEXT NOT NULL DEFAULT '',- storage_key TEXT NOT NULL DEFAULT '',- attributes TEXT NOT NULL DEFAULT '{}',- distributed_trace_id TEXT DEFAULT NULL,- is_root INTEGER NOT NULL DEFAULT 1,- conversation_id TEXT NOT NULL DEFAULT '',- tool_call_count INTEGER NOT NULL DEFAULT 0,- tool_names TEXT NOT NULL DEFAULT '',- flagged INTEGER NOT NULL DEFAULT 0,- flagged_terms TEXT NOT NULL DEFAULT ''-);-CREATE INDEX IF NOT EXISTS idx_ai_traces_project_recorded ON ai_traces(project_id, recorded_at);-CREATE INDEX IF NOT EXISTS idx_ai_traces_project_conversation ON ai_traces(project_id, conversation_id);--CREATE TABLE IF NOT EXISTS exception_stack_traces (- id TEXT NOT NULL,- project_id TEXT NOT NULL,- trace_id TEXT,- trace_type TEXT NOT NULL DEFAULT 'endpoint',- exception_hash TEXT NOT NULL DEFAULT '',- stack_trace TEXT NOT NULL DEFAULT '',- recorded_at DATETIME NOT NULL,- attributes TEXT NOT NULL DEFAULT '{}',- app_version TEXT NOT NULL DEFAULT '',- server_name TEXT NOT NULL DEFAULT '',- is_message INTEGER NOT NULL DEFAULT 0,- distributed_trace_id TEXT DEFAULT NULL,- session_id TEXT DEFAULT NULL-);-CREATE INDEX IF NOT EXISTS idx_exceptions_project_recorded ON exception_stack_traces(project_id, recorded_at);-CREATE INDEX IF NOT EXISTS idx_exceptions_project_hash ON exception_stack_traces(project_id, exception_hash);--CREATE TABLE IF NOT EXISTS spans (- id TEXT NOT NULL,- trace_id TEXT NOT NULL,- project_id TEXT NOT NULL,- name TEXT NOT NULL DEFAULT '',- start_time DATETIME NOT NULL,- duration INTEGER NOT NULL DEFAULT 0,- recorded_at DATETIME NOT NULL,- parent_span_id TEXT DEFAULT NULL,- attributes TEXT NOT NULL DEFAULT '{}'-);-CREATE INDEX IF NOT EXISTS idx_spans_project_trace ON spans(project_id, trace_id);--CREATE TABLE IF NOT EXISTS metric_points (- project_id TEXT NOT NULL,- name TEXT NOT NULL DEFAULT '',- value REAL NOT NULL DEFAULT 0,- tags TEXT NOT NULL DEFAULT '{}',- recorded_at DATETIME NOT NULL-);-CREATE INDEX IF NOT EXISTS idx_metric_points_project_name ON metric_points(project_id, name, recorded_at);--CREATE TABLE IF NOT EXISTS session_recordings (- id TEXT NOT NULL,- project_id TEXT NOT NULL,- exception_id TEXT NOT NULL,- file_path TEXT NOT NULL DEFAULT '',- recorded_at DATETIME NOT NULL,- session_id TEXT DEFAULT NULL,- segment_index INTEGER NOT NULL DEFAULT 0-);-CREATE INDEX IF NOT EXISTS idx_session_recordings_project_exception ON session_recordings(project_id, exception_id);-CREATE INDEX IF NOT EXISTS idx_session_recordings_session_id ON session_recordings(session_id);--CREATE TABLE IF NOT EXISTS archived_exceptions (- project_id TEXT NOT NULL,- exception_hash TEXT NOT NULL,- archived_at DATETIME NOT NULL DEFAULT (datetime('now')),- UNIQUE(project_id, exception_hash)-);--CREATE TABLE IF NOT EXISTS slow_endpoints (- project_id TEXT NOT NULL,- endpoint TEXT NOT NULL,- offset_ms INTEGER NOT NULL DEFAULT 0,- reason TEXT NOT NULL DEFAULT '',- UNIQUE(project_id, endpoint)-);--CREATE TABLE IF NOT EXISTS fired_notifications (- project_id TEXT NOT NULL,- rule_id INTEGER NOT NULL DEFAULT 0,- rule_type TEXT NOT NULL DEFAULT '',- rule_name TEXT NOT NULL DEFAULT '',- channel_type TEXT NOT NULL DEFAULT '',- channel_name TEXT NOT NULL DEFAULT '',- severity TEXT NOT NULL DEFAULT '',- subject TEXT NOT NULL DEFAULT '',- body TEXT NOT NULL DEFAULT '',- status TEXT NOT NULL DEFAULT '',- error_message TEXT NOT NULL DEFAULT '',- endpoint TEXT NOT NULL DEFAULT '',- url TEXT NOT NULL DEFAULT '',- fired_at DATETIME NOT NULL-);--CREATE TABLE IF NOT EXISTS profiling_stacks (- project_id TEXT NOT NULL,- service_name TEXT NOT NULL DEFAULT '',- stack_hash INTEGER NOT NULL,- stack TEXT NOT NULL DEFAULT '[]',- last_seen DATETIME NOT NULL,- UNIQUE(project_id, service_name, stack_hash)-);--CREATE TABLE IF NOT EXISTS profiling_samples (- project_id TEXT NOT NULL,- profile_id TEXT NOT NULL,- service_name TEXT NOT NULL DEFAULT '',- type TEXT NOT NULL DEFAULT '',- unit TEXT NOT NULL DEFAULT '',- is_gauge INTEGER NOT NULL DEFAULT 0,- start_time DATETIME NOT NULL,- end_time DATETIME NOT NULL,- stack_hash INTEGER NOT NULL,- value INTEGER NOT NULL DEFAULT 0,- labels TEXT NOT NULL DEFAULT '{}',- server_name TEXT NOT NULL DEFAULT '',- app_version TEXT NOT NULL DEFAULT '',- trace_id TEXT NOT NULL DEFAULT '',- span_id TEXT NOT NULL DEFAULT ''-);-CREATE INDEX IF NOT EXISTS idx_profiling_samples_query ON profiling_samples(project_id, type, service_name, start_time);--CREATE TABLE IF NOT EXISTS profiles (- id TEXT NOT NULL,- project_id TEXT NOT NULL,- recorded_at DATETIME NOT NULL,- duration INTEGER NOT NULL DEFAULT 0,- service_name TEXT NOT NULL DEFAULT '',- profile_type TEXT NOT NULL DEFAULT '',- unit TEXT NOT NULL DEFAULT '',- is_gauge INTEGER NOT NULL DEFAULT 0,- sample_count INTEGER NOT NULL DEFAULT 0,- total_value INTEGER NOT NULL DEFAULT 0,- server_name TEXT NOT NULL DEFAULT '',- app_version TEXT NOT NULL DEFAULT '',- attributes TEXT NOT NULL DEFAULT '{}',- storage_key TEXT NOT NULL DEFAULT '',- trace_id TEXT NOT NULL DEFAULT '',- span_id TEXT NOT NULL DEFAULT '',- distributed_trace_id TEXT DEFAULT NULL-);-CREATE INDEX IF NOT EXISTS idx_profiles_project_recorded ON profiles(project_id, recorded_at);--CREATE TABLE IF NOT EXISTS log_records (- id TEXT NOT NULL,- project_id TEXT NOT NULL,- timestamp DATETIME NOT NULL,- trace_id TEXT NOT NULL DEFAULT '',- span_id TEXT NOT NULL DEFAULT '',- trace_flags INTEGER NOT NULL DEFAULT 0,- severity_text TEXT NOT NULL DEFAULT '',- severity_number INTEGER NOT NULL DEFAULT 0,- service_name TEXT NOT NULL DEFAULT '',- body TEXT NOT NULL DEFAULT '',- resource_schema_url TEXT NOT NULL DEFAULT '',- resource_attributes TEXT NOT NULL DEFAULT '{}',- scope_schema_url TEXT NOT NULL DEFAULT '',- scope_name TEXT NOT NULL DEFAULT '',- scope_version TEXT NOT NULL DEFAULT '',- scope_attributes TEXT NOT NULL DEFAULT '{}',- log_attributes TEXT NOT NULL DEFAULT '{}'-);-CREATE INDEX IF NOT EXISTS idx_log_records_project_timestamp ON log_records(project_id, timestamp);--CREATE TABLE IF NOT EXISTS sessions (- id TEXT NOT NULL PRIMARY KEY,- project_id TEXT NOT NULL,- started_at DATETIME NOT NULL,- ended_at DATETIME,- duration INTEGER NOT NULL DEFAULT 0,- client_ip TEXT NOT NULL DEFAULT '',- attributes TEXT NOT NULL DEFAULT '{}',- app_version TEXT NOT NULL DEFAULT '',- server_name TEXT NOT NULL DEFAULT '',- distributed_trace_id TEXT DEFAULT NULL-);-CREATE INDEX IF NOT EXISTS idx_sessions_project_started ON sessions(project_id, started_at);-`-- statements := splitStatements(ddl)- for _, stmt := range statements {- if _, err := telemetryDB.Exec(stmt); err != nil {- return err- }- }- return nil-}--func splitStatements(sql string) []string {- var stmts []string- current := ""- for _, line := range splitLines(sql) {- trimmed := trimSpace(line)- if trimmed == "" {- continue- }- current += line + "\n"- if len(trimmed) > 0 && trimmed[len(trimmed)-1] == ';' {- stmts = append(stmts, current)- current = ""- }- }- if trimSpace(current) != "" {- stmts = append(stmts, current)- }- return stmts-}--func splitLines(s string) []string {- var lines []string- start := 0- for i := 0; i < len(s); i++ {- if s[i] == '\n' {- lines = append(lines, s[start:i])- start = i + 1- }- }- if start < len(s) {- lines = append(lines, s[start:])- }- return lines-}--func trimSpace(s string) string {- start, end := 0, len(s)- for start < end && (s[start] == ' ' || s[start] == '\t' || s[start] == '\r' || s[start] == '\n') {- start++- }- for end > start && (s[end-1] == ' ' || s[end-1] == '\t' || s[end-1] == '\r' || s[end-1] == '\n') {- end--- }- return s[start:end]+ dbtest.SetupSQLite(t)}