The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
fix: Distributed-trace log search on sqlite, closes #353
narrow-adventure97d66fe22c09
diff --git a/backend/app/repositories/telemetry/log_record_repository_test.go b/backend/app/repositories/telemetry/log_record_repository_test.goindex 1d717e4f..ac18ae3a 100644--- a/backend/app/repositories/telemetry/log_record_repository_test.go+++ b/backend/app/repositories/telemetry/log_record_repository_test.go@@ -2,6 +2,7 @@ package telemetryimport ("context"+ "fmt""testing""time"@@ -105,3 +106,120 @@ func TestLogRecordRepository_SearchPaginationTotal(t *testing.T) {t.Errorf("expected no rows and total 0, got %d rows total %d", len(rows), total)}}++func searchBodies(t *testing.T, params shared.LogSearchParams) ([]string, int64) {+ t.Helper()+ rows, total, err := LogRecordRepository.Search(context.Background(), params)+ if err != nil {+ t.Fatalf("Search page %d failed: %v", params.Page, err)+ }+ bodies := make([]string, 0, len(rows))+ for _, r := range rows {+ bodies = append(bodies, r.Body)+ }+ return bodies, total+}++func assertBodies(t *testing.T, label string, got, want []string) {+ t.Helper()+ if fmt.Sprint(got) != fmt.Sprint(want) {+ t.Errorf("%s: expected %v, got %v", label, want, got)+ }+}++func TestLogRecordRepository_SearchByTraceIdsMergesAndPaginates(t *testing.T) {+ setupTestDB(t)+ ctx := context.Background()+ projectId := uuid.New()+ now := truncateMs(time.Now().UTC())++ var records []models.LogRecord+ for i := 0; i < 7; i++ {+ rec := makeLogRecord(projectId, 9, fmt.Sprintf("row %d", i), now.Add(time.Duration(i)*time.Second))+ rec.TraceId = []string{"aaaa", "bbbb"}[i%2]+ records = append(records, rec)+ }+ unrelated := makeLogRecord(projectId, 9, "unrelated", now.Add(10*time.Second))+ unrelated.TraceId = "cccc"+ records = append(records, unrelated)+ if err := LogRecordRepository.InsertAsync(ctx, records); err != nil {+ t.Fatalf("InsertAsync failed: %v", err)+ }++ base := shared.LogSearchParams{+ ProjectId: projectId,+ FromDate: now.Add(-time.Hour),+ ToDate: now.Add(time.Hour),+ TraceIds: []string{"aaaa", "bbbb", "dddd"},+ PageSize: 3,+ }++ var got []string+ for page := 1; page <= 3; page++ {+ params := base+ params.Page = page+ bodies, total := searchBodies(t, params)+ if total != 7 {+ t.Errorf("page %d: expected total 7, got %d", page, total)+ }+ got = append(got, bodies...)+ }+ assertBodies(t, "desc pages", got, []string{"row 6", "row 5", "row 4", "row 3", "row 2", "row 1", "row 0"})++ params := base+ params.Page = 2+ params.SortDirection = "asc"+ bodies, _ := searchBodies(t, params)+ assertBodies(t, "asc page 2", bodies, []string{"row 3", "row 4", "row 5"})++ params = base+ params.Page = 1+ params.TraceIds = []string{"dddd"}+ bodies, total := searchBodies(t, params)+ if total != 0 || len(bodies) != 0 {+ t.Errorf("unknown trace id: expected no rows, got total %d rows %v", total, bodies)+ }+}++func TestLogRecordRepository_SearchByTraceIdsPastBranchCap(t *testing.T) {+ setupTestDB(t)+ ctx := context.Background()+ projectId := uuid.New()+ now := truncateMs(time.Now().UTC())++ records := []models.LogRecord{+ makeLogRecord(projectId, 9, "older", now),+ makeLogRecord(projectId, 9, "newer", now.Add(time.Second)),+ makeLogRecord(projectId, 9, "unrelated", now.Add(2*time.Second)),+ }+ records[0].TraceId = "aaaa"+ records[1].TraceId = "aaaa"+ records[2].TraceId = "cccc"+ if err := LogRecordRepository.InsertAsync(ctx, records); err != nil {+ t.Fatalf("InsertAsync failed: %v", err)+ }++ traceIds := make([]string, 0, 300)+ for i := 0; i < 299; i++ {+ traceIds = append(traceIds, fmt.Sprintf("%04x", i))+ }+ traceIds = append(traceIds, "aaaa")++ params := shared.LogSearchParams{+ ProjectId: projectId,+ FromDate: now.Add(-time.Hour),+ ToDate: now.Add(time.Hour),+ TraceIds: traceIds,+ PageSize: 1,+ Page: 1,+ }+ bodies, total := searchBodies(t, params)+ if total != 2 {+ t.Errorf("expected total 2, got %d", total)+ }+ assertBodies(t, "page 1", bodies, []string{"newer"})++ params.Page = 2+ bodies, _ = searchBodies(t, params)+ assertBodies(t, "page 2", bodies, []string{"older"})+}diff --git a/backend/app/repositories/telemetry/sqlite/log_record.repository.go b/backend/app/repositories/telemetry/sqlite/log_record.repository.goindex dcdbafc7..995f7cbb 100644--- a/backend/app/repositories/telemetry/sqlite/log_record.repository.go+++ b/backend/app/repositories/telemetry/sqlite/log_record.repository.go@@ -95,6 +95,16 @@ func (r *logRecord) toModel() models.LogRecord {type logRecordRepository struct{}+const logRecordColumns = `id, project_id, timestamp, trace_id, span_id, trace_flags,+ severity_text, severity_number, service_name, body,+ resource_schema_url, resource_attributes,+ scope_schema_url, scope_name, scope_version, scope_attributes,+ log_attributes`++// SQLite caps a compound SELECT at 500 terms; past this many trace ids the+// page query falls back to the IN form.+const maxTraceIdBranches = 100+func (r *logRecordRepository) InsertAsync(ctx context.Context, records []models.LogRecord) error {if len(records) == 0 {return nil@@ -118,8 +128,13 @@ func (r *logRecordRepository) InsertAsync(ctx context.Context, records []models.func (r *logRecordRepository) Search(ctx context.Context, params shared.LogSearchParams) ([]models.LogRecord, int64, error) {where, args := r.buildWhere(params)+ traceIdIn := ""+ if len(params.TraceIds) > 0 {+ traceIdIn = " AND " + traceIdInClause(params.TraceIds, args)+ }+countResult, err := lit.SelectSingleNamed[models.CountResult](db.TelemetryDB,- "SELECT COUNT(*) AS count FROM log_records WHERE "+where, args)+ "SELECT COUNT(*) AS count FROM log_records WHERE "+where+traceIdIn, args)if err != nil {return nil, 0, err}@@ -148,15 +163,18 @@ func (r *logRecordRepository) Search(ctx context.Context, params shared.LogSearcargs["limit"] = params.PageSizeargs["offset"] = offset- query := fmt.Sprintf(`SELECT id, project_id, timestamp, trace_id, span_id, trace_flags,- severity_text, severity_number, service_name, body,- resource_schema_url, resource_attributes,- scope_schema_url, scope_name, scope_version, scope_attributes,- log_attributes- FROM log_records- WHERE %s- ORDER BY %s %s, id- LIMIT :limit OFFSET :offset`, where, orderBy, direction)+ var query string+ switch {+ case len(params.TraceIds) == 0:+ query = fmt.Sprintf(`SELECT %s FROM log_records WHERE %s ORDER BY %s %s, id LIMIT :limit OFFSET :offset`,+ logRecordColumns, where, orderBy, direction)+ case len(params.TraceIds) <= maxTraceIdBranches:+ args["branch_limit"] = offset + params.PageSize+ query = perTraceIdPageQuery(where, len(params.TraceIds), orderBy, direction)+ default:+ query = fmt.Sprintf(`SELECT %s FROM log_records WHERE %s ORDER BY +%s %s, id LIMIT :limit OFFSET :offset`,+ logRecordColumns, where+traceIdIn, orderBy, direction)+ }rows, err := lit.SelectNamed[logRecord](db.TelemetryDB, query, args)if err != nil {@@ -172,12 +190,7 @@ func (r *logRecordRepository) Search(ctx context.Context, params shared.LogSearcfunc (r *logRecordRepository) FindByTraceId(ctx context.Context, projectId uuid.UUID, traceId string) ([]models.LogRecord, error) {rows, err := lit.SelectNamed[logRecord](db.TelemetryDB,- `SELECT id, project_id, timestamp, trace_id, span_id, trace_flags,- severity_text, severity_number, service_name, body,- resource_schema_url, resource_attributes,- scope_schema_url, scope_name, scope_version, scope_attributes,- log_attributes- FROM log_records+ `SELECT `+logRecordColumns+` FROM log_recordsWHERE project_id = :project_id AND trace_id = :trace_idORDER BY timestamp ASC`,lit.P{"project_id": projectId, "trace_id": traceId})@@ -208,15 +221,7 @@ func (r *logRecordRepository) buildWhere(params shared.LogSearchParams) (string,clauses = append(clauses, "service_name = :service_name")args["service_name"] = params.ServiceName}- if len(params.TraceIds) > 0 {- placeholders := make([]string, len(params.TraceIds))- for i, tid := range params.TraceIds {- key := fmt.Sprintf("tid%d", i)- placeholders[i] = ":" + key- args[key] = tid- }- clauses = append(clauses, "trace_id IN ("+strings.Join(placeholders, ", ")+")")- } else if params.TraceId != "" {+ if params.TraceId != "" {clauses = append(clauses, "trace_id = :trace_id")args["trace_id"] = params.TraceId}@@ -282,6 +287,35 @@ func (r *logRecordRepository) buildWhere(params shared.LogSearchParams) (string,return strings.Join(clauses, " AND "), args}+func traceIdKey(i int) string {+ return fmt.Sprintf("tid%d", i)+}++func traceIdInClause(traceIds []string, args lit.P) string {+ placeholders := make([]string, len(traceIds))+ for i, tid := range traceIds {+ key := traceIdKey(i)+ placeholders[i] = ":" + key+ args[key] = tid+ }+ return "trace_id IN (" + strings.Join(placeholders, ", ") + ")"+}++// An IN list over trace ids is several disjoint ranges in the trace index, so+// the planner cannot take ORDER BY timestamp from it and, without sqlite_stat1,+// walks the whole time window instead (#353). One branch per id is a single+// seek in index order whatever the stats say; each branch carries enough rows+// to reach the requested offset before the outer sort merges them.+func perTraceIdPageQuery(where string, traceIdCount int, orderBy, direction string) string {+ branches := make([]string, traceIdCount)+ for i := range branches {+ branches[i] = fmt.Sprintf(`SELECT * FROM (SELECT %s FROM log_records WHERE %s AND trace_id = :%s ORDER BY %s %s, id LIMIT :branch_limit)`,+ logRecordColumns, where, traceIdKey(i), orderBy, direction)+ }+ return strings.Join(branches, " UNION ALL ") ++ fmt.Sprintf(" ORDER BY %s %s, id LIMIT :limit OFFSET :offset", orderBy, direction)+}+func attrColumn(scope string) string {switch scope {case "resource":