Skip to content

The only tool you need to know what is happening and how to fix it.

fix: Distributed-trace log search on sqlite, closes #353

narrow-adventure97d66fe22c09
diff --git a/backend/app/repositories/telemetry/log_record_repository_test.go b/backend/app/repositories/telemetry/log_record_repository_test.go
index 1d717e4f..ac18ae3a 100644
--- a/backend/app/repositories/telemetry/log_record_repository_test.go
+++ b/backend/app/repositories/telemetry/log_record_repository_test.go
@@ -2,6 +2,7 @@ package telemetry
import (
"context"
+ "fmt"
"testing"
"time"
@@ -105,3 +106,120 @@ func TestLogRecordRepository_SearchPaginationTotal(t *testing.T) {
t.Errorf("expected no rows and total 0, got %d rows total %d", len(rows), total)
}
}
+
+func searchBodies(t *testing.T, params shared.LogSearchParams) ([]string, int64) {
+ t.Helper()
+ rows, total, err := LogRecordRepository.Search(context.Background(), params)
+ if err != nil {
+ t.Fatalf("Search page %d failed: %v", params.Page, err)
+ }
+ bodies := make([]string, 0, len(rows))
+ for _, r := range rows {
+ bodies = append(bodies, r.Body)
+ }
+ return bodies, total
+}
+
+func assertBodies(t *testing.T, label string, got, want []string) {
+ t.Helper()
+ if fmt.Sprint(got) != fmt.Sprint(want) {
+ t.Errorf("%s: expected %v, got %v", label, want, got)
+ }
+}
+
+func TestLogRecordRepository_SearchByTraceIdsMergesAndPaginates(t *testing.T) {
+ setupTestDB(t)
+ ctx := context.Background()
+ projectId := uuid.New()
+ now := truncateMs(time.Now().UTC())
+
+ var records []models.LogRecord
+ for i := 0; i < 7; i++ {
+ rec := makeLogRecord(projectId, 9, fmt.Sprintf("row %d", i), now.Add(time.Duration(i)*time.Second))
+ rec.TraceId = []string{"aaaa", "bbbb"}[i%2]
+ records = append(records, rec)
+ }
+ unrelated := makeLogRecord(projectId, 9, "unrelated", now.Add(10*time.Second))
+ unrelated.TraceId = "cccc"
+ records = append(records, unrelated)
+ if err := LogRecordRepository.InsertAsync(ctx, records); err != nil {
+ t.Fatalf("InsertAsync failed: %v", err)
+ }
+
+ base := shared.LogSearchParams{
+ ProjectId: projectId,
+ FromDate: now.Add(-time.Hour),
+ ToDate: now.Add(time.Hour),
+ TraceIds: []string{"aaaa", "bbbb", "dddd"},
+ PageSize: 3,
+ }
+
+ var got []string
+ for page := 1; page <= 3; page++ {
+ params := base
+ params.Page = page
+ bodies, total := searchBodies(t, params)
+ if total != 7 {
+ t.Errorf("page %d: expected total 7, got %d", page, total)
+ }
+ got = append(got, bodies...)
+ }
+ assertBodies(t, "desc pages", got, []string{"row 6", "row 5", "row 4", "row 3", "row 2", "row 1", "row 0"})
+
+ params := base
+ params.Page = 2
+ params.SortDirection = "asc"
+ bodies, _ := searchBodies(t, params)
+ assertBodies(t, "asc page 2", bodies, []string{"row 3", "row 4", "row 5"})
+
+ params = base
+ params.Page = 1
+ params.TraceIds = []string{"dddd"}
+ bodies, total := searchBodies(t, params)
+ if total != 0 || len(bodies) != 0 {
+ t.Errorf("unknown trace id: expected no rows, got total %d rows %v", total, bodies)
+ }
+}
+
+func TestLogRecordRepository_SearchByTraceIdsPastBranchCap(t *testing.T) {
+ setupTestDB(t)
+ ctx := context.Background()
+ projectId := uuid.New()
+ now := truncateMs(time.Now().UTC())
+
+ records := []models.LogRecord{
+ makeLogRecord(projectId, 9, "older", now),
+ makeLogRecord(projectId, 9, "newer", now.Add(time.Second)),
+ makeLogRecord(projectId, 9, "unrelated", now.Add(2*time.Second)),
+ }
+ records[0].TraceId = "aaaa"
+ records[1].TraceId = "aaaa"
+ records[2].TraceId = "cccc"
+ if err := LogRecordRepository.InsertAsync(ctx, records); err != nil {
+ t.Fatalf("InsertAsync failed: %v", err)
+ }
+
+ traceIds := make([]string, 0, 300)
+ for i := 0; i < 299; i++ {
+ traceIds = append(traceIds, fmt.Sprintf("%04x", i))
+ }
+ traceIds = append(traceIds, "aaaa")
+
+ params := shared.LogSearchParams{
+ ProjectId: projectId,
+ FromDate: now.Add(-time.Hour),
+ ToDate: now.Add(time.Hour),
+ TraceIds: traceIds,
+ PageSize: 1,
+ Page: 1,
+ }
+ bodies, total := searchBodies(t, params)
+ if total != 2 {
+ t.Errorf("expected total 2, got %d", total)
+ }
+ assertBodies(t, "page 1", bodies, []string{"newer"})
+
+ params.Page = 2
+ bodies, _ = searchBodies(t, params)
+ assertBodies(t, "page 2", bodies, []string{"older"})
+}
diff --git a/backend/app/repositories/telemetry/sqlite/log_record.repository.go b/backend/app/repositories/telemetry/sqlite/log_record.repository.go
index dcdbafc7..995f7cbb 100644
--- a/backend/app/repositories/telemetry/sqlite/log_record.repository.go
+++ b/backend/app/repositories/telemetry/sqlite/log_record.repository.go
@@ -95,6 +95,16 @@ func (r *logRecord) toModel() models.LogRecord {
type logRecordRepository struct{}
+const logRecordColumns = `id, project_id, timestamp, trace_id, span_id, trace_flags,
+ severity_text, severity_number, service_name, body,
+ resource_schema_url, resource_attributes,
+ scope_schema_url, scope_name, scope_version, scope_attributes,
+ log_attributes`
+
+// SQLite caps a compound SELECT at 500 terms; past this many trace ids the
+// page query falls back to the IN form.
+const maxTraceIdBranches = 100
+
func (r *logRecordRepository) InsertAsync(ctx context.Context, records []models.LogRecord) error {
if len(records) == 0 {
return nil
@@ -118,8 +128,13 @@ func (r *logRecordRepository) InsertAsync(ctx context.Context, records []models.
func (r *logRecordRepository) Search(ctx context.Context, params shared.LogSearchParams) ([]models.LogRecord, int64, error) {
where, args := r.buildWhere(params)
+ traceIdIn := ""
+ if len(params.TraceIds) > 0 {
+ traceIdIn = " AND " + traceIdInClause(params.TraceIds, args)
+ }
+
countResult, err := lit.SelectSingleNamed[models.CountResult](db.TelemetryDB,
- "SELECT COUNT(*) AS count FROM log_records WHERE "+where, args)
+ "SELECT COUNT(*) AS count FROM log_records WHERE "+where+traceIdIn, args)
if err != nil {
return nil, 0, err
}
@@ -148,15 +163,18 @@ func (r *logRecordRepository) Search(ctx context.Context, params shared.LogSearc
args["limit"] = params.PageSize
args["offset"] = offset
- query := fmt.Sprintf(`SELECT id, project_id, timestamp, trace_id, span_id, trace_flags,
- severity_text, severity_number, service_name, body,
- resource_schema_url, resource_attributes,
- scope_schema_url, scope_name, scope_version, scope_attributes,
- log_attributes
- FROM log_records
- WHERE %s
- ORDER BY %s %s, id
- LIMIT :limit OFFSET :offset`, where, orderBy, direction)
+ var query string
+ switch {
+ case len(params.TraceIds) == 0:
+ query = fmt.Sprintf(`SELECT %s FROM log_records WHERE %s ORDER BY %s %s, id LIMIT :limit OFFSET :offset`,
+ logRecordColumns, where, orderBy, direction)
+ case len(params.TraceIds) <= maxTraceIdBranches:
+ args["branch_limit"] = offset + params.PageSize
+ query = perTraceIdPageQuery(where, len(params.TraceIds), orderBy, direction)
+ default:
+ query = fmt.Sprintf(`SELECT %s FROM log_records WHERE %s ORDER BY +%s %s, id LIMIT :limit OFFSET :offset`,
+ logRecordColumns, where+traceIdIn, orderBy, direction)
+ }
rows, err := lit.SelectNamed[logRecord](db.TelemetryDB, query, args)
if err != nil {
@@ -172,12 +190,7 @@ func (r *logRecordRepository) Search(ctx context.Context, params shared.LogSearc
func (r *logRecordRepository) FindByTraceId(ctx context.Context, projectId uuid.UUID, traceId string) ([]models.LogRecord, error) {
rows, err := lit.SelectNamed[logRecord](db.TelemetryDB,
- `SELECT id, project_id, timestamp, trace_id, span_id, trace_flags,
- severity_text, severity_number, service_name, body,
- resource_schema_url, resource_attributes,
- scope_schema_url, scope_name, scope_version, scope_attributes,
- log_attributes
- FROM log_records
+ `SELECT `+logRecordColumns+` FROM log_records
WHERE project_id = :project_id AND trace_id = :trace_id
ORDER BY timestamp ASC`,
lit.P{"project_id": projectId, "trace_id": traceId})
@@ -208,15 +221,7 @@ func (r *logRecordRepository) buildWhere(params shared.LogSearchParams) (string,
clauses = append(clauses, "service_name = :service_name")
args["service_name"] = params.ServiceName
}
- if len(params.TraceIds) > 0 {
- placeholders := make([]string, len(params.TraceIds))
- for i, tid := range params.TraceIds {
- key := fmt.Sprintf("tid%d", i)
- placeholders[i] = ":" + key
- args[key] = tid
- }
- clauses = append(clauses, "trace_id IN ("+strings.Join(placeholders, ", ")+")")
- } else if params.TraceId != "" {
+ if params.TraceId != "" {
clauses = append(clauses, "trace_id = :trace_id")
args["trace_id"] = params.TraceId
}
@@ -282,6 +287,35 @@ func (r *logRecordRepository) buildWhere(params shared.LogSearchParams) (string,
return strings.Join(clauses, " AND "), args
}
+func traceIdKey(i int) string {
+ return fmt.Sprintf("tid%d", i)
+}
+
+func traceIdInClause(traceIds []string, args lit.P) string {
+ placeholders := make([]string, len(traceIds))
+ for i, tid := range traceIds {
+ key := traceIdKey(i)
+ placeholders[i] = ":" + key
+ args[key] = tid
+ }
+ return "trace_id IN (" + strings.Join(placeholders, ", ") + ")"
+}
+
+// An IN list over trace ids is several disjoint ranges in the trace index, so
+// the planner cannot take ORDER BY timestamp from it and, without sqlite_stat1,
+// walks the whole time window instead (#353). One branch per id is a single
+// seek in index order whatever the stats say; each branch carries enough rows
+// to reach the requested offset before the outer sort merges them.
+func perTraceIdPageQuery(where string, traceIdCount int, orderBy, direction string) string {
+ branches := make([]string, traceIdCount)
+ for i := range branches {
+ branches[i] = fmt.Sprintf(`SELECT * FROM (SELECT %s FROM log_records WHERE %s AND trace_id = :%s ORDER BY %s %s, id LIMIT :branch_limit)`,
+ logRecordColumns, where, traceIdKey(i), orderBy, direction)
+ }
+ return strings.Join(branches, " UNION ALL ") +
+ fmt.Sprintf(" ORDER BY %s %s, id LIMIT :limit OFFSET :offset", orderBy, direction)
+}
+
func attrColumn(scope string) string {
switch scope {
case "resource":