Skip to content

The only tool you need to know what is happening and how to fix it.

making sqlite faster

ddux9293b981167c
diff --git a/backend/app/controllers/metric_query.controller.go b/backend/app/controllers/metric_query.controller.go
index f2d5afcb..b6fb6953 100644
--- a/backend/app/controllers/metric_query.controller.go
+++ b/backend/app/controllers/metric_query.controller.go
@@ -78,10 +78,19 @@ func (c *metricQueryController) Query(ctx *gin.Context) {
agg = "avg"
}
+ spanName := fmt.Sprintf("query metric: %s agg=%s", q.Name, agg)
+ if q.GroupBy != "" {
+ spanName += " group=" + q.GroupBy
+ }
+ if len(q.TagFilters) > 0 {
+ spanName += fmt.Sprintf(" filters=%d", len(q.TagFilters))
+ }
+ span := traceway.StartSpan(ctx, spanName)
series, err := repositories.MetricPointRepository.QueryTimeSeries(
ctx, projectId, q.Name, req.From, req.To,
req.IntervalMinutes, agg, q.TagFilters, q.GroupBy,
)
+ span.End()
if err != nil {
ctx.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("failed to query metric %s: %w", q.Name, err))
return
@@ -127,7 +136,9 @@ func (c *metricQueryController) Discover(ctx *gin.Context) {
}
}
+ span := traceway.StartSpan(ctx, "discover metrics")
discovered, err := repositories.MetricPointRepository.DiscoverMetrics(ctx, projectId, from, to)
+ span.End()
if err != nil {
ctx.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("failed to discover metrics: %w", err))
return
@@ -173,7 +184,9 @@ func (c *metricQueryController) DiscoverTags(ctx *gin.Context) {
from := time.Now().AddDate(0, 0, -7)
to := time.Now()
+ span := traceway.StartSpan(ctx, fmt.Sprintf("discover tag values: %s.%s", name, key))
values, err := repositories.MetricPointRepository.DiscoverTagValues(ctx, projectId, name, key, from, to)
+ span.End()
if err != nil {
ctx.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("failed to discover tag values: %w", err))
return
diff --git a/backend/app/db/db_sqlite.go b/backend/app/db/db_sqlite.go
index 5f99dbef..e4bd1063 100644
--- a/backend/app/db/db_sqlite.go
+++ b/backend/app/db/db_sqlite.go
@@ -26,7 +26,7 @@ func initSQLite() error {
path = "./traceway.db"
}
- mainDB, err := openSQLite(path, true)
+ mainDB, err := openSQLite(path, false)
if err != nil {
return err
}
@@ -38,7 +38,7 @@ func initSQLite() error {
if path == ":memory:" {
telemetryPath = ":memory:"
}
- telDB, err := openSQLite(telemetryPath, false)
+ telDB, err := openSQLite(telemetryPath, true)
if err != nil {
return err
}
@@ -48,22 +48,61 @@ func initSQLite() error {
return nil
}
-func openSQLite(path string, foreignKeys bool) (*sql.DB, error) {
- d, err := sql.Open("sqlite", path)
+// openSQLite opens a SQLite database with pragmas tuned for its role.
+// The main DB gets foreign-key enforcement and the safest fsync mode; the
+// telemetry DB relaxes synchronous to NORMAL and grows the page cache because
+// it is append-only and cheaply re-derivable from clients on a hard crash.
+//
+// Pragmas go on the DSN (modernc.org/sqlite supports `_pragma=name(value)`)
+// rather than via db.Exec because Exec only configures the first pooled
+// connection — once SetMaxOpenConns goes above 1 new connections would
+// otherwise boot without the pragmas applied.
+func openSQLite(path string, telemetry bool) (*sql.DB, error) {
+ var dsn string
+ if path == ":memory:" {
+ dsn = path
+ } else {
+ params := []string{
+ "_pragma=journal_mode(WAL)",
+ "_pragma=busy_timeout(5000)",
+ }
+ if telemetry {
+ params = append(params,
+ "_pragma=synchronous(NORMAL)",
+ "_pragma=cache_size(-65536)",
+ "_pragma=temp_store(MEMORY)",
+ )
+ } else {
+ params = append(params, "_pragma=foreign_keys(ON)")
+ }
+ dsn = "file:" + path + "?" + strings.Join(params, "&")
+ }
+
+ d, err := sql.Open("sqlite", dsn)
if err != nil {
return nil, fmt.Errorf("failed to open sqlite at %s: %w", path, err)
}
if err := d.Ping(); err != nil {
return nil, fmt.Errorf("failed to ping sqlite at %s: %w", path, err)
}
- if foreignKeys {
- if _, err := d.Exec("PRAGMA foreign_keys = ON"); err != nil {
- return nil, fmt.Errorf("failed to enable foreign keys: %w", err)
+
+ if path == ":memory:" {
+ if !telemetry {
+ if _, err := d.Exec("PRAGMA foreign_keys = ON"); err != nil {
+ return nil, fmt.Errorf("failed to enable foreign keys: %w", err)
+ }
+ }
+ if _, err := d.Exec("PRAGMA journal_mode = WAL"); err != nil {
+ return nil, fmt.Errorf("failed to set WAL mode: %w", err)
}
}
- if _, err := d.Exec("PRAGMA journal_mode = WAL"); err != nil {
- return nil, fmt.Errorf("failed to set WAL mode: %w", err)
+
+ if telemetry {
+ // WAL allows concurrent readers; SQLite still serializes writes at the
+ // file level and busy_timeout absorbs short contention windows.
+ d.SetMaxOpenConns(4)
+ } else {
+ d.SetMaxOpenConns(1)
}
- d.SetMaxOpenConns(1)
return d, nil
}
diff --git a/backend/app/migrations/sqlite_telemetry/0006_metric_points_covering_index.up.sql b/backend/app/migrations/sqlite_telemetry/0006_metric_points_covering_index.up.sql
new file mode 100644
index 00000000..3fcea344
--- /dev/null
+++ b/backend/app/migrations/sqlite_telemetry/0006_metric_points_covering_index.up.sql
@@ -0,0 +1,2 @@
+DROP INDEX IF EXISTS idx_metric_points_project_name;
+CREATE INDEX IF NOT EXISTS idx_metric_points_project_name_value ON metric_points(project_id, name, recorded_at, value);
diff --git a/backend/app/repositories/metric_point.repository_sqlite.go b/backend/app/repositories/metric_point.repository_sqlite.go
index bd2a62c0..59e6464c 100644
--- a/backend/app/repositories/metric_point.repository_sqlite.go
+++ b/backend/app/repositories/metric_point.repository_sqlite.go
@@ -4,6 +4,7 @@ package repositories
import (
"context"
+ "database/sql"
"fmt"
"strings"
"time"
@@ -42,10 +43,10 @@ func (r *metricPointRepository) InsertAsync(ctx context.Context, points []models
query, args, err := lit.ParseNamedQuery(db.Driver,
"INSERT INTO metric_points (project_id, name, value, tags, recorded_at) VALUES (:project_id, :name, :value, :tags, :recorded_at)",
lit.P{
- "project_id": p.ProjectId,
- "name": p.Name,
- "value": p.Value,
- "tags": tagsVal,
+ "project_id": p.ProjectId,
+ "name": p.Name,
+ "value": p.Value,
+ "tags": tagsVal,
"recorded_at": NewSQLiteTime(p.RecordedAt),
})
if err != nil {
@@ -143,45 +144,47 @@ func (r *metricPointRepository) QueryTimeSeries(ctx context.Context, projectId u
}
func (r *metricPointRepository) DiscoverMetrics(ctx context.Context, projectId uuid.UUID, from, to time.Time) ([]models.DiscoveredMetric, error) {
- type nameRow struct {
- Name string `lit:"name"`
- }
- lit.RegisterModel[nameRow](db.Driver)
-
- names, err := lit.SelectNamed[nameRow](db.TelemetryDB,
- "SELECT DISTINCT name FROM metric_points WHERE project_id = :project_id AND recorded_at >= :from AND recorded_at <= :to ORDER BY name ASC",
+ query, args, err := lit.ParseNamedQuery(db.Driver,
+ `SELECT name, j.key AS tag_key
+ FROM metric_points
+ LEFT JOIN json_each(tags) j
+ WHERE project_id = :project_id AND recorded_at >= :from AND recorded_at <= :to
+ GROUP BY name, j.key
+ ORDER BY name ASC, j.key ASC`,
lit.P{"project_id": projectId, "from": NewSQLiteTime(from), "to": NewSQLiteTime(to)})
if err != nil {
return nil, err
}
- var metrics []models.DiscoveredMetric
- for _, n := range names {
- type keyRow struct {
- Key string `lit:"key"`
- }
- lit.RegisterModel[keyRow](db.Driver)
+ rows, err := db.TelemetryDB.QueryContext(ctx, query, args...)
+ if err != nil {
+ return nil, err
+ }
+ defer rows.Close()
- tagKeys, err := lit.SelectNamed[keyRow](db.TelemetryDB,
- `SELECT DISTINCT j.key FROM metric_points, json_each(tags) j
- WHERE project_id = :project_id AND name = :name AND recorded_at >= :from AND recorded_at <= :to
- ORDER BY j.key ASC`,
- lit.P{"project_id": projectId, "name": n.Name, "from": NewSQLiteTime(from), "to": NewSQLiteTime(to)})
- if err != nil {
+ byName := make(map[string]*models.DiscoveredMetric)
+ order := make([]string, 0)
+ for rows.Next() {
+ var name string
+ var tagKey sql.NullString
+ if err := rows.Scan(&name, &tagKey); err != nil {
return nil, err
}
-
- keys := make([]string, 0, len(tagKeys))
- for _, k := range tagKeys {
- keys = append(keys, k.Key)
+ m, ok := byName[name]
+ if !ok {
+ m = &models.DiscoveredMetric{Name: name, TagKeys: []string{}}
+ byName[name] = m
+ order = append(order, name)
+ }
+ if tagKey.Valid && tagKey.String != "" {
+ m.TagKeys = append(m.TagKeys, tagKey.String)
}
-
- metrics = append(metrics, models.DiscoveredMetric{
- Name: n.Name,
- TagKeys: keys,
- })
}
+ metrics := make([]models.DiscoveredMetric, 0, len(order))
+ for _, n := range order {
+ metrics = append(metrics, *byName[n])
+ }
return metrics, nil
}