Skip to content

The only tool you need to know what is happening and how to fix it.

Log each migration at boot and add a Helm startupProbe (#351)

FrameAutomataac5670df73c7
diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
index 08e06f42..f4826db6 100644
--- a/.github/workflows/backend.yml
+++ b/.github/workflows/backend.yml
@@ -291,7 +291,9 @@ jobs:
# what opts the tagged suite in -- without them the job would report ok
# while running nothing. app/chdb is included because batch_retry_test.go
# is `//go:build telemetry_ch` and would otherwise run in no job at all;
- # it uses fakes and needs no live ClickHouse.
+ # it uses fakes and needs no live ClickHouse. app/migrations is included
+ # for the same reason: migrate_logger_test.go is tagged
+ # `transactional_pg || telemetry_ch` and needs no database either.
#
# -json is written to a file for the tally below and replayed through jq
# so the log still reads like ordinary `go test` output. pipefail is set
@@ -310,7 +312,7 @@ jobs:
run: |
set -o pipefail
go test -json -tags "transactional_pg telemetry_ch" -count=1 \
- ./app/repositories/... ./app/chdb/... \
+ ./app/repositories/... ./app/chdb/... ./app/migrations/... \
| tee "$RUNNER_TEMP/pgch-tests.jsonl" \
| jq -j 'select(.Action == "output") | .Output'
diff --git a/Dockerfile b/Dockerfile
index 17c78e9b..4fcc1595 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -105,7 +105,9 @@ VOLUME ["/var/lib/clickhouse", "/var/lib/postgresql/data"]
EXPOSE 80 8082
-HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \
+# 300s, the Helm chart's default startupProbe budget: the server binds only after
+# migrations run, and Docker ends the start period at the first healthy probe.
+HEALTHCHECK --interval=30s --timeout=10s --start-period=300s --retries=3 \
CMD curl -f http://localhost/health || exit 1
CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf"]
diff --git a/Dockerfile.browser b/Dockerfile.browser
index eba79d6e..0457e42b 100644
--- a/Dockerfile.browser
+++ b/Dockerfile.browser
@@ -112,7 +112,9 @@ ENV GIN_MODE=release \
EXPOSE 80 8082
-HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
+# 300s, the Helm chart's default startupProbe budget: the server binds only after
+# migrations run, and Docker ends the start period at the first healthy probe.
+HEALTHCHECK --interval=30s --timeout=3s --start-period=300s --retries=3 \
CMD wget -qO- http://localhost/health || exit 1
ENTRYPOINT ["/usr/local/bin/traceway"]
diff --git a/Dockerfile.duckdb b/Dockerfile.duckdb
index c178d907..17a460e9 100644
--- a/Dockerfile.duckdb
+++ b/Dockerfile.duckdb
@@ -76,7 +76,9 @@ ENV GIN_MODE=release \
EXPOSE 80 8082
-HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
+# 300s, the Helm chart's default startupProbe budget: the server binds only after
+# migrations run, and Docker ends the start period at the first healthy probe.
+HEALTHCHECK --interval=30s --timeout=3s --start-period=300s --retries=3 \
CMD wget -qO- http://localhost/health || exit 1
ENTRYPOINT ["/usr/local/bin/traceway"]
diff --git a/Dockerfile.minimal b/Dockerfile.minimal
index 857dd77c..e6895dd1 100644
--- a/Dockerfile.minimal
+++ b/Dockerfile.minimal
@@ -80,7 +80,9 @@ ENV GIN_MODE=release
EXPOSE 80 8082
# Health check using wget (alpine-native, no curl needed)
-HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
+# 300s, the Helm chart's default startupProbe budget: the server binds only after
+# migrations run, and Docker ends the start period at the first healthy probe.
+HEALTHCHECK --interval=30s --timeout=3s --start-period=300s --retries=3 \
CMD wget -qO- http://localhost/health || exit 1
# Run the binary directly (no systemd needed)
diff --git a/Dockerfile.sqlite b/Dockerfile.sqlite
index 4b925a8f..45f1fc1b 100644
--- a/Dockerfile.sqlite
+++ b/Dockerfile.sqlite
@@ -78,7 +78,9 @@ ENV GIN_MODE=release \
EXPOSE 80 8082
-HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
+# 300s, the Helm chart's default startupProbe budget: the server binds only after
+# migrations run, and Docker ends the start period at the first healthy probe.
+HEALTHCHECK --interval=30s --timeout=3s --start-period=300s --retries=3 \
CMD wget -qO- http://localhost/health || exit 1
ENTRYPOINT ["/usr/local/bin/traceway"]
diff --git a/backend/app/migrations/migrate_logger.go b/backend/app/migrations/migrate_logger.go
new file mode 100644
index 00000000..c18ef332
--- /dev/null
+++ b/backend/app/migrations/migrate_logger.go
@@ -0,0 +1,36 @@
+//go:build transactional_pg || telemetry_ch
+
+package migrations
+
+import (
+ "fmt"
+ "strings"
+
+ "github.com/tracewayapp/traceway/backend/app/config"
+)
+
+// golang-migrate announces the migration it is about to run only in verbose
+// mode, and that is the line an operator needs while a slow index build holds
+// up boot. Verbose also emits the read-ahead goroutine's buffering lines, which
+// name the *next* migration while the current one is still running and would
+// read as progress that is not happening, so they are dropped here.
+var migrateReadAheadPrefixes = []string{"Start buffering", "Scheduled"}
+
+// migrateLogger gives the ClickHouse and Postgres paths the per-migration
+// progress that runMigrationsOn logs itself on the embedded stores: one line
+// naming each migration as it starts, one more with its elapsed time.
+type migrateLogger struct {
+ store string
+}
+
+func (l migrateLogger) Printf(format string, v ...any) {
+ msg := strings.TrimRight(fmt.Sprintf(format, v...), "\n")
+ for _, prefix := range migrateReadAheadPrefixes {
+ if strings.HasPrefix(msg, prefix) {
+ return
+ }
+ }
+ config.Logf("migrations: %s %s", l.store, msg)
+}
+
+func (migrateLogger) Verbose() bool { return true }
diff --git a/backend/app/migrations/migrate_logger_test.go b/backend/app/migrations/migrate_logger_test.go
new file mode 100644
index 00000000..302e88fd
--- /dev/null
+++ b/backend/app/migrations/migrate_logger_test.go
@@ -0,0 +1,66 @@
+//go:build transactional_pg || telemetry_ch
+
+package migrations
+
+import (
+ "bytes"
+ "strings"
+ "testing"
+ "testing/fstest"
+
+ "github.com/golang-migrate/migrate/v4"
+ "github.com/golang-migrate/migrate/v4/database/stub"
+ "github.com/golang-migrate/migrate/v4/source/iofs"
+)
+
+// What the operator needs while an index build holds up boot is a line naming
+// the migration that is *running*, which golang-migrate emits only in verbose
+// mode — along with read-ahead lines naming the next migration, which the
+// logger drops. Both halves depend on golang-migrate's own wording, so drive
+// the real Up() through the stub driver rather than asserting on transcribed
+// format strings: an upgrade that renames either one fails here.
+func TestMigrateLoggerReportsEachMigrationAsItRuns(t *testing.T) {
+ source, err := iofs.New(fstest.MapFS{
+ "m/1_first.up.sql": &fstest.MapFile{Data: []byte("SELECT 1")},
+ "m/2_second.up.sql": &fstest.MapFile{Data: []byte("SELECT 2")},
+ }, "m")
+ if err != nil {
+ t.Fatalf("failed to create migration source: %v", err)
+ }
+
+ driver, err := stub.WithInstance(nil, &stub.Config{})
+ if err != nil {
+ t.Fatalf("failed to create stub driver: %v", err)
+ }
+
+ m, err := migrate.NewWithInstance("iofs", source, "stub", driver)
+ if err != nil {
+ t.Fatalf("failed to create migrate instance: %v", err)
+ }
+
+ var buf bytes.Buffer
+ t.Cleanup(captureLog(&buf))
+
+ m.Log = migrateLogger{store: "postgres"}
+ if err := m.Up(); err != nil {
+ t.Fatalf("migrate up failed: %v", err)
+ }
+ got := buf.String()
+
+ for _, want := range []string{
+ "migrations: postgres Read and execute 1/u first",
+ "migrations: postgres Read and execute 2/u second",
+ "Finished 1/u first",
+ "Finished 2/u second",
+ } {
+ if !strings.Contains(got, want) {
+ t.Fatalf("expected %q in the migration log, got:\n%s", want, got)
+ }
+ }
+
+ for _, unwanted := range migrateReadAheadPrefixes {
+ if strings.Contains(got, unwanted) {
+ t.Fatalf("expected read-ahead line %q to be dropped, got:\n%s", unwanted, got)
+ }
+ }
+}
diff --git a/backend/app/migrations/migrations.go b/backend/app/migrations/migrations.go
index 71416233..f23d1a02 100644
--- a/backend/app/migrations/migrations.go
+++ b/backend/app/migrations/migrations.go
@@ -4,8 +4,12 @@ import (
"database/sql"
"embed"
"fmt"
+ "io/fs"
"sort"
"strings"
+ "time"
+
+ "github.com/tracewayapp/traceway/backend/app/config"
)
type ExtensionMigration struct {
@@ -24,12 +28,12 @@ const sqliteTrackingDDL = `CREATE TABLE IF NOT EXISTS %s (
applied_at DATETIME DEFAULT (datetime('now'))
)`
-func runMigrationsOn(target *sql.DB, fsys embed.FS, dir, trackingTable, createTrackingSQL string) error {
+func runMigrationsOn(target *sql.DB, fsys fs.FS, dir, trackingTable, createTrackingSQL string) error {
if _, err := target.Exec(fmt.Sprintf(createTrackingSQL, trackingTable)); err != nil {
return fmt.Errorf("failed to create %s table: %w", trackingTable, err)
}
- entries, err := fsys.ReadDir(dir)
+ entries, err := fs.ReadDir(fsys, dir)
if err != nil {
return fmt.Errorf("failed to read migrations dir %s: %w", dir, err)
}
@@ -54,11 +58,14 @@ func runMigrationsOn(target *sql.DB, fsys embed.FS, dir, trackingTable, createTr
continue
}
- content, err := fsys.ReadFile(dir + "/" + file)
+ content, err := fs.ReadFile(fsys, dir+"/"+file)
if err != nil {
return fmt.Errorf("failed to read migration file %s: %w", file, err)
}
+ config.Logf("migrations: applying %s/%s", dir, version)
+ started := time.Now()
+
statements := splitStatements(string(content))
for _, stmt := range statements {
stmt = strings.TrimSpace(stmt)
@@ -73,6 +80,8 @@ func runMigrationsOn(target *sql.DB, fsys embed.FS, dir, trackingTable, createTr
if _, err := target.Exec(fmt.Sprintf("INSERT INTO %s (version) VALUES (?)", trackingTable), version); err != nil {
return fmt.Errorf("failed to record migration version %s: %w", version, err)
}
+
+ config.Logf("migrations: applied %s/%s in %s", dir, version, time.Since(started).Round(time.Millisecond))
}
return nil
diff --git a/backend/app/migrations/migrations_log_test.go b/backend/app/migrations/migrations_log_test.go
new file mode 100644
index 00000000..20179651
--- /dev/null
+++ b/backend/app/migrations/migrations_log_test.go
@@ -0,0 +1,96 @@
+package migrations
+
+import (
+ "bytes"
+ "database/sql"
+ "log"
+ "strings"
+ "testing"
+ "testing/fstest"
+
+ _ "modernc.org/sqlite"
+)
+
+// openMemorySQLite returns a fresh in-memory database pinned to one connection:
+// every new connection to ":memory:" opens a separate, empty database, so a
+// second connection would see none of the migrated schema.
+func openMemorySQLite(t *testing.T) *sql.DB {
+ t.Helper()
+ target, err := sql.Open("sqlite", ":memory:")
+ if err != nil {
+ t.Fatalf("failed to open in-memory sqlite: %v", err)
+ }
+ t.Cleanup(func() { target.Close() })
+ target.SetMaxOpenConns(1)
+ return target
+}
+
+// captureLog redirects the standard logger into buf with its flags cleared, so
+// no timestamp precedes a line and assertions can match from the line start.
+// The returned func restores both.
+func captureLog(buf *bytes.Buffer) func() {
+ out, flags := log.Writer(), log.Flags()
+ log.SetOutput(buf)
+ log.SetFlags(0)
+ return func() {
+ log.SetOutput(out)
+ log.SetFlags(flags)
+ }
+}
+
+// assertLogLines checks buf holds exactly these lines in order. Each is matched
+// as a prefix so the elapsed-time suffix does not pin a duration.
+func assertLogLines(t *testing.T, buf *bytes.Buffer, want ...string) {
+ t.Helper()
+ got := strings.Split(strings.TrimSuffix(buf.String(), "\n"), "\n")
+ if len(got) != len(want) {
+ t.Fatalf("want %d log lines, got %d:\n%s", len(want), len(got), buf.String())
+ }
+ for i := range want {
+ if !strings.HasPrefix(got[i], want[i]) {
+ t.Errorf("line %d: want prefix %q, got %q", i, want[i], got[i])
+ }
+ }
+}
+
+// The announcement has to land before the statements run, since naming a
+// migration only once it finishes is the silence this replaced. A failing
+// migration is what makes that visible: it leaves the announcement with no
+// completion. And an already-applied migration has to stay quiet, or every
+// restart reprints the full history.
+func TestRunMigrationsOnAnnouncesEachMigrationBeforeRunningIt(t *testing.T) {
+ var buf bytes.Buffer
+ t.Cleanup(captureLog(&buf))
+ target := openMemorySQLite(t)
+
+ fsys := fstest.MapFS{
+ "m/0001_first.up.sql": {Data: []byte("CREATE TABLE first (id INTEGER)")},
+ "m/0002_second.up.sql": {Data: []byte("CREATE TABLE second (")},
+ }
+ if err := runMigrationsOn(target, fsys, "m", "schema_migrations", sqliteTrackingDDL); err == nil {
+ t.Fatal("expected the broken migration to fail")
+ }
+ assertLogLines(t, &buf,
+ "[tracewaybackend] migrations: applying m/0001_first",
+ "[tracewaybackend] migrations: applied m/0001_first in ",
+ "[tracewaybackend] migrations: applying m/0002_second",
+ )
+
+ fsys["m/0002_second.up.sql"] = &fstest.MapFile{Data: []byte("CREATE TABLE second (id INTEGER)")}
+ buf.Reset()
+ if err := runMigrationsOn(target, fsys, "m", "schema_migrations", sqliteTrackingDDL); err != nil {
+ t.Fatalf("migrations failed: %v", err)
+ }
+ assertLogLines(t, &buf,
+ "[tracewaybackend] migrations: applying m/0002_second",
+ "[tracewaybackend] migrations: applied m/0002_second in ",
+ )
+
+ buf.Reset()
+ if err := runMigrationsOn(target, fsys, "m", "schema_migrations", sqliteTrackingDDL); err != nil {
+ t.Fatalf("third run failed: %v", err)
+ }
+ if buf.Len() != 0 {
+ t.Errorf("re-running applied migrations logged %q, want silence", buf.String())
+ }
+}
diff --git a/backend/app/migrations/migrations_telemetry_ch.go b/backend/app/migrations/migrations_telemetry_ch.go
index 103e4233..30e234eb 100644
--- a/backend/app/migrations/migrations_telemetry_ch.go
+++ b/backend/app/migrations/migrations_telemetry_ch.go
@@ -42,6 +42,8 @@ func runMigrationsClickhouse(connStr string) error {
return err
}
+ m.Log = migrateLogger{store: "clickhouse"}
+
if err := m.Up(); err != nil && err != migrate.ErrNoChange {
return err
}
diff --git a/backend/app/migrations/migrations_transactional_pg.go b/backend/app/migrations/migrations_transactional_pg.go
index 67a38fd9..951ed5cd 100644
--- a/backend/app/migrations/migrations_transactional_pg.go
+++ b/backend/app/migrations/migrations_transactional_pg.go
@@ -42,6 +42,8 @@ func runMigrationsPostgres(connStr string) error {
return fmt.Errorf("failed to create migrate instance: %w", err)
}
+ m.Log = migrateLogger{store: "postgres"}
+
if err := m.Up(); err != nil && err != migrate.ErrNoChange {
return fmt.Errorf("postgres migration failed: %w", err)
}
@@ -78,6 +80,8 @@ func runExtensionMigrations(db *sql.DB, ext ExtensionMigration) error {
return fmt.Errorf("failed to create extension migrate instance: %w", err)
}
+ m.Log = migrateLogger{store: "postgres extension " + tableName}
+
if err := m.Up(); err != nil && err != migrate.ErrNoChange {
return fmt.Errorf("extension postgres migration failed: %w", err)
}
diff --git a/backend/app/migrations/telemetry_group_index_test.go b/backend/app/migrations/telemetry_group_index_test.go
index ba76bb77..21ca6e76 100644
--- a/backend/app/migrations/telemetry_group_index_test.go
+++ b/backend/app/migrations/telemetry_group_index_test.go
@@ -2,36 +2,16 @@
package migrations
-import (
- "database/sql"
- "slices"
- "strings"
- "testing"
+import "testing"
- "github.com/tracewayapp/traceway/backend/app/retention"
- _ "modernc.org/sqlite"
-)
-
-// Every per-group telemetry read pairs a group column with the table's time
-// column: as a second filter on the grouped lists, as the ORDER BY on the
-// per-group detail reads. An index that stops at the group column leaves SQLite
-// to either re-filter the group across the whole window once per group, which
-// is quadratic in the number of groups, or seek the group and sort its entire
-// history. Both are unbounded and invisible to functional tests.
-//
-// The first loop guards the indexes behind known reads so a later migration
-// cannot narrow or drop one unnoticed. The second inverts the question: every
-// index that leads with project_id must reach its table's time column, so a
-// short index fails here the day it lands rather than when somebody remembers
-// to list it. Indexes that stop short on purpose are named with the reason,
-// and each entry is itself checked so it cannot outlive the index it excuses.
-func TestTelemetryGroupIndexesCoverTimeColumn(t *testing.T) {
- telemetryDB, err := sql.Open("sqlite", ":memory:")
- if err != nil {
- t.Fatalf("failed to open in-memory sqlite: %v", err)
- }
- defer telemetryDB.Close()
- telemetryDB.SetMaxOpenConns(1)
+// The grouped-list reads for these three tables filter
+// (project_id, <group>, recorded_at). An index that stops at the group column
+// leaves SQLite to seek the (project_id, recorded_at) index and re-filter the
+// group column across the whole window once per group, which is quadratic in
+// the number of groups and invisible to functional tests. Guard the covering
+// indexes so a later migration cannot narrow or drop them unnoticed.
+func TestTelemetryGroupIndexesCoverRecordedAt(t *testing.T) {
+ telemetryDB := openMemorySQLite(t)
if err := runMigrationsOn(telemetryDB, migrationsSqliteTelemetryFS, "sqlite_telemetry", "schema_migrations", sqliteTrackingDDL); err != nil {
t.Fatalf("telemetry migrations failed: %v", err)
diff --git a/docker-compose.browser.yml b/docker-compose.browser.yml
index 4fc455b0..c48a9064 100644
--- a/docker-compose.browser.yml
+++ b/docker-compose.browser.yml
@@ -54,12 +54,6 @@ services:
volumes:
- traceway-data:/data
restart: unless-stopped
- healthcheck:
- test: ["CMD", "wget", "-qO-", "http://localhost/health"]
- interval: 30s
- timeout: 3s
- start_period: 5s
- retries: 3
volumes:
traceway-data:
diff --git a/docker-compose.duckdb.yml b/docker-compose.duckdb.yml
index 030e634b..c3cf4b72 100644
--- a/docker-compose.duckdb.yml
+++ b/docker-compose.duckdb.yml
@@ -42,12 +42,6 @@ services:
volumes:
- traceway-data:/data
restart: unless-stopped
- healthcheck:
- test: ["CMD", "wget", "-qO-", "http://localhost/health"]
- interval: 30s
- timeout: 3s
- start_period: 5s
- retries: 3
volumes:
traceway-data:
diff --git a/docker-compose.sqlite.yml b/docker-compose.sqlite.yml
index 7c32be60..4f5e8855 100644
--- a/docker-compose.sqlite.yml
+++ b/docker-compose.sqlite.yml
@@ -32,12 +32,6 @@ services:
volumes:
- traceway-data:/data
restart: unless-stopped
- healthcheck:
- test: ["CMD", "wget", "-qO-", "http://localhost/health"]
- interval: 30s
- timeout: 3s
- start_period: 5s
- retries: 3
volumes:
traceway-data:
diff --git a/helm/traceway/templates/deployment.yaml b/helm/traceway/templates/deployment.yaml
index 0494f497..fc83c282 100644
--- a/helm/traceway/templates/deployment.yaml
+++ b/helm/traceway/templates/deployment.yaml
@@ -8,6 +8,9 @@ spec:
{{- if not .Values.autoscaling.enabled }}
replicas: {{ .Values.replicaCount }}
{{- end }}
+ {{- $probe := .Values.startupProbe | default dict }}
+ {{- $probeBudget := mul ($probe.periodSeconds | default 0) ($probe.failureThreshold | default 0) }}
+ progressDeadlineSeconds: {{ .Values.progressDeadlineSeconds | default (max 600 (add $probeBudget 60)) }}
selector:
matchLabels:
{{- include "traceway.selectorLabels" . | nindent 6 }}
@@ -51,6 +54,10 @@ spec:
name: {{ include "traceway.fullname" . }}
- secretRef:
name: {{ include "traceway.secretName" . }}
+ {{- with .Values.startupProbe }}
+ startupProbe:
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
livenessProbe:
httpGet:
path: /health
diff --git a/helm/traceway/values.yaml b/helm/traceway/values.yaml
index 6557c87e..1e4538b5 100644
--- a/helm/traceway/values.yaml
+++ b/helm/traceway/values.yaml
@@ -40,6 +40,33 @@ ingress:
resources: {}
+# Startup probe. The liveness budget only starts once this succeeds, so a slow
+# first boot is not mistaken for a hung process. The server binds its ports only
+# after migrations finish, and an upgrade that builds indexes on a large
+# telemetry database can take minutes — without this the kubelet would kill the
+# pod mid-migration and restart it into the same stall on every retry. The
+# default budget (periodSeconds x failureThreshold = 300s) is the same one the
+# Docker images give their HEALTHCHECK start period. Raise failureThreshold if
+# an upgrade needs longer; the backend logs each migration with its elapsed
+# time (boot phases before migrations, such as DuckDB WAL replay, are not yet
+# timed). The Deployment's progressDeadlineSeconds follows this budget, but a
+# deploy tool's own timeout does not: `helm upgrade --wait` defaults to 5m, and
+# `--atomic` (or a Flux HelmRelease remediation) rolls back when it expires,
+# scaling the migrating pod away — so raise `--timeout` / `spec.timeout` in
+# step with the budget.
+startupProbe:
+ httpGet:
+ path: /health
+ port: http
+ periodSeconds: 5
+ timeoutSeconds: 3
+ failureThreshold: 60
+
+# Deployment progress deadline. Derived from the startup probe budget plus a
+# minute for scheduling and image pull, never below the Kubernetes default of
+# 600s; set to override.
+# progressDeadlineSeconds: 900
+
autoscaling:
enabled: false
minReplicas: 1