The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
Log each migration at boot and add a Helm startupProbe (#351)
FrameAutomataac5670df73c7
diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.ymlindex 08e06f42..f4826db6 100644--- a/.github/workflows/backend.yml+++ b/.github/workflows/backend.yml@@ -291,7 +291,9 @@ jobs:# what opts the tagged suite in -- without them the job would report ok# while running nothing. app/chdb is included because batch_retry_test.go# is `//go:build telemetry_ch` and would otherwise run in no job at all;- # it uses fakes and needs no live ClickHouse.+ # it uses fakes and needs no live ClickHouse. app/migrations is included+ # for the same reason: migrate_logger_test.go is tagged+ # `transactional_pg || telemetry_ch` and needs no database either.## -json is written to a file for the tally below and replayed through jq# so the log still reads like ordinary `go test` output. pipefail is set@@ -310,7 +312,7 @@ jobs:run: |set -o pipefailgo test -json -tags "transactional_pg telemetry_ch" -count=1 \- ./app/repositories/... ./app/chdb/... \+ ./app/repositories/... ./app/chdb/... ./app/migrations/... \| tee "$RUNNER_TEMP/pgch-tests.jsonl" \| jq -j 'select(.Action == "output") | .Output'diff --git a/Dockerfile b/Dockerfileindex 17c78e9b..4fcc1595 100644--- a/Dockerfile+++ b/Dockerfile@@ -105,7 +105,9 @@ VOLUME ["/var/lib/clickhouse", "/var/lib/postgresql/data"]EXPOSE 80 8082-HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \+# 300s, the Helm chart's default startupProbe budget: the server binds only after+# migrations run, and Docker ends the start period at the first healthy probe.+HEALTHCHECK --interval=30s --timeout=10s --start-period=300s --retries=3 \CMD curl -f http://localhost/health || exit 1CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf"]diff --git a/Dockerfile.browser b/Dockerfile.browserindex eba79d6e..0457e42b 100644--- a/Dockerfile.browser+++ b/Dockerfile.browser@@ -112,7 +112,9 @@ ENV GIN_MODE=release \EXPOSE 80 8082-HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \+# 300s, the Helm chart's default startupProbe budget: the server binds only after+# migrations run, and Docker ends the start period at the first healthy probe.+HEALTHCHECK --interval=30s --timeout=3s --start-period=300s --retries=3 \CMD wget -qO- http://localhost/health || exit 1ENTRYPOINT ["/usr/local/bin/traceway"]diff --git a/Dockerfile.duckdb b/Dockerfile.duckdbindex c178d907..17a460e9 100644--- a/Dockerfile.duckdb+++ b/Dockerfile.duckdb@@ -76,7 +76,9 @@ ENV GIN_MODE=release \EXPOSE 80 8082-HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \+# 300s, the Helm chart's default startupProbe budget: the server binds only after+# migrations run, and Docker ends the start period at the first healthy probe.+HEALTHCHECK --interval=30s --timeout=3s --start-period=300s --retries=3 \CMD wget -qO- http://localhost/health || exit 1ENTRYPOINT ["/usr/local/bin/traceway"]diff --git a/Dockerfile.minimal b/Dockerfile.minimalindex 857dd77c..e6895dd1 100644--- a/Dockerfile.minimal+++ b/Dockerfile.minimal@@ -80,7 +80,9 @@ ENV GIN_MODE=releaseEXPOSE 80 8082# Health check using wget (alpine-native, no curl needed)-HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \+# 300s, the Helm chart's default startupProbe budget: the server binds only after+# migrations run, and Docker ends the start period at the first healthy probe.+HEALTHCHECK --interval=30s --timeout=3s --start-period=300s --retries=3 \CMD wget -qO- http://localhost/health || exit 1# Run the binary directly (no systemd needed)diff --git a/Dockerfile.sqlite b/Dockerfile.sqliteindex 4b925a8f..45f1fc1b 100644--- a/Dockerfile.sqlite+++ b/Dockerfile.sqlite@@ -78,7 +78,9 @@ ENV GIN_MODE=release \EXPOSE 80 8082-HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \+# 300s, the Helm chart's default startupProbe budget: the server binds only after+# migrations run, and Docker ends the start period at the first healthy probe.+HEALTHCHECK --interval=30s --timeout=3s --start-period=300s --retries=3 \CMD wget -qO- http://localhost/health || exit 1ENTRYPOINT ["/usr/local/bin/traceway"]diff --git a/backend/app/migrations/migrate_logger.go b/backend/app/migrations/migrate_logger.gonew file mode 100644index 00000000..c18ef332--- /dev/null+++ b/backend/app/migrations/migrate_logger.go@@ -0,0 +1,36 @@+//go:build transactional_pg || telemetry_ch++package migrations++import (+ "fmt"+ "strings"++ "github.com/tracewayapp/traceway/backend/app/config"+)++// golang-migrate announces the migration it is about to run only in verbose+// mode, and that is the line an operator needs while a slow index build holds+// up boot. Verbose also emits the read-ahead goroutine's buffering lines, which+// name the *next* migration while the current one is still running and would+// read as progress that is not happening, so they are dropped here.+var migrateReadAheadPrefixes = []string{"Start buffering", "Scheduled"}++// migrateLogger gives the ClickHouse and Postgres paths the per-migration+// progress that runMigrationsOn logs itself on the embedded stores: one line+// naming each migration as it starts, one more with its elapsed time.+type migrateLogger struct {+ store string+}++func (l migrateLogger) Printf(format string, v ...any) {+ msg := strings.TrimRight(fmt.Sprintf(format, v...), "\n")+ for _, prefix := range migrateReadAheadPrefixes {+ if strings.HasPrefix(msg, prefix) {+ return+ }+ }+ config.Logf("migrations: %s %s", l.store, msg)+}++func (migrateLogger) Verbose() bool { return true }diff --git a/backend/app/migrations/migrate_logger_test.go b/backend/app/migrations/migrate_logger_test.gonew file mode 100644index 00000000..302e88fd--- /dev/null+++ b/backend/app/migrations/migrate_logger_test.go@@ -0,0 +1,66 @@+//go:build transactional_pg || telemetry_ch++package migrations++import (+ "bytes"+ "strings"+ "testing"+ "testing/fstest"++ "github.com/golang-migrate/migrate/v4"+ "github.com/golang-migrate/migrate/v4/database/stub"+ "github.com/golang-migrate/migrate/v4/source/iofs"+)++// What the operator needs while an index build holds up boot is a line naming+// the migration that is *running*, which golang-migrate emits only in verbose+// mode — along with read-ahead lines naming the next migration, which the+// logger drops. Both halves depend on golang-migrate's own wording, so drive+// the real Up() through the stub driver rather than asserting on transcribed+// format strings: an upgrade that renames either one fails here.+func TestMigrateLoggerReportsEachMigrationAsItRuns(t *testing.T) {+ source, err := iofs.New(fstest.MapFS{+ "m/1_first.up.sql": &fstest.MapFile{Data: []byte("SELECT 1")},+ "m/2_second.up.sql": &fstest.MapFile{Data: []byte("SELECT 2")},+ }, "m")+ if err != nil {+ t.Fatalf("failed to create migration source: %v", err)+ }++ driver, err := stub.WithInstance(nil, &stub.Config{})+ if err != nil {+ t.Fatalf("failed to create stub driver: %v", err)+ }++ m, err := migrate.NewWithInstance("iofs", source, "stub", driver)+ if err != nil {+ t.Fatalf("failed to create migrate instance: %v", err)+ }++ var buf bytes.Buffer+ t.Cleanup(captureLog(&buf))++ m.Log = migrateLogger{store: "postgres"}+ if err := m.Up(); err != nil {+ t.Fatalf("migrate up failed: %v", err)+ }+ got := buf.String()++ for _, want := range []string{+ "migrations: postgres Read and execute 1/u first",+ "migrations: postgres Read and execute 2/u second",+ "Finished 1/u first",+ "Finished 2/u second",+ } {+ if !strings.Contains(got, want) {+ t.Fatalf("expected %q in the migration log, got:\n%s", want, got)+ }+ }++ for _, unwanted := range migrateReadAheadPrefixes {+ if strings.Contains(got, unwanted) {+ t.Fatalf("expected read-ahead line %q to be dropped, got:\n%s", unwanted, got)+ }+ }+}diff --git a/backend/app/migrations/migrations.go b/backend/app/migrations/migrations.goindex 71416233..f23d1a02 100644--- a/backend/app/migrations/migrations.go+++ b/backend/app/migrations/migrations.go@@ -4,8 +4,12 @@ import ("database/sql""embed""fmt"+ "io/fs""sort""strings"+ "time"++ "github.com/tracewayapp/traceway/backend/app/config")type ExtensionMigration struct {@@ -24,12 +28,12 @@ const sqliteTrackingDDL = `CREATE TABLE IF NOT EXISTS %s (applied_at DATETIME DEFAULT (datetime('now')))`-func runMigrationsOn(target *sql.DB, fsys embed.FS, dir, trackingTable, createTrackingSQL string) error {+func runMigrationsOn(target *sql.DB, fsys fs.FS, dir, trackingTable, createTrackingSQL string) error {if _, err := target.Exec(fmt.Sprintf(createTrackingSQL, trackingTable)); err != nil {return fmt.Errorf("failed to create %s table: %w", trackingTable, err)}- entries, err := fsys.ReadDir(dir)+ entries, err := fs.ReadDir(fsys, dir)if err != nil {return fmt.Errorf("failed to read migrations dir %s: %w", dir, err)}@@ -54,11 +58,14 @@ func runMigrationsOn(target *sql.DB, fsys embed.FS, dir, trackingTable, createTrcontinue}- content, err := fsys.ReadFile(dir + "/" + file)+ content, err := fs.ReadFile(fsys, dir+"/"+file)if err != nil {return fmt.Errorf("failed to read migration file %s: %w", file, err)}+ config.Logf("migrations: applying %s/%s", dir, version)+ started := time.Now()+statements := splitStatements(string(content))for _, stmt := range statements {stmt = strings.TrimSpace(stmt)@@ -73,6 +80,8 @@ func runMigrationsOn(target *sql.DB, fsys embed.FS, dir, trackingTable, createTrif _, err := target.Exec(fmt.Sprintf("INSERT INTO %s (version) VALUES (?)", trackingTable), version); err != nil {return fmt.Errorf("failed to record migration version %s: %w", version, err)}++ config.Logf("migrations: applied %s/%s in %s", dir, version, time.Since(started).Round(time.Millisecond))}return nildiff --git a/backend/app/migrations/migrations_log_test.go b/backend/app/migrations/migrations_log_test.gonew file mode 100644index 00000000..20179651--- /dev/null+++ b/backend/app/migrations/migrations_log_test.go@@ -0,0 +1,96 @@+package migrations++import (+ "bytes"+ "database/sql"+ "log"+ "strings"+ "testing"+ "testing/fstest"++ _ "modernc.org/sqlite"+)++// openMemorySQLite returns a fresh in-memory database pinned to one connection:+// every new connection to ":memory:" opens a separate, empty database, so a+// second connection would see none of the migrated schema.+func openMemorySQLite(t *testing.T) *sql.DB {+ t.Helper()+ target, err := sql.Open("sqlite", ":memory:")+ if err != nil {+ t.Fatalf("failed to open in-memory sqlite: %v", err)+ }+ t.Cleanup(func() { target.Close() })+ target.SetMaxOpenConns(1)+ return target+}++// captureLog redirects the standard logger into buf with its flags cleared, so+// no timestamp precedes a line and assertions can match from the line start.+// The returned func restores both.+func captureLog(buf *bytes.Buffer) func() {+ out, flags := log.Writer(), log.Flags()+ log.SetOutput(buf)+ log.SetFlags(0)+ return func() {+ log.SetOutput(out)+ log.SetFlags(flags)+ }+}++// assertLogLines checks buf holds exactly these lines in order. Each is matched+// as a prefix so the elapsed-time suffix does not pin a duration.+func assertLogLines(t *testing.T, buf *bytes.Buffer, want ...string) {+ t.Helper()+ got := strings.Split(strings.TrimSuffix(buf.String(), "\n"), "\n")+ if len(got) != len(want) {+ t.Fatalf("want %d log lines, got %d:\n%s", len(want), len(got), buf.String())+ }+ for i := range want {+ if !strings.HasPrefix(got[i], want[i]) {+ t.Errorf("line %d: want prefix %q, got %q", i, want[i], got[i])+ }+ }+}++// The announcement has to land before the statements run, since naming a+// migration only once it finishes is the silence this replaced. A failing+// migration is what makes that visible: it leaves the announcement with no+// completion. And an already-applied migration has to stay quiet, or every+// restart reprints the full history.+func TestRunMigrationsOnAnnouncesEachMigrationBeforeRunningIt(t *testing.T) {+ var buf bytes.Buffer+ t.Cleanup(captureLog(&buf))+ target := openMemorySQLite(t)++ fsys := fstest.MapFS{+ "m/0001_first.up.sql": {Data: []byte("CREATE TABLE first (id INTEGER)")},+ "m/0002_second.up.sql": {Data: []byte("CREATE TABLE second (")},+ }+ if err := runMigrationsOn(target, fsys, "m", "schema_migrations", sqliteTrackingDDL); err == nil {+ t.Fatal("expected the broken migration to fail")+ }+ assertLogLines(t, &buf,+ "[tracewaybackend] migrations: applying m/0001_first",+ "[tracewaybackend] migrations: applied m/0001_first in ",+ "[tracewaybackend] migrations: applying m/0002_second",+ )++ fsys["m/0002_second.up.sql"] = &fstest.MapFile{Data: []byte("CREATE TABLE second (id INTEGER)")}+ buf.Reset()+ if err := runMigrationsOn(target, fsys, "m", "schema_migrations", sqliteTrackingDDL); err != nil {+ t.Fatalf("migrations failed: %v", err)+ }+ assertLogLines(t, &buf,+ "[tracewaybackend] migrations: applying m/0002_second",+ "[tracewaybackend] migrations: applied m/0002_second in ",+ )++ buf.Reset()+ if err := runMigrationsOn(target, fsys, "m", "schema_migrations", sqliteTrackingDDL); err != nil {+ t.Fatalf("third run failed: %v", err)+ }+ if buf.Len() != 0 {+ t.Errorf("re-running applied migrations logged %q, want silence", buf.String())+ }+}diff --git a/backend/app/migrations/migrations_telemetry_ch.go b/backend/app/migrations/migrations_telemetry_ch.goindex 103e4233..30e234eb 100644--- a/backend/app/migrations/migrations_telemetry_ch.go+++ b/backend/app/migrations/migrations_telemetry_ch.go@@ -42,6 +42,8 @@ func runMigrationsClickhouse(connStr string) error {return err}+ m.Log = migrateLogger{store: "clickhouse"}+if err := m.Up(); err != nil && err != migrate.ErrNoChange {return err}diff --git a/backend/app/migrations/migrations_transactional_pg.go b/backend/app/migrations/migrations_transactional_pg.goindex 67a38fd9..951ed5cd 100644--- a/backend/app/migrations/migrations_transactional_pg.go+++ b/backend/app/migrations/migrations_transactional_pg.go@@ -42,6 +42,8 @@ func runMigrationsPostgres(connStr string) error {return fmt.Errorf("failed to create migrate instance: %w", err)}+ m.Log = migrateLogger{store: "postgres"}+if err := m.Up(); err != nil && err != migrate.ErrNoChange {return fmt.Errorf("postgres migration failed: %w", err)}@@ -78,6 +80,8 @@ func runExtensionMigrations(db *sql.DB, ext ExtensionMigration) error {return fmt.Errorf("failed to create extension migrate instance: %w", err)}+ m.Log = migrateLogger{store: "postgres extension " + tableName}+if err := m.Up(); err != nil && err != migrate.ErrNoChange {return fmt.Errorf("extension postgres migration failed: %w", err)}diff --git a/backend/app/migrations/telemetry_group_index_test.go b/backend/app/migrations/telemetry_group_index_test.goindex ba76bb77..21ca6e76 100644--- a/backend/app/migrations/telemetry_group_index_test.go+++ b/backend/app/migrations/telemetry_group_index_test.go@@ -2,36 +2,16 @@package migrations-import (- "database/sql"- "slices"- "strings"- "testing"+import "testing"- "github.com/tracewayapp/traceway/backend/app/retention"- _ "modernc.org/sqlite"-)--// Every per-group telemetry read pairs a group column with the table's time-// column: as a second filter on the grouped lists, as the ORDER BY on the-// per-group detail reads. An index that stops at the group column leaves SQLite-// to either re-filter the group across the whole window once per group, which-// is quadratic in the number of groups, or seek the group and sort its entire-// history. Both are unbounded and invisible to functional tests.-//-// The first loop guards the indexes behind known reads so a later migration-// cannot narrow or drop one unnoticed. The second inverts the question: every-// index that leads with project_id must reach its table's time column, so a-// short index fails here the day it lands rather than when somebody remembers-// to list it. Indexes that stop short on purpose are named with the reason,-// and each entry is itself checked so it cannot outlive the index it excuses.-func TestTelemetryGroupIndexesCoverTimeColumn(t *testing.T) {- telemetryDB, err := sql.Open("sqlite", ":memory:")- if err != nil {- t.Fatalf("failed to open in-memory sqlite: %v", err)- }- defer telemetryDB.Close()- telemetryDB.SetMaxOpenConns(1)+// The grouped-list reads for these three tables filter+// (project_id, <group>, recorded_at). An index that stops at the group column+// leaves SQLite to seek the (project_id, recorded_at) index and re-filter the+// group column across the whole window once per group, which is quadratic in+// the number of groups and invisible to functional tests. Guard the covering+// indexes so a later migration cannot narrow or drop them unnoticed.+func TestTelemetryGroupIndexesCoverRecordedAt(t *testing.T) {+ telemetryDB := openMemorySQLite(t)if err := runMigrationsOn(telemetryDB, migrationsSqliteTelemetryFS, "sqlite_telemetry", "schema_migrations", sqliteTrackingDDL); err != nil {t.Fatalf("telemetry migrations failed: %v", err)diff --git a/docker-compose.browser.yml b/docker-compose.browser.ymlindex 4fc455b0..c48a9064 100644--- a/docker-compose.browser.yml+++ b/docker-compose.browser.yml@@ -54,12 +54,6 @@ services:volumes:- traceway-data:/datarestart: unless-stopped- healthcheck:- test: ["CMD", "wget", "-qO-", "http://localhost/health"]- interval: 30s- timeout: 3s- start_period: 5s- retries: 3volumes:traceway-data:diff --git a/docker-compose.duckdb.yml b/docker-compose.duckdb.ymlindex 030e634b..c3cf4b72 100644--- a/docker-compose.duckdb.yml+++ b/docker-compose.duckdb.yml@@ -42,12 +42,6 @@ services:volumes:- traceway-data:/datarestart: unless-stopped- healthcheck:- test: ["CMD", "wget", "-qO-", "http://localhost/health"]- interval: 30s- timeout: 3s- start_period: 5s- retries: 3volumes:traceway-data:diff --git a/docker-compose.sqlite.yml b/docker-compose.sqlite.ymlindex 7c32be60..4f5e8855 100644--- a/docker-compose.sqlite.yml+++ b/docker-compose.sqlite.yml@@ -32,12 +32,6 @@ services:volumes:- traceway-data:/datarestart: unless-stopped- healthcheck:- test: ["CMD", "wget", "-qO-", "http://localhost/health"]- interval: 30s- timeout: 3s- start_period: 5s- retries: 3volumes:traceway-data:diff --git a/helm/traceway/templates/deployment.yaml b/helm/traceway/templates/deployment.yamlindex 0494f497..fc83c282 100644--- a/helm/traceway/templates/deployment.yaml+++ b/helm/traceway/templates/deployment.yaml@@ -8,6 +8,9 @@ spec:{{- if not .Values.autoscaling.enabled }}replicas: {{ .Values.replicaCount }}{{- end }}+ {{- $probe := .Values.startupProbe | default dict }}+ {{- $probeBudget := mul ($probe.periodSeconds | default 0) ($probe.failureThreshold | default 0) }}+ progressDeadlineSeconds: {{ .Values.progressDeadlineSeconds | default (max 600 (add $probeBudget 60)) }}selector:matchLabels:{{- include "traceway.selectorLabels" . | nindent 6 }}@@ -51,6 +54,10 @@ spec:name: {{ include "traceway.fullname" . }}- secretRef:name: {{ include "traceway.secretName" . }}+ {{- with .Values.startupProbe }}+ startupProbe:+ {{- toYaml . | nindent 12 }}+ {{- end }}livenessProbe:httpGet:path: /healthdiff --git a/helm/traceway/values.yaml b/helm/traceway/values.yamlindex 6557c87e..1e4538b5 100644--- a/helm/traceway/values.yaml+++ b/helm/traceway/values.yaml@@ -40,6 +40,33 @@ ingress:resources: {}+# Startup probe. The liveness budget only starts once this succeeds, so a slow+# first boot is not mistaken for a hung process. The server binds its ports only+# after migrations finish, and an upgrade that builds indexes on a large+# telemetry database can take minutes — without this the kubelet would kill the+# pod mid-migration and restart it into the same stall on every retry. The+# default budget (periodSeconds x failureThreshold = 300s) is the same one the+# Docker images give their HEALTHCHECK start period. Raise failureThreshold if+# an upgrade needs longer; the backend logs each migration with its elapsed+# time (boot phases before migrations, such as DuckDB WAL replay, are not yet+# timed). The Deployment's progressDeadlineSeconds follows this budget, but a+# deploy tool's own timeout does not: `helm upgrade --wait` defaults to 5m, and+# `--atomic` (or a Flux HelmRelease remediation) rolls back when it expires,+# scaling the migrating pod away — so raise `--timeout` / `spec.timeout` in+# step with the budget.+startupProbe:+ httpGet:+ path: /health+ port: http+ periodSeconds: 5+ timeoutSeconds: 3+ failureThreshold: 60++# Deployment progress deadline. Derived from the startup probe budget plus a+# minute for scheduling and image pull, never below the Kubernetes default of+# 600s; set to override.+# progressDeadlineSeconds: 900+autoscaling:enabled: falseminReplicas: 1