🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(topics): combine pending Topics schema migrations (#19513)
Lysander Kiesel632ffcfaf9dc
diff --git a/packages/shared/clickhouse/migrations/canonical/0055_update_topic_tables.down.sql b/packages/shared/clickhouse/migrations/canonical/0055_update_topic_tables.down.sqlnew file mode 100644index 0000000000..41125f2b00--- /dev/null+++ b/packages/shared/clickhouse/migrations/canonical/0055_update_topic_tables.down.sql@@ -0,0 +1,2 @@+ALTER TABLE topic_assignments {CLICKHOUSE_CLUSTER_CLAUSE} DROP INDEX IF EXISTS idx_trace_id{CLICKHOUSE_CLUSTERED_ONLY: SETTINGS alter_sync = 2};+ALTER TABLE topic_facet_summaries {CLICKHOUSE_CLUSTER_CLAUSE} MODIFY COLUMN embedding Array(Float32) SETTINGS mutations_sync = 2{CLICKHOUSE_CLUSTERED_ONLY:, alter_sync = 2};diff --git a/packages/shared/clickhouse/migrations/canonical/0055_update_topic_tables.up.sql b/packages/shared/clickhouse/migrations/canonical/0055_update_topic_tables.up.sqlnew file mode 100644index 0000000000..b7f1d0c780--- /dev/null+++ b/packages/shared/clickhouse/migrations/canonical/0055_update_topic_tables.up.sql@@ -0,0 +1,5 @@+-- Store summary embeddings as BFloat16, halving their size. Cosine distance on normalized vectors is unaffected.+ALTER TABLE topic_facet_summaries {CLICKHOUSE_CLUSTER_CLAUSE} MODIFY COLUMN embedding Array(BFloat16) SETTINGS mutations_sync = 2{CLICKHOUSE_CLUSTERED_ONLY:, alter_sync = 2};+-- Point lookups by trace_id (trace views, API, agent) otherwise scan every granule in the time window. The table is small, so existing parts are materialized.+ALTER TABLE topic_assignments {CLICKHOUSE_CLUSTER_CLAUSE} ADD INDEX IF NOT EXISTS idx_trace_id trace_id TYPE bloom_filter(0.001) GRANULARITY 1{CLICKHOUSE_CLUSTERED_ONLY: SETTINGS alter_sync = 2};+ALTER TABLE topic_assignments {CLICKHOUSE_CLUSTER_CLAUSE} MATERIALIZE INDEX IF EXISTS idx_trace_id{CLICKHOUSE_CLUSTERED_ONLY: SETTINGS mutations_sync = 2};diff --git a/packages/shared/prisma/migrations/20261030120000_add_topic_rule_scoped_runs/migration.sql b/packages/shared/prisma/migrations/20261030120000_add_topic_rule_scoped_runs/migration.sqlnew file mode 100644index 0000000000..62e55ab77c--- /dev/null+++ b/packages/shared/prisma/migrations/20261030120000_add_topic_rule_scoped_runs/migration.sql@@ -0,0 +1,6 @@+-- Topics rule of a continuous map, or the batch action of a one-off analysis. Nullable: existing runs and writers that predate the columns leave them null.+ALTER TABLE "topic_clustering_runs" ADD COLUMN "rule_id" TEXT;+ALTER TABLE "topic_clustering_runs" ADD COLUMN "batch_action_id" TEXT;++CREATE INDEX "topic_clustering_runs_rule_serving_map_idx" ON "topic_clustering_runs"("project_id", "rule_id", "facet_id", "status");+CREATE INDEX "topic_clustering_runs_project_id_batch_action_id_idx" ON "topic_clustering_runs"("project_id", "batch_action_id");diff --git a/packages/shared/prisma/schema.prisma b/packages/shared/prisma/schema.prismaindex 3a259c6640..2fbe156b30 100644--- a/packages/shared/prisma/schema.prisma+++ b/packages/shared/prisma/schema.prisma@@ -2257,7 +2257,12 @@ model TopicClusteringRun {startedAt DateTime? @map("started_at") @db.Timestamptz(3)finishedAt DateTime? @map("finished_at") @db.Timestamptz(3)topicVersionIds String[] @default([]) @map("topic_version_ids")+ // Topics rule of a continuous map, or the batch action of a one-off analysis; null on runs written before either existed.+ ruleId String? @map("rule_id")+ batchActionId String? @map("batch_action_id")@@index([projectId, facetId, status, facetVersion(sort: Desc), createdAt(sort: Desc), id(sort: Desc)], map: "topic_clustering_runs_serving_map_idx")+ @@index([projectId, ruleId, facetId, status], map: "topic_clustering_runs_rule_serving_map_idx")+ @@index([projectId, batchActionId])@@map("topic_clustering_runs")}diff --git a/worker/src/features/topics/README.md b/worker/src/features/topics/README.mdindex 686d2a99c3..37dbd0c593 100644--- a/worker/src/features/topics/README.md+++ b/worker/src/features/topics/README.md@@ -480,6 +480,9 @@ Naming calls use short member labels; accepted evidence references store typedfacet-version and trace-or-session fields under the definition's project.A trace reference omits its parent session. Those references identify the currentsource result, not a historical summary snapshot.++Summary embeddings are stored as `Array(BFloat16)` (migration `0054_update_topic_tables`). JSON inserts truncate each element to BFloat16 and reads return plain numbers, so vectors loaded from storage differ slightly from the model output that online assignment classified. Classifier inputs are normalized and compared by cosine distance, which this rounding barely moves. Topic reuse is stricter: `sameTopicGeometry` only tolerates Float64 roundoff, so centroids recomputed from summaries written as Float32 before the BFloat16 migration never match, and the first update of such a map names every topic again and creates new topic versions.+Definitions use RowBinary inserts with `Float64` geometry to preserve classifier precision and`ReplacingMergeTree(created_at)` partitioned by creation month and ordered byproject, creation date and version ID to deduplicate identical retry writes.