:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
chore(metrics): check the capture Avro schema against the ClickHouse Kafka table (#125938)
Daniel Viscac7e52d2e4b19
diff --git a/posthog/test/repo_invariants/test_metrics_avro_contract.py b/posthog/test/repo_invariants/test_metrics_avro_contract.pynew file mode 100644index 000000000000..d3d16230842b--- /dev/null+++ b/posthog/test/repo_invariants/test_metrics_avro_contract.py@@ -0,0 +1,99 @@+# ClickHouse reads `clickhouse_metrics` with `input_format_avro_allow_missing_fields = 1`, so a field+# capture stops sending is read as the default. A NULL `series_fingerprint` makes the metrics4 MV drop+# the row, and a type mismatch makes `kafka_skip_broken_messages` skip the block. Neither raises.++import re+import json+from pathlib import Path++REPO_ROOT = Path(__file__).resolve().parents[3]+AVRO_SCHEMA_RS = REPO_ROOT / "rust/capture-logs/src/metrics_avro_schema.rs"+KAFKA_TABLE_HCL = REPO_ROOT / "posthog/clickhouse/hcl/roles/apm/shared/metrics4.hcl"+KAFKA_TABLE = "kafka_metrics_avro4"++# The ClickHouse column types that read each Avro type correctly.+AVRO_TO_CLICKHOUSE: dict[str, set[str]] = {+ "string": {"String"},+ "bytes": {"String"},+ "int": {"Int32"},+ "long": {"Int64"},+ "long/timestamp-micros": {"DateTime64(6)"},+ "double": {"Float64"},+ "boolean": {"UInt8", "Bool"},+ "array<double>": {"Array(Float64)"},+ "array<long>": {"Array(Int64)"},+ "map<string>": {"Map(String, String)"},+}+++def avro_fields(schema_rs: str) -> dict[str, str]:+ schema = json.loads(schema_rs[schema_rs.index('r#"') + 3 : schema_rs.index('"#')])+ return {field["name"]: _avro_type(field["type"]) for field in schema["fields"]}+++def _avro_type(avro_type: object) -> str:+ if isinstance(avro_type, list):+ (non_null,) = [t for t in avro_type if t != "null"]+ return _avro_type(non_null)+ if isinstance(avro_type, str):+ return avro_type+ assert isinstance(avro_type, dict)+ if "logicalType" in avro_type:+ return f"{avro_type['type']}/{avro_type['logicalType']}"+ if avro_type["type"] == "array":+ return f"array<{avro_type['items']}>"+ if avro_type["type"] == "map":+ return f"map<{avro_type['values']}>"+ return avro_type["type"]+++def kafka_table_columns(hcl: str, table: str) -> dict[str, str]:+ block = hcl[hcl.index(f'table "{table}" {{') :]+ block = block[: block.index('engine "kafka"')]+ columns: dict[str, str] = {}+ for name, body in re.findall(r'column\s+"(\w+)"\s*\{([^}]*)\}', block):+ column_type = re.search(r'\btype\s*=\s*"([^"]+)"', body)+ assert column_type, f"{KAFKA_TABLE} column {name!r} has no type"+ columns[name] = _strip_nullable(column_type.group(1))+ # A declaration the pattern misses would drop out of the reverse check without an error.+ assert len(columns) == len(re.findall(r'\bcolumn\s+"', block)), f"Could not parse every column of {table}"+ return columns+++def _strip_nullable(column_type: str) -> str:+ match = re.fullmatch(r"Nullable\((.+)\)", column_type)+ return match.group(1) if match else column_type+++def contract_errors(fields: dict[str, str], columns: dict[str, str]) -> list[str]:+ errors = [f"Avro field {name!r} has no column in {KAFKA_TABLE}" for name in sorted(fields.keys() - columns.keys())]+ errors += [f"{KAFKA_TABLE} column {name!r} has no Avro field" for name in sorted(columns.keys() - fields.keys())]+ for name in sorted(fields.keys() & columns.keys()):+ allowed = AVRO_TO_CLICKHOUSE.get(fields[name], set())+ if columns[name] not in allowed:+ errors.append(f"{name!r}: Avro {fields[name]} cannot be read as {columns[name]}")+ return errors+++def test_capture_avro_schema_matches_the_clickhouse_kafka_table() -> None:+ fields = avro_fields(AVRO_SCHEMA_RS.read_text())+ columns = kafka_table_columns(KAFKA_TABLE_HCL.read_text(), KAFKA_TABLE)++ assert "series_fingerprint" in fields+ assert contract_errors(fields, columns) == []+++def test_kafka_table_columns_reads_every_declaration_layout() -> None:+ hcl = """+ table "kafka_metrics_avro4" {+ column "uuid" { type = "String" }+ column "count" {+ type = "Nullable(Int64)"+ }+ column "retention_days" { type = "Int32" comment = "days" }+ engine "kafka" {+ }+ }+"""++ assert kafka_table_columns(hcl, KAFKA_TABLE) == {"uuid": "String", "count": "Int64", "retention_days": "Int32"}