🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
perf(clickhouse): gzip query responses from ClickHouse (#1310)
Goldflag3fc6af8e119a
diff --git a/server/src/db/clickhouse/client.test.ts b/server/src/db/clickhouse/client.test.tsindex fbcbe29bf..03703ca3f 100644--- a/server/src/db/clickhouse/client.test.ts+++ b/server/src/db/clickhouse/client.test.ts@@ -16,4 +16,17 @@ describe("clickhouse write client", () => {};expect(writeClientOptions.clickhouse_settings).toMatchObject({ date_time_input_format: "best_effort" });});++ // Results are gzipped on the wire; inserts stay uncompressed so batches don't+ // cost gzip CPU on the ingestion path. The query user may be readonly=1 when+ // created by hand, which rejects enable_http_compression, so it stays off.+ it("compresses query responses but not inserts or user-authored SQL", async () => {+ await import("./client.js");++ const [writeClientOptions, queryClientOptions] = mocks.createClient.mock.calls.map(+ call => call[0] as { compression?: { request?: boolean; response?: boolean } }+ );+ expect(writeClientOptions.compression).toEqual({ response: true, request: false });+ expect(queryClientOptions.compression).toBeUndefined();+ });});diff --git a/server/src/db/clickhouse/client.ts b/server/src/db/clickhouse/client.tsindex 21f80889d..d3b8fc69a 100644--- a/server/src/db/clickhouse/client.ts+++ b/server/src/db/clickhouse/client.ts@@ -8,6 +8,13 @@ export const clickhouse = createClient({database: process.env.CLICKHOUSE_DB,password: process.env.CLICKHOUSE_PASSWORD,request_timeout: CLICKHOUSE_REQUEST_TIMEOUT_MS,+ // Gzip query results on the wire (sends enable_http_compression=1 and+ // Accept-Encoding: gzip). Dashboard results such as session lists and replay+ // events can be MBs of JSON, and ClickHouse usually runs on another host.+ // Inserts stay uncompressed: gzipping every batch would cost Node CPU on the+ // ingestion path. The main user is never readonly=1 (it inserts), so it can+ // set enable_http_compression.+ compression: { response: true, request: false },clickhouse_settings: {// Writers serialise every instant as ISO-8601 with an explicit `Z` (see// dateTime.ts). The default `basic` parser rejects the offset; best_effort@@ -26,6 +33,8 @@ export const clickhouse = createClient({// bypass must still land inside these grants and the caller's sites.export const CLICKHOUSE_QUERY_USER = process.env.CLICKHOUSE_QUERY_USER || "rybbit_query";+// No response compression here: results are capped at 1,000 rows, and a+// hand-created readonly=1 user would reject the enable_http_compression setting.export const clickhouseQuery = createClient({url: process.env.CLICKHOUSE_HOST,database: process.env.CLICKHOUSE_DB,