Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

perf(clickhouse): gzip query responses from ClickHouse (#1310)

Goldflag3fc6af8e119a
diff --git a/server/src/db/clickhouse/client.test.ts b/server/src/db/clickhouse/client.test.ts
index fbcbe29bf..03703ca3f 100644
--- a/server/src/db/clickhouse/client.test.ts
+++ b/server/src/db/clickhouse/client.test.ts
@@ -16,4 +16,17 @@ describe("clickhouse write client", () => {
};
expect(writeClientOptions.clickhouse_settings).toMatchObject({ date_time_input_format: "best_effort" });
});
+
+ // Results are gzipped on the wire; inserts stay uncompressed so batches don't
+ // cost gzip CPU on the ingestion path. The query user may be readonly=1 when
+ // created by hand, which rejects enable_http_compression, so it stays off.
+ it("compresses query responses but not inserts or user-authored SQL", async () => {
+ await import("./client.js");
+
+ const [writeClientOptions, queryClientOptions] = mocks.createClient.mock.calls.map(
+ call => call[0] as { compression?: { request?: boolean; response?: boolean } }
+ );
+ expect(writeClientOptions.compression).toEqual({ response: true, request: false });
+ expect(queryClientOptions.compression).toBeUndefined();
+ });
});
diff --git a/server/src/db/clickhouse/client.ts b/server/src/db/clickhouse/client.ts
index 21f80889d..d3b8fc69a 100644
--- a/server/src/db/clickhouse/client.ts
+++ b/server/src/db/clickhouse/client.ts
@@ -8,6 +8,13 @@ export const clickhouse = createClient({
database: process.env.CLICKHOUSE_DB,
password: process.env.CLICKHOUSE_PASSWORD,
request_timeout: CLICKHOUSE_REQUEST_TIMEOUT_MS,
+ // Gzip query results on the wire (sends enable_http_compression=1 and
+ // Accept-Encoding: gzip). Dashboard results such as session lists and replay
+ // events can be MBs of JSON, and ClickHouse usually runs on another host.
+ // Inserts stay uncompressed: gzipping every batch would cost Node CPU on the
+ // ingestion path. The main user is never readonly=1 (it inserts), so it can
+ // set enable_http_compression.
+ compression: { response: true, request: false },
clickhouse_settings: {
// Writers serialise every instant as ISO-8601 with an explicit `Z` (see
// dateTime.ts). The default `basic` parser rejects the offset; best_effort
@@ -26,6 +33,8 @@ export const clickhouse = createClient({
// bypass must still land inside these grants and the caller's sites.
export const CLICKHOUSE_QUERY_USER = process.env.CLICKHOUSE_QUERY_USER || "rybbit_query";
+// No response compression here: results are capped at 1,000 rows, and a
+// hand-created readonly=1 user would reject the enable_http_compression setting.
export const clickhouseQuery = createClient({
url: process.env.CLICKHOUSE_HOST,
database: process.env.CLICKHOUSE_DB,