Umami is a privacy-first web and product analytics platform. Traffic, campaigns, behavior, conversions, and revenue in one place. No cookies, no surveillance, self-hosted or in the cloud.
TypeScript analyticsaudience-segmentationchartscohort-analysisgoogle-analyticsproduct-analyticsstatisticsuser-journeyweb-analytics
add clickhouse+s3 workflow for heatmaps
Francis Cao07e40db5f4f7
diff --git a/db/clickhouse/migrations/12_add_heatmap.sql b/db/clickhouse/migrations/12_add_heatmap.sqlindex ba5f93cb7..db44666b3 100644--- a/db/clickhouse/migrations/12_add_heatmap.sql+++ b/db/clickhouse/migrations/12_add_heatmap.sql@@ -26,3 +26,25 @@ ENGINE = MergeTreePARTITION BY toYYYYMM(created_at)ORDER BY (website_id, url_path, event_type, created_at)SETTINGS index_granularity = 8192;++-- Create heatmap_snapshot+CREATE TABLE umami.heatmap_snapshot+(+ snapshot_id UUID,+ website_id UUID,+ url_path String,+ viewport_w UInt32,+ viewport_h UInt32,+ page_w UInt32,+ page_h UInt32,+ status UInt8,+ mime_type LowCardinality(String),+ object_key String,+ image_size Nullable(UInt32),+ error Nullable(String),+ created_at DateTime('UTC')+)+ENGINE = MergeTree+ PARTITION BY toYYYYMM(created_at)+ ORDER BY (website_id, url_path, viewport_w, viewport_h, created_at)+ SETTINGS index_granularity = 8192;diff --git a/db/clickhouse/schema.sql b/db/clickhouse/schema.sqlindex a075977f1..75943488c 100644--- a/db/clickhouse/schema.sql+++ b/db/clickhouse/schema.sql@@ -428,3 +428,25 @@ ENGINE = MergeTreePARTITION BY toYYYYMM(created_at)ORDER BY (website_id, url_path, event_type, created_at)SETTINGS index_granularity = 8192;++-- Create heatmap_snapshot+CREATE TABLE umami.heatmap_snapshot+(+ snapshot_id UUID,+ website_id UUID,+ url_path String,+ viewport_w UInt32,+ viewport_h UInt32,+ page_w UInt32,+ page_h UInt32,+ status UInt8,+ mime_type LowCardinality(String),+ object_key String,+ image_size Nullable(UInt32),+ error Nullable(String),+ created_at DateTime('UTC')+)+ENGINE = MergeTree+ PARTITION BY toYYYYMM(created_at)+ ORDER BY (website_id, url_path, viewport_w, viewport_h, created_at)+ SETTINGS index_granularity = 8192;diff --git a/package.json b/package.jsonindex fd263afcf..72f9a47b5 100644--- a/package.json+++ b/package.json@@ -53,6 +53,7 @@".next/cache"],"dependencies": {+ "@aws-sdk/client-s3": "^3.924.0","@clickhouse/client": "^1.18.5","@date-fns/utc": "^2.1.1","@dicebear/collection": "^9.4.2",diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex c4f918851..7bf76d6af 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -8,6 +8,9 @@ importers:.:dependencies:+ '@aws-sdk/client-s3':+ specifier: ^3.924.0+ version: 3.1050.0'@clickhouse/client':specifier: ^1.18.5version: 1.18.5@@ -352,6 +355,125 @@ packages:'@asamuzakjp/[email protected]':resolution: {integrity: sha512-n8GuYSrI9bF7FFZ/SjhwevlHc8xaVlb/7HmHelnc/PZXBD2ZR49NnN9sMMuDdEGPeeRQ5d0hqlSlEpgCX3Wl0Q==}+ '@aws-crypto/[email protected]':+ resolution: {integrity: sha512-nLbCWqQNgUiwwtFsen1AdzAtvuLRsQS8rYgMuxCrdKf9kOssamGLuPwyTY9wyYblNr9+1XM8v6zoDTPPSIeANg==}+ engines: {node: '>=16.0.0'}++ '@aws-crypto/[email protected]':+ resolution: {integrity: sha512-+iWb8qaHLYKrNvGRbiYRHSdKRWhto5XlZUEBwDjYNf+ly5SVYG6zEoYIdxvf5R3zyeP16w4PLBn3rH1xc74Rag==}++ '@aws-crypto/[email protected]':+ resolution: {integrity: sha512-OH6lveCFfcDjX4dbAvCFSYUjJZjDr/3XJ3xHtjn3Oj5b9RjojQo8npoLeA/bNwkOkrSQ0wgrHzXk4tDRxGKJeg==}++ '@aws-crypto/[email protected]':+ resolution: {integrity: sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw==}++ '@aws-crypto/[email protected]':+ resolution: {integrity: sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA==}+ engines: {node: '>=16.0.0'}++ '@aws-crypto/[email protected]':+ resolution: {integrity: sha512-iAvUotm021kM33eCdNfwIN//F77/IADDSs58i+MDaOqFrVjZo9bAal0NK7HurRuWLLpF1iLX7gbWrjHjeo+YFg==}++ '@aws-crypto/[email protected]':+ resolution: {integrity: sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==}++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-9kgtv+bXZQrOIJT2INPPBCezrJu1FlgGrzEat/ut4A4V53IT00LynsBZgp12eFKbjJuNCeTo7iPSKjPsX8ub+A==}+ engines: {node: '>=20.0.0'}++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-qrqgioqYFjwR6LatVNS1L2Vk++EwRIxqSQXPKNv5Ofux2D8UNgqMQ1znnMyEImXquVPTtbf71fc128pvmU6y9A==}+ engines: {node: '>=20.0.0'}++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-fVfUCL/Xh2zINYMPZvj+iBn6XWouQf0DAnjaWCI9MkmqXzL2Iy5FoQB8O7syFe6gN6AH1ecDDU58T51Ou0kFkA==}+ engines: {node: '>=20.0.0'}++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-m3WjZEgPtioMhPmwqUt+DhlTJ2i9ufR6DhfkyXojb9puEvfR+ur2U5shavu5/Cc9WHHsDCvALi6UFHgcqjhQ5w==}+ engines: {node: '>=20.0.0'}++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-D78L/m2Dr6cJnnSvWoAudPhQmCwmJ7j6APXsPYmFpPaKfQTfCSu0rdm8j14Np+VmXF9z8Aj8HE3xFpsrwtfgeg==}+ engines: {node: '>=20.0.0'}++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-Mu5ESvFXeinafVM8jTIvRqcvK2Ehj4kz3auT39yUcHwu1Vfxo6xRlmUafdKLW4tusjAJukQwK09sCSMgOm7OKg==}+ engines: {node: '>=20.0.0'}++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-O6WkZga3kf0yqyJYd1dbeJqVhEgJx/x1UaLgtbR+XuL/YP+K5y6QTxQKL7ka9z3jnQASESKGAPnRyt4D5hQrxA==}+ engines: {node: '>=20.0.0'}++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-D/DJmbrWRP5BXEO3FH+ar4el+2n6OlGofiud7dQun2jES+AQEJjczenp1jBb4MBN7CpGpS8nsWGQLtuzc9tQbA==}+ engines: {node: '>=20.0.0'}++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-EnbYVajGgbkb24s0K1eo4VNAPV5mHIET7LSvirTaFCwkfrfaOJxtSE+wY/tJdKDS21cEYkZs2ruCaAm+W4iblg==}+ engines: {node: '>=20.0.0'}++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-RVV/9NbFwI8ZHEH5dn39lGyFmSbSVj1+orZdr6QsOe1mW9DCglmlen0cFaNZmCcqkqc7erNRHNBduxbeZuHAnw==}+ engines: {node: '>=20.0.0'}++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-/67fXX0ddllD4u2Nujc5PvT4byHgpMUfz6+RxIKi/0nFIckeorm7JvXgzBuDyVKw0s58EbofmETDWUf9vTEuHQ==}+ engines: {node: '>=20.0.0'}++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-Aaj0d+xbo1jJquBWJP0/9V/XZRYukO3LWIRp3dOLHmoFrYKb4YZ0aLefgVHfGcNOVBS2ZTq7L/n5JcrE7DaC+Q==}+ engines: {node: '>=20.0.0'}++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-dA5pKTom/Ls9mgeyeaRBNQrRIVOLVjv4AmKOB0/e4yaiXEUy0gSz2d3liP8JHtYoCAEWySU1jWnyzwLOREN+4g==}+ engines: {node: '>=20.0.0'}++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-NdnMVQCR1YjIcqFAiNLdBiOwr2DyQDB2IiXQrBhzolKOv32ae4d4Ll7IzLMi04eMHiq/o/Y/GjFuVjF9HuG0QA==}+ engines: {node: '>=20.0.0'}++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-rI3NZvJcEvjoD0+0PI0iUAwlPw2IlSlhyvgBK/3WkKJQE/YiKFedd9dMN2lVacdNxPNhxL/jzQaKQdrGtQagjQ==}+ engines: {node: '>=20.0.0'}++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-M4T2I2WPuH5WQpU8Tsp+u2bcO29zGRkU14ATzuqb9I4xh8tzsLqtp4hzaJM5aO2dhMZnHDzyQwSFVgc3XbnoGg==}+ engines: {node: '>=20.0.0'}++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-Gli9A0u8EVVb+5bFDGS/QbSVg28w/wpEidg1ggVcSj65BDTdGR6punsOcVjqdiu1i42WHWo51MCvARPIIz9juw==}+ engines: {node: '>=20.0.0'}++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-FtQ/Bt327peZJuyo4WZSOLVUTw9ujRxntepiC7L65FxA2P82Xlq0g14T22BuqBUeMjDoxa9nvwiMHjLIfP3eUg==}+ engines: {node: '>=20.0.0'}++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-0Phbz4t6HI3D3skxvG2uI+VWU034/nSIw1T8d+FPzzQG9EQTrw94o9mOKO2Gv3n3Oc8P7JD7RAUxkoneLWv5Eg==}+ engines: {node: '>=20.0.0'}++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-r7+d0lQMTHKypkmaF5jRTBYLYHCUHzt3gaVoN9SidLhQeWhCmHk3AKrboDTpPF5b7Pt7vKu3+oeMjznM2Eu1ow==}+ engines: {node: '>=20.0.0'}++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-gjlAdtHMbtR9X5iIhVUvbVcy55KnznpC6bkDUWW9z915bi0ckdUr5cjf16Kp6xq0bP5HBD2xzgbL9F9Quv5vUw==}+ engines: {node: '>=20.0.0'}++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-WhlJNNINQB+9qtLtZJcpQdgZw3SCDCpXdUJP7cToGwHbCWCnRckGlc6Bx/OhWwIYFNAn+FIydY8SZ0QmVu3xTQ==}+ engines: {node: '>=20.0.0'}++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-V8z5YcDPfsvzrBlj0xR1vhRtocblhYbqdreCJB/voGd4Sr5zjNAeWxexbnqVtskTJe0vFb5KMqbSL++ePl+zRw==}+ engines: {node: '>=20.0.0'}++ '@aws/[email protected]':+ resolution: {integrity: sha512-iY8yvjE0y651BixKNPgmv1WrQc+GZ142sb0z4gYnChDDY2YqI4P/jsSopBWrKfAt7LOJAkOXt7rC/hms+WclQQ==}+ engines: {node: '>=18.0.0'}+'@babel/[email protected]':resolution: {integrity: sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg==}engines: {node: '>=6.9.0'}@@ -1753,6 +1875,9 @@ packages:cpu: [x64]os: [win32]+ '@nodable/[email protected]':+ resolution: {integrity: sha512-nyT7T3nbMyBI/lvr6L5TyWbFJAI9FTgVRakNoBqCD+PmID8DzFrrNdLLtHMwMszOtqZa8PAOV24ZqDnQrhQINA==}+'@nodelib/[email protected]':resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==}engines: {node: '>= 8'}@@ -2430,6 +2555,42 @@ packages:resolution: {integrity: sha512-LtoMMhxAlorcGhmFYI+LhPgbPZCkgP6ra1YL604EeF6U98pLlQ3iWIGMdWSC+vWmPBWBNgmDBAhnAobLROJmwg==}engines: {node: '>=18'}+ '@smithy/[email protected]':+ resolution: {integrity: sha512-Ep/7tPamGY8mgESE3LyLKtxJyy6U52WWAqr/3wial47Sj4u3PiIF73AOGI27UyLy9duTkhZbgzodOfLV4TduZg==}+ engines: {node: '>=18.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-I2Bti0DKFo2IJyN28ijCsx51BAumEYR4/1yZ1FXyBygy9MqbnMqCev4JPth/MbpRfBSRAX35hITSnAdJRo1u5w==}+ engines: {node: '>=18.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-F+DRf8IJazRJgYog2A/yJK7eYVc0rqTlRzO+5ZxjJd4WkZoKz0IJRncf7G6t1pdVT3kryJcwuTFhN1c5m6N47A==}+ engines: {node: '>=18.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==}+ engines: {node: '>=14.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-/jPhevcTFPMVl6KNjbaI47iOg1zxC7IsnX4PQDGVZKMFceOXtB8IEYaB7a9VvkP/3oC60WzTeKocvSI7vLT0vA==}+ engines: {node: '>=18.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-53+75QuPl6DL+ct6vVEB51FDO5oulXr20TPV46VvJZg76lIlXNWfxi8j+G2V/t0I2qxCBOa3vX/8bmjrpFVo9g==}+ engines: {node: '>=18.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-P+otAxbV4CqBybp7EkcJCrig63yE2E7PuNVOmilVMRcx/O+QDzGULTrKsq4DV13gSfak9ObPrWaHl/9bL5YcWw==}+ engines: {node: '>=18.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==}+ engines: {node: '>=14.0.0'}++ '@smithy/[email protected]':+ resolution: {integrity: sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==}+ engines: {node: '>=14.0.0'}+'@standard-schema/[email protected]':resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==}@@ -2906,6 +3067,9 @@ packages:resolution: {integrity: sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==}+ resolution: {integrity: sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==}+resolution: {integrity: sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==}@@ -3551,6 +3715,13 @@ packages:resolution: {integrity: sha512-rLV8JHxTyhVmFYhBJuMujcrHqOT2cnO5Zxj37qROj23CP39GXubJRBUFF0z8KFK77Uc0SukZUf7JZhsVEQ6n8w==}+ resolution: {integrity: sha512-00aAWieqff+ZJhsXA4g1g7M8k+7AYoMUUHF+/zFb5U6Uv/P0Vl4QZo84/IcufzYalLuEj9928bXN9PbbFzMF0Q==}++ resolution: {integrity: sha512-C0AaNuC+mscy6vrAQKAc/rMq+zAPHodfHGZu4sGVehvAQt/JLG1O5zEcYcXSY5zSqr4YVgxsB+pHXTq0i7eDlg==}+ hasBin: true+resolution: {integrity: sha512-GwLTyxkCXjXbxqIhTsMI2Nui8huMPtnxg7krajPJAjnEG/iiOS7i+zCtWGZR9G0NBKbXKh6X9m9UIsYX/N6vvQ==}@@ -4512,6 +4683,10 @@ packages:resolution: {integrity: sha512-b7uo2UCUOYZcnF/3ID0lulOJi/bafxa1xPe7ZPsammBSpjSWQkjNxlt635YGS2MiR9GjvuXCtz2emr3jbsz98g==}+ resolution: {integrity: sha512-cbrerZV+6rvdQrrD+iGMcZFEiiSrbv9Tfdkvnusy6y0x0GKBXREFg/Y65GhIfm0tnLntThhzCnfKwp1WRjeCyQ==}+ engines: {node: '>=14.0.0'}+resolution: {integrity: sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==}engines: {node: '>=0.10.0'}@@ -5615,6 +5790,9 @@ packages:resolution: {integrity: sha512-laJTa3Jb+VQpaC6DseHhF7dXVqHTfJPCRDaEbid/drOhgitgYku/letMUqOXFoWV0zIIUbjpdH2t+tYj4bQMRQ==}engines: {node: '>=8'}+ resolution: {integrity: sha512-ums3KNd42PGyx5xaoVTO1mjU1bH3NpY4vsrVlnv9PNGqQj8wd7rJ6nEypLrJ7z5vxK5RP0yMLo6J/Gsm62DI5Q==}+resolution: {integrity: sha512-IezA2qp+vcdlhJaVm5SOdPPTUu0FCEqfNSli2vRuSIBbu5Nq5UvygTk/VzeCqfLz2Atj3dVII5QBKGZRZ0edzw==}@@ -6065,6 +6243,10 @@ packages:resolution: {integrity: sha512-EvGK8EJ3DhaHfbRlETOWAS5pO9MZITeauHKJyb8wyajUfQUenkIg2MvLDTZ4T/TgIcm3HU0TFBgWWboAZ30UHg==}engines: {node: '>=18'}+ resolution: {integrity: sha512-k8KO9hrMyNk6tUWqUfkTEZbezRRpONVOzUTnc97VnCvyj6Tf9lyUR9EDAIeiVLv56jsMcoXEwjW8Kv5yPY52lw==}+ engines: {node: '>=16.0.0'}+resolution: {integrity: sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==}@@ -6152,6 +6334,271 @@ snapshots:'@asamuzakjp/[email protected]': {}+ '@aws-crypto/[email protected]':+ dependencies:+ '@aws-crypto/util': 5.2.0+ '@aws-sdk/types': 3.973.8+ tslib: 2.8.1++ '@aws-crypto/[email protected]':+ dependencies:+ '@aws-crypto/util': 5.2.0+ '@aws-sdk/types': 3.973.8+ tslib: 2.8.1++ '@aws-crypto/[email protected]':+ dependencies:+ '@aws-crypto/supports-web-crypto': 5.2.0+ '@aws-crypto/util': 5.2.0+ '@aws-sdk/types': 3.973.8+ '@aws-sdk/util-locate-window': 3.965.5+ '@smithy/util-utf8': 2.3.0+ tslib: 2.8.1++ '@aws-crypto/[email protected]':+ dependencies:+ '@aws-crypto/sha256-js': 5.2.0+ '@aws-crypto/supports-web-crypto': 5.2.0+ '@aws-crypto/util': 5.2.0+ '@aws-sdk/types': 3.973.8+ '@aws-sdk/util-locate-window': 3.965.5+ '@smithy/util-utf8': 2.3.0+ tslib: 2.8.1++ '@aws-crypto/[email protected]':+ dependencies:+ '@aws-crypto/util': 5.2.0+ '@aws-sdk/types': 3.973.8+ tslib: 2.8.1++ '@aws-crypto/[email protected]':+ dependencies:+ tslib: 2.8.1++ '@aws-crypto/[email protected]':+ dependencies:+ '@aws-sdk/types': 3.973.8+ '@smithy/util-utf8': 2.3.0+ tslib: 2.8.1++ '@aws-sdk/[email protected]':+ dependencies:+ '@aws-crypto/sha1-browser': 5.2.0+ '@aws-crypto/sha256-browser': 5.2.0+ '@aws-crypto/sha256-js': 5.2.0+ '@aws-sdk/core': 3.974.12+ '@aws-sdk/credential-provider-node': 3.972.43+ '@aws-sdk/middleware-bucket-endpoint': 3.972.14+ '@aws-sdk/middleware-expect-continue': 3.972.12+ '@aws-sdk/middleware-flexible-checksums': 3.974.20+ '@aws-sdk/middleware-location-constraint': 3.972.10+ '@aws-sdk/middleware-sdk-s3': 3.972.41+ '@aws-sdk/middleware-ssec': 3.972.10+ '@aws-sdk/signature-v4-multi-region': 3.996.27+ '@aws-sdk/types': 3.973.8+ '@smithy/core': 3.24.3+ '@smithy/fetch-http-handler': 5.4.3+ '@smithy/node-http-handler': 4.7.3+ '@smithy/types': 4.14.2+ tslib: 2.8.1++ '@aws-sdk/[email protected]':+ dependencies:+ '@aws-sdk/types': 3.973.8+ '@aws-sdk/xml-builder': 3.972.24+ '@aws/lambda-invoke-store': 0.2.4+ '@smithy/core': 3.24.3+ '@smithy/signature-v4': 5.4.3+ '@smithy/types': 4.14.2+ bowser: 2.14.1+ tslib: 2.8.1++ '@aws-sdk/[email protected]':+ dependencies:+ '@smithy/types': 4.14.2+ tslib: 2.8.1++ '@aws-sdk/[email protected]':+ dependencies:+ '@aws-sdk/core': 3.974.12+ '@aws-sdk/types': 3.973.8+ '@smithy/core': 3.24.3+ '@smithy/types': 4.14.2+ tslib: 2.8.1++ '@aws-sdk/[email protected]':+ dependencies:+ '@aws-sdk/core': 3.974.12+ '@aws-sdk/types': 3.973.8+ '@smithy/core': 3.24.3+ '@smithy/fetch-http-handler': 5.4.3+ '@smithy/node-http-handler': 4.7.3+ '@smithy/types': 4.14.2+ tslib: 2.8.1++ '@aws-sdk/[email protected]':+ dependencies:+ '@aws-sdk/core': 3.974.12+ '@aws-sdk/credential-provider-env': 3.972.38+ '@aws-sdk/credential-provider-http': 3.972.40+ '@aws-sdk/credential-provider-login': 3.972.42+ '@aws-sdk/credential-provider-process': 3.972.38+ '@aws-sdk/credential-provider-sso': 3.972.42+ '@aws-sdk/credential-provider-web-identity': 3.972.42+ '@aws-sdk/nested-clients': 3.997.10+ '@aws-sdk/types': 3.973.8+ '@smithy/core': 3.24.3+ '@smithy/credential-provider-imds': 4.3.3+ '@smithy/types': 4.14.2+ tslib: 2.8.1++ '@aws-sdk/[email protected]':+ dependencies:+ '@aws-sdk/core': 3.974.12+ '@aws-sdk/nested-clients': 3.997.10+ '@aws-sdk/types': 3.973.8+ '@smithy/core': 3.24.3+ '@smithy/types': 4.14.2+ tslib: 2.8.1++ '@aws-sdk/[email protected]':+ dependencies:+ '@aws-sdk/credential-provider-env': 3.972.38+ '@aws-sdk/credential-provider-http': 3.972.40+ '@aws-sdk/credential-provider-ini': 3.972.42+ '@aws-sdk/credential-provider-process': 3.972.38+ '@aws-sdk/credential-provider-sso': 3.972.42+ '@aws-sdk/credential-provider-web-identity': 3.972.42+ '@aws-sdk/types': 3.973.8+ '@smithy/core': 3.24.3+ '@smithy/credential-provider-imds': 4.3.3+ '@smithy/types': 4.14.2+ tslib: 2.8.1++ '@aws-sdk/[email protected]':+ dependencies:+ '@aws-sdk/core': 3.974.12+ '@aws-sdk/types': 3.973.8+ '@smithy/core': 3.24.3+ '@smithy/types': 4.14.2+ tslib: 2.8.1++ '@aws-sdk/[email protected]':+ dependencies:+ '@aws-sdk/core': 3.974.12+ '@aws-sdk/nested-clients': 3.997.10+ '@aws-sdk/token-providers': 3.1049.0+ '@aws-sdk/types': 3.973.8+ '@smithy/core': 3.24.3+ '@smithy/types': 4.14.2+ tslib: 2.8.1++ '@aws-sdk/[email protected]':+ dependencies:+ '@aws-sdk/core': 3.974.12+ '@aws-sdk/nested-clients': 3.997.10+ '@aws-sdk/types': 3.973.8+ '@smithy/core': 3.24.3+ '@smithy/types': 4.14.2+ tslib: 2.8.1++ '@aws-sdk/[email protected]':+ dependencies:+ '@aws-sdk/core': 3.974.12+ '@aws-sdk/types': 3.973.8+ '@smithy/core': 3.24.3+ '@smithy/types': 4.14.2+ tslib: 2.8.1++ '@aws-sdk/[email protected]':+ dependencies:+ '@aws-sdk/types': 3.973.8+ '@smithy/core': 3.24.3+ '@smithy/types': 4.14.2+ tslib: 2.8.1++ '@aws-sdk/[email protected]':+ dependencies:+ '@aws-crypto/crc32': 5.2.0+ '@aws-crypto/crc32c': 5.2.0+ '@aws-crypto/util': 5.2.0+ '@aws-sdk/core': 3.974.12+ '@aws-sdk/crc64-nvme': 3.972.8+ '@aws-sdk/types': 3.973.8+ '@smithy/core': 3.24.3+ '@smithy/types': 4.14.2+ tslib: 2.8.1++ '@aws-sdk/[email protected]':+ dependencies:+ '@aws-sdk/types': 3.973.8+ '@smithy/types': 4.14.2+ tslib: 2.8.1++ '@aws-sdk/[email protected]':+ dependencies:+ '@aws-sdk/core': 3.974.12+ '@aws-sdk/signature-v4-multi-region': 3.996.27+ '@aws-sdk/types': 3.973.8+ '@smithy/core': 3.24.3+ '@smithy/signature-v4': 5.4.3+ '@smithy/types': 4.14.2+ tslib: 2.8.1++ '@aws-sdk/[email protected]':+ dependencies:+ '@aws-sdk/types': 3.973.8+ '@smithy/types': 4.14.2+ tslib: 2.8.1++ '@aws-sdk/[email protected]':+ dependencies:+ '@aws-crypto/sha256-browser': 5.2.0+ '@aws-crypto/sha256-js': 5.2.0+ '@aws-sdk/core': 3.974.12+ '@aws-sdk/signature-v4-multi-region': 3.996.27+ '@aws-sdk/types': 3.973.8+ '@smithy/core': 3.24.3+ '@smithy/fetch-http-handler': 5.4.3+ '@smithy/node-http-handler': 4.7.3+ '@smithy/types': 4.14.2+ tslib: 2.8.1++ '@aws-sdk/[email protected]':+ dependencies:+ '@aws-sdk/types': 3.973.8+ '@smithy/core': 3.24.3+ '@smithy/signature-v4': 5.4.3+ '@smithy/types': 4.14.2+ tslib: 2.8.1++ '@aws-sdk/[email protected]':+ dependencies:+ '@aws-sdk/core': 3.974.12+ '@aws-sdk/nested-clients': 3.997.10+ '@aws-sdk/types': 3.973.8+ '@smithy/core': 3.24.3+ '@smithy/types': 4.14.2+ tslib: 2.8.1++ '@aws-sdk/[email protected]':+ dependencies:+ '@smithy/types': 4.14.2+ tslib: 2.8.1++ '@aws-sdk/[email protected]':+ dependencies:+ tslib: 2.8.1++ '@aws-sdk/[email protected]':+ dependencies:+ '@nodable/entities': 2.1.0+ '@smithy/types': 4.14.2+ fast-xml-parser: 5.7.3+ tslib: 2.8.1++ '@aws/[email protected]': {}+'@babel/[email protected]':dependencies:'@babel/helper-validator-identifier': 7.27.1@@ -7381,6 +7828,8 @@ snapshots:'@next/[email protected]':optional: true+ '@nodable/[email protected]': {}+'@nodelib/[email protected]':dependencies:'@nodelib/fs.stat': 2.0.5@@ -7941,6 +8390,54 @@ snapshots:'@sindresorhus/[email protected]': {}+ '@smithy/[email protected]':+ dependencies:+ '@aws-crypto/crc32': 5.2.0+ '@smithy/types': 4.14.2+ tslib: 2.8.1++ '@smithy/[email protected]':+ dependencies:+ '@smithy/core': 3.24.3+ '@smithy/types': 4.14.2+ tslib: 2.8.1++ '@smithy/[email protected]':+ dependencies:+ '@smithy/core': 3.24.3+ '@smithy/types': 4.14.2+ tslib: 2.8.1++ '@smithy/[email protected]':+ dependencies:+ tslib: 2.8.1++ '@smithy/[email protected]':+ dependencies:+ '@smithy/core': 3.24.3+ '@smithy/types': 4.14.2+ tslib: 2.8.1++ '@smithy/[email protected]':+ dependencies:+ '@smithy/core': 3.24.3+ '@smithy/types': 4.14.2+ tslib: 2.8.1++ '@smithy/[email protected]':+ dependencies:+ tslib: 2.8.1++ '@smithy/[email protected]':+ dependencies:+ '@smithy/is-array-buffer': 2.2.0+ tslib: 2.8.1++ '@smithy/[email protected]':+ dependencies:+ '@smithy/util-buffer-from': 2.2.0+ tslib: 2.8.1+'@standard-schema/[email protected]': {}'@svgr/[email protected](@babel/[email protected])':@@ -8412,6 +8909,8 @@ snapshots:+ [email protected]: {}+dependencies:balanced-match: 1.0.2@@ -9169,6 +9668,18 @@ snapshots:dependencies:fast-string-width: 3.0.2+ dependencies:+ path-expression-matcher: 1.5.0+ xml-naming: 0.1.0++ dependencies:+ '@nodable/entities': 2.1.0+ fast-xml-builder: 1.2.0+ path-expression-matcher: 1.5.0+ strnum: 2.3.0+dependencies:reusify: 1.1.0@@ -10109,6 +10620,8 @@ snapshots:+ [email protected]: {}+@@ -11342,6 +11855,8 @@ snapshots:dependencies:min-indent: 1.0.1+ [email protected]: {}+@@ -11786,6 +12301,8 @@ snapshots:+ [email protected]: {}+diff --git a/src/lib/heatmap-s3.ts b/src/lib/heatmap-s3.tsnew file mode 100644index 000000000..ccb234d47--- /dev/null+++ b/src/lib/heatmap-s3.ts@@ -0,0 +1,56 @@+import { GetObjectCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3';++let client: S3Client | null = null;++function getBucket() {+ const bucket = process.env.S3_HEATMAP_BUCKET;++ if (!bucket) {+ throw new Error('S3_HEATMAP_BUCKET is not set.');+ }++ return bucket;+}++function getClient() {+ if (!client) {+ client = new S3Client({+ region: process.env.AWS_REGION,+ });+ }++ return client;+}++export async function putHeatmapSnapshot(+ objectKey: string,+ imageData: Buffer,+ mimeType: string,+) {+ await getClient().send(+ new PutObjectCommand({+ Bucket: getBucket(),+ Key: objectKey,+ Body: imageData,+ ContentType: mimeType,+ }),+ );+}++export async function getHeatmapSnapshot(objectKey: string) {+ const response = await getClient().send(+ new GetObjectCommand({+ Bucket: getBucket(),+ Key: objectKey,+ }),+ );++ if (!response.Body) {+ return null;+ }++ return {+ mimeType: response.ContentType || 'application/octet-stream',+ imageData: Buffer.from(await response.Body.transformToByteArray()),+ };+}diff --git a/src/queries/sql/heatmap/ensureHeatmapSnapshot.ts b/src/queries/sql/heatmap/ensureHeatmapSnapshot.tsindex 90496563a..49beb663a 100644--- a/src/queries/sql/heatmap/ensureHeatmapSnapshot.ts+++ b/src/queries/sql/heatmap/ensureHeatmapSnapshot.ts@@ -1,5 +1,7 @@+import clickhouse from '@/lib/clickhouse';import prisma from '@/lib/prisma';import { uuid } from '@/lib/crypto';+import { getHeatmapSnapshot, putHeatmapSnapshot } from '@/lib/heatmap-s3';import { getWebsite } from '@/queries/prisma';const SNAPSHOT_STATUS = {@@ -10,7 +12,6 @@ const SNAPSHOT_STATUS = {const SNAPSHOT_RETRY_DELAY_MS = 15 * 60 * 1000;const SNAPSHOT_PENDING_WINDOW_MS = 30 * 1000;-export type HeatmapSnapshotStatus = (typeof SNAPSHOT_STATUS)[keyof typeof SNAPSHOT_STATUS];export interface HeatmapSnapshotImage {@@ -35,6 +36,7 @@ interface SnapshotRecord {pageH: number;status: HeatmapSnapshotStatus;mimeType: string | null;+ objectKey: string | null;imageSize: number | null;error: string | null;hasImage: boolean;@@ -57,6 +59,12 @@ interface CaptureResult {pageH: number;}+const CLICKHOUSE_SNAPSHOT_STATUS = {+ pending: 0,+ ready: 1,+ failed: 2,+} as const;+async function measurePage(page: any) {return page.evaluate(() => {const doc = document.documentElement;@@ -143,6 +151,19 @@ async function findSnapshot(urlPath: string,viewportW: number,viewportH: number,+): Promise<SnapshotRecord | null> {+ if (clickhouse.enabled) {+ return findClickhouseSnapshot(websiteId, urlPath, viewportW, viewportH);+ }++ return findRelationalSnapshot(websiteId, urlPath, viewportW, viewportH);+}++async function findRelationalSnapshot(+ websiteId: string,+ urlPath: string,+ viewportW: number,+ viewportH: number,): Promise<SnapshotRecord | null> {const rows = await prisma.rawQuery(`@@ -156,6 +177,7 @@ async function findSnapshot(page_h as "pageH",status,mime_type as "mimeType",+ null as "objectKey",image_size as "imageSize",error,image_data is not null as "hasImage",@@ -174,6 +196,78 @@ async function findSnapshot(return rows?.[0] ?? null;}+async function findClickhouseSnapshot(+ websiteId: string,+ urlPath: string,+ viewportW: number,+ viewportH: number,+): Promise<SnapshotRecord | null> {+ const rows = await clickhouse.rawQuery<+ {+ id: string;+ websiteId: string;+ urlPath: string;+ viewportW: number;+ viewportH: number;+ pageW: number;+ pageH: number;+ status: number;+ mimeType: string | null;+ objectKey: string;+ imageSize: number | null;+ error: string | null;+ createdAt: string;+ }[]+ >(+ `+ select+ snapshot_id as id,+ website_id as websiteId,+ url_path as urlPath,+ viewport_w as viewportW,+ viewport_h as viewportH,+ page_w as pageW,+ page_h as pageH,+ status,+ mime_type as mimeType,+ object_key as objectKey,+ image_size as imageSize,+ error,+ created_at as createdAt+ from heatmap_snapshot+ where website_id = {websiteId:UUID}+ and url_path = {urlPath:String}+ and viewport_w = {viewportW:UInt32}+ and viewport_h = {viewportH:UInt32}+ order by created_at desc+ limit 1+ `,+ { websiteId, urlPath, viewportW, viewportH },+ 'findHeatmapSnapshot',+ );++ const row = rows?.[0];++ if (!row) {+ return null;+ }++ const status = Object.entries(CLICKHOUSE_SNAPSHOT_STATUS).find(([, value]) => value === row.status)?.[0];++ if (!status) {+ return null;+ }++ return {+ ...row,+ status: SNAPSHOT_STATUS[status as keyof typeof SNAPSHOT_STATUS],+ mimeType: row.mimeType || null,+ objectKey: row.objectKey || null,+ hasImage: row.status === CLICKHOUSE_SNAPSHOT_STATUS.ready && Boolean(row.objectKey),+ updatedAt: row.createdAt,+ };+}+function getSnapshotImageUrl(websiteId: string, snapshotId: string) {return `/api/websites/${websiteId}/heatmaps/snapshots/${snapshotId}`;}@@ -231,6 +325,65 @@ export function shouldSkipSnapshot(urlPath: string) {}async function upsertSnapshotRecord({+ id,+ websiteId,+ urlPath,+ viewportW,+ viewportH,+ pageW,+ pageH,+ status,+ mimeType,+ imageData,+ objectKey,+ error,+}: {+ id: string;+ websiteId: string;+ urlPath: string;+ viewportW: number;+ viewportH: number;+ pageW: number;+ pageH: number;+ status: HeatmapSnapshotStatus;+ mimeType: string | null;+ imageData: Buffer | null;+ objectKey?: string | null;+ error: string | null;+}) {+ if (clickhouse.enabled) {+ return insertClickhouseSnapshotRecord({+ id,+ websiteId,+ urlPath,+ viewportW,+ viewportH,+ pageW,+ pageH,+ status,+ mimeType,+ objectKey: objectKey ?? null,+ imageSize: imageData?.byteLength ?? null,+ error,+ });+ }++ return upsertRelationalSnapshotRecord({+ id,+ websiteId,+ urlPath,+ viewportW,+ viewportH,+ pageW,+ pageH,+ status,+ mimeType,+ imageData,+ error,+ });+}++async function upsertRelationalSnapshotRecord({id,websiteId,urlPath,@@ -317,6 +470,56 @@ async function upsertSnapshotRecord({);}+async function insertClickhouseSnapshotRecord({+ id,+ websiteId,+ urlPath,+ viewportW,+ viewportH,+ pageW,+ pageH,+ status,+ mimeType,+ objectKey,+ imageSize,+ error,+}: {+ id: string;+ websiteId: string;+ urlPath: string;+ viewportW: number;+ viewportH: number;+ pageW: number;+ pageH: number;+ status: HeatmapSnapshotStatus;+ mimeType: string | null;+ objectKey: string | null;+ imageSize: number | null;+ error: string | null;+}) {+ return clickhouse.insert('heatmap_snapshot', [+ {+ snapshot_id: id,+ website_id: websiteId,+ url_path: urlPath,+ viewport_w: viewportW,+ viewport_h: viewportH,+ page_w: pageW,+ page_h: pageH,+ status: CLICKHOUSE_SNAPSHOT_STATUS[status],+ mime_type: mimeType || '',+ object_key: objectKey || '',+ image_size: imageSize,+ error,+ created_at: clickhouse.getUTCString(),+ },+ ]);+}++function getSnapshotObjectKey(websiteId: string, snapshotId: string, viewportW: number, viewportH: number) {+ return `heatmaps/${websiteId}/${viewportW}x${viewportH}/${snapshotId}.png`;+}+async function captureSnapshot(url: string,viewportW: number,@@ -456,6 +659,14 @@ export async function ensureHeatmapSnapshot({try {const capture = await captureSnapshot(captureUrl, viewportW, viewportH, pageW);+ const objectKey =+ clickhouse.enabled+ ? getSnapshotObjectKey(websiteId, snapshotId, viewportW, viewportH)+ : null;++ if (objectKey) {+ await putHeatmapSnapshot(objectKey, capture.imageData, capture.mimeType);+ }await upsertSnapshotRecord({id: snapshotId,@@ -467,7 +678,8 @@ export async function ensureHeatmapSnapshot({pageH: capture.pageH,status: SNAPSHOT_STATUS.ready,mimeType: capture.mimeType,- imageData: capture.imageData,+ imageData: clickhouse.enabled ? null : capture.imageData,+ objectKey,error: null,});} catch (error) {@@ -495,6 +707,39 @@ export async function getHeatmapSnapshotImage(websiteId: string,snapshotId: string,): Promise<{ mimeType: string; imageData: Buffer } | null> {+ if (clickhouse.enabled) {+ const rows = await clickhouse.rawQuery<+ { mimeType: string; objectKey: string }[]+ >(+ `+ select+ mime_type as mimeType,+ object_key as objectKey+ from heatmap_snapshot+ where snapshot_id = {snapshotId:UUID}+ and website_id = {websiteId:UUID}+ and status = {status:UInt8}+ and object_key != ''+ order by created_at desc+ limit 1+ `,+ {+ websiteId,+ snapshotId,+ status: CLICKHOUSE_SNAPSHOT_STATUS.ready,+ },+ 'getHeatmapSnapshotImage',+ );++ const row = rows?.[0];++ if (!row?.objectKey) {+ return null;+ }++ return getHeatmapSnapshot(row.objectKey);+ }+const rows = await prisma.rawQuery(`select