Skip to content

armgit / umami

Public Imported from GitHub
0

Umami is a privacy-first web and product analytics platform. Traffic, campaigns, behavior, conversions, and revenue in one place. No cookies, no surveillance, self-hosted or in the cloud.

add clickhouse+s3 workflow for heatmaps

Francis Cao07e40db5f4f7
diff --git a/db/clickhouse/migrations/12_add_heatmap.sql b/db/clickhouse/migrations/12_add_heatmap.sql
index ba5f93cb7..db44666b3 100644
--- a/db/clickhouse/migrations/12_add_heatmap.sql
+++ b/db/clickhouse/migrations/12_add_heatmap.sql
@@ -26,3 +26,25 @@ ENGINE = MergeTree
PARTITION BY toYYYYMM(created_at)
ORDER BY (website_id, url_path, event_type, created_at)
SETTINGS index_granularity = 8192;
+
+-- Create heatmap_snapshot
+CREATE TABLE umami.heatmap_snapshot
+(
+ snapshot_id UUID,
+ website_id UUID,
+ url_path String,
+ viewport_w UInt32,
+ viewport_h UInt32,
+ page_w UInt32,
+ page_h UInt32,
+ status UInt8,
+ mime_type LowCardinality(String),
+ object_key String,
+ image_size Nullable(UInt32),
+ error Nullable(String),
+ created_at DateTime('UTC')
+)
+ENGINE = MergeTree
+ PARTITION BY toYYYYMM(created_at)
+ ORDER BY (website_id, url_path, viewport_w, viewport_h, created_at)
+ SETTINGS index_granularity = 8192;
diff --git a/db/clickhouse/schema.sql b/db/clickhouse/schema.sql
index a075977f1..75943488c 100644
--- a/db/clickhouse/schema.sql
+++ b/db/clickhouse/schema.sql
@@ -428,3 +428,25 @@ ENGINE = MergeTree
PARTITION BY toYYYYMM(created_at)
ORDER BY (website_id, url_path, event_type, created_at)
SETTINGS index_granularity = 8192;
+
+-- Create heatmap_snapshot
+CREATE TABLE umami.heatmap_snapshot
+(
+ snapshot_id UUID,
+ website_id UUID,
+ url_path String,
+ viewport_w UInt32,
+ viewport_h UInt32,
+ page_w UInt32,
+ page_h UInt32,
+ status UInt8,
+ mime_type LowCardinality(String),
+ object_key String,
+ image_size Nullable(UInt32),
+ error Nullable(String),
+ created_at DateTime('UTC')
+)
+ENGINE = MergeTree
+ PARTITION BY toYYYYMM(created_at)
+ ORDER BY (website_id, url_path, viewport_w, viewport_h, created_at)
+ SETTINGS index_granularity = 8192;
diff --git a/package.json b/package.json
index fd263afcf..72f9a47b5 100644
--- a/package.json
+++ b/package.json
@@ -53,6 +53,7 @@
".next/cache"
],
"dependencies": {
+ "@aws-sdk/client-s3": "^3.924.0",
"@clickhouse/client": "^1.18.5",
"@date-fns/utc": "^2.1.1",
"@dicebear/collection": "^9.4.2",
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index c4f918851..7bf76d6af 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -8,6 +8,9 @@ importers:
.:
dependencies:
+ '@aws-sdk/client-s3':
+ specifier: ^3.924.0
+ version: 3.1050.0
'@clickhouse/client':
specifier: ^1.18.5
version: 1.18.5
@@ -352,6 +355,125 @@ packages:
'@asamuzakjp/[email protected]':
resolution: {integrity: sha512-n8GuYSrI9bF7FFZ/SjhwevlHc8xaVlb/7HmHelnc/PZXBD2ZR49NnN9sMMuDdEGPeeRQ5d0hqlSlEpgCX3Wl0Q==}
+ '@aws-crypto/[email protected]':
+ resolution: {integrity: sha512-nLbCWqQNgUiwwtFsen1AdzAtvuLRsQS8rYgMuxCrdKf9kOssamGLuPwyTY9wyYblNr9+1XM8v6zoDTPPSIeANg==}
+ engines: {node: '>=16.0.0'}
+
+ '@aws-crypto/[email protected]':
+ resolution: {integrity: sha512-+iWb8qaHLYKrNvGRbiYRHSdKRWhto5XlZUEBwDjYNf+ly5SVYG6zEoYIdxvf5R3zyeP16w4PLBn3rH1xc74Rag==}
+
+ '@aws-crypto/[email protected]':
+ resolution: {integrity: sha512-OH6lveCFfcDjX4dbAvCFSYUjJZjDr/3XJ3xHtjn3Oj5b9RjojQo8npoLeA/bNwkOkrSQ0wgrHzXk4tDRxGKJeg==}
+
+ '@aws-crypto/[email protected]':
+ resolution: {integrity: sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw==}
+
+ '@aws-crypto/[email protected]':
+ resolution: {integrity: sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA==}
+ engines: {node: '>=16.0.0'}
+
+ '@aws-crypto/[email protected]':
+ resolution: {integrity: sha512-iAvUotm021kM33eCdNfwIN//F77/IADDSs58i+MDaOqFrVjZo9bAal0NK7HurRuWLLpF1iLX7gbWrjHjeo+YFg==}
+
+ '@aws-crypto/[email protected]':
+ resolution: {integrity: sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==}
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-9kgtv+bXZQrOIJT2INPPBCezrJu1FlgGrzEat/ut4A4V53IT00LynsBZgp12eFKbjJuNCeTo7iPSKjPsX8ub+A==}
+ engines: {node: '>=20.0.0'}
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-qrqgioqYFjwR6LatVNS1L2Vk++EwRIxqSQXPKNv5Ofux2D8UNgqMQ1znnMyEImXquVPTtbf71fc128pvmU6y9A==}
+ engines: {node: '>=20.0.0'}
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-fVfUCL/Xh2zINYMPZvj+iBn6XWouQf0DAnjaWCI9MkmqXzL2Iy5FoQB8O7syFe6gN6AH1ecDDU58T51Ou0kFkA==}
+ engines: {node: '>=20.0.0'}
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-m3WjZEgPtioMhPmwqUt+DhlTJ2i9ufR6DhfkyXojb9puEvfR+ur2U5shavu5/Cc9WHHsDCvALi6UFHgcqjhQ5w==}
+ engines: {node: '>=20.0.0'}
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-D78L/m2Dr6cJnnSvWoAudPhQmCwmJ7j6APXsPYmFpPaKfQTfCSu0rdm8j14Np+VmXF9z8Aj8HE3xFpsrwtfgeg==}
+ engines: {node: '>=20.0.0'}
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-Mu5ESvFXeinafVM8jTIvRqcvK2Ehj4kz3auT39yUcHwu1Vfxo6xRlmUafdKLW4tusjAJukQwK09sCSMgOm7OKg==}
+ engines: {node: '>=20.0.0'}
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-O6WkZga3kf0yqyJYd1dbeJqVhEgJx/x1UaLgtbR+XuL/YP+K5y6QTxQKL7ka9z3jnQASESKGAPnRyt4D5hQrxA==}
+ engines: {node: '>=20.0.0'}
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-D/DJmbrWRP5BXEO3FH+ar4el+2n6OlGofiud7dQun2jES+AQEJjczenp1jBb4MBN7CpGpS8nsWGQLtuzc9tQbA==}
+ engines: {node: '>=20.0.0'}
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-EnbYVajGgbkb24s0K1eo4VNAPV5mHIET7LSvirTaFCwkfrfaOJxtSE+wY/tJdKDS21cEYkZs2ruCaAm+W4iblg==}
+ engines: {node: '>=20.0.0'}
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-RVV/9NbFwI8ZHEH5dn39lGyFmSbSVj1+orZdr6QsOe1mW9DCglmlen0cFaNZmCcqkqc7erNRHNBduxbeZuHAnw==}
+ engines: {node: '>=20.0.0'}
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-/67fXX0ddllD4u2Nujc5PvT4byHgpMUfz6+RxIKi/0nFIckeorm7JvXgzBuDyVKw0s58EbofmETDWUf9vTEuHQ==}
+ engines: {node: '>=20.0.0'}
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-Aaj0d+xbo1jJquBWJP0/9V/XZRYukO3LWIRp3dOLHmoFrYKb4YZ0aLefgVHfGcNOVBS2ZTq7L/n5JcrE7DaC+Q==}
+ engines: {node: '>=20.0.0'}
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-dA5pKTom/Ls9mgeyeaRBNQrRIVOLVjv4AmKOB0/e4yaiXEUy0gSz2d3liP8JHtYoCAEWySU1jWnyzwLOREN+4g==}
+ engines: {node: '>=20.0.0'}
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-NdnMVQCR1YjIcqFAiNLdBiOwr2DyQDB2IiXQrBhzolKOv32ae4d4Ll7IzLMi04eMHiq/o/Y/GjFuVjF9HuG0QA==}
+ engines: {node: '>=20.0.0'}
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-rI3NZvJcEvjoD0+0PI0iUAwlPw2IlSlhyvgBK/3WkKJQE/YiKFedd9dMN2lVacdNxPNhxL/jzQaKQdrGtQagjQ==}
+ engines: {node: '>=20.0.0'}
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-M4T2I2WPuH5WQpU8Tsp+u2bcO29zGRkU14ATzuqb9I4xh8tzsLqtp4hzaJM5aO2dhMZnHDzyQwSFVgc3XbnoGg==}
+ engines: {node: '>=20.0.0'}
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-Gli9A0u8EVVb+5bFDGS/QbSVg28w/wpEidg1ggVcSj65BDTdGR6punsOcVjqdiu1i42WHWo51MCvARPIIz9juw==}
+ engines: {node: '>=20.0.0'}
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-FtQ/Bt327peZJuyo4WZSOLVUTw9ujRxntepiC7L65FxA2P82Xlq0g14T22BuqBUeMjDoxa9nvwiMHjLIfP3eUg==}
+ engines: {node: '>=20.0.0'}
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-0Phbz4t6HI3D3skxvG2uI+VWU034/nSIw1T8d+FPzzQG9EQTrw94o9mOKO2Gv3n3Oc8P7JD7RAUxkoneLWv5Eg==}
+ engines: {node: '>=20.0.0'}
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-r7+d0lQMTHKypkmaF5jRTBYLYHCUHzt3gaVoN9SidLhQeWhCmHk3AKrboDTpPF5b7Pt7vKu3+oeMjznM2Eu1ow==}
+ engines: {node: '>=20.0.0'}
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-gjlAdtHMbtR9X5iIhVUvbVcy55KnznpC6bkDUWW9z915bi0ckdUr5cjf16Kp6xq0bP5HBD2xzgbL9F9Quv5vUw==}
+ engines: {node: '>=20.0.0'}
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-WhlJNNINQB+9qtLtZJcpQdgZw3SCDCpXdUJP7cToGwHbCWCnRckGlc6Bx/OhWwIYFNAn+FIydY8SZ0QmVu3xTQ==}
+ engines: {node: '>=20.0.0'}
+
+ '@aws-sdk/[email protected]':
+ resolution: {integrity: sha512-V8z5YcDPfsvzrBlj0xR1vhRtocblhYbqdreCJB/voGd4Sr5zjNAeWxexbnqVtskTJe0vFb5KMqbSL++ePl+zRw==}
+ engines: {node: '>=20.0.0'}
+
+ resolution: {integrity: sha512-iY8yvjE0y651BixKNPgmv1WrQc+GZ142sb0z4gYnChDDY2YqI4P/jsSopBWrKfAt7LOJAkOXt7rC/hms+WclQQ==}
+ engines: {node: '>=18.0.0'}
+
resolution: {integrity: sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg==}
engines: {node: '>=6.9.0'}
@@ -1753,6 +1875,9 @@ packages:
cpu: [x64]
os: [win32]
+ '@nodable/[email protected]':
+ resolution: {integrity: sha512-nyT7T3nbMyBI/lvr6L5TyWbFJAI9FTgVRakNoBqCD+PmID8DzFrrNdLLtHMwMszOtqZa8PAOV24ZqDnQrhQINA==}
+
'@nodelib/[email protected]':
resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==}
engines: {node: '>= 8'}
@@ -2430,6 +2555,42 @@ packages:
resolution: {integrity: sha512-LtoMMhxAlorcGhmFYI+LhPgbPZCkgP6ra1YL604EeF6U98pLlQ3iWIGMdWSC+vWmPBWBNgmDBAhnAobLROJmwg==}
engines: {node: '>=18'}
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-Ep/7tPamGY8mgESE3LyLKtxJyy6U52WWAqr/3wial47Sj4u3PiIF73AOGI27UyLy9duTkhZbgzodOfLV4TduZg==}
+ engines: {node: '>=18.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-I2Bti0DKFo2IJyN28ijCsx51BAumEYR4/1yZ1FXyBygy9MqbnMqCev4JPth/MbpRfBSRAX35hITSnAdJRo1u5w==}
+ engines: {node: '>=18.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-F+DRf8IJazRJgYog2A/yJK7eYVc0rqTlRzO+5ZxjJd4WkZoKz0IJRncf7G6t1pdVT3kryJcwuTFhN1c5m6N47A==}
+ engines: {node: '>=18.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==}
+ engines: {node: '>=14.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-/jPhevcTFPMVl6KNjbaI47iOg1zxC7IsnX4PQDGVZKMFceOXtB8IEYaB7a9VvkP/3oC60WzTeKocvSI7vLT0vA==}
+ engines: {node: '>=18.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-53+75QuPl6DL+ct6vVEB51FDO5oulXr20TPV46VvJZg76lIlXNWfxi8j+G2V/t0I2qxCBOa3vX/8bmjrpFVo9g==}
+ engines: {node: '>=18.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-P+otAxbV4CqBybp7EkcJCrig63yE2E7PuNVOmilVMRcx/O+QDzGULTrKsq4DV13gSfak9ObPrWaHl/9bL5YcWw==}
+ engines: {node: '>=18.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==}
+ engines: {node: '>=14.0.0'}
+
+ '@smithy/[email protected]':
+ resolution: {integrity: sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==}
+ engines: {node: '>=14.0.0'}
+
'@standard-schema/[email protected]':
resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==}
@@ -2906,6 +3067,9 @@ packages:
resolution: {integrity: sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==}
+ resolution: {integrity: sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==}
+
resolution: {integrity: sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==}
@@ -3551,6 +3715,13 @@ packages:
resolution: {integrity: sha512-rLV8JHxTyhVmFYhBJuMujcrHqOT2cnO5Zxj37qROj23CP39GXubJRBUFF0z8KFK77Uc0SukZUf7JZhsVEQ6n8w==}
+ resolution: {integrity: sha512-00aAWieqff+ZJhsXA4g1g7M8k+7AYoMUUHF+/zFb5U6Uv/P0Vl4QZo84/IcufzYalLuEj9928bXN9PbbFzMF0Q==}
+
+ resolution: {integrity: sha512-C0AaNuC+mscy6vrAQKAc/rMq+zAPHodfHGZu4sGVehvAQt/JLG1O5zEcYcXSY5zSqr4YVgxsB+pHXTq0i7eDlg==}
+ hasBin: true
+
resolution: {integrity: sha512-GwLTyxkCXjXbxqIhTsMI2Nui8huMPtnxg7krajPJAjnEG/iiOS7i+zCtWGZR9G0NBKbXKh6X9m9UIsYX/N6vvQ==}
@@ -4512,6 +4683,10 @@ packages:
resolution: {integrity: sha512-b7uo2UCUOYZcnF/3ID0lulOJi/bafxa1xPe7ZPsammBSpjSWQkjNxlt635YGS2MiR9GjvuXCtz2emr3jbsz98g==}
+ resolution: {integrity: sha512-cbrerZV+6rvdQrrD+iGMcZFEiiSrbv9Tfdkvnusy6y0x0GKBXREFg/Y65GhIfm0tnLntThhzCnfKwp1WRjeCyQ==}
+ engines: {node: '>=14.0.0'}
+
resolution: {integrity: sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==}
engines: {node: '>=0.10.0'}
@@ -5615,6 +5790,9 @@ packages:
resolution: {integrity: sha512-laJTa3Jb+VQpaC6DseHhF7dXVqHTfJPCRDaEbid/drOhgitgYku/letMUqOXFoWV0zIIUbjpdH2t+tYj4bQMRQ==}
engines: {node: '>=8'}
+ resolution: {integrity: sha512-ums3KNd42PGyx5xaoVTO1mjU1bH3NpY4vsrVlnv9PNGqQj8wd7rJ6nEypLrJ7z5vxK5RP0yMLo6J/Gsm62DI5Q==}
+
resolution: {integrity: sha512-IezA2qp+vcdlhJaVm5SOdPPTUu0FCEqfNSli2vRuSIBbu5Nq5UvygTk/VzeCqfLz2Atj3dVII5QBKGZRZ0edzw==}
@@ -6065,6 +6243,10 @@ packages:
resolution: {integrity: sha512-EvGK8EJ3DhaHfbRlETOWAS5pO9MZITeauHKJyb8wyajUfQUenkIg2MvLDTZ4T/TgIcm3HU0TFBgWWboAZ30UHg==}
engines: {node: '>=18'}
+ resolution: {integrity: sha512-k8KO9hrMyNk6tUWqUfkTEZbezRRpONVOzUTnc97VnCvyj6Tf9lyUR9EDAIeiVLv56jsMcoXEwjW8Kv5yPY52lw==}
+ engines: {node: '>=16.0.0'}
+
resolution: {integrity: sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==}
@@ -6152,6 +6334,271 @@ snapshots:
'@asamuzakjp/[email protected]': {}
+ '@aws-crypto/[email protected]':
+ dependencies:
+ '@aws-crypto/util': 5.2.0
+ '@aws-sdk/types': 3.973.8
+ tslib: 2.8.1
+
+ '@aws-crypto/[email protected]':
+ dependencies:
+ '@aws-crypto/util': 5.2.0
+ '@aws-sdk/types': 3.973.8
+ tslib: 2.8.1
+
+ '@aws-crypto/[email protected]':
+ dependencies:
+ '@aws-crypto/supports-web-crypto': 5.2.0
+ '@aws-crypto/util': 5.2.0
+ '@aws-sdk/types': 3.973.8
+ '@aws-sdk/util-locate-window': 3.965.5
+ '@smithy/util-utf8': 2.3.0
+ tslib: 2.8.1
+
+ '@aws-crypto/[email protected]':
+ dependencies:
+ '@aws-crypto/sha256-js': 5.2.0
+ '@aws-crypto/supports-web-crypto': 5.2.0
+ '@aws-crypto/util': 5.2.0
+ '@aws-sdk/types': 3.973.8
+ '@aws-sdk/util-locate-window': 3.965.5
+ '@smithy/util-utf8': 2.3.0
+ tslib: 2.8.1
+
+ '@aws-crypto/[email protected]':
+ dependencies:
+ '@aws-crypto/util': 5.2.0
+ '@aws-sdk/types': 3.973.8
+ tslib: 2.8.1
+
+ '@aws-crypto/[email protected]':
+ dependencies:
+ tslib: 2.8.1
+
+ '@aws-crypto/[email protected]':
+ dependencies:
+ '@aws-sdk/types': 3.973.8
+ '@smithy/util-utf8': 2.3.0
+ tslib: 2.8.1
+
+ '@aws-sdk/[email protected]':
+ dependencies:
+ '@aws-crypto/sha1-browser': 5.2.0
+ '@aws-crypto/sha256-browser': 5.2.0
+ '@aws-crypto/sha256-js': 5.2.0
+ '@aws-sdk/core': 3.974.12
+ '@aws-sdk/credential-provider-node': 3.972.43
+ '@aws-sdk/middleware-bucket-endpoint': 3.972.14
+ '@aws-sdk/middleware-expect-continue': 3.972.12
+ '@aws-sdk/middleware-flexible-checksums': 3.974.20
+ '@aws-sdk/middleware-location-constraint': 3.972.10
+ '@aws-sdk/middleware-sdk-s3': 3.972.41
+ '@aws-sdk/middleware-ssec': 3.972.10
+ '@aws-sdk/signature-v4-multi-region': 3.996.27
+ '@aws-sdk/types': 3.973.8
+ '@smithy/core': 3.24.3
+ '@smithy/fetch-http-handler': 5.4.3
+ '@smithy/node-http-handler': 4.7.3
+ '@smithy/types': 4.14.2
+ tslib: 2.8.1
+
+ '@aws-sdk/[email protected]':
+ dependencies:
+ '@aws-sdk/types': 3.973.8
+ '@aws-sdk/xml-builder': 3.972.24
+ '@aws/lambda-invoke-store': 0.2.4
+ '@smithy/core': 3.24.3
+ '@smithy/signature-v4': 5.4.3
+ '@smithy/types': 4.14.2
+ bowser: 2.14.1
+ tslib: 2.8.1
+
+ '@aws-sdk/[email protected]':
+ dependencies:
+ '@smithy/types': 4.14.2
+ tslib: 2.8.1
+
+ '@aws-sdk/[email protected]':
+ dependencies:
+ '@aws-sdk/core': 3.974.12
+ '@aws-sdk/types': 3.973.8
+ '@smithy/core': 3.24.3
+ '@smithy/types': 4.14.2
+ tslib: 2.8.1
+
+ '@aws-sdk/[email protected]':
+ dependencies:
+ '@aws-sdk/core': 3.974.12
+ '@aws-sdk/types': 3.973.8
+ '@smithy/core': 3.24.3
+ '@smithy/fetch-http-handler': 5.4.3
+ '@smithy/node-http-handler': 4.7.3
+ '@smithy/types': 4.14.2
+ tslib: 2.8.1
+
+ '@aws-sdk/[email protected]':
+ dependencies:
+ '@aws-sdk/core': 3.974.12
+ '@aws-sdk/credential-provider-env': 3.972.38
+ '@aws-sdk/credential-provider-http': 3.972.40
+ '@aws-sdk/credential-provider-login': 3.972.42
+ '@aws-sdk/credential-provider-process': 3.972.38
+ '@aws-sdk/credential-provider-sso': 3.972.42
+ '@aws-sdk/credential-provider-web-identity': 3.972.42
+ '@aws-sdk/nested-clients': 3.997.10
+ '@aws-sdk/types': 3.973.8
+ '@smithy/core': 3.24.3
+ '@smithy/credential-provider-imds': 4.3.3
+ '@smithy/types': 4.14.2
+ tslib: 2.8.1
+
+ '@aws-sdk/[email protected]':
+ dependencies:
+ '@aws-sdk/core': 3.974.12
+ '@aws-sdk/nested-clients': 3.997.10
+ '@aws-sdk/types': 3.973.8
+ '@smithy/core': 3.24.3
+ '@smithy/types': 4.14.2
+ tslib: 2.8.1
+
+ '@aws-sdk/[email protected]':
+ dependencies:
+ '@aws-sdk/credential-provider-env': 3.972.38
+ '@aws-sdk/credential-provider-http': 3.972.40
+ '@aws-sdk/credential-provider-ini': 3.972.42
+ '@aws-sdk/credential-provider-process': 3.972.38
+ '@aws-sdk/credential-provider-sso': 3.972.42
+ '@aws-sdk/credential-provider-web-identity': 3.972.42
+ '@aws-sdk/types': 3.973.8
+ '@smithy/core': 3.24.3
+ '@smithy/credential-provider-imds': 4.3.3
+ '@smithy/types': 4.14.2
+ tslib: 2.8.1
+
+ '@aws-sdk/[email protected]':
+ dependencies:
+ '@aws-sdk/core': 3.974.12
+ '@aws-sdk/types': 3.973.8
+ '@smithy/core': 3.24.3
+ '@smithy/types': 4.14.2
+ tslib: 2.8.1
+
+ '@aws-sdk/[email protected]':
+ dependencies:
+ '@aws-sdk/core': 3.974.12
+ '@aws-sdk/nested-clients': 3.997.10
+ '@aws-sdk/token-providers': 3.1049.0
+ '@aws-sdk/types': 3.973.8
+ '@smithy/core': 3.24.3
+ '@smithy/types': 4.14.2
+ tslib: 2.8.1
+
+ '@aws-sdk/[email protected]':
+ dependencies:
+ '@aws-sdk/core': 3.974.12
+ '@aws-sdk/nested-clients': 3.997.10
+ '@aws-sdk/types': 3.973.8
+ '@smithy/core': 3.24.3
+ '@smithy/types': 4.14.2
+ tslib: 2.8.1
+
+ '@aws-sdk/[email protected]':
+ dependencies:
+ '@aws-sdk/core': 3.974.12
+ '@aws-sdk/types': 3.973.8
+ '@smithy/core': 3.24.3
+ '@smithy/types': 4.14.2
+ tslib: 2.8.1
+
+ '@aws-sdk/[email protected]':
+ dependencies:
+ '@aws-sdk/types': 3.973.8
+ '@smithy/core': 3.24.3
+ '@smithy/types': 4.14.2
+ tslib: 2.8.1
+
+ '@aws-sdk/[email protected]':
+ dependencies:
+ '@aws-crypto/crc32': 5.2.0
+ '@aws-crypto/crc32c': 5.2.0
+ '@aws-crypto/util': 5.2.0
+ '@aws-sdk/core': 3.974.12
+ '@aws-sdk/crc64-nvme': 3.972.8
+ '@aws-sdk/types': 3.973.8
+ '@smithy/core': 3.24.3
+ '@smithy/types': 4.14.2
+ tslib: 2.8.1
+
+ '@aws-sdk/[email protected]':
+ dependencies:
+ '@aws-sdk/types': 3.973.8
+ '@smithy/types': 4.14.2
+ tslib: 2.8.1
+
+ '@aws-sdk/[email protected]':
+ dependencies:
+ '@aws-sdk/core': 3.974.12
+ '@aws-sdk/signature-v4-multi-region': 3.996.27
+ '@aws-sdk/types': 3.973.8
+ '@smithy/core': 3.24.3
+ '@smithy/signature-v4': 5.4.3
+ '@smithy/types': 4.14.2
+ tslib: 2.8.1
+
+ '@aws-sdk/[email protected]':
+ dependencies:
+ '@aws-sdk/types': 3.973.8
+ '@smithy/types': 4.14.2
+ tslib: 2.8.1
+
+ '@aws-sdk/[email protected]':
+ dependencies:
+ '@aws-crypto/sha256-browser': 5.2.0
+ '@aws-crypto/sha256-js': 5.2.0
+ '@aws-sdk/core': 3.974.12
+ '@aws-sdk/signature-v4-multi-region': 3.996.27
+ '@aws-sdk/types': 3.973.8
+ '@smithy/core': 3.24.3
+ '@smithy/fetch-http-handler': 5.4.3
+ '@smithy/node-http-handler': 4.7.3
+ '@smithy/types': 4.14.2
+ tslib: 2.8.1
+
+ '@aws-sdk/[email protected]':
+ dependencies:
+ '@aws-sdk/types': 3.973.8
+ '@smithy/core': 3.24.3
+ '@smithy/signature-v4': 5.4.3
+ '@smithy/types': 4.14.2
+ tslib: 2.8.1
+
+ '@aws-sdk/[email protected]':
+ dependencies:
+ '@aws-sdk/core': 3.974.12
+ '@aws-sdk/nested-clients': 3.997.10
+ '@aws-sdk/types': 3.973.8
+ '@smithy/core': 3.24.3
+ '@smithy/types': 4.14.2
+ tslib: 2.8.1
+
+ '@aws-sdk/[email protected]':
+ dependencies:
+ '@smithy/types': 4.14.2
+ tslib: 2.8.1
+
+ '@aws-sdk/[email protected]':
+ dependencies:
+ tslib: 2.8.1
+
+ '@aws-sdk/[email protected]':
+ dependencies:
+ '@nodable/entities': 2.1.0
+ '@smithy/types': 4.14.2
+ fast-xml-parser: 5.7.3
+ tslib: 2.8.1
+
+ '@aws/[email protected]': {}
+
dependencies:
'@babel/helper-validator-identifier': 7.27.1
@@ -7381,6 +7828,8 @@ snapshots:
optional: true
+ '@nodable/[email protected]': {}
+
'@nodelib/[email protected]':
dependencies:
'@nodelib/fs.stat': 2.0.5
@@ -7941,6 +8390,54 @@ snapshots:
'@sindresorhus/[email protected]': {}
+ '@smithy/[email protected]':
+ dependencies:
+ '@aws-crypto/crc32': 5.2.0
+ '@smithy/types': 4.14.2
+ tslib: 2.8.1
+
+ '@smithy/[email protected]':
+ dependencies:
+ '@smithy/core': 3.24.3
+ '@smithy/types': 4.14.2
+ tslib: 2.8.1
+
+ '@smithy/[email protected]':
+ dependencies:
+ '@smithy/core': 3.24.3
+ '@smithy/types': 4.14.2
+ tslib: 2.8.1
+
+ '@smithy/[email protected]':
+ dependencies:
+ tslib: 2.8.1
+
+ '@smithy/[email protected]':
+ dependencies:
+ '@smithy/core': 3.24.3
+ '@smithy/types': 4.14.2
+ tslib: 2.8.1
+
+ '@smithy/[email protected]':
+ dependencies:
+ '@smithy/core': 3.24.3
+ '@smithy/types': 4.14.2
+ tslib: 2.8.1
+
+ '@smithy/[email protected]':
+ dependencies:
+ tslib: 2.8.1
+
+ '@smithy/[email protected]':
+ dependencies:
+ '@smithy/is-array-buffer': 2.2.0
+ tslib: 2.8.1
+
+ '@smithy/[email protected]':
+ dependencies:
+ '@smithy/util-buffer-from': 2.2.0
+ tslib: 2.8.1
+
'@standard-schema/[email protected]': {}
@@ -8412,6 +8909,8 @@ snapshots:
+
dependencies:
balanced-match: 1.0.2
@@ -9169,6 +9668,18 @@ snapshots:
dependencies:
fast-string-width: 3.0.2
+ dependencies:
+ path-expression-matcher: 1.5.0
+ xml-naming: 0.1.0
+
+ dependencies:
+ '@nodable/entities': 2.1.0
+ fast-xml-builder: 1.2.0
+ path-expression-matcher: 1.5.0
+ strnum: 2.3.0
+
dependencies:
reusify: 1.1.0
@@ -10109,6 +10620,8 @@ snapshots:
+
@@ -11342,6 +11855,8 @@ snapshots:
dependencies:
min-indent: 1.0.1
+
@@ -11786,6 +12301,8 @@ snapshots:
+
diff --git a/src/lib/heatmap-s3.ts b/src/lib/heatmap-s3.ts
new file mode 100644
index 000000000..ccb234d47
--- /dev/null
+++ b/src/lib/heatmap-s3.ts
@@ -0,0 +1,56 @@
+import { GetObjectCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
+
+let client: S3Client | null = null;
+
+function getBucket() {
+ const bucket = process.env.S3_HEATMAP_BUCKET;
+
+ if (!bucket) {
+ throw new Error('S3_HEATMAP_BUCKET is not set.');
+ }
+
+ return bucket;
+}
+
+function getClient() {
+ if (!client) {
+ client = new S3Client({
+ region: process.env.AWS_REGION,
+ });
+ }
+
+ return client;
+}
+
+export async function putHeatmapSnapshot(
+ objectKey: string,
+ imageData: Buffer,
+ mimeType: string,
+) {
+ await getClient().send(
+ new PutObjectCommand({
+ Bucket: getBucket(),
+ Key: objectKey,
+ Body: imageData,
+ ContentType: mimeType,
+ }),
+ );
+}
+
+export async function getHeatmapSnapshot(objectKey: string) {
+ const response = await getClient().send(
+ new GetObjectCommand({
+ Bucket: getBucket(),
+ Key: objectKey,
+ }),
+ );
+
+ if (!response.Body) {
+ return null;
+ }
+
+ return {
+ mimeType: response.ContentType || 'application/octet-stream',
+ imageData: Buffer.from(await response.Body.transformToByteArray()),
+ };
+}
diff --git a/src/queries/sql/heatmap/ensureHeatmapSnapshot.ts b/src/queries/sql/heatmap/ensureHeatmapSnapshot.ts
index 90496563a..49beb663a 100644
--- a/src/queries/sql/heatmap/ensureHeatmapSnapshot.ts
+++ b/src/queries/sql/heatmap/ensureHeatmapSnapshot.ts
@@ -1,5 +1,7 @@
+import clickhouse from '@/lib/clickhouse';
import prisma from '@/lib/prisma';
import { uuid } from '@/lib/crypto';
+import { getHeatmapSnapshot, putHeatmapSnapshot } from '@/lib/heatmap-s3';
import { getWebsite } from '@/queries/prisma';
const SNAPSHOT_STATUS = {
@@ -10,7 +12,6 @@ const SNAPSHOT_STATUS = {
const SNAPSHOT_RETRY_DELAY_MS = 15 * 60 * 1000;
const SNAPSHOT_PENDING_WINDOW_MS = 30 * 1000;
-
export type HeatmapSnapshotStatus = (typeof SNAPSHOT_STATUS)[keyof typeof SNAPSHOT_STATUS];
export interface HeatmapSnapshotImage {
@@ -35,6 +36,7 @@ interface SnapshotRecord {
pageH: number;
status: HeatmapSnapshotStatus;
mimeType: string | null;
+ objectKey: string | null;
imageSize: number | null;
error: string | null;
hasImage: boolean;
@@ -57,6 +59,12 @@ interface CaptureResult {
pageH: number;
}
+const CLICKHOUSE_SNAPSHOT_STATUS = {
+ pending: 0,
+ ready: 1,
+ failed: 2,
+} as const;
+
async function measurePage(page: any) {
return page.evaluate(() => {
const doc = document.documentElement;
@@ -143,6 +151,19 @@ async function findSnapshot(
urlPath: string,
viewportW: number,
viewportH: number,
+): Promise<SnapshotRecord | null> {
+ if (clickhouse.enabled) {
+ return findClickhouseSnapshot(websiteId, urlPath, viewportW, viewportH);
+ }
+
+ return findRelationalSnapshot(websiteId, urlPath, viewportW, viewportH);
+}
+
+async function findRelationalSnapshot(
+ websiteId: string,
+ urlPath: string,
+ viewportW: number,
+ viewportH: number,
): Promise<SnapshotRecord | null> {
const rows = await prisma.rawQuery(
`
@@ -156,6 +177,7 @@ async function findSnapshot(
page_h as "pageH",
status,
mime_type as "mimeType",
+ null as "objectKey",
image_size as "imageSize",
error,
image_data is not null as "hasImage",
@@ -174,6 +196,78 @@ async function findSnapshot(
return rows?.[0] ?? null;
}
+async function findClickhouseSnapshot(
+ websiteId: string,
+ urlPath: string,
+ viewportW: number,
+ viewportH: number,
+): Promise<SnapshotRecord | null> {
+ const rows = await clickhouse.rawQuery<
+ {
+ id: string;
+ websiteId: string;
+ urlPath: string;
+ viewportW: number;
+ viewportH: number;
+ pageW: number;
+ pageH: number;
+ status: number;
+ mimeType: string | null;
+ objectKey: string;
+ imageSize: number | null;
+ error: string | null;
+ createdAt: string;
+ }[]
+ >(
+ `
+ select
+ snapshot_id as id,
+ website_id as websiteId,
+ url_path as urlPath,
+ viewport_w as viewportW,
+ viewport_h as viewportH,
+ page_w as pageW,
+ page_h as pageH,
+ status,
+ mime_type as mimeType,
+ object_key as objectKey,
+ image_size as imageSize,
+ error,
+ created_at as createdAt
+ from heatmap_snapshot
+ where website_id = {websiteId:UUID}
+ and url_path = {urlPath:String}
+ and viewport_w = {viewportW:UInt32}
+ and viewport_h = {viewportH:UInt32}
+ order by created_at desc
+ limit 1
+ `,
+ { websiteId, urlPath, viewportW, viewportH },
+ 'findHeatmapSnapshot',
+ );
+
+ const row = rows?.[0];
+
+ if (!row) {
+ return null;
+ }
+
+ const status = Object.entries(CLICKHOUSE_SNAPSHOT_STATUS).find(([, value]) => value === row.status)?.[0];
+
+ if (!status) {
+ return null;
+ }
+
+ return {
+ ...row,
+ status: SNAPSHOT_STATUS[status as keyof typeof SNAPSHOT_STATUS],
+ mimeType: row.mimeType || null,
+ objectKey: row.objectKey || null,
+ hasImage: row.status === CLICKHOUSE_SNAPSHOT_STATUS.ready && Boolean(row.objectKey),
+ updatedAt: row.createdAt,
+ };
+}
+
function getSnapshotImageUrl(websiteId: string, snapshotId: string) {
return `/api/websites/${websiteId}/heatmaps/snapshots/${snapshotId}`;
}
@@ -231,6 +325,65 @@ export function shouldSkipSnapshot(urlPath: string) {
}
async function upsertSnapshotRecord({
+ id,
+ websiteId,
+ urlPath,
+ viewportW,
+ viewportH,
+ pageW,
+ pageH,
+ status,
+ mimeType,
+ imageData,
+ objectKey,
+ error,
+}: {
+ id: string;
+ websiteId: string;
+ urlPath: string;
+ viewportW: number;
+ viewportH: number;
+ pageW: number;
+ pageH: number;
+ status: HeatmapSnapshotStatus;
+ mimeType: string | null;
+ imageData: Buffer | null;
+ objectKey?: string | null;
+ error: string | null;
+}) {
+ if (clickhouse.enabled) {
+ return insertClickhouseSnapshotRecord({
+ id,
+ websiteId,
+ urlPath,
+ viewportW,
+ viewportH,
+ pageW,
+ pageH,
+ status,
+ mimeType,
+ objectKey: objectKey ?? null,
+ imageSize: imageData?.byteLength ?? null,
+ error,
+ });
+ }
+
+ return upsertRelationalSnapshotRecord({
+ id,
+ websiteId,
+ urlPath,
+ viewportW,
+ viewportH,
+ pageW,
+ pageH,
+ status,
+ mimeType,
+ imageData,
+ error,
+ });
+}
+
+async function upsertRelationalSnapshotRecord({
id,
websiteId,
urlPath,
@@ -317,6 +470,56 @@ async function upsertSnapshotRecord({
);
}
+async function insertClickhouseSnapshotRecord({
+ id,
+ websiteId,
+ urlPath,
+ viewportW,
+ viewportH,
+ pageW,
+ pageH,
+ status,
+ mimeType,
+ objectKey,
+ imageSize,
+ error,
+}: {
+ id: string;
+ websiteId: string;
+ urlPath: string;
+ viewportW: number;
+ viewportH: number;
+ pageW: number;
+ pageH: number;
+ status: HeatmapSnapshotStatus;
+ mimeType: string | null;
+ objectKey: string | null;
+ imageSize: number | null;
+ error: string | null;
+}) {
+ return clickhouse.insert('heatmap_snapshot', [
+ {
+ snapshot_id: id,
+ website_id: websiteId,
+ url_path: urlPath,
+ viewport_w: viewportW,
+ viewport_h: viewportH,
+ page_w: pageW,
+ page_h: pageH,
+ status: CLICKHOUSE_SNAPSHOT_STATUS[status],
+ mime_type: mimeType || '',
+ object_key: objectKey || '',
+ image_size: imageSize,
+ error,
+ created_at: clickhouse.getUTCString(),
+ },
+ ]);
+}
+
+function getSnapshotObjectKey(websiteId: string, snapshotId: string, viewportW: number, viewportH: number) {
+ return `heatmaps/${websiteId}/${viewportW}x${viewportH}/${snapshotId}.png`;
+}
+
async function captureSnapshot(
url: string,
viewportW: number,
@@ -456,6 +659,14 @@ export async function ensureHeatmapSnapshot({
try {
const capture = await captureSnapshot(captureUrl, viewportW, viewportH, pageW);
+ const objectKey =
+ clickhouse.enabled
+ ? getSnapshotObjectKey(websiteId, snapshotId, viewportW, viewportH)
+ : null;
+
+ if (objectKey) {
+ await putHeatmapSnapshot(objectKey, capture.imageData, capture.mimeType);
+ }
await upsertSnapshotRecord({
id: snapshotId,
@@ -467,7 +678,8 @@ export async function ensureHeatmapSnapshot({
pageH: capture.pageH,
status: SNAPSHOT_STATUS.ready,
mimeType: capture.mimeType,
- imageData: capture.imageData,
+ imageData: clickhouse.enabled ? null : capture.imageData,
+ objectKey,
error: null,
});
} catch (error) {
@@ -495,6 +707,39 @@ export async function getHeatmapSnapshotImage(
websiteId: string,
snapshotId: string,
): Promise<{ mimeType: string; imageData: Buffer } | null> {
+ if (clickhouse.enabled) {
+ const rows = await clickhouse.rawQuery<
+ { mimeType: string; objectKey: string }[]
+ >(
+ `
+ select
+ mime_type as mimeType,
+ object_key as objectKey
+ from heatmap_snapshot
+ where snapshot_id = {snapshotId:UUID}
+ and website_id = {websiteId:UUID}
+ and status = {status:UInt8}
+ and object_key != ''
+ order by created_at desc
+ limit 1
+ `,
+ {
+ websiteId,
+ snapshotId,
+ status: CLICKHOUSE_SNAPSHOT_STATUS.ready,
+ },
+ 'getHeatmapSnapshotImage',
+ );
+
+ const row = rows?.[0];
+
+ if (!row?.objectKey) {
+ return null;
+ }
+
+ return getHeatmapSnapshot(row.objectKey);
+ }
+
const rows = await prisma.rawQuery(
`
select