Skip to content

Papermark is the open-source DocSend alternative and secure data rooms with built-in analytics and custom domains.

update tinybird pipes and datasource

Marc Seitz6dc59b1f3f26
diff --git a/lib/tinybird/datasources/page_views.datasource b/lib/tinybird/datasources/page_views.datasource
index 3088b5a9..32477b7d 100644
--- a/lib/tinybird/datasources/page_views.datasource
+++ b/lib/tinybird/datasources/page_views.datasource
@@ -1,4 +1,4 @@
-VERSION 1
+VERSION 2
DESCRIPTION >
Page views are events when a user views a document
@@ -8,6 +8,7 @@ SCHEMA >
`linkId` String `json:$.linkId`,
`documentId` String `json:$.documentId`,
`viewId` String `json:$.viewId`,
+ `versionNumber` UInt16 `json:$.versionNumber`,
# Unix timestamp
`time` Int64 `json:$.time`,
`duration` UInt32 `json:$.duration`,
@@ -35,5 +36,5 @@ SCHEMA >
ENGINE "MergeTree"
-ENGINE_SORTING_KEY "linkId,documentId,viewId,pageNumber,time,id"
+ENGINE_SORTING_KEY "linkId,documentId,viewId,versionNumber,pageNumber,time,id"
diff --git a/lib/tinybird/endpoints/get_page_duration_per_view.pipe b/lib/tinybird/endpoints/get_page_duration_per_view.pipe
index b6b89e18..a2c2cf11 100644
--- a/lib/tinybird/endpoints/get_page_duration_per_view.pipe
+++ b/lib/tinybird/endpoints/get_page_duration_per_view.pipe
@@ -1,4 +1,4 @@
-VERSION 2
+VERSION 3
NODE endpoint
SQL >
@@ -7,7 +7,7 @@ SQL >
pageNumber,
SUM(duration) AS sum_duration
FROM
- page_views__v1
+ page_views__v2
WHERE
documentId = {{ String(documentId, required=True) }}
AND viewId = {{ String(viewId, required=True) }}
diff --git a/lib/tinybird/endpoints/get_total_average_page_duration.pipe b/lib/tinybird/endpoints/get_total_average_page_duration.pipe
index ce69e85e..6cbd0e2a 100644
--- a/lib/tinybird/endpoints/get_total_average_page_duration.pipe
+++ b/lib/tinybird/endpoints/get_total_average_page_duration.pipe
@@ -1,17 +1,18 @@
-VERSION 1
+VERSION 2
NODE endpoint
SQL >
%
SELECT
+ versionNumber,
pageNumber,
AVG(duration) AS avg_duration
FROM
- page_views__v1
+ page_views__v2
WHERE
documentId = {{ String(documentId, required=True) }}
AND time >= {{ Int64(since, required=True) }}
GROUP BY
- pageNumber
+ versionNumber, pageNumber
ORDER BY
- pageNumber ASC
+ versionNumber ASC, pageNumber ASC
diff --git a/lib/tinybird/endpoints/tb_backfill_populate.pipe b/lib/tinybird/endpoints/tb_backfill_populate.pipe
new file mode 100644
index 00000000..13b510fa
--- /dev/null
+++ b/lib/tinybird/endpoints/tb_backfill_populate.pipe
@@ -0,0 +1,10 @@
+NODE temp_0
+SQL >
+ SELECT
+ * ,
+ 1 AS versionNumber
+ FROM page_views__v1
+ WHERE fromUnixTimestamp64Milli(time) <= '2023-10-14 19:25:00.000'
+
+TYPE materialized
+DATASOURCE page_views__v2
\ No newline at end of file
diff --git a/lib/tinybird/endpoints/tb_materialize_until_change_ingest.pipe b/lib/tinybird/endpoints/tb_materialize_until_change_ingest.pipe
new file mode 100644
index 00000000..8940902e
--- /dev/null
+++ b/lib/tinybird/endpoints/tb_materialize_until_change_ingest.pipe
@@ -0,0 +1,10 @@
+NODE temp_0
+SQL >
+ SELECT
+ * ,
+ 1 AS versionNumber
+ FROM page_views__v1
+ WHERE fromUnixTimestamp64Milli(time) > '2023-10-14 19:25:00.000'
+
+TYPE materialized
+DATASOURCE page_views__v2
\ No newline at end of file
diff --git a/lib/tinybird/pipes.ts b/lib/tinybird/pipes.ts
index ae892958..710f306e 100644
--- a/lib/tinybird/pipes.ts
+++ b/lib/tinybird/pipes.ts
@@ -5,19 +5,20 @@ import { Tinybird } from "@chronark/zod-bird";
const tb = new Tinybird({ token: process.env.TINYBIRD_TOKEN! });
export const getTotalAvgPageDuration = tb.buildPipe({
- pipe: "get_total_average_page_duration__v1",
+ pipe: "get_total_average_page_duration__v2",
parameters: z.object({
documentId: z.string(),
since: z.number(),
}),
data: z.object({
+ versionNumber: z.number().int(),
pageNumber: z.string(),
avg_duration: z.number(),
}),
});
export const getViewPageDuration = tb.buildPipe({
- pipe: "get_page_duration_per_view__v2",
+ pipe: "get_page_duration_per_view__v3",
parameters: z.object({
documentId: z.string(),
viewId: z.string(),
diff --git a/lib/tinybird/publish.ts b/lib/tinybird/publish.ts
index 8395e580..04558613 100644
--- a/lib/tinybird/publish.ts
+++ b/lib/tinybird/publish.ts
@@ -5,12 +5,13 @@ import { Tinybird } from "@chronark/zod-bird";
const tb = new Tinybird({ token: process.env.TINYBIRD_TOKEN! });
export const publishPageView = tb.buildIngestEndpoint({
- datasource: "page_views__v1",
+ datasource: "page_views__v2",
event: z.object({
id: z.string(),
linkId: z.string(),
documentId: z.string(),
viewId: z.string(),
+ versionNumber: z.number().int().min(1).max(65535).optional().default(1),
time: z.number().int(),
duration: z.number().int(),
pageNumber: z.string(),