Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): route change tracking through the store with optional Bigtable double-write (#4484)

Gergő Móriczc88d0940f897
diff --git a/apps/api/package.json b/apps/api/package.json
index bf67e0f86..b580e71c4 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -86,6 +86,7 @@
"@bull-board/express": "^6.21.3",
"@clickhouse/client": "^1.8.1",
"@dqbd/tiktoken": "^1.0.22",
+ "@google-cloud/bigtable": "^7.2.0",
"@google-cloud/storage": "^7.21.0",
"@mendable/firecrawl-rs": "workspace:*",
"@openrouter/ai-sdk-provider": "^2.9.1",
diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yaml
index 5562a4bb7..f02af0445 100644
--- a/apps/api/pnpm-lock.yaml
+++ b/apps/api/pnpm-lock.yaml
@@ -41,8 +41,11 @@ overrides:
js-cookie@<3.0.7: 3.0.7
shell-quote@<=1.8.4: 1.10.0
esbuild@>=0.17.0 <0.28.1: 0.28.1
- '@opentelemetry/core@>=2.0.0 <2.8.0': 2.8.0
query-string@>=9.0.0 <10.0.0: '>=9.5.0 <10.0.0'
+ '@google-cloud/opentelemetry-cloud-monitoring-exporter': ^0.22.0
+ '@opentelemetry/core@<2.9.0': 2.9.0
+ '@opentelemetry/resources@<2.9.0': 2.9.0
+ '@opentelemetry/sdk-metrics@<2.9.0': 2.9.0
patchedDependencies:
[email protected]: cf2058381802eaf328e1cb26b185762ba41e2157a5545e7ed9bba56b7de0c49b
@@ -87,6 +90,9 @@ importers:
'@dqbd/tiktoken':
specifier: ^1.0.22
version: 1.0.22
+ '@google-cloud/bigtable':
+ specifier: ^7.2.0
+ version: 7.2.0(@opentelemetry/[email protected](@opentelemetry/[email protected]))
'@google-cloud/storage':
specifier: ^7.21.0
version: 7.21.0
@@ -101,7 +107,7 @@ importers:
version: 2.58.2
'@sentry/node':
specifier: ^10.67.0
- version: 10.67.0(@opentelemetry/[email protected](@opentelemetry/[email protected]))
+ version: 10.67.0(@opentelemetry/[email protected](@opentelemetry/[email protected]))
'@types/ws':
specifier: ^8.5.12
version: 8.5.12
@@ -786,22 +792,85 @@ packages:
'@noble/hashes':
optional: true
+ '@google-cloud/[email protected]':
+ resolution: {integrity: sha512-sw9cLk0Tho+/3hmCuvf2Pj0LL2vt5w6EaC+GzVlcvrc42RnhMuvH0zXrCh0x6ddHS7iQeqr5JIMQg29BlRwnBw==}
+ engines: {node: '>=22'}
+
+ '@google-cloud/[email protected]':
+ resolution: {integrity: sha512-9FM/1aMgb2xATXpkaJNEZQdzcLr+E+Vpw/1pBEs9J206KfsMbfGp+YAzq2uAWgWLzRqZoCjpblCsD3am/UX4dg==}
+ engines: {node: '>=22'}
+
+ '@google-cloud/[email protected]':
+ resolution: {integrity: sha512-11CltWBC0rMUw8dw+t1npCF2u4+NuESmbrPoV3p1Unu3C4HRjkmPePmIBmonX7IlhwCNBc/xFB+HWLEpcZWKvA==}
+ engines: {node: '>=22'}
+
+ '@google-cloud/[email protected]':
+ resolution: {integrity: sha512-98q1nvs/qa8Oh3F4FuSyUniWphIOD4w4r0+Ko/Fh83eGv/AXP18zXvAM9KRj8cvvQ7Km9whvzOuxJU3vO3HTLQ==}
+ engines: {node: '>=18'}
+ deprecated: Google Cloud Platform now supports native OpenTelemetry Protocol (OTLP) endpoints via the Telemetry API. Please follow https://github.com/GoogleCloudPlatform/opentelemetry-operations-js/blob/main/MIGRATION.md
+ peerDependencies:
+ '@opentelemetry/api': ^1.9.0
+ '@opentelemetry/core': 2.9.0
+ '@opentelemetry/resources': 2.9.0
+ '@opentelemetry/sdk-metrics': 2.9.0
+
+ '@google-cloud/[email protected]':
+ resolution: {integrity: sha512-/7ujlMoKtDtrbQlJihCjQnm31n2s2RTlvJqcSbt2jV3OkCzPAdo3u31Q13HNugqtIRUSk7bUoLx6AzhURkhW4w==}
+ engines: {node: '>=14'}
+ deprecated: Google Cloud Platform now supports native OpenTelemetry Protocol (OTLP) endpoints via the Telemetry API. Please follow https://github.com/GoogleCloudPlatform/opentelemetry-operations-js/blob/main/MIGRATION.md
+ peerDependencies:
+ '@opentelemetry/resources': 2.9.0
+
+ '@google-cloud/[email protected]':
+ resolution: {integrity: sha512-dTvbFl+q9ac0Me8+GaDrFdwDBia/L8Qxxi/NY/jeHaUogDIpqOtLoqcXV9HdgiLxy5vjEGaJMqir1nVra1qG7w==}
+ engines: {node: '>=18'}
+ deprecated: Google Cloud Platform now supports native OpenTelemetry Protocol (OTLP) endpoints via the Telemetry API. Please follow https://github.com/GoogleCloudPlatform/opentelemetry-operations-js/blob/main/MIGRATION.md
+ peerDependencies:
+ '@opentelemetry/api': ^1.0.0
+ '@opentelemetry/core': 2.9.0
+ '@opentelemetry/resources': 2.9.0
+
'@google-cloud/[email protected]':
resolution: {integrity: sha512-DJS3s0OVH4zFDB1PzjxAsHqJT6sKVbRwwML0ZBP9PbU7Yebtu/7SWMRzvO2J3nUi9pRNITCfu4LJeooM2w4pjg==}
engines: {node: '>=14.0.0'}
+ '@google-cloud/[email protected]':
+ resolution: {integrity: sha512-1TUx3KdaU3cN7nfCdNf+UVqA/PSX29Cjcox3fZZBtINlRrXVTmUkQnCKv2MbBUbCopbK4olAT1IHl76uZyCiVA==}
+ engines: {node: '>=14.0.0'}
+
+ '@google-cloud/[email protected]':
+ resolution: {integrity: sha512-FmPFgn9x11vtE26VyqmoL+aunGyi5PACJxaQ0rbCUNJ1DrsDc6RlsILOjnHgj7HrsbEQBCJLNZGmDSU1hWW7ig==}
+ engines: {node: '>=22'}
+
'@google-cloud/[email protected]':
resolution: {integrity: sha512-MmaX6HeSvyPbWGwFq7mXdo0uQZLGBYCwziiLIGq5JVX+/bdI3SAq6bP98trV5eTWfLuvsMcIC1YJOF2vfteLFA==}
engines: {node: '>=14.0.0'}
+ '@google-cloud/[email protected]':
+ resolution: {integrity: sha512-zA3o4l87UJ56odT/e9PcT4n2bqVnrp6dg9vI75DoGGbl0/YJemX8Fta7454htECt07eALxxc4iaAbrX3T+QulA==}
+ engines: {node: '>=22'}
+
'@google-cloud/[email protected]':
resolution: {integrity: sha512-Orxzlfb9c67A15cq2JQEyVc7wEsmFBmHjZWZYQMUyJ1qivXyMwdyNOs9odi79hze+2zqdTtu1E19IM/FtqZ10g==}
engines: {node: '>=14'}
+ '@google-cloud/[email protected]':
+ resolution: {integrity: sha512-lpN2AtoQ/iimp7jjm5zJFWbpW7OJc5qWmQdt59CI4ENeoIRIx+yf2ShSR2kanQfjFOshA74eSbimXXuRaSCUVg==}
+ engines: {node: '>=22'}
+
'@google-cloud/[email protected]':
resolution: {integrity: sha512-l+IFTkd+6Y5LoAuXyYCKNAKtw/Ci+rAMqgdTB1jv4iZiLhw0rtq+0qjIRbBizXkNzEFmXiXUW0H7sZQQvk1ffA==}
engines: {node: '>=14'}
+ resolution: {integrity: sha512-k9Dj3DV/itK9D06Y8f190Qgop7/Ui+D0njFV3LHMPwPT75DpXLQohE9Wmz0QElrJnzsjB7KPWiKJbOl7IPDArQ==}
+ engines: {node: '>=12.10.0'}
+
+ resolution: {integrity: sha512-wtF6h+DY6M3YaDBPAmvuuA6jV8Sif9MjtOI5euKFWRgCDl5PeDpPsHR9u2l6St5ceY8AZgoNDww5+HvEsXFsGg==}
+ engines: {node: '>=6'}
+ hasBin: true
+
'@inquirer/[email protected]':
resolution: {integrity: sha512-3eTuUO1vH2cZm2ZKHeQxnOqlTi9EfZDGgIe3BL3I4u+rJHocr9Fz86M4fjYABPvFnQG/gGK551HqDiIcETwU6Q==}
engines: {node: '>=23.5.0 || ^22.13.0 || ^20.17.0'}
@@ -946,6 +1015,9 @@ packages:
'@jridgewell/[email protected]':
resolution: {integrity: sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==}
+ '@js-sdsl/[email protected]':
+ resolution: {integrity: sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==}
+
'@mixmark-io/[email protected]':
resolution: {integrity: sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==}
@@ -1435,12 +1507,6 @@ packages:
resolution: {integrity: sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==}
engines: {node: '>=8.0.0'}
- '@opentelemetry/[email protected]':
- resolution: {integrity: sha512-hd1Lfh8p545nNz+jq1Ejfz+Mn1hyLuxYn1YzTfFNrxr8urEWMNQLPf1Th8kjOH+HxwawCrtgBp8JpBUR4ZSgww==}
- engines: {node: ^18.19.0 || >=20.6.0}
- peerDependencies:
- '@opentelemetry/api': '>=1.0.0 <1.10.0'
-
'@opentelemetry/[email protected]':
resolution: {integrity: sha512-m2nckMT80NnmjTYSPjJQObBJ+8dgkoajEOUbznL8AHZ3T3yHRk2P7gI1PhEBc1+lOnrYE9UWrWHqJDsmqjmNbw==}
engines: {node: ^18.19.0 || >=20.6.0}
@@ -1459,6 +1525,12 @@ packages:
peerDependencies:
'@opentelemetry/api': '>=1.3.0 <1.10.0'
+ '@opentelemetry/[email protected]':
+ resolution: {integrity: sha512-Xx8RGS4H5XEBl01WuCreMIpiah9cCXMbSkeuIePPdD2cUpq/vUzYmj8E/MK1OsbOc93FuAD4jfn2WOacKwLn7Q==}
+ engines: {node: ^18.19.0 || >=20.6.0}
+ peerDependencies:
+ '@opentelemetry/api': '>=1.9.0 <1.10.0'
+
'@opentelemetry/[email protected]':
resolution: {integrity: sha512-cp9zmTl62R8PJrpvFcmc8N2JQU/xfa0S+61q511Nji+QxCfZ8Ifvg7H27G8cANe4crg4RTrWsVvanHiXjSp6ag==}
engines: {node: ^18.19.0 || >=20.6.0}
@@ -1632,6 +1704,33 @@ packages:
resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==}
engines: {node: '>=14'}
+ '@protobufjs/[email protected]':
+ resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==}
+
+ '@protobufjs/[email protected]':
+ resolution: {integrity: sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==}
+
+ '@protobufjs/[email protected]':
+ resolution: {integrity: sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==}
+
+ '@protobufjs/[email protected]':
+ resolution: {integrity: sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==}
+
+ '@protobufjs/[email protected]':
+ resolution: {integrity: sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==}
+
+ '@protobufjs/[email protected]':
+ resolution: {integrity: sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==}
+
+ '@protobufjs/[email protected]':
+ resolution: {integrity: sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==}
+
+ '@protobufjs/[email protected]':
+ resolution: {integrity: sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==}
+
+ '@protobufjs/[email protected]':
+ resolution: {integrity: sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug==}
+
'@react-email/[email protected]':
resolution: {integrity: sha512-wDaMy27xAq1cJHtSFptp0DTKPuV2GYhloqia95ub/DH9Dea1aWYsbdM918MOc/b/HvVS3w1z8DWzfAk13bGStQ==}
engines: {node: '>=18.0.0'}
@@ -1805,7 +1904,7 @@ packages:
engines: {node: '>=18'}
peerDependencies:
'@opentelemetry/api': ^1.9.0
- '@opentelemetry/core': 2.8.0
+ '@opentelemetry/core': 2.9.0
'@opentelemetry/exporter-trace-otlp-http': '>=0.57.0 <1'
'@opentelemetry/instrumentation': '>=0.57.1 <1'
'@opentelemetry/sdk-trace-base': ^1.30.1 || ^2.1.0
@@ -1830,7 +1929,7 @@ packages:
engines: {node: '>=18'}
peerDependencies:
'@opentelemetry/api': ^1.9.0
- '@opentelemetry/core': 2.8.0
+ '@opentelemetry/core': 2.9.0
'@opentelemetry/sdk-trace-base': ^1.30.1 || ^2.1.0
'@sentry/[email protected]':
@@ -1905,6 +2004,9 @@ packages:
resolution: {integrity: sha512-jsxagdikDiDBeIRaPYtArcT8my4tN1og7MtMRquFT3XNA6axxyHDRUemqDz/taRDdOUn0GnGHRCuff4q48sW9A==}
+ '@types/[email protected]':
+ resolution: {integrity: sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==}
+
resolution: {integrity: sha512-ymXWVrDiCxTBE3+RIrrP533E70eA+9qu7zdWoHuOmGujkYtzf4HQF96b8nwHLqhuf4ykX61IGRIB38CC6/sImQ==}
@@ -2256,6 +2358,9 @@ packages:
resolution: {integrity: sha512-JocpCSOixzy5XFJi2ub6IMmV/G9i8Lrm2lZvwBv9xPdglmZM0ufDVBbjbrfU/zuLvBfD7Bv2eYxz9i+OHTgkew==}
deprecated: pkg version number incorrect
+ resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==}
+
resolution: {integrity: sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==}
@@ -2351,6 +2456,10 @@ packages:
peerDependencies:
typanion: '*'
+ resolution: {integrity: sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==}
+ engines: {node: '>=12'}
+
resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==}
engines: {node: '>=0.10.0'}
@@ -2543,6 +2652,10 @@ packages:
resolution: {integrity: sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==}
+ resolution: {integrity: sha512-xCbB8IN3IT+tdgoEPOnJmYTNJDrygGFOmiQEiVa5eAD+JEB1vTgMNhVGRnN5Eex/6amck7cdcrixb1qN9Go+GQ==}
+ engines: {node: '>=10'}
+
resolution: {integrity: sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==}
engines: {node: '>=12'}
@@ -2761,9 +2874,17 @@ packages:
engines: {node: '>=18'}
hasBin: true
+ resolution: {integrity: sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==}
+ engines: {node: '>=6'}
+
resolution: {integrity: sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==}
+ resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==}
+ engines: {node: '>=10'}
+
resolution: {integrity: sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g==}
engines: {node: '>=0.10'}
@@ -2821,6 +2942,9 @@ packages:
resolution: {integrity: sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==}
+ resolution: {integrity: sha512-LIREwygxtxzHF11oLJ4xIVKu/ZWNgrj/QaGvaSD8ZggIsgCyCtSYevlrpWVqNau57ZwezV8K1HFBSjQ7FcRbTQ==}
+
resolution: {integrity: sha512-bCK/2Z4zLidyB4ReuIsvALH6w31YfAQDmXMqMx6FyfHqvBxtjC0eRumeSu4Bs3XtXwpyIywtSTrVT99BxY1f9w==}
@@ -2878,6 +3002,9 @@ packages:
resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==}
engines: {node: ^12.20 || >= 14.13}
+ resolution: {integrity: sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==}
+
resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==}
engines: {node: '>=8'}
@@ -2957,6 +3084,10 @@ packages:
resolution: {integrity: sha512-YGGyuEdVIjqxkxVH1pUTMY/XtmmsApXrCVv5EU25iX6inEPbV+VakJfLealkBtJN69AQmh1eGOdCl9Sm1UP6XQ==}
engines: {node: '>=18'}
+ resolution: {integrity: sha512-kB3rzJV7d9juLZh8/56QTXCwQfxyhdOMdyYk1HdQKFtF8TJTDTZQJtixWIwXdE9Jji91mC41DUNpjleo4L4eAQ==}
+ engines: {node: '>=18'}
+
resolution: {integrity: sha512-a4tiq7E0/5fTjxPAaH4jpjkSv/uCaU2p5KC6HVGrvl0cDjA8iBZv4vv1gyzlmK0ZUKqwpOyQMKzZQe3lTit77A==}
engines: {node: '>=14'}
@@ -2965,10 +3096,18 @@ packages:
resolution: {integrity: sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg==}
engines: {node: '>=18'}
+ resolution: {integrity: sha512-2YYnIlHaKBGT2IPg3G2M57hia9Galz15zsEOvw9T3oRf0lSn6KN6VcHQLqby7x8ksYKnjXvp3rp2KJyLCN6zfQ==}
+ engines: {node: '>=22'}
+
resolution: {integrity: sha512-L8IF951YhQl7FQWqamDmdwo0WNNPDO3KmHTb9ehUUtnRteOptNpznwy2D3jr1ib9fSaO0u+7beSrtj52XpdVgA==}
engines: {node: '>=10.20.0'}
+ resolution: {integrity: sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==}
+ engines: {node: 6.* || 8.* || >= 10.*}
+
resolution: {integrity: sha512-R1QfovbPsKmosqTnPoRFiJ7CF9MLRgb53ChvMZm+r4p76/+8yKDy17qLL2PKInORy2RkZZekuK0efYgmzTkXyQ==}
engines: {node: '>=18'}
@@ -2997,10 +3136,18 @@ packages:
resolution: {integrity: sha512-7ABviyMOlX5hIVD60YOfHw4/CxOfBhyduaYB+wbFWCWoni4N7SLcV46hrVRktuBbZjFC9ONyqamZITN7q3n32w==}
engines: {node: '>=18'}
+ resolution: {integrity: sha512-vzpgPutxrghPsnjrjpzLX2bdv8IOL719Rh0oEjGnQu8YCIbnbMuTTQ5zU9LcKvLdOPgCxBwppbvnhgW90Qna5Q==}
+ engines: {node: '>=22'}
+
resolution: {integrity: sha512-Jb6Z0+nvECVz+2lzSMt9u98UsoakXxA2HGHMCxh+so3n90XgYWkq5dur19JAJV7ONiJY22yBTyJB1TSkvPq9Ng==}
engines: {node: '>=14'}
+ resolution: {integrity: sha512-PN84pgPw6USmNfy8hlMkifsII+VzCTddRODjPqGZ0JKQuB3WYMEMMepTypmE7ymMXFYiOe3Oi0dHskQ4jkOZfg==}
+ engines: {node: '>=22'}
+
resolution: {integrity: sha512-NEgUnEcBiP5HrPzufUkBzJOD/Sxsco3rLNo1F1TNf7ieU8ryUzBhqba8r756CjLX7rn3fHl6iLEwPYuqpoKgQQ==}
engines: {node: '>=14'}
@@ -3009,10 +3156,26 @@ packages:
resolution: {integrity: sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA==}
engines: {node: '>=14'}
+ resolution: {integrity: sha512-HMhaQghlOTvbcb3c4T5jmmOMtG3JUF1iOQMezaJXL86CDS+Tm2vHd0IeLFRAx3+ewd+bo9E1HFHoy17X5aJa9A==}
+ engines: {node: '>=22'}
+
+ resolution: {integrity: sha512-7g1yzQKx0mmNTjiK0H9dJ8eqKqDBveES9vLHeg5neb3BMQy/d1oQefIMhIpOVT8a+f+LOcixMEdRbFIW/cQUJw==}
+ engines: {node: '>=18'}
+
+ resolution: {integrity: sha512-xEJJYLZ4qeenVyfzispNfRjCe9bsv7CzBv5zYFLvScOze9snJ8S9W6hjQ729CWPQt5mvn/JrcRaCHzQiukt0ng==}
+ engines: {node: '>=18'}
+
resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==}
engines: {node: '>= 0.4'}
+ resolution: {integrity: sha512-I3GNn9ONy5Vrm6XdjgOu7296DnN9vudnjv7dXKW2jAogfS20YMYZAzfZNkJUMgdpQTtLJ9t+DQCzYgGBpdYuwA==}
+ engines: {node: '>=12'}
+
resolution: {integrity: sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==}
engines: {node: '>=14.0.0'}
@@ -3073,6 +3236,10 @@ packages:
resolution: {integrity: sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==}
engines: {node: '>= 6'}
+ resolution: {integrity: sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==}
+ engines: {node: '>= 14'}
+
resolution: {integrity: sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==}
engines: {node: '>= 6'}
@@ -3150,6 +3317,10 @@ packages:
resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==}
engines: {node: '>=0.12.0'}
+ resolution: {integrity: sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==}
+ engines: {node: '>=8'}
+
resolution: {integrity: sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==}
@@ -3160,6 +3331,13 @@ packages:
resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==}
engines: {node: '>=8'}
+ resolution: {integrity: sha512-rMYPYvCzsXywIsldgLaSoPlw5PfoB/ssr7hY4pLfcodrA5M/eArza1a9VmTiNIBNMjOGr1Ow9mTyU2o69U6U9Q==}
+
+ resolution: {integrity: sha512-a2xr4E3s1PjDS8ORcGgXpWx6V+liNs+O3JRD2mb9aeugD7rtkkZ0zgLdYgw0tWsKhsdiezGYptSiMlVazCBTuQ==}
+ engines: {node: '>= 0.4'}
+
resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==}
@@ -3355,6 +3533,9 @@ packages:
resolution: {integrity: sha512-0aeh5HHHgmq1KRdMMDHfhMWQmIT/m7nRDTlxlFqni2Sp0had9baqsjJRvDGdlvgd6NmPE0nPloOipiQJGFtTHQ==}
engines: {node: '>=20.0.0'}
+ resolution: {integrity: sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==}
+
resolution: {integrity: sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==}
@@ -3376,6 +3557,9 @@ packages:
resolution: {integrity: sha512-eWsgQW89IewS95ZOcr15HHCX6FVDxq3f2PNUIng3fyzsPev9imFQxIYdFZ6crl8L56UR6ZlGDLcEb3RZsCSSqw==}
+ resolution: {integrity: sha512-QZ1d4xoBHYUeuouhEq3lk3Uq7ldgyFXGBhg04+oRLnIz8o9T65Eh+8YdroUwn846zchkA9yDsDl5CVVaV2nqYw==}
+
resolution: {integrity: sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==}
@@ -3391,6 +3575,12 @@ packages:
resolution: {integrity: sha512-CdaO738xRapbKIMVn2m4F6KTj4j7ooJ8POVnebSgKo3KBz5axNXRAL7ZdRjIV6NOr2Uf4vjtRkxrFETOioCqSA==}
engines: {node: '>= 12.0.0'}
+ resolution: {integrity: sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==}
+
+ resolution: {integrity: sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==}
+
resolution: {integrity: sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==}
hasBin: true
@@ -3537,6 +3727,9 @@ packages:
resolution: {integrity: sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==}
+ resolution: {integrity: sha512-ibPK3iA+vaY1eEjESkQkM0BbCqFOaZMiXRTtdB0u7b4djtY6JnsjvPdUHVMg6xQt3B8fpTTWHI9A+ADjM9frzg==}
+
resolution: {integrity: sha512-qN8v/s2PAJwGUtr1/hYTpNKlD6Y9rc4p8KSmJXyGdYGZsDGKXrGThikLFP9OCHFeLeEpQzPwiAtdIvBLqm//Hw==}
@@ -3590,6 +3783,10 @@ packages:
resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==}
engines: {node: '>=0.10.0'}
+ resolution: {integrity: sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==}
+ engines: {node: '>= 6'}
+
resolution: {integrity: sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==}
engines: {node: '>= 0.4'}
@@ -3652,10 +3849,18 @@ packages:
resolution: {integrity: sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==}
engines: {node: '>=8'}
+ resolution: {integrity: sha512-aNZ+VfjobsWryoiPnEApGGmf5WmNsCo9xu8dfaYamG5qaLP7ClhLN6NgsFe6SwJ2UbLEBK5dv9x8Mn5+RVhMWQ==}
+ engines: {node: '>=18'}
+
resolution: {integrity: sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==}
engines: {node: '>=8'}
+ resolution: {integrity: sha512-MyIV3ZA/PmyBN/ud8vV9XzwTrNtR4jFrObymZYnZqMmW0zA8Z17vnT0rBgFE/TlohB+YCHqXMgZzb3Csp49vqg==}
+ engines: {node: '>=14.16'}
+
resolution: {integrity: sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==}
@@ -3804,6 +4009,14 @@ packages:
resolution: {integrity: sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA==}
+ resolution: {integrity: sha512-VUQlb/J2WjOG8BKhdLwJUz+MKxSMtBGYh+TaNmRtiYVhlIzXjzeAGIx7gtc3lXlA2iURzfF2+cXaQvC2ofwzpg==}
+ engines: {node: '>=22'}
+
+ resolution: {integrity: sha512-dYDWdjSl5RNb7SgPxGQcRU+GtvP7s2fpkrY0r432PcOIaZ0/rBcxEZnQN67iJhFuQiVw754JDoPruPCNdGsbjg==}
+ engines: {node: '>=12.0.0'}
+
resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==}
engines: {node: '>= 0.10'}
@@ -3823,6 +4036,12 @@ packages:
resolution: {integrity: sha512-JZd3gMVBAVQkSs6HdNZo9Sdo0LNcQeMNP3CozBJb3JYC/QUYZTnKxP+f8oWRX4rHP5EurWxqAHTSwUCjlNKa1w==}
+ resolution: {integrity: sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==}
+
+ resolution: {integrity: sha512-m7KOje7jZxrmutanlkS1daj1dS6z6BgslzOXmcSEpIlCxM3VJH7lG5QLeck/6hgF6F4crFf01UtQmNsJfweTAw==}
+
resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==}
engines: {node: '>=6'}
@@ -3895,6 +4114,10 @@ packages:
resolution: {integrity: sha512-2RDWXg5jgRptDrB1w9O/JgSZC0j7y4SlaXnor93H/UJm/QyDiFgBKNtrh0TI6oCXqYSaSoXxFh6Sd3VtYfhRXw==}
engines: {node: '>=12'}
+ resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==}
+ engines: {node: '>=0.10.0'}
+
resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==}
engines: {node: '>=0.10.0'}
@@ -3925,6 +4148,10 @@ packages:
resolution: {integrity: sha512-dUOvLMJ0/JJYEn8NrpOaGNE7X3vpI5XlZS/u0ANjqtcZVKnIxP7IgCFwrKTxENw29emmwug53awKtaMm4i9g5w==}
engines: {node: '>=14'}
+ resolution: {integrity: sha512-4kGHEBl0ME6gR+8QB1sPMyg58jUxLUcCZNeqLDrfqzk0eWQN8XJFmeu7fmZCjlZCRW0S2u64Ik1HkO/0sWgCYQ==}
+ engines: {node: '>=22'}
+
resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==}
engines: {node: '>= 4'}
@@ -4078,6 +4305,10 @@ packages:
resolution: {integrity: sha512-VE0SOVEHCk7Qc8ulkWw3ntAzXuqf7S2lvwQaDLRnUeIEaKNQJzV6BwmLKhOqT61aGhfUMrXeaBk+oDGCzvhcug==}
+ resolution: {integrity: sha512-FUTYXpLroqytNKWIfHzlDWoy9E4tmBB/RklNMy6w3VJs+/XEYAHgbiylg4SS43iOk/9bM0BlJ2EDpFAGT66IoQ==}
+ engines: {node: '>=4'}
+
resolution: {integrity: sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg==}
@@ -4174,6 +4405,10 @@ packages:
resolution: {integrity: sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==}
+ resolution: {integrity: sha512-bNr5j2YjSdajgCVsp+8JVjRf1uHIbpNISLeKp/7V1NnVMX3Gh6H/kECNGlm2Q3I68ACODQ0iv6aZ3Rvm8WgLGA==}
+ engines: {node: '>=22'}
+
resolution: {integrity: sha512-resvxdc6Mgb7YEThw6G6bExlXKkv6+YbuzGg9xuXxSgxJF7Ozs+o8Y9+2R3sArdWdW8nOokoQb1yrpFB0pQK2g==}
engines: {node: '>=14'}
@@ -4318,6 +4553,9 @@ packages:
resolution: {integrity: sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==}
+ resolution: {integrity: sha512-XdVKMF4SJ0nP/O7XIPB0JwAEuT9lDIYnNsK8yGVe43y0AWoKeJNdv3ZNWh7ksJ6KqQFjOO6ox/VEitLnaVNufw==}
+
resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==}
@@ -4532,6 +4770,10 @@ packages:
resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==}
engines: {node: '>=0.4'}
+ resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==}
+ engines: {node: '>=10'}
+
resolution: {integrity: sha512-AvbaCLOO2Otw/lW5bmh9d/WEdcDFdQp2Z2ZUH3pX9U2ihyUY0nvLv7J6TrWowklRGPYbB/IuIMfYgxaCPg5Bpg==}
engines: {node: '>= 14.6'}
@@ -4542,6 +4784,14 @@ packages:
engines: {node: '>= 14.6'}
hasBin: true
+ resolution: {integrity: sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==}
+ engines: {node: '>=12'}
+
+ resolution: {integrity: sha512-GZtjxm/J/4TSxuL3FNYjCmLktBTnIw/rVmKSIyKeYAZpmJB2ig9VauCC5xsa82GNKVKDAqpOn3KVzNt0zmrU0g==}
+ engines: {node: '>=12'}
+
resolution: {integrity: sha512-jIH9yLR9wqr0wOS0TpBvo/g/2UgZH5qePVbjgRliiF0BYvOZyaBknKsF+x9Iht0O6sqgnB93rCICdOZFecJuDw==}
engines: {node: '>=12'}
@@ -4920,15 +5170,101 @@ snapshots:
optionalDependencies:
'@noble/hashes': 1.8.0
+ '@google-cloud/[email protected]':
+ dependencies:
+ google-gax: 6.1.0
+ transitivePeerDependencies:
+ - supports-color
+
+ '@google-cloud/[email protected](@opentelemetry/[email protected](@opentelemetry/[email protected]))':
+ dependencies:
+ '@google-cloud/bigtable-api': 0.3.0
+ '@google-cloud/monitoring': 6.0.0
+ '@google-cloud/opentelemetry-cloud-monitoring-exporter': 0.22.0(@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))
+ '@google-cloud/opentelemetry-resource-util': 2.4.0(@opentelemetry/[email protected](@opentelemetry/[email protected]))
+ '@google-cloud/precise-date': 6.0.1
+ '@google-cloud/projectify': 6.0.1
+ '@google-cloud/promisify': 6.0.1
+ '@opentelemetry/api': 1.9.1
+ '@opentelemetry/resources': 2.9.0(@opentelemetry/[email protected])
+ '@opentelemetry/sdk-metrics': 2.9.0(@opentelemetry/[email protected])
+ '@types/long': 4.0.2
+ abort-controller: 3.0.0
+ arrify: 2.0.1
+ concat-stream: 2.0.0
+ dot-prop: 6.0.0
+ escape-string-regexp: 4.0.0
+ extend: 3.0.2
+ fast-crc32c: 2.0.0
+ google-gax: 6.1.0
+ grpc-gcp: 1.1.1
+ is: 3.3.2
+ is-utf8: 0.2.1
+ lodash.snakecase: 4.1.1
+ long: 4.0.0
+ p-queue: 8.1.1
+ pumpify: 2.0.1
+ stream-events: 1.0.5
+ transitivePeerDependencies:
+ - '@opentelemetry/core'
+ - encoding
+ - supports-color
+
+ '@google-cloud/[email protected]':
+ dependencies:
+ google-gax: 6.1.0
+ transitivePeerDependencies:
+ - supports-color
+
+ '@google-cloud/[email protected](@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))':
+ dependencies:
+ '@google-cloud/opentelemetry-resource-util': 3.1.0(@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))
+ '@google-cloud/precise-date': 4.0.0
+ '@opentelemetry/api': 1.9.1
+ '@opentelemetry/core': 2.9.0(@opentelemetry/[email protected])
+ '@opentelemetry/resources': 2.9.0(@opentelemetry/[email protected])
+ '@opentelemetry/sdk-metrics': 2.9.0(@opentelemetry/[email protected])
+ google-auth-library: 10.5.0
+ googleapis: 173.0.0
+ transitivePeerDependencies:
+ - supports-color
+
+ '@google-cloud/[email protected](@opentelemetry/[email protected](@opentelemetry/[email protected]))':
+ dependencies:
+ '@opentelemetry/resources': 2.9.0(@opentelemetry/[email protected])
+ '@opentelemetry/semantic-conventions': 1.43.0
+ gcp-metadata: 6.1.1
+ transitivePeerDependencies:
+ - encoding
+ - supports-color
+
+ '@google-cloud/[email protected](@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))':
+ dependencies:
+ '@opentelemetry/api': 1.9.1
+ '@opentelemetry/core': 2.9.0(@opentelemetry/[email protected])
+ '@opentelemetry/resources': 2.9.0(@opentelemetry/[email protected])
+ '@opentelemetry/semantic-conventions': 1.43.0
+ gcp-metadata: 9.0.3
+ transitivePeerDependencies:
+ - supports-color
+
'@google-cloud/[email protected]':
dependencies:
arrify: 2.0.1
extend: 3.0.2
+ '@google-cloud/[email protected]': {}
+
+ '@google-cloud/[email protected]': {}
+
'@google-cloud/[email protected]': {}
+ '@google-cloud/[email protected]': {}
+
'@google-cloud/[email protected]': {}
+ '@google-cloud/[email protected]': {}
+
'@google-cloud/[email protected]':
dependencies:
'@google-cloud/paginator': 5.0.2
@@ -4949,6 +5285,18 @@ snapshots:
- encoding
- supports-color
+ dependencies:
+ '@grpc/proto-loader': 0.8.1
+ '@js-sdsl/ordered-map': 4.4.2
+
+ dependencies:
+ lodash.camelcase: 4.3.0
+ long: 5.3.2
+ protobufjs: 7.6.6
+ yargs: 17.7.3
+
'@inquirer/[email protected]': {}
@@ -5081,6 +5429,8 @@ snapshots:
'@jridgewell/[email protected]': {}
+ '@js-sdsl/[email protected]': {}
+
'@mixmark-io/[email protected]': {}
'@msgpackr-extract/[email protected]':
@@ -5476,11 +5826,6 @@ snapshots:
'@opentelemetry/[email protected]': {}
- '@opentelemetry/[email protected](@opentelemetry/[email protected])':
- dependencies:
- '@opentelemetry/api': 1.9.1
- '@opentelemetry/semantic-conventions': 1.43.0
-
'@opentelemetry/[email protected](@opentelemetry/[email protected])':
dependencies:
'@opentelemetry/api': 1.9.1
@@ -5501,6 +5846,12 @@ snapshots:
'@opentelemetry/core': 2.9.0(@opentelemetry/[email protected])
'@opentelemetry/semantic-conventions': 1.43.0
+ '@opentelemetry/[email protected](@opentelemetry/[email protected])':
+ dependencies:
+ '@opentelemetry/api': 1.9.1
+ '@opentelemetry/core': 2.9.0(@opentelemetry/[email protected])
+ '@opentelemetry/resources': 2.9.0(@opentelemetry/[email protected])
+
'@opentelemetry/[email protected](@opentelemetry/[email protected])':
dependencies:
'@opentelemetry/api': 1.9.1
@@ -5612,6 +5963,26 @@ snapshots:
optional: true
+ '@protobufjs/[email protected]': {}
+
+ '@protobufjs/[email protected]': {}
+
+ '@protobufjs/[email protected]': {}
+
+ '@protobufjs/[email protected]': {}
+
+ '@protobufjs/[email protected]':
+ dependencies:
+ '@protobufjs/aspromise': 1.1.2
+
+ '@protobufjs/[email protected]': {}
+
+ '@protobufjs/[email protected]': {}
+
+ '@protobufjs/[email protected]': {}
+
+ '@protobufjs/[email protected]': {}
+
dependencies:
html-to-text: 9.0.5
@@ -5726,27 +6097,27 @@ snapshots:
dependencies:
'@sentry/conventions': 0.16.0
- '@sentry/[email protected](@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))':
+ '@sentry/[email protected](@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))':
dependencies:
'@sentry/conventions': 0.16.0
'@sentry/core': 10.67.0
- '@sentry/opentelemetry': 10.67.0(@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))
+ '@sentry/opentelemetry': 10.67.0(@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))
import-in-the-middle: 3.3.2
optionalDependencies:
'@opentelemetry/api': 1.9.1
- '@opentelemetry/core': 2.8.0(@opentelemetry/[email protected])
+ '@opentelemetry/core': 2.9.0(@opentelemetry/[email protected])
'@opentelemetry/instrumentation': 0.220.0(@opentelemetry/[email protected])
'@opentelemetry/sdk-trace-base': 2.9.0(@opentelemetry/[email protected])
- '@sentry/[email protected](@opentelemetry/[email protected](@opentelemetry/[email protected]))':
+ '@sentry/[email protected](@opentelemetry/[email protected](@opentelemetry/[email protected]))':
dependencies:
'@opentelemetry/api': 1.9.1
'@opentelemetry/instrumentation': 0.220.0(@opentelemetry/[email protected])
'@opentelemetry/sdk-trace-base': 2.9.0(@opentelemetry/[email protected])
'@sentry/conventions': 0.16.0
'@sentry/core': 10.67.0
- '@sentry/node-core': 10.67.0(@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))
- '@sentry/opentelemetry': 10.67.0(@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))
+ '@sentry/node-core': 10.67.0(@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))
+ '@sentry/opentelemetry': 10.67.0(@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))
'@sentry/server-utils': 10.67.0
import-in-the-middle: 3.3.2
transitivePeerDependencies:
@@ -5754,10 +6125,10 @@ snapshots:
- '@opentelemetry/exporter-trace-otlp-http'
- supports-color
- '@sentry/[email protected](@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))':
+ '@sentry/[email protected](@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))':
dependencies:
'@opentelemetry/api': 1.9.1
- '@opentelemetry/core': 2.8.0(@opentelemetry/[email protected])
+ '@opentelemetry/core': 2.9.0(@opentelemetry/[email protected])
'@opentelemetry/sdk-trace-base': 2.9.0(@opentelemetry/[email protected])
'@sentry/conventions': 0.16.0
'@sentry/core': 10.67.0
@@ -5846,6 +6217,8 @@ snapshots:
'@types/[email protected]': {}
+ '@types/[email protected]': {}
+
'@types/[email protected]': {}
'@types/[email protected]': {}
@@ -6138,6 +6511,11 @@ snapshots:
+ dependencies:
+ file-uri-to-path: 1.0.0
+ optional: true
+
@@ -6264,6 +6642,12 @@ snapshots:
dependencies:
typanion: 3.14.0
+ dependencies:
+ string-width: 4.2.3
+ strip-ansi: 6.0.1
+ wrap-ansi: 7.0.0
+
@@ -6427,6 +6811,10 @@ snapshots:
domelementtype: 2.3.0
domhandler: 5.0.3
+ dependencies:
+ is-obj: 2.0.0
+
@@ -6540,8 +6928,12 @@ snapshots:
'@esbuild/win32-ia32': 0.28.1
'@esbuild/win32-x64': 0.28.1
+
+
dependencies:
estraverse: 5.3.0
@@ -6621,6 +7013,10 @@ snapshots:
+ optionalDependencies:
+ sse4_crc32: 6.0.1
+
@@ -6680,6 +7076,9 @@ snapshots:
node-domexception: 1.0.0
web-streams-polyfill: 3.3.3
+ optional: true
+
dependencies:
to-regex-range: 5.0.1
@@ -6774,6 +7173,14 @@ snapshots:
transitivePeerDependencies:
- supports-color
+ dependencies:
+ extend: 3.0.2
+ https-proxy-agent: 7.0.6
+ node-fetch: 3.3.2
+ transitivePeerDependencies:
+ - supports-color
+
dependencies:
gaxios: 6.7.1
@@ -6791,6 +7198,14 @@ snapshots:
transitivePeerDependencies:
- supports-color
+ dependencies:
+ gaxios: 7.1.3
+ google-logging-utils: 2.0.1
+ json-bigint: 1.0.0
+ transitivePeerDependencies:
+ - supports-color
+
dependencies:
async: 2.6.4
@@ -6798,6 +7213,8 @@ snapshots:
ip-address: 6.4.0
yauzl: 3.4.0
+
@@ -6847,6 +7264,17 @@ snapshots:
transitivePeerDependencies:
- supports-color
+ dependencies:
+ base64-js: 1.5.1
+ ecdsa-sig-formatter: 1.0.11
+ gaxios: 7.3.1
+ gcp-metadata: 9.0.3
+ google-logging-utils: 2.0.1
+ jws: 4.0.1
+ transitivePeerDependencies:
+ - supports-color
+
dependencies:
base64-js: 1.5.1
@@ -6859,12 +7287,53 @@ snapshots:
- encoding
- supports-color
+ dependencies:
+ '@grpc/grpc-js': 1.14.4
+ '@grpc/proto-loader': 0.8.1
+ '@opentelemetry/api': 1.9.1
+ duplexify: 4.1.3
+ google-auth-library: 11.0.2
+ google-logging-utils: 2.0.1
+ node-fetch: 3.3.2
+ object-hash: 3.0.0
+ proto3-json-serializer: 4.0.2
+ protobufjs: 7.6.6
+ retry-request: 9.0.1
+ transitivePeerDependencies:
+ - supports-color
+
+
+ dependencies:
+ extend: 3.0.2
+ gaxios: 7.1.3
+ google-auth-library: 10.5.0
+ google-logging-utils: 1.1.3
+ qs: 6.15.2
+ url-template: 2.0.8
+ transitivePeerDependencies:
+ - supports-color
+
+ dependencies:
+ google-auth-library: 10.5.0
+ googleapis-common: 8.0.3
+ transitivePeerDependencies:
+ - supports-color
+
+ dependencies:
+ '@grpc/grpc-js': 1.14.4
+ protobufjs: 7.6.6
+
dependencies:
gaxios: 6.7.1
@@ -6946,6 +7415,13 @@ snapshots:
transitivePeerDependencies:
- supports-color
+ dependencies:
+ agent-base: 7.1.3
+ debug: 4.4.3
+ transitivePeerDependencies:
+ - supports-color
+
dependencies:
agent-base: 6.0.2
@@ -7028,12 +7504,18 @@ snapshots:
+
+
+
@@ -7224,6 +7706,8 @@ snapshots:
rfdc: 1.4.1
wrap-ansi: 9.0.0
+
@@ -7238,6 +7722,8 @@ snapshots:
+
@@ -7266,6 +7752,10 @@ snapshots:
safe-stable-stringify: 2.4.3
triple-beam: 1.4.1
+
+
dependencies:
js-tokens: 4.0.0
@@ -7372,6 +7862,9 @@ snapshots:
+ optional: true
+
@@ -7418,6 +7911,8 @@ snapshots:
+
@@ -7493,10 +7988,17 @@ snapshots:
eventemitter3: 4.0.7
p-timeout: 3.2.0
+ dependencies:
+ eventemitter3: 5.0.1
+ p-timeout: 6.1.4
+
dependencies:
p-finally: 1.0.0
+
@@ -7623,6 +8125,24 @@ snapshots:
+ dependencies:
+ protobufjs: 7.6.6
+
+ dependencies:
+ '@protobufjs/aspromise': 1.1.2
+ '@protobufjs/base64': 1.1.2
+ '@protobufjs/codegen': 2.0.5
+ '@protobufjs/eventemitter': 1.1.1
+ '@protobufjs/fetch': 1.1.1
+ '@protobufjs/float': 1.0.2
+ '@protobufjs/path': 1.1.2
+ '@protobufjs/pool': 1.1.0
+ '@protobufjs/utf8': 1.1.2
+ '@types/node': 22.19.1
+ long: 5.3.2
+
dependencies:
forwarded: 0.2.0
@@ -7640,6 +8160,17 @@ snapshots:
dependencies:
punycode: 2.3.1
+ dependencies:
+ end-of-stream: 1.4.5
+ once: 1.4.0
+
+ dependencies:
+ duplexify: 4.1.3
+ inherits: 2.0.4
+ pump: 3.0.4
+
@@ -7721,6 +8252,8 @@ snapshots:
dependencies:
node-abort-controller: 3.1.1
+
@@ -7760,6 +8293,13 @@ snapshots:
- encoding
- supports-color
+ dependencies:
+ extend: 3.0.2
+ teeny-request: 11.0.1
+ transitivePeerDependencies:
+ - supports-color
+
@@ -7932,6 +8472,12 @@ snapshots:
+ dependencies:
+ bindings: 1.5.0
+ node-addon-api: 1.7.2
+ optional: true
+
@@ -8029,6 +8575,15 @@ snapshots:
dependencies:
bintrees: 1.0.2
+ dependencies:
+ http-proxy-agent: 7.0.2
+ https-proxy-agent: 7.0.6
+ node-fetch: 3.3.2
+ stream-events: 1.0.5
+ transitivePeerDependencies:
+ - supports-color
+
dependencies:
http-proxy-agent: 5.0.0
@@ -8176,6 +8731,8 @@ snapshots:
querystringify: 2.2.0
requires-port: 1.0.0
+
@@ -8327,10 +8884,24 @@ snapshots:
+
+
+ dependencies:
+ cliui: 8.0.1
+ escalade: 3.2.0
+ get-caller-file: 2.0.5
+ require-directory: 2.1.1
+ string-width: 4.2.3
+ y18n: 5.0.8
+ yargs-parser: 21.1.1
+
dependencies:
pend: 1.2.0
diff --git a/apps/api/pnpm-workspace.yaml b/apps/api/pnpm-workspace.yaml
index 705e03274..1ab706025 100644
--- a/apps/api/pnpm-workspace.yaml
+++ b/apps/api/pnpm-workspace.yaml
@@ -47,8 +47,15 @@ overrides:
"js-cookie@<3.0.7": "3.0.7"
"shell-quote@<=1.8.4": "1.10.0"
"esbuild@>=0.17.0 <0.28.1": "0.28.1"
- "@opentelemetry/core@>=2.0.0 <2.8.0": "2.8.0"
"query-string@>=9.0.0 <10.0.0": ">=9.5.0 <10.0.0"
+ # GHSA-8988-4f7v-96qf (@opentelemetry/core): only patched on the 2.x
+ # line. @google-cloud/bigtable@7 resolves its metrics chain onto OTel
+ # 1.x; Google's own exporter 0.22 already peers with 2.x, so force the
+ # trio onto the patched line the rest of the tree already uses.
+ "@google-cloud/opentelemetry-cloud-monitoring-exporter": "^0.22.0"
+ "@opentelemetry/core@<2.9.0": "2.9.0"
+ "@opentelemetry/resources@<2.9.0": "2.9.0"
+ "@opentelemetry/sdk-metrics@<2.9.0": "2.9.0"
patchedDependencies:
allowBuilds:
@@ -56,6 +63,10 @@ allowBuilds:
"@sentry-internal/node-cpu-profiler": true
"@sentry/cli": true
bigint-buffer: true
+ # Optional native accelerator for fast-crc32c (pulled in by
+ # @google-cloud/bigtable); the pure-JS fallback is fine for our payload
+ # sizes, so no native compilation.
+ sse4_crc32: false
bufferutil: true
esbuild: true
foundationdb: true
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index 62b1309f4..129532cf8 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -235,6 +235,17 @@ const configSchema = z.object({
PARSE_UPLOAD_REF_SECRET: emptyStringAsUndefined(z.string().trim().min(1)),
PARSE_UPLOAD_PUBLIC_BASE_URL: z.string().url().optional(),
+ // Cloud Bigtable (change tracking bookkeeping store). The client
+ // auto-detects BIGTABLE_EMULATOR_HOST, so local dev only needs the
+ // emulator plus these vars. BIGTABLE_CREDENTIALS mirrors
+ // GCS_CREDENTIALS: base64-encoded service-account JSON; unset falls
+ // back to Application Default Credentials.
+ BIGTABLE_PROJECT_ID: z.string().optional(),
+ BIGTABLE_INSTANCE_ID: z.string().optional(),
+ BIGTABLE_APP_PROFILE_ID: z.string().optional(),
+ BIGTABLE_CHANGE_TRACKING_TABLE: z.string().optional(),
+ BIGTABLE_CREDENTIALS: z.string().optional(),
+
// ClickHouse (Search Analytics)
CLICKHOUSE_ANALYTICS_URL: z.string().optional(),
CLICKHOUSE_ANALYTICS_DATABASE: z.string().optional(),
diff --git a/apps/api/src/lib/change-tracking-store.ts b/apps/api/src/lib/change-tracking-store.ts
new file mode 100644
index 000000000..2a02afc69
--- /dev/null
+++ b/apps/api/src/lib/change-tracking-store.ts
@@ -0,0 +1,205 @@
+import type { Bigtable, Table } from "@google-cloud/bigtable";
+import crypto from "crypto";
+import { config } from "../config";
+import { logger } from "./logger";
+import {
+ changeTrackingInsertScrape as changeTrackingInsertScrapeRpc,
+ diffGetLastScrape,
+} from "../db/rpc";
+
+// Change tracking bookkeeping. Migration step 1 (#4475 is the intended
+// endpoint): Postgres stays the primary store -- writes go to
+// change_tracking_scrapes, reads come from it -- and, when Bigtable is
+// configured, every write is double-written to the Bigtable layout the
+// endpoint uses: one row per (team_id, url, tag) holding the latest
+// scrape's job id, cell timestamp = date_added, family GC max_versions=1.
+// The Bigtable side is never read yet; its writes are detached and
+// best-effort.
+//
+// Row key: team_id || sha256(url) || sha256(tag)
+// - team_id first: team-scoped prefix operations (bulk delete on cleanup)
+// stay a single range.
+// - url before tag: the dominant write mix (every plain-markdown scrape)
+// carries tag=null, so the first hashed position must hold the
+// high-entropy component; a near-constant tag there would funnel each
+// team's bulk traffic into one contiguous sub-band.
+// - tag null is encoded injectively (0x00 vs 0x01+tag) so null and ""
+// stay distinct keys, mirroring the Postgres semantics.
+
+const FAMILY = "m";
+const QUALIFIER = "job";
+const TABLE_ID = config.BIGTABLE_CHANGE_TRACKING_TABLE || "change_tracking";
+
+function sha256(data: crypto.BinaryLike): Buffer {
+ return crypto.createHash("sha256").update(data).digest();
+}
+
+function tagHash(tag: string | null): Buffer {
+ if (tag === null) {
+ return sha256(Buffer.from([0x00]));
+ }
+ return sha256(Buffer.concat([Buffer.from([0x01]), Buffer.from(tag, "utf8")]));
+}
+
+function changeTrackingRowKey(
+ teamId: string,
+ url: string,
+ tag: string | null,
+): Buffer {
+ // Fixed-width url/tag digests make the key unambiguously splittable at
+ // len - 64 regardless of what bytes appear in team_id/url/tag.
+ return Buffer.concat([
+ Buffer.from(teamId, "utf8"),
+ sha256(Buffer.from(url, "utf8")),
+ tagHash(tag),
+ ]);
+}
+
+function bigtableConfigured(): boolean {
+ return !!config.BIGTABLE_INSTANCE_ID;
+}
+
+let bigtableClient: Bigtable | null = null;
+let table: Table | null = null;
+
+// The Bigtable package is loaded lazily: log_job imports this module on
+// the primary scrape-logging path, so a broken or slow package load must
+// never take that path down -- failures surface only in the double-write.
+async function getChangeTrackingTable(): Promise<Table> {
+ if (!bigtableClient) {
+ const { Bigtable } = await import("@google-cloud/bigtable");
+ bigtableClient = new Bigtable({
+ projectId: config.BIGTABLE_PROJECT_ID,
+ ...(config.BIGTABLE_APP_PROFILE_ID
+ ? { appProfileId: config.BIGTABLE_APP_PROFILE_ID }
+ : {}),
+ // Mirrors GCS_CREDENTIALS: base64-encoded service-account JSON.
+ // Parsed here (not at module load) so a malformed value can only
+ // fail the non-fatal double-write, never the primary path. Unset
+ // falls back to Application Default Credentials.
+ ...(config.BIGTABLE_CREDENTIALS
+ ? {
+ credentials: JSON.parse(atob(config.BIGTABLE_CREDENTIALS)),
+ }
+ : {}),
+ // The client's Cloud Monitoring metrics handler requires the
+ // OTel 1.x line, which GHSA-8988-4f7v-96qf only patches on 2.x;
+ // we don't consume client-side metrics, so disable the handler
+ // and let the dependency overrides move the tree to 2.x.
+ metricsEnabled: false,
+ });
+ }
+ if (!table) {
+ table = bigtableClient
+ .instance(config.BIGTABLE_INSTANCE_ID!)
+ .table(TABLE_ID);
+ }
+ return table;
+}
+
+// The double-write is best-effort: it runs detached with a bounded
+// timeout so a slow or hung Bigtable can never add failure latency to
+// the already-successful Postgres write.
+const BIGTABLE_DOUBLE_WRITE_TIMEOUT_MS = 10_000;
+
+function doubleWriteToBigtable(params: {
+ team_id: string;
+ url: string;
+ job_id: string;
+ tag: string | null;
+ date_added: Date;
+}): void {
+ void (async () => {
+ let timer: NodeJS.Timeout | undefined;
+ try {
+ const write = getChangeTrackingTable().then(table =>
+ // Runtime accepts Buffer keys (converted verbatim); the .d.ts
+ // only declares string.
+ table.mutate([
+ {
+ key: changeTrackingRowKey(
+ params.team_id,
+ params.url,
+ params.tag,
+ ) as unknown as string,
+ // Required by Mutation.parse -- without it the entry
+ // carries no setCell mutations.
+ method: "insert" as const,
+ data: {
+ [FAMILY]: {
+ [QUALIFIER]: {
+ value: params.job_id,
+ timestamp: params.date_added,
+ },
+ },
+ },
+ },
+ ]),
+ );
+ // Keep the write handled in case the race below settles on the
+ // timeout first and it rejects later.
+ write.catch(() => {});
+ await Promise.race([
+ write,
+ new Promise<never>((_, reject) => {
+ timer = setTimeout(
+ () =>
+ reject(
+ new Error(
+ `Bigtable change tracking double-write timed out after ${BIGTABLE_DOUBLE_WRITE_TIMEOUT_MS}ms`,
+ ),
+ ),
+ BIGTABLE_DOUBLE_WRITE_TIMEOUT_MS,
+ );
+ timer.unref();
+ }),
+ ]);
+ } catch (error) {
+ logger.warn("Error inserting change tracking record into Bigtable", {
+ error,
+ teamId: params.team_id,
+ });
+ } finally {
+ if (timer) clearTimeout(timer);
+ }
+ })();
+}
+
+export async function changeTrackingInsertScrape(params: {
+ team_id: string;
+ url: string;
+ job_id: string;
+ tag: string | null;
+ /** When the scrape was logged; becomes the cell timestamp. */
+ date_added: Date;
+}): Promise<void> {
+ await changeTrackingInsertScrapeRpc({
+ team_id: params.team_id,
+ url: params.url,
+ job_id: params.job_id,
+ change_tracking_tag: params.tag,
+ date_added: params.date_added.toISOString(),
+ });
+
+ if (bigtableConfigured()) {
+ doubleWriteToBigtable(params);
+ }
+}
+
+/**
+ * Point lookup of the latest scrape for (team_id, url, tag). Returns null
+ * when the team never scraped this url+tag combination.
+ */
+export async function changeTrackingGetLastScrape(params: {
+ team_id: string;
+ url: string;
+ tag: string | null;
+}): Promise<{ job_id: string; date_added: string } | null> {
+ const rows = await diffGetLastScrape(params.team_id, params.url, params.tag);
+ const row = rows[0];
+ if (!row) return null;
+ return {
+ job_id: row.o_job_id,
+ date_added: new Date(row.o_date_added).toISOString(),
+ };
+}
diff --git a/apps/api/src/scraper/scrapeURL/transformers/diff.ts b/apps/api/src/scraper/scrapeURL/transformers/diff.ts
index 4004bb766..49fc3d5e3 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/diff.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/diff.ts
@@ -1,4 +1,4 @@
-import { diffGetLastScrape } from "../../../db/rpc";
+import { changeTrackingGetLastScrape } from "../../../lib/change-tracking-store";
import { Document } from "../../../controllers/v1/types";
import { Meta } from "../index";
import { generateCompletions } from "./llmExtract";
@@ -88,13 +88,13 @@ export async function deriveDiff(
}
const start = Date.now();
- let resData: { o_job_id: string; o_date_added: string }[];
+ let last: Awaited<ReturnType<typeof changeTrackingGetLastScrape>>;
try {
- resData = await diffGetLastScrape(
- meta.internalOptions.teamId!,
- document.metadata.sourceURL ?? meta.rewrittenUrl ?? meta.url,
- changeTrackingFormat?.tag ?? null,
- );
+ last = await changeTrackingGetLastScrape({
+ team_id: meta.internalOptions.teamId!,
+ url: document.metadata.sourceURL ?? meta.rewrittenUrl ?? meta.url,
+ tag: changeTrackingFormat?.tag ?? null,
+ });
} catch (error) {
meta.logger.error("Error fetching previous scrape", { error });
document.warning =
@@ -113,25 +113,17 @@ export async function deriveDiff(
});
}
- const data:
- | {
- o_job_id: string;
- o_date_added: string;
- }
- | undefined
- | null = resData[0];
-
- const rawJob = data?.o_job_id ? await getJobFromGCS(data.o_job_id) : null;
+ const rawJob = last?.job_id ? await getJobFromGCS(last.job_id) : null;
const job: Document | null = rawJob?.[0] ?? null;
meta.logger.debug("Change tracking debugging", {
- isDataPresent: !!data,
- data,
+ isDataPresent: !!last,
+ data: last,
isRawJobPresent: !!rawJob,
isJobPresent: !!job,
});
- if (data && job) {
+ if (last && job) {
const previousMarkdown = job.markdown!;
const currentMarkdown = document.markdown!;
@@ -149,7 +141,7 @@ export async function deriveDiff(
: "same";
document.changeTracking = {
- previousScrapeAt: data.o_date_added,
+ previousScrapeAt: last.date_added,
changeStatus,
visibility: meta.internalOptions.urlInvisibleInCurrentCrawl
? "hidden"
diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.ts
index fb85f5c83..f81f9011a 100644
--- a/apps/api/src/services/logging/log_job.ts
+++ b/apps/api/src/services/logging/log_job.ts
@@ -1,6 +1,6 @@
import { db } from "../../db/connection";
import * as schema from "../../db/schema";
-import { changeTrackingInsertScrape } from "../../db/rpc";
+import { changeTrackingInsertScrape } from "../../lib/change-tracking-store";
import { config } from "../../config";
import "dotenv/config";
import { logger as _logger } from "../../lib/logger";
@@ -379,8 +379,8 @@ export async function logScrape(scrape: LoggedScrape, force: boolean = false) {
team_id: scrape.team_id,
url: scrape.url,
job_id: scrape.id,
- change_tracking_tag: hasChangeTracking ? hasChangeTracking.tag : null,
- date_added: new Date().toISOString(),
+ tag: hasChangeTracking ? hasChangeTracking.tag : null,
+ date_added: new Date(),
});
_logger.debug("Change tracking record inserted successfully");
} catch (error) {