Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
wip: add frontend distributed otel (#5188)
Wolf Mermelstein2a29d67aec21
diff --git a/.cargo/config.toml b/.cargo/config.tomlindex 8cfa9bf0f6..dce456f5bd 100644--- a/.cargo/config.toml+++ b/.cargo/config.toml@@ -16,3 +16,9 @@# (e.g. `cargo x cache-wasm` never builds guppy/hakari/aws-sdk/tokio).[alias]x = "run -p xtask --"++# worker-rs-otel (wasm32-unknown-unknown) pulls getrandom 0.3 via OTLP types;+# getrandom 0.3 needs its backend selected at+# build time. Harmless for wasm crates that don't use getrandom 0.3.+[target.wasm32-unknown-unknown]+rustflags = ['--cfg', 'getrandom_backend="wasm_js"']diff --git a/.config/hakari.toml b/.config/hakari.tomlindex f0b54a5ff9..742fa1f783 100644--- a/.config/hakari.toml+++ b/.config/hakari.toml@@ -59,6 +59,7 @@ workspace-members = ["macro_sync_service_jwt","sync_service","sync_service_bebop_schema",+ "worker-rs-otel",# maybe_send: zero-dep marker traits used by cache-core; must stay# wasm-safe (workspace-hack pulls native-only deps like mio into wasm32)."maybe_send",diff --git a/.github/workflows/deploy_preview.yml b/.github/workflows/deploy_preview.ymlindex 711e919d23..636e2271b6 100644--- a/.github/workflows/deploy_preview.yml+++ b/.github/workflows/deploy_preview.yml@@ -42,8 +42,6 @@ jobs:- name: Buildrun: just build-devenv:- VITE_DD_WEB_APP_TOKEN: ${{ secrets.DD_WEB_APP_TOKEN }}- VITE_DD_HASH: ${{ github.sha }}VITE_SEGMENT_WRITE_KEY: ${{ secrets.SEGMENT_WRITE_KEY_PRODUCTION }}VITE_POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}working-directory: apps/webdiff --git a/.github/workflows/deploy_web_app.yml b/.github/workflows/deploy_web_app.ymlindex da462bae72..802c6b05da 100644--- a/.github/workflows/deploy_web_app.yml+++ b/.github/workflows/deploy_web_app.yml@@ -22,8 +22,6 @@ on:required: trueDD_API_KEY:required: true- DD_WEB_APP_TOKEN:- required: trueSEGMENT_WRITE_KEY:required: truePOSTHOG_API_KEY:@@ -64,8 +62,6 @@ jobs:- name: Buildrun: just build-${{ inputs.environment }}env:- VITE_DD_WEB_APP_TOKEN: ${{ secrets.DD_WEB_APP_TOKEN }}- VITE_DD_HASH: ${{ github.sha }}VITE_SEGMENT_WRITE_KEY: ${{ secrets.SEGMENT_WRITE_KEY }}VITE_POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}working-directory: apps/webdiff --git a/.github/workflows/deploy_web_app_dev_push.yml b/.github/workflows/deploy_web_app_dev_push.ymlindex 6e5b2a91c0..fd972eaf30 100644--- a/.github/workflows/deploy_web_app_dev_push.yml+++ b/.github/workflows/deploy_web_app_dev_push.yml@@ -33,7 +33,6 @@ jobs:PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }}DD_APP_KEY: ${{ secrets.DD_APP_KEY }}DD_API_KEY: ${{ secrets.DD_API_KEY }}- DD_WEB_APP_TOKEN: ${{ secrets.DD_WEB_APP_TOKEN }}SEGMENT_WRITE_KEY: ${{ secrets.SEGMENT_WRITE_KEY_PRODUCTION }}POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}concurrency:diff --git a/.github/workflows/release-production.yml b/.github/workflows/release-production.ymlindex f5a6680c73..021ecfe604 100644--- a/.github/workflows/release-production.yml+++ b/.github/workflows/release-production.yml@@ -59,8 +59,6 @@ jobs:working-directory: apps/webrun: just build-prodenv:- VITE_DD_WEB_APP_TOKEN: ${{ secrets.DD_WEB_APP_TOKEN }}- VITE_DD_HASH: ${{ github.sha }}VITE_SEGMENT_WRITE_KEY: ${{ secrets.SEGMENT_WRITE_KEY_PRODUCTION }}VITE_POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}- name: Upload Build Artifactdiff --git a/.github/workspace-dep-closures.json b/.github/workspace-dep-closures.jsonindex 898d4abc37..3a900a9559 100644--- a/.github/workspace-dep-closures.json+++ b/.github/workspace-dep-closures.json@@ -5570,6 +5570,7 @@],"sync_service": ["crates/macro_sync_service_jwt",+ "crates/worker-rs-otel","services/sync-service","services/sync-service/bebop-schema"],@@ -6046,6 +6047,9 @@"crates/webhook","crates/workspace-hack"],+ "worker-rs-otel": [+ "crates/worker-rs-otel"+ ],"worker_trigger": ["crates/macro_aws_config","crates/macro_entrypoint",diff --git a/Cargo.lock b/Cargo.lockindex 7ea064e6ed..88abf3dfb6 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -3238,6 +3238,18 @@ dependencies = ["web-sys",]+[[package]]+name = "const-hex"+version = "1.19.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "33e2a781ebdf4467d1428dc4593067825fb646f6871475098d8577421af73558"+dependencies = [+ "cfg-if",+ "cpufeatures 0.2.17",+ "proptest",+ "serde_core",+]+[[package]]name = "const-oid"version = "0.9.6"@@ -8785,10 +8797,15 @@ name = "macro_tower_layers"version = "0.1.0"dependencies = ["http 1.4.0",+ "opentelemetry",+ "opentelemetry-http",+ "opentelemetry_sdk","tokio","tower 0.5.3","tower-http","tracing",+ "tracing-opentelemetry",+ "tracing-subscriber","workspace-hack",]@@ -10511,9 +10528,13 @@ version = "0.31.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "a7175df06de5eaee9909d4805a3d07e28bb752c34cab57fa9cff549da596b30f"dependencies = [+ "base64 0.22.1",+ "const-hex","opentelemetry","opentelemetry_sdk","prost 0.14.3",+ "serde",+ "serde_json","tonic","tonic-prost",]@@ -11485,6 +11506,21 @@ dependencies = ["workspace-hack",]+[[package]]+name = "proptest"+version = "1.11.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744"+dependencies = [+ "bitflags 2.11.1",+ "num-traits",+ "rand 0.9.4",+ "rand_chacha 0.9.0",+ "rand_xorshift",+ "regex-syntax",+ "unarray",+]+[[package]]name = "prost"version = "0.12.6"@@ -11794,6 +11830,15 @@ dependencies = ["rand_core 0.5.1",]+[[package]]+name = "rand_xorshift"+version = "0.4.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a"+dependencies = [+ "rand_core 0.9.5",+]+[[package]]name = "rand_xoshiro"version = "0.6.0"@@ -14338,6 +14383,7 @@ name = "sync_service"version = "0.1.0"dependencies = ["bebop",+ "getrandom 0.3.4","loro","macro_sync_service_jwt","matchit 0.8.4",@@ -14352,6 +14398,7 @@ dependencies = ["wasm-bindgen","web-time","worker",+ "worker-rs-otel",][[package]]@@ -15294,6 +15341,12 @@ version = "0.1.7"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971"+[[package]]+name = "unarray"+version = "0.1.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94"+[[package]]name = "unfurl"version = "0.1.0"@@ -16550,6 +16603,23 @@ dependencies = ["worker-sys",]+[[package]]+name = "worker-rs-otel"+version = "0.1.0"+dependencies = [+ "getrandom 0.3.4",+ "http 1.4.0",+ "opentelemetry",+ "opentelemetry-proto",+ "opentelemetry_sdk",+ "serde_json",+ "tracing",+ "tracing-subscriber",+ "uuid",+ "web-time",+ "worker",+]+[[package]]name = "worker-sys"version = "0.8.1"@@ -16660,6 +16730,7 @@ dependencies = ["num-traits","once_cell","opentelemetry",+ "opentelemetry-http","opentelemetry_sdk","percent-encoding","phf 0.13.1",@@ -16715,6 +16786,8 @@ dependencies = ["tower-http","tracing","tracing-core",+ "tracing-log",+ "tracing-subscriber","triomphe","tungstenite 0.28.0","typenum",diff --git a/Cargo.toml b/Cargo.tomlindex 6809e8f9b3..0117574552 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -48,6 +48,7 @@ members = ["crates/soup_realtime","crates/task_dedup","crates/unfurl",+ "crates/worker-rs-otel","crates/workspace-hack","services/ai_projections_refresh_handler","services/authentication_service",@@ -141,6 +142,7 @@ dotenvy = "0.15.7"ego-tree = "0.6.2"either = "1"futures = "0.3.31"+getrandom = "0.3"glob = "0.3.3"# Pinned engine for the generated dependency artifacts (used by the xtask# graph/deps commands). This version lives in Cargo.lock, so dev machines and@@ -172,7 +174,8 @@ lol_html = "2.4.0"lru = "0.16"mockall = "0.13"nom = "8"-opentelemetry = "0.31.0"+opentelemetry = { version = "0.31.0", default-features = false }+opentelemetry-http = { version = "0.31.0", default-features = false }opentelemetry-otlp = { version = "0.31.0", default-features = false, features = ["grpc-tonic","internal-logs",@@ -180,7 +183,7 @@ opentelemetry-otlp = { version = "0.31.0", default-features = false, features ="metrics","trace",] }-opentelemetry_sdk = { version = "0.31.0", features = ["rt-tokio"] }+opentelemetry_sdk = { version = "0.31.0", default-features = false }ordered-float = "5"paste = "1"pdfium-render = { version = "0.8.37", default-features = false, features = [@@ -263,6 +266,8 @@ uuid = { version = "1.15.1", features = ["fast-rng", "serde", "v4", "v5", "v7"]# anything older than 0.2.123, so keep this floor above that.wasm-bindgen = "0.2.123"wasm-bindgen-futures = "0.4"+web-time = "1.1.0"+worker = "0.8.1"# this is pinned to match axum 0.7.9 dependency requirementsasync-stripe = { version = "0.41", default-features = false, features = ["billing",diff --git a/apps/web/package.json b/apps/web/package.jsonindex 1c8e360d30..cc648c3c43 100644--- a/apps/web/package.json+++ b/apps/web/package.json@@ -29,8 +29,6 @@"knip": "bunx --bun knip"},"dependencies": {- "@datadog/browser-logs": "^6.22.0",- "@dinero.js/currencies": "2.0.0-alpha.8","@aws-crypto/sha256-js": "^5.2.0","@codemirror/lang-cpp": "^6.0.3","@codemirror/lang-css": "^6.3.1",@@ -43,6 +41,7 @@"@corvu/dialog": "^0.2.4","@corvu/drawer": "^0.2.4","@date-fns/tz": "^1.4.1",+ "@dinero.js/currencies": "2.0.0-alpha.8","@floating-ui/dom": "^1.7.4","@fontsource-variable/inter": "^5.2.8","@fontsource-variable/playfair-display": "^5.2.8",@@ -67,7 +66,9 @@"@lukemorales/query-key-factory": "^1.3.4","@macro-inc/collaboration": "workspace:*","@macro-inc/lexical-core": "workspace:*",+ "@macro-inc/observability": "workspace:*","@normy/query-core": "^0.21.0",+ "@opentelemetry/api": "^1.9.1","@phosphor-icons/core": "^2.1.1","@solid-primitives/broadcast-channel": "^0.1.1","@solid-primitives/context": "^0.3.2",@@ -98,10 +99,10 @@"async-mutex": "^0.5.0","codemirror": "^6.0.2","colorjs.io": "^0.5.2",- "date-fns": "^4.1.0","d3-array": "^3.2.4","d3-scale": "^4.0.2","d3-shape": "^3.2.0",+ "date-fns": "^4.1.0","detect-browser": "^5.3.0","dinero.js": "2.0.0-alpha.8","email-validator": "^2.0.4",@@ -116,13 +117,13 @@"loro-crdt": "catalog:","nanoid": "^3.3.7","neverthrow": "^8.2.0",- "posthog-js": "^1.387.0","pdfjs-dist": "github:macro-inc/pdf.js#v2.16.52-web",+ "posthog-js": "^1.387.0","quill": "^2.0.3","short-uuid": "^5.2.0","signature_pad": "^5.0.10",- "solid-gesture": "^0.0.3","solid-custom-scrollbars": "^0.1.7",+ "solid-gesture": "^0.0.3","solid-js": "^1.9.7","solid-styled-components": "^0.28.5","solid-transition-group": "^0.2.3",@@ -130,8 +131,8 @@"tabbable": "^6.2.0","tailwind-merge": "^3.4.0","tailwindcss": "^4.2.4",- "tauri-plugin-safe-area-insets": "^0.1.0","tauri-plugin-device-info-api": "^1.0.1",+ "tauri-plugin-safe-area-insets": "^0.1.0","ts-pattern": "^5.9.0","unicode-emoji-json": "^0.8.0","uuid": "^10.0.0",diff --git a/apps/web/src/components/app/FatalError.tsx b/apps/web/src/components/app/FatalError.tsxindex 40dbfef8a8..4d6d51367a 100644--- a/apps/web/src/components/app/FatalError.tsx+++ b/apps/web/src/components/app/FatalError.tsx@@ -1,4 +1,5 @@-import { logger } from '@observability';+import { Telemetry } from '@macro-inc/observability';+import ResetIcon from '@phosphor/arrow-clockwise.svg';import HomeIcon from '@phosphor/house.svg';import { Button, Dialog, Surface } from '@ui';@@ -10,7 +11,7 @@ interface FatalErrorProps {}export function FatalError(props: FatalErrorProps) {- logger.error(props.error || 'Unknown error', {+ Telemetry.error(props.error || 'Unknown error', {url: window.location.href,});diff --git a/apps/web/src/features/block-email/component/BaseInput.tsx b/apps/web/src/features/block-email/component/BaseInput.tsxindex 7ca5b64abd..2d082353db 100644--- a/apps/web/src/features/block-email/component/BaseInput.tsx+++ b/apps/web/src/features/block-email/component/BaseInput.tsx@@ -44,7 +44,7 @@ import {$appendWatermarkNodeToLast,$removeAllWatermarkNodes,} from '@macro-inc/lexical-core';-import { logger } from '@observability';+import { Telemetry } from '@macro-inc/observability';import ChevronDown from '@phosphor/caret-down.svg';import CaretRight from '@phosphor/caret-right.svg';import DotsThree from '@phosphor/dots-three.svg';@@ -691,7 +691,7 @@ export function BaseInput(props: {$removeAllWatermarkNodes(editor());const prepared = prepareEmailBody(editor());if (!prepared) {- logger.error(+ Telemetry.error(new Error('Unable to prepare email body for draft collection.'));return null;@@ -740,12 +740,12 @@ export function BaseInput(props: {const newMessage = props.newMessage ?? false;if (!currentThread && !newMessage) {- logger.error(new Error('Failed to save draft: thread not found'));+ Telemetry.error(new Error('Failed to save draft: thread not found'));return;}if (newMessage && currentThread) {- logger.error(+ Telemetry.error(new Error('Failed to save draft: new message and current thread cannot be provided together')@@ -986,14 +986,14 @@ export function BaseInput(props: {const newMessage = props.newMessage ?? false;if (!currentThread && !newMessage) {- logger.error(new Error("Can't send email, no email thread found"));+ Telemetry.error(new Error("Can't send email, no email thread found"));toast.failure('Email failed to send');return;}if (newMessage && currentThread) {toast.failure('Email failed to send');- logger.error('New message and thread cannot be provided together');+ Telemetry.error('New message and thread cannot be provided together');return;}@@ -1006,14 +1006,14 @@ export function BaseInput(props: {if (emailLinksQuery.isError) {toast.failure('Email failed to send: Could not load email accounts');- logger.error('Failed to load email links');+ Telemetry.error('Failed to load email links');return;}const linksData = emailLinksQuery.data;if (!linksData || linksData.links.length < 1) {toast.failure('Email failed to send: No email account connected');- logger.error('No links found');+ Telemetry.error('No links found');return;}linkId = primaryLinkId() ?? linksData.links[0].id;diff --git a/apps/web/src/features/block-email/component/MessageContainer.tsx b/apps/web/src/features/block-email/component/MessageContainer.tsxindex eb8a201696..601cbbf99a 100644--- a/apps/web/src/features/block-email/component/MessageContainer.tsx+++ b/apps/web/src/features/block-email/component/MessageContainer.tsx@@ -13,8 +13,7 @@ import { UserIcon, type UserIconProps } from '@core/component/UserIcon';import { VideoPreview } from '@core/component/VideoPreview';import { fileTypeToBlockName } from '@core/constant/allBlocks';import { isMobile } from '@core/mobile/isMobile';--import { logger } from '@observability';+import { Telemetry } from '@macro-inc/observability';import { refetchSoupEntity } from '@queries/soup/cache';import { emailClient } from '@service-email/client';import type { ApiMessage, Attachment } from '@service-email/generated/schemas';@@ -168,11 +167,11 @@ export function MessageContainer(props: MessageContainerProps) {});if (response.isErr()) {toast.failure('Failed to get attachment. Please try again.');- return logger.error('Failed to get or create attachment document id', {- error: new Error(+ return Telemetry.error(+ new Error('Failed to get or create attachment document id: ' + response.error- ),- });+ )+ );}const { document_id } = response.value;@@ -182,14 +181,11 @@ export function MessageContainer(props: MessageContainerProps) {});if (maybeDocumentMetadata.isErr()) {toast.failure('Failed to get attachment. Please try again.');- return logger.error(- 'Failed to get or create attachment document metadata',- {- error: new Error(- 'Failed to get or create attachment document metadata: ' +- maybeDocumentMetadata.error- ),- }+ return Telemetry.error(+ new Error(+ 'Failed to get or create attachment document metadata: ' ++ maybeDocumentMetadata.error+ ));}diff --git a/apps/web/src/features/block-email/component/compose/Compose.tsx b/apps/web/src/features/block-email/component/compose/Compose.tsxindex bb797af6d8..31ae28015b 100644--- a/apps/web/src/features/block-email/component/compose/Compose.tsx+++ b/apps/web/src/features/block-email/component/compose/Compose.tsx@@ -49,7 +49,8 @@ import {$appendWatermarkNodeToLast,$removeAllWatermarkNodes,} from '@macro-inc/lexical-core';-import { logger } from '@observability/logger';+import { Telemetry } from '@macro-inc/observability';+import ArrowCounterClockwise from '@phosphor-icons/core/regular/arrow-counter-clockwise.svg?component-solid';import { queryClient } from '@queries/client';import {@@ -235,7 +236,7 @@ export function EmailCompose(props: EmailComposeProps) {$removeAllWatermarkNodes(editor());const prepared = prepareEmailBody(editor());if (!prepared) {- logger.error(+ Telemetry.error(new Error('Unable to prepare email body for draft collection.'));return null;diff --git a/apps/web/src/features/block-email/component/compose/ComposeBody.tsx b/apps/web/src/features/block-email/component/compose/ComposeBody.tsxindex d3122a219d..3c0faa0543 100644--- a/apps/web/src/features/block-email/component/compose/ComposeBody.tsx+++ b/apps/web/src/features/block-email/component/compose/ComposeBody.tsx@@ -8,7 +8,8 @@ import { MarkdownTextarea } from '@core/component/LexicalMarkdown/component/coreimport { createFilesReadyHandler } from '@core/component/LexicalMarkdown/utils/fileUploadUtils';import { fileFolderDrop } from '@core/directive/fileFolderDrop';import { handleFileFolderDrop } from '@core/util/upload';-import { logger } from '@observability/logger';+import { Telemetry } from '@macro-inc/observability';+import { cn, Scroll } from '@ui';import type { LexicalEditor } from 'lexical';import {@@ -88,7 +89,7 @@ export function ComposeBody(props: {const logComposeBody = (event: string, details?: Record<string, unknown>) => {if (!props.debugName) return;- logger.log(`[ComposeBody] ${event}`, {+ Telemetry.info(`[ComposeBody] ${event}`, {debugName: props.debugName,...details,});diff --git a/apps/web/src/features/block-email/util/makeAttachmentPublic.ts b/apps/web/src/features/block-email/util/makeAttachmentPublic.tsindex 811d340ef4..2aaae06638 100644--- a/apps/web/src/features/block-email/util/makeAttachmentPublic.ts+++ b/apps/web/src/features/block-email/util/makeAttachmentPublic.ts@@ -1,7 +1,8 @@import { analytics } from '@app/lib/analytics';import { toast } from '@core/component/Toast/Toast';-import { logger } from '@observability';+import { Telemetry } from '@macro-inc/observability';+import { storageServiceClient } from '@service-storage/client';export const makeAttachmentPublic = async (attachmentId: string) => {@@ -35,6 +36,8 @@ export const makeAttachmentPublic = async (attachmentId: string) => {toast.alert('Recipients may not be able to view this file', {subtext: 'Please consult the document owner to change share permissions',});- logger.error('Failed to make attachment public', { errors: result.error });+ Telemetry.error('Failed to make attachment public', {+ errors: JSON.stringify(result.error),+ });}};diff --git a/apps/web/src/features/block-md/component/Block.tsx b/apps/web/src/features/block-md/component/Block.tsxindex a09bf215e9..fdec9f6570 100644--- a/apps/web/src/features/block-md/component/Block.tsx+++ b/apps/web/src/features/block-md/component/Block.tsx@@ -20,6 +20,7 @@ import {LORO_WAL_DB_NAME,} from '@macro-inc/collaboration/collab/wal';import { MARKDOWN_LORO_SCHEMA } from '@macro-inc/lexical-core/markdown-loro-schema';+import type { Span } from '@macro-inc/observability';import { DocumentDebouncedNotificationReadMarker } from '@notifications';import { useInstructionsMdIdQuery } from '@queries/storage/instructions-md';import { storageServiceClient } from '@service-storage/client';@@ -35,6 +36,7 @@ import {import type { MarkdownData } from '../definition';import { HistoryProvider } from '../history/HistoryContext';import { OldOverlay } from '../history/OldOverlay';+import { resumeDocumentSpan, stampLoroSnapshotState } from '../observability';import { blockDataSignal, mdStore } from '../signal/markdownBlockData';import { FindAndReplace } from './FindAndReplace';import { MarkdownNameProvider, useMarkdownName } from './MarkdownNameProvider';@@ -51,16 +53,78 @@ export interface BlockMarkdownProps {}type MarkdownLoroManager = LoroManager<typeof MARKDOWN_LORO_SCHEMA>;+type SnapshotResult = {+ outcome: 'seeded' | 'discarded' | 'unavailable' | 'error';+ bytes?: number;+};++const snapshotStepNames = {+ optimistic: 'doc.snapshot.optimistic',+ local: 'doc.snapshot.local-cache',+ s3: 'doc.snapshot.s3-cache',+ remote: 'doc.snapshot.remote-sync',+} as const;++type SnapshotSource = keyof typeof snapshotStepNames;++function startSnapshotIngest(+ parentSpan: Span | undefined,+ source: SnapshotSource,+ loroManager: MarkdownLoroManager,+ ingest: () => Promise<SnapshotResult>+): void {+ parentSpan?.event('doc.snapshot.attempt', {+ 'snapshot.source': source,+ });++ const operation = parentSpan+ ? parentSpan.span(snapshotStepNames[source], async (snapshotSpan) => {+ snapshotSpan.setAttr('snapshot.source', source);+ try {+ const result = await ingest();+ snapshotSpan.setAttr('outcome', result.outcome);+ if (result.bytes !== undefined) {+ snapshotSpan.setAttr('snapshot.bytes', result.bytes);+ }+ if (result.outcome === 'seeded') {+ stampLoroSnapshotState(snapshotSpan, loroManager.doc);+ } else if (result.outcome === 'error') {+ snapshotSpan.error('snapshot ingestion failed');+ }+ return result;+ } catch (error) {+ snapshotSpan.error(error);+ snapshotSpan.setAttr('outcome', 'error');+ throw error;+ }+ })+ : ingest();++ void operation+ .then(({ outcome, bytes }) => {+ parentSpan?.event('doc.snapshot.result', {+ 'snapshot.source': source,+ ...(bytes !== undefined && { 'snapshot.bytes': bytes }),+ outcome,+ });+ })+ .catch(() => {+ parentSpan?.event('doc.snapshot.result', {+ 'snapshot.source': source,+ outcome: 'error',+ });+ });+}async function ingestLocalSnapshot(loroManager: MarkdownLoroManager,snapshotStore: IDBSnapshotStore<RawUpdate>,walStore: BrowserWALStore<RawUpdate>-) {+): Promise<SnapshotResult> {const localSnapshot = await snapshotStore.load();- if (!localSnapshot) return;+ if (!localSnapshot) return { outcome: 'unavailable' };const walEntries = await walStore.getAll();- await loroManager.ingest({+ const seeded = await loroManager.ingest({kind: 'local',snapshot: localSnapshot,walUpdates: walEntries.map((entry) => entry.update),@@ -79,32 +143,47 @@ async function ingestLocalSnapshot(});await snapshotStore.save(snapshot);}+ return {+ outcome: seeded ? 'seeded' : 'discarded',+ bytes: localSnapshot.length,+ };}+/**+ * Ingest the sync-service initial sync. `discarded` means the snapshot+ * arrived fine but lost the seed race.+ */async function ingestRemoteSnapshot(loroManager: MarkdownLoroManager,doInitialSync: MarkdownData['doInitialSync']-): Promise<boolean> {+): Promise<SnapshotResult> {const sync = await doInitialSync();if (sync.isErr()) {console.error('Failed to receive initial sync', sync.error);- return loroManager.initialized;+ return { outcome: 'error' };}- await loroManager.ingest({+ const bytes = sync.value.snapshot.length;+ const seeded = await loroManager.ingest({kind: 'dss',snapshot: sync.value.snapshot,});- return true;+ return { outcome: seeded ? 'seeded' : 'discarded', bytes };}async function ingestS3Snapshot(loroManager: MarkdownLoroManager,blockId: string-) {+): Promise<SnapshotResult> {const result = await storageServiceClient.fetchCachedSnapshot(blockId);- if (result.isOk()) {- await loroManager.ingest({ kind: 's3', snapshot: result.value });- }+ if (result.isErr()) return { outcome: 'unavailable' };+ const seeded = await loroManager.ingest({+ kind: 's3',+ snapshot: result.value,+ });+ return {+ outcome: seeded ? 'seeded' : 'discarded',+ bytes: result.value.length,+ };}export default function BlockMarkdown(props: BlockMarkdownProps) {@@ -142,17 +221,32 @@ function BlockMarkdownContent({ optimisticSnapshot }: BlockMarkdownProps) {}setBlockError(null);+ // optimistic is the "hard coded" snapshot like the golden one when you create a+ // new document+ const span = resumeDocumentSpan(blockId);if (optimisticSnapshot) {- loroManager.ingest({- kind: 'optimistic',- snapshot: optimisticSnapshot,+ startSnapshotIngest(span, 'optimistic', loroManager, async () => {+ const seeded = await loroManager.ingest({+ kind: 'optimistic',+ snapshot: optimisticSnapshot,+ });+ return {+ outcome: seeded ? 'seeded' : 'discarded',+ bytes: optimisticSnapshot.length,+ };});}// First one wins automatically (loro manager takes care of ignoring the rest)- ingestLocalSnapshot(loroManager, snapshotStore, walStore);- ingestS3Snapshot(loroManager, blockId);- ingestRemoteSnapshot(loroManager, data.doInitialSync);+ startSnapshotIngest(span, 'local', loroManager, () =>+ ingestLocalSnapshot(loroManager, snapshotStore, walStore)+ );+ startSnapshotIngest(span, 's3', loroManager, () =>+ ingestS3Snapshot(loroManager, blockId)+ );+ startSnapshotIngest(span, 'remote', loroManager, () =>+ ingestRemoteSnapshot(loroManager, data.doInitialSync)+ );}));diff --git a/apps/web/src/features/block-md/component/MarkdownCollabProvider.tsx b/apps/web/src/features/block-md/component/MarkdownCollabProvider.tsxindex cab5166639..5c7adefea5 100644--- a/apps/web/src/features/block-md/component/MarkdownCollabProvider.tsx+++ b/apps/web/src/features/block-md/component/MarkdownCollabProvider.tsx@@ -68,6 +68,7 @@ import {onCleanup,type Setter,} from 'solid-js';+import { endDocumentSpan, resumeDocumentSpan } from '../observability';import { CollabStatus } from './CollabStatus';type MutatedNodes = UpdateListenerPayload['mutatedNodes'];@@ -459,7 +460,7 @@ export function MarkdownCollabProvider(props: MarkdownCollabProviderProps) {const state = loroManager.state;const empty = state? isStateEmpty(state.state as unknown as SerializedEditorState)- : null;+ : undefined;logSyncService({documentId: syncSource()!.documentId,@@ -478,6 +479,7 @@ export function MarkdownCollabProvider(props: MarkdownCollabProviderProps) {message:'editor init: no state from loroManager — editor will NOT become ready (skeleton stays)',});+ endDocumentSpan(syncSource()!.documentId);return;}@@ -514,6 +516,7 @@ export function MarkdownCollabProvider(props: MarkdownCollabProviderProps) {'editor init: initializeEditorWithVersionedState failed',});props.setEditorError(initError);+ endDocumentSpan(syncSource()!.documentId);return;}}@@ -521,6 +524,9 @@ export function MarkdownCollabProvider(props: MarkdownCollabProviderProps) {// Start the sync enginestartSync();props.setEditorReady(true);+ const documentId = syncSource()!.documentId;+ resumeDocumentSpan(documentId)?.event('editor.ready');+ endDocumentSpan(documentId);logSyncService({documentId: syncSource()!.documentId,level: 'info',@@ -551,6 +557,8 @@ export function MarkdownCollabProvider(props: MarkdownCollabProviderProps) {onCleanup(() => {syncEngine.stop();walSyncer.destroy();+ const documentId = syncSource()?.documentId;+ if (documentId) endDocumentSpan(documentId);});return (diff --git a/apps/web/src/features/block-md/definition.ts b/apps/web/src/features/block-md/definition.tsindex dd54763baf..5656d60edc 100644--- a/apps/web/src/features/block-md/definition.ts+++ b/apps/web/src/features/block-md/definition.ts@@ -5,6 +5,7 @@ import {loadResult,} from '@core/block';import { ENABLE_MARKDOWN_LIVE_COLLABORATION } from '@core/constant/featureFlags';+import { ThrownResultError } from '@core/util/result';import {fetchDocumentLocation,waitForDocumentSyncServiceReady,@@ -14,6 +15,12 @@ import { makeFileFromBlob } from '@service-storage/util/makeFileFromBlob';import { createSyncServiceSource } from '@service-sync/source';import { err, ok } from 'neverthrow';import MarkdownBlock from './component/Block';+import {+ endDocumentSpan,+ registerDocumentSpan,+ resumeDocumentSpan,+ startDocumentSpan,+} from './observability';import type { MarkdownRewriteOutput } from './signal/rewriteSignal';export const definition = defineBlock({@@ -38,72 +45,120 @@ export const definition = defineBlock({});}- const [maybeBundle, maybeLocation] = await Promise.all([- fetchDocumentLoadBundle(documentId),- loadResult(fetchDocumentLocation({ documentId })),- ]);- if (maybeBundle.isErr()) return err(maybeBundle.error);- const { token, documentMetadata, userAccessLevel } = maybeBundle.value;-- if (maybeLocation.isErr()) return err(maybeLocation.error);+ let rootSpan = resumeDocumentSpan(documentId);+ if (!rootSpan) {+ rootSpan = startDocumentSpan('doc.open');+ rootSpan.setAttr('doc.type', 'md');+ rootSpan.setAttr('document.id', documentId);+ registerDocumentSpan(documentId, rootSpan);+ }+ return rootSpan.span('doc.load', async (loadSpan) => {+ const loadBundle = () =>+ loadSpan.span('doc.load.bundle', async (bundleSpan) => {+ const result = await fetchDocumentLoadBundle(documentId);+ if (result.isErr()) {+ bundleSpan.error(new ThrownResultError(result.error));+ }+ return result;+ });- let location = maybeLocation.value;- if (- location.type === 'presignedUrl' &&- location.content.state === 'pending'- ) {- location = await waitForDocumentSyncServiceReady({ documentId }).catch(- (error) => {- console.error(- 'Failed waiting for markdown sync-service location',- error+ // The location span covers the sync-service readiness wait too, so it+ // reflects when the document actually became loadable.+ const loadLocation = () =>+ loadSpan.span('doc.load.location', async (locationSpan) => {+ const result = await loadResult(+ fetchDocumentLocation({ documentId }));- return location;- }- );- }+ if (result.isErr()) {+ locationSpan.error(new ThrownResultError(result.error));+ return result;+ }+ let location = result.value;+ if (+ location.type === 'presignedUrl' &&+ location.content.state === 'pending'+ ) {+ location = await waitForDocumentSyncServiceReady({+ documentId,+ }).catch((error) => {+ console.error(+ 'Failed waiting for markdown sync-service location',+ error+ );+ return location;+ });+ locationSpan.setAttr('location.pending', true);+ return ok(location);+ }+ return ok(location);+ });- // Markdown initialization and lifecycle persistence are backend-owned.- // If a markdown document still resolves to object storage here, opening- // it would require a backend repair/backfill path rather than a frontend- // sync-service mutation that leaves DB content metadata inconsistent.- if (location.type !== 'syncServiceContent') {- console.error(- 'Markdown document is not available in sync-service',- documentId,- location.content- );- return LoadErrors.INVALID;- }+ const [maybeBundle, maybeLocation] = await Promise.all([+ loadBundle(),+ loadLocation(),+ ]);+ if (maybeBundle.isErr()) {+ loadSpan.error('load bundle failed');+ rootSpan.error('load bundle failed');+ endDocumentSpan(documentId);+ return err(maybeBundle.error);+ }+ const { token, documentMetadata, userAccessLevel } = maybeBundle.value;++ if (maybeLocation.isErr()) {+ loadSpan.error('load location failed');+ rootSpan.error('load location failed');+ endDocumentSpan(documentId);+ return err(maybeLocation.error);+ }- const { source: syncSource, doInitialSync } = createSyncServiceSource(- source.id,- token- );+ const location = maybeLocation.value;- // HACK: unfortunately, most blocks still rely on a dssFile for things like- // metadata and fileName. so I'm creating an empty blob file to get around that.- const fileWithoutBlob = await makeFileFromBlob({- blob: new Blob([]),- documentKeyParts: {- owner: documentMetadata.owner,- documentId: documentMetadata.documentId,- documentVersionId: documentMetadata.documentVersionId.toString(),+ // Markdown initialization and lifecycle persistence are backend-owned.+ // If a markdown document still resolves to object storage here, opening+ // it would require a backend repair/backfill path rather than a frontend+ // sync-service mutation that leaves DB content metadata inconsistent.+ if (location.type !== 'syncServiceContent') {+ console.error(+ 'Markdown document is not available in sync-service',+ documentId,+ location.content+ );+ loadSpan.error('markdown document not in sync-service');+ rootSpan.error('markdown document not in sync-service');+ endDocumentSpan(documentId);+ return LoadErrors.INVALID;+ }++ const { source: syncSource, doInitialSync } = createSyncServiceSource(+ source.id,+ token+ );++ // HACK: unfortunately, most blocks still rely on a dssFile for things like+ // metadata and fileName. so I'm creating an empty blob file to get around that.+ const fileWithoutBlob = await makeFileFromBlob({+ blob: new Blob([]),+ documentKeyParts: {+ owner: documentMetadata.owner,+ documentId: documentMetadata.documentId,+ documentVersionId: documentMetadata.documentVersionId.toString(),+ // @ts-ignore: TODO: fix / replace @macro-inc/document-processing-job-types+ fileType: 'md',+ },+ fileName: documentMetadata.documentName,+ mimeType: definition.accepted['md']!,// @ts-ignore: TODO: fix / replace @macro-inc/document-processing-job-types- fileType: 'md',- },- fileName: documentMetadata.documentName,- mimeType: definition.accepted['md']!,- // @ts-ignore: TODO: fix / replace @macro-inc/document-processing-job-types- metadata: documentMetadata,- });+ metadata: documentMetadata,+ });- return ok({- dssFile: fileWithoutBlob,- userAccessLevel,- syncSource,- doInitialSync,- documentMetadata,+ return ok({+ dssFile: fileWithoutBlob,+ userAccessLevel,+ syncSource,+ doInitialSync,+ documentMetadata,+ });});}return LoadErrors.INVALID;diff --git a/apps/web/src/features/block-md/history/HistoryContext.tsx b/apps/web/src/features/block-md/history/HistoryContext.tsxindex 3180206b3a..75f1b6ff78 100644--- a/apps/web/src/features/block-md/history/HistoryContext.tsx+++ b/apps/web/src/features/block-md/history/HistoryContext.tsx@@ -1,5 +1,6 @@import { MACRO_AGENT_BOT_ID } from '@core/constant/macroAgent';import { tryMacroId, useDisplayName } from '@core/user';+import { ThrownResultError } from '@core/util/result';import { isAiPeer } from '@macro-inc/collaboration/collab/ai-peer';import {buildDiffState,@@ -101,7 +102,7 @@ export function HistoryProvider(props: {() => (isOpen() || shouldLoad() ? props.documentId() : undefined),async (documentId) => {const result = await syncServiceClient.getSnapshot({ documentId });- if (result.isErr()) throw new Error(String(result.error));+ if (result.isErr()) throw new ThrownResultError(result.error);const doc = new LoroDoc();doc.import(result.value);return doc;diff --git a/apps/web/src/features/block-md/observability.ts b/apps/web/src/features/block-md/observability.tsnew file mode 100644index 0000000000..21412f4696--- /dev/null+++ b/apps/web/src/features/block-md/observability.ts@@ -0,0 +1,41 @@+import { type Span, Telemetry } from '@macro-inc/observability';+import type { LoroDoc } from 'loro-crdt';++const documentSpans = new Map<string, Span>();++export function startDocumentSpan(name: string): Span {+ return Telemetry.span(name);+}++export function registerDocumentSpan(documentId: string, span: Span): void {+ const previous = documentSpans.get(documentId);+ if (previous !== span) previous?.end();+ documentSpans.set(documentId, span);+}++export function resumeDocumentSpan(documentId: string): Span | undefined {+ return documentSpans.get(documentId);+}++export function endDocumentSpan(documentId: string): void {+ const span = documentSpans.get(documentId);+ if (!span) return;+ documentSpans.delete(documentId);+ span.end();+}++export function endTrackedDocumentSpan(span: Span): void {+ for (const [documentId, candidate] of documentSpans) {+ if (candidate === span) documentSpans.delete(documentId);+ }+ span.end();+}++export function stampLoroSnapshotState(span: Span, doc: LoroDoc): void {+ const version = doc.oplogVersion();++ span.setAttr('snapshot.op_count', doc.opCount());+ span.setAttr('snapshot.peer_count', version.length());+ span.setAttr('snapshot.frontier_count', doc.oplogFrontiers().length);+ span.setAttr('snapshot.is_shallow', doc.isShallow());+}diff --git a/apps/web/src/features/command/Launcher.tsx b/apps/web/src/features/command/Launcher.tsxindex 232b5e5d84..c788c7a922 100644--- a/apps/web/src/features/command/Launcher.tsx+++ b/apps/web/src/features/command/Launcher.tsx@@ -1,6 +1,11 @@import { useFeatureFlag } from '@app/lib/analytics/posthog';import { setAutomationComposerOpen } from '@block-automation/component';import { EMAIL_COMPOSE_TO_INPUT_ID } from '@block-email/constants';+import {+ endTrackedDocumentSpan,+ registerDocumentSpan,+ startDocumentSpan,+} from '@block-md/observability';import { openNewChannelModal } from '@channel/CreateChannelModal';import { useSplitLayout } from '@components/app/split-layout/layout';import type { BlockAlias, BlockName } from '@core/block';@@ -51,6 +56,7 @@ import { AnimatedTaskIcon } from '@icon/wide-task';import WideTask from '@icon/wide-task.svg';import { Dialog } from '@kobalte/core/dialog';import { getMarkdownGoldenBytes } from '@macro-inc/lexical-core/markdown-golden';+import type { Span } from '@macro-inc/observability';import ArrowRight from '@phosphor/arrow-right.svg';import { createProject } from '@queries/storage/projects';import { CommandMenuShell, cn, Hotkey, Layer } from '@ui';@@ -72,9 +78,11 @@ const createBlock = async (spec: {createFn: () => Promise<string | undefined>;loading?: boolean;shouldInsert?: boolean;+ /** Active creation span; registered by document id after creation. */+ span?: Span;}) => {const { openWithSplit } = useSplitLayout();- const { blockName, createFn, loading } = spec;+ const { blockName, createFn, loading, span } = spec;setCreateMenuOpen(false, false);@@ -96,10 +104,15 @@ const createBlock = async (spec: {const id = await createFn();if (!id) {+ span?.setAttr('error', true);+ span?.end();split?.goBack();return;}+ span?.setAttr('document.id', id);+ if (span) registerDocumentSpan(id, span);+// If we are creating a new markdown document "from scratch" then we can let// them instantly start editingconst createMdParams =@@ -130,6 +143,8 @@ const createBlock = async (spec: {});}++ span?.event('doc.navigated');};const createComponent = async (spec: {@@ -168,20 +183,33 @@ export function runCreateAction(const source = options.source ?? 'create_menu';switch (blockName) {- case 'md':- createBlock({- blockName: 'md',- loading: true,- createFn: () =>- createMarkdownFile({- title: '',- content: '',- projectId: undefined,- source,- }),- shouldInsert,- });+ case 'md': {+ const span = startDocumentSpan('doc.create');+ span.setAttr('doc.type', 'md');+ span.setAttr('doc.source', source);+ void span+ .run(() =>+ createBlock({+ blockName: 'md',+ loading: true,+ span,+ createFn: () =>+ createMarkdownFile({+ title: '',+ content: '',+ projectId: undefined,+ source,+ }),+ shouldInsert,+ })+ )+ .catch((error) => {+ span.error(error);+ endTrackedDocumentSpan(span);+ throw error;+ });return;+ }case 'canvas':createBlock({blockName: 'canvas',diff --git a/apps/web/src/index.tsx b/apps/web/src/index.tsxindex f21a381854..4c9e8dccad 100644--- a/apps/web/src/index.tsx+++ b/apps/web/src/index.tsx@@ -1,4 +1,6 @@+import { initializeBrowserObservability } from './observability/browser';import './index.css';+import '@fontsource-variable/inter';import '@fontsource-variable/roboto-mono';import '@fontsource-variable/playfair-display';@@ -85,21 +87,13 @@ const renderApp = () => {render(() => <Root />, root);};-function main() {+async function main() {+ await initializeBrowserObservability();+console.log('App Version ', import.meta.env.__APP_VERSION__);// during `vite dev` (but not dev builds), don't inject analytics/observabilityif (!import.meta.hot) {- const scheduleIdleTask =- window.requestIdleCallback ?? ((cb: () => void) => setTimeout(cb, 1));-- // Lazy load and init observability (Datadog) to reduce initial bundle- scheduleIdleTask(() => {- import('@observability').then((Observability) => {- Observability.init(import.meta.env.__APP_VERSION__);- });- });-// this event is emitted when dynamically loading a module fails// for example when you're using the app and a new version is deployedwindow.addEventListener('vite:preloadError', () =>@@ -110,4 +104,5 @@ function main() {renderApp();}+// unawaitedmain();diff --git a/apps/web/src/lib/core/component/LexicalMarkdown/utils/fileUploadUtils.ts b/apps/web/src/lib/core/component/LexicalMarkdown/utils/fileUploadUtils.tsindex 4e2ea2541b..6a91bb15e2 100644--- a/apps/web/src/lib/core/component/LexicalMarkdown/utils/fileUploadUtils.ts+++ b/apps/web/src/lib/core/component/LexicalMarkdown/utils/fileUploadUtils.ts@@ -11,7 +11,8 @@ import {type UploadInput,uploadFiles,} from '@core/util/upload';-import { logger } from '@observability';+import { Telemetry } from '@macro-inc/observability';+import { fileExtension } from '@service-storage/util/filename';import type { LexicalEditor } from 'lexical';import {@@ -131,9 +132,7 @@ async function onFilesReady(const item = await documentUploadToItem(result);if (!item) {toast.failure('Document upload failed or timed out');- logger.error('Document upload failed or timed out', {- cause: new Error(),- });+ Telemetry.error(new Error('Document upload failed or timed out'));continue;}uploadedItemIds.push(item.id);@@ -149,9 +148,7 @@ async function onFilesReady(const item = await documentUploadToItem(result);if (!item) {toast.failure('Folder upload failed or timed out');- logger.error('Folder upload failed or timed out', {- cause: new Error(),- });+ Telemetry.error(new Error('Folder upload failed or timed out'));continue;}uploadedItemIds.push(item.id);diff --git a/apps/web/src/lib/core/hotkey/hotkeys.ts b/apps/web/src/lib/core/hotkey/hotkeys.tsindex 92d6ece56e..60b63ac6fc 100644--- a/apps/web/src/lib/core/hotkey/hotkeys.ts+++ b/apps/web/src/lib/core/hotkey/hotkeys.ts@@ -1,7 +1,8 @@import { IS_MAC } from '@core/constant/isMac';import { isTouchDevice } from '@core/mobile/isTouchDevice';import { isEditableInput } from '@core/util/isEditableInput';-import { logger } from '@observability';+import { Telemetry } from '@macro-inc/observability';+import { onCleanup, onMount, untrack } from 'solid-js';import {EVENT_MODIFIER_KEYS,@@ -145,16 +146,17 @@ export function registerHotkey(};if (!scopeId) {- logger.error('Scope ID is required for hotkey registration.', {- error: new Error('No scope ID provided'),- scopeId,- });+ Telemetry.error(+ new Error('Scope ID is required for hotkey registration.'),+ {+ scopeId,+ }+ );return noopDisposer;}const scopeNode = hotkeyScopeTree.get(scopeId);if (!scopeNode) {- logger.error('Scope ID not found.', {- error: new Error('Scope ID not found'),+ Telemetry.error(new Error('Scope ID not found.'), {scopeId,});return noopDisposer;@@ -175,15 +177,12 @@ export function registerHotkey(existingHotkeys.length !== hotkeys?.length ||!existingHotkeys.every((el, i) => el === hotkeys[i])) {- logger.log(+ Telemetry.warn(`Hotkey token "${hotkeyToken}" is already registered with a different command. ` +`Existing: ${existingCommand.description}, New: ${description}. ` +`This is likely a bug, please fix it. `,{- level: 'warn',- error: new Error(- 'Hotkey token already registered with a different command'- ),+ error: 'Hotkey token already registered with a different command',});}@@ -194,15 +193,13 @@ export function registerHotkey(if (providedCommandScopeId) {const existingScope = hotkeyScopeTree.get(providedCommandScopeId);if (!existingScope) {- logger.error('Provided command scope ID not found.', {- error: new Error('Provided command scope ID not found'),+ Telemetry.error(new Error('Provided command scope ID not found.'), {providedCommandScopeId,});return noopDisposer;}if (existingScope.type !== 'command') {- logger.error('Provided scope is not a command scope.', {- error: new Error('Provided scope is not a command scope'),+ Telemetry.error(new Error('Provided scope is not a command scope.'), {providedCommandScopeId,scopeType: existingScope.type,});@@ -250,11 +247,10 @@ export function registerHotkey(hotkeys?.forEach((h) => {const existingHandlers = scopeNode.hotkeyCommands.get(h);if (existingHandlers && existingHandlers.length > 0) {- logger.log(+ Telemetry.warn(`Hotkey ${h} already registered in scope ${scopeId}. Previous hotkey is being overwritten.`,{- level: 'warn',- error: new Error('Hotkey already registered in scope'),+ error: 'Hotkey already registered in scope',});}@@ -516,7 +512,7 @@ export function useHotkeyDOMScope(const currentDataAttribute = el.getAttribute('data-hotkey-scope');if (currentDataAttribute) {- logger.error(+ Telemetry.error(`Attempting to attach ${scopeId} hotkey scope, but the specified element already has a hotkey scope attached to it. This is an error and will break the hotkey scope tree. Please use that scope, ${currentDataAttribute}, instead of inventing your own.`);}@@ -553,7 +549,7 @@ export function attachGlobalDOMScope(el: Element) {const currentDataAttribute = el.getAttribute('data-hotkey-scope');if (currentDataAttribute) {- logger.error(+ Telemetry.error(`Attempting to attach global hotkey scope, but the specified element already has a hotkey scope attached to it. This is an error and will break the hotkey scope tree. Please use that scope, ${currentDataAttribute}, instead of inventing your own.`);}@@ -660,10 +656,12 @@ export function useHotKeyRoot() {}const isEditableFocused = isEditableInput(document.activeElement);if (!currentScopeId) {- return logger.error(`Could not find current hotkey scope.`, {- error: new Error('Could not find current hotkey scope'),- currentScopeId,- });+ return Telemetry.error(+ new Error('Could not find current hotkey scope.'),+ {+ currentScopeId,+ }+ );}const pressedKeysString = getKeyString(currentPressedKeys);diff --git a/apps/web/src/lib/core/hotkey/utils.ts b/apps/web/src/lib/core/hotkey/utils.tsindex b57d213c89..975cdd7c7d 100644--- a/apps/web/src/lib/core/hotkey/utils.ts+++ b/apps/web/src/lib/core/hotkey/utils.ts@@ -1,5 +1,6 @@import { IS_MAC } from '@core/constant/isMac';-import { logger } from '@observability/logger';+import { Telemetry } from '@macro-inc/observability';+import { createEffect, createMemo } from 'solid-js';import {macOptionReverse,@@ -347,12 +348,10 @@ export function getActiveCommandByToken(commandScope.originalParentScopeId);if (!originalParentScope) {- logger.error('Original parent scope not found for command scope:', {- error: new Error(- 'Original parent scope not found for command scope'- ),- commandScopeId: commandScope?.scopeId,- });+ Telemetry.error(+ new Error('Original parent scope not found for command scope'),+ { commandScopeId: commandScope?.scopeId }+ );return [];}diff --git a/apps/web/src/lib/core/util/fetchWithToken.ts b/apps/web/src/lib/core/util/fetchWithToken.tsindex e6db7c2fa0..cf177a9f01 100644--- a/apps/web/src/lib/core/util/fetchWithToken.ts+++ b/apps/web/src/lib/core/util/fetchWithToken.ts@@ -1,6 +1,7 @@import { ENABLE_BEARER_TOKEN_AUTH } from '@core/constant/featureFlags';import { SERVER_HOSTS } from '@core/constant/servers';-import { logger } from '@observability';+import { Telemetry } from '@macro-inc/observability';+import { fetchWithAuth } from '@service-auth/fetch';import { err, ok, type Result } from 'neverthrow';import type { ObjectLike, ResultError } from './result';@@ -143,10 +144,10 @@ export async function fetchWithToken<if (ENABLE_BEARER_TOKEN_AUTH) {const result = await fetchWithAuth<T, CustomErrorCode>(input, init);if (result.isErr()) {- logger.error('fetchWithToken: fetchWithAuth failed', {- input,- init,- errors: result.error,+ Telemetry.error('fetchWithToken: fetchWithAuth failed', {+ input: typeof input === 'string' ? input : input.url,+ method: init?.method,+ errors: JSON.stringify(result.error),});}diff --git a/apps/web/src/lib/core/util/result.spec.ts b/apps/web/src/lib/core/util/result.spec.tsindex 4223adb878..451bd394f2 100644--- a/apps/web/src/lib/core/util/result.spec.ts+++ b/apps/web/src/lib/core/util/result.spec.ts@@ -10,11 +10,14 @@ describe('throwOnErr', () => {});it('throws a ThrownResultError for err values', async () => {- const error = [{ code: 'ERROR', message: 'Test error' }];+ const error = [+ { code: 'ERROR', message: 'First error' },+ { code: 'ERROR', message: 'Second error' },+ ];await expect(throwOnErr(async () => err(error))).rejects.toMatchObject({errors: error,- message: 'Test error',+ message: 'First error, Second error',});});});diff --git a/apps/web/src/lib/core/util/safeFetch.spec.ts b/apps/web/src/lib/core/util/safeFetch.spec.tsindex 3a2b6ef648..ad23224484 100644--- a/apps/web/src/lib/core/util/safeFetch.spec.ts+++ b/apps/web/src/lib/core/util/safeFetch.spec.ts@@ -129,6 +129,25 @@ describe('safeFetch', () => {}});+ test('keeps an expected 404 as a NOT_FOUND result', async () => {+ mockFetch.mockImplementationOnce(() =>+ Promise.resolve({+ ok: false,+ status: 404,+ } as Response)+ );++ const result = await safeFetch<{ data: string }>(+ 'https://localhost/data',+ { trace: { expectedStatusCodes: [404] } }+ );++ expect(result).toMatchObject({+ error: [{ code: 'NOT_FOUND' }],+ });+ expect(mockFetch.mock.calls[0]?.[1]).not.toHaveProperty('trace');+ });+test('handle 401 Unauthorized', async () => {mockFetch.mockImplementationOnce(() =>Promise.resolve({diff --git a/apps/web/src/lib/core/util/safeFetch.ts b/apps/web/src/lib/core/util/safeFetch.tsindex dddb10dcd6..6aac644423 100644--- a/apps/web/src/lib/core/util/safeFetch.ts+++ b/apps/web/src/lib/core/util/safeFetch.ts@@ -1,8 +1,84 @@+import { SERVER_HOSTS, SYNC_SERVICE_HOSTS } from '@core/constant/servers';+import { Telemetry } from '@macro-inc/observability';import { err, ok, type Result } from 'neverthrow';import { platformFetch } from './platformFetch';import type { ObjectLike, ResultError } from './result';import { sleep } from './sleep';+const tracedOrigins: ReadonlySet<string> = (() => {+ const origins = new Set<string>();+ for (const host of [+ ...Object.values(SERVER_HOSTS),+ SYNC_SERVICE_HOSTS.worker,+ ]) {+ try {+ origins.add(new URL(host).origin);+ } catch {}+ }+ if (typeof window !== 'undefined') origins.add(window.location.origin);+ return origins;+})();++function isTracedOrigin(url: URL): boolean {+ return tracedOrigins.has(url.origin);+}++function requestUrl(input: RequestInfo): URL | undefined {+ const raw = typeof input === 'string' ? input : input.url;+ try {+ return new URL(+ raw,+ typeof window === 'undefined' ? undefined : window.location.origin+ );+ } catch {+ return undefined;+ }+}++function tracedFetch(+ input: RequestInfo,+ init: RequestInit & { headers: Record<string, string> },+ traceOptions?: SafeFetchTraceOptions+): Promise<Response> {+ const url = requestUrl(input);+ if (!url) return platformFetch(input, init);+ const method = (init.method ?? 'GET').toUpperCase();+ const span = Telemetry.clientSpan(`http ${method} ${url.pathname}`);+ span.setAttr('http.method', method);+ // Path only: query strings can carry tokens.+ span.setAttr('http.url', `${url.origin}${url.pathname}`);+ return span.run(async () => {+ try {+ if (isTracedOrigin(url)) {+ span.injectTraceHeaders(init.headers);+ }+ const response = await platformFetch(input, init);+ span.setAttr('http.status_code', response.status);+ if (!response.ok) {+ const expected = traceOptions?.expectedStatusCodes?.includes(+ response.status+ );+ if (expected) {+ span.setAttr('http.expected_status', true);+ return response;+ }+ const message = `HTTP ${response.status} for ${method} ${url.pathname}`;+ span.error({+ name: 'HttpError',+ message,+ stack: new Error(message).stack,+ });+ }+ return response;+ } catch (error) {+ span.error(error);+ throw error;+ } finally {+ span.end();+ }+ });+}+/*** Base error codes for fetch operations.*/@@ -61,11 +137,18 @@ export interface RetryConfig {delay?: 'exponential' | number;}+/** Trace-only classification for responses that are expected by the caller. */+export interface SafeFetchTraceOptions {+ /** Non-OK responses that should not mark the HTTP span as failed. */+ expectedStatusCodes?: readonly number[];+}+/*** Extended RequestInit interface that includes retry configuration.*/export interface SafeFetchInit extends RequestInit {retry?: RetryConfig;+ trace?: SafeFetchTraceOptions;}export type TextResponse = { contentType: 'text/plain'; body: string };@@ -160,7 +243,7 @@ export async function safeFetch<init?: SafeFetchInit,errorResponseHandler?: ErrorResponseHandler<CustomErrorCode>): Promise<Result<T, ResultError<BaseFetchErrorCode | CustomErrorCode>[]>> {- const { retry, ...fetchInit } = init || {};+ const { retry, trace, ...fetchInit } = init || {};const maxTries = retry?.maxTries ?? 1;const delay = retry?.delay ?? 0;type ErrorCode = BaseFetchErrorCode | CustomErrorCode;@@ -170,20 +253,24 @@ export async function safeFetch<for (let attempt = 1; attempt <= maxTries; attempt++) {try {- const response = await platformFetch(input, {- ...fetchInit,- headers: {- ...(fetchInit?.method !== 'GET' &&- fetchInit?.method !== 'HEAD' &&- !(fetchInit?.body instanceof FormData) && {- 'Content-Type':- (fetchInit?.headers as Record<string, string> | undefined)?.[- 'Content-Type'- ] ?? 'application/json',- }),- ...fetchInit?.headers,- } as Record<string, string>,- });+ const response = await tracedFetch(+ input,+ {+ ...fetchInit,+ headers: {+ ...(fetchInit?.method !== 'GET' &&+ fetchInit?.method !== 'HEAD' &&+ !(fetchInit?.body instanceof FormData) && {+ 'Content-Type':+ (fetchInit?.headers as Record<string, string> | undefined)?.[+ 'Content-Type'+ ] ?? 'application/json',+ }),+ ...fetchInit?.headers,+ } as Record<string, string>,+ },+ trace+ );if (!response.ok) {if (errorResponseHandler) {diff --git a/apps/web/src/lib/observability/index.ts b/apps/web/src/lib/observability/index.tsdeleted file mode 100644index 1d41e57efc..0000000000--- a/apps/web/src/lib/observability/index.ts+++ /dev/null@@ -1,80 +0,0 @@-/// <reference types="../../vite-env.d.ts" />--import { datadogLogs } from '@datadog/browser-logs';-import { isInitialized, setInitialized } from './shared';--const clientToken = import.meta.env.VITE_DD_WEB_APP_TOKEN;-const env = import.meta.env.MODE === 'production' ? 'prod' : 'dev';-const service = 'web-app';-const site = 'us5.datadoghq.com';--// Route intake through the first-party analytics proxy (Cloudflare Worker) so-// ad blockers / tracking protection don't drop logs the way they block-// requests sent straight to *.datadoghq.com. The worker maps the `/i/dd`-// prefix to the us5 browser intake origin; see services/analytics-proxy.-const proxy = (options: { path: string; parameters: string }) =>- `https://macro-prox.macroverse.workers.dev/i/dd${options.path}?${options.parameters}`;--interface User {- id: string;- email: string;- [key: string]: any;-}--// init() is deferred to an idle callback, so setUser() can run before the SDK-// is ready. Buffer the latest user and apply it once initialized.-let pendingUser: User | undefined;--export function init(version = import.meta.env.__APP_VERSION__) {- if (import.meta.hot || isInitialized()) return;-- datadogLogs.init({- clientToken,- env,- version,- service,- site,- proxy,- // Catch exceptions without RUM: forwards uncaught exceptions, unhandled- // promise rejections, and failed network requests (XHR/fetch) to Datadog- // as error-level logs (with stack traces).- forwardErrorsToLogs: true,- // Also forward explicit console.error() calls.- forwardConsoleLogs: ['error'],- // Forward browser Reporting API entries (CSP violations, deprecations,- // interventions).- forwardReports: 'all',- telemetrySampleRate: 0,- beforeSend: (event, _context) => {- if (event.message.includes('unfurl-service')) return false;-- // these are from VList and can be ignored: https://github.com/inokawa/virtua?tab=readme-ov-file#what-is-resizeobserver-loop-completed-with-undelivered-notifications-error- if (- event.message.includes(- 'ResizeObserver loop completed with undelivered notifications'- )- )- return false;-- return true;- },- });-- setInitialized(true);-- if (pendingUser) datadogLogs.setUser(pendingUser);-}--export function setUser(user: User) {- pendingUser = user;- if (isInitialized()) datadogLogs.setUser(user);-}--// Drop the user from log context on logout so logs aren't attributed to a-// signed-out user. Mirrors the analytics.reset() in the logout flow.-export function clearUser() {- pendingUser = undefined;- if (isInitialized()) datadogLogs.clearUser();-}--export { error, log, logger } from './logger';diff --git a/apps/web/src/lib/observability/logger.ts b/apps/web/src/lib/observability/logger.tsdeleted file mode 100644index 0eed3eea8f..0000000000--- a/apps/web/src/lib/observability/logger.ts+++ /dev/null@@ -1,79 +0,0 @@-import { datadogLogs } from '@datadog/browser-logs';-import { isInitialized } from './shared';--interface Context {- error?: Error;- level?: 'debug' | 'info' | 'warn' | 'error';- [key: string]: any;-}--/**- * Log a message to Datadog.- * @param messsage - The message to log.- * @param context - The context of the log such as error, level, etc.- */-export function log(messsage: string, context?: Context) {- if (import.meta.hot || !isInitialized())- return console.log(messsage, context);-- const { error, level, ...messageContext } = {- error: undefined,- level: 'info',- ...context,- } as const;-- datadogLogs.logger.log(messsage, messageContext, level, error);-}--interface ErrorContext extends Context {- cause?: Error;-}--/**- * Log a warning to Datadog.- * @param message - The message to log.- * @param context - The context of the log such as error, level, etc.- */-function warn(message: string, context?: Context) {- if (import.meta.hot || !isInitialized())- return console.warn(message, context);-- const { error, level, ...messageContext } = {- error: undefined,- level: 'warn',- ...context,- } as const;-- datadogLogs.logger.log(message, messageContext, level, error);-}--/**- * Log an error to Datadog.- * @param errorMessage - The Error object or message to log.- * @param context - The context of the log such as cause, level, etc.- *- * If used in a catch block, set the context.cause to the error thrown.- */-export function error(errorMessage: Error | string, context?: ErrorContext) {- if (import.meta.hot || !isInitialized())- return console.error(errorMessage, context);-- const { error, ...errorContext } = {- error:- typeof errorMessage === 'string' ? new Error(errorMessage) : errorMessage,- level: 'error',- ...context,- } as const;-- if (context?.cause) {- error.cause = context.cause;- }-- datadogLogs.logger.error(error.message || error.name, errorContext, error);-}--export const logger = {- log,- warn,- error,-};diff --git a/apps/web/src/lib/observability/shared.ts b/apps/web/src/lib/observability/shared.tsdeleted file mode 100644index 50e670a034..0000000000--- a/apps/web/src/lib/observability/shared.ts+++ /dev/null@@ -1,8 +0,0 @@-let initialized = false;-export function isInitialized() {- return initialized;-}--export function setInitialized(value: boolean) {- initialized = value;-}diff --git a/apps/web/src/lib/queries/soup/reactive-items.ts b/apps/web/src/lib/queries/soup/reactive-items.tsindex 3826b835c1..13d536a8f0 100644--- a/apps/web/src/lib/queries/soup/reactive-items.ts+++ b/apps/web/src/lib/queries/soup/reactive-items.ts@@ -11,7 +11,7 @@*/import { createQuerySignal } from '@graphql-cache/solid/create-query-signal';-import { logger } from '@observability/logger';+import { Telemetry } from '@macro-inc/observability';import { useInstructionsMdIdQuery } from '@queries/storage/instructions-md';import {SoupDocument,@@ -124,7 +124,7 @@ export function useReactiveSoupAstItemsQuery(createComputed(on(error, (queryError) => {if (queryError) {- logger.error(queryError, { graphqlOperation: 'Soup' });+ Telemetry.error(queryError, { graphqlOperation: 'Soup' });}}));diff --git a/apps/web/src/lib/queries/sync/document-peers.ts b/apps/web/src/lib/queries/sync/document-peers.tsindex e45b75f87d..156b2cafb3 100644--- a/apps/web/src/lib/queries/sync/document-peers.ts+++ b/apps/web/src/lib/queries/sync/document-peers.ts@@ -1,3 +1,4 @@+import { ThrownResultError } from '@core/util/result';import { syncServiceClient } from '@service-sync/client';import { useQuery } from '@tanstack/solid-query';import type { Accessor } from 'solid-js';@@ -7,7 +8,7 @@ async function fetchDocumentPeers(documentId: string): Promise<Map<string, string>> {const result = await syncServiceClient.getDocumentMetadata({ documentId });- if (result.isErr()) throw new Error(String(result.error));+ if (result.isErr()) throw new ThrownResultError(result.error);return new Map(result.value.peers.map((p) => [String(p.peer_id), p.user_id]));}diff --git a/apps/web/src/lib/service-clients/service-auth/client.ts b/apps/web/src/lib/service-clients/service-auth/client.tsindex d41a911940..341e9508d3 100644--- a/apps/web/src/lib/service-clients/service-auth/client.ts+++ b/apps/web/src/lib/service-clients/service-auth/client.ts@@ -8,7 +8,8 @@ import {type SafeFetchInit,safeFetch,} from '@core/util/safeFetch';-import { logger } from '@observability';+import { Telemetry } from '@macro-inc/observability';+import { makePersisted } from '@solid-primitives/storage';import { err, ok } from 'neverthrow';import { createSignal } from 'solid-js';@@ -124,7 +125,7 @@ async function getAccessToken(): Promise<string | null> {return null;}} catch (error) {- logger.error('Error refreshing access token', { error });+ Telemetry.error('Error refreshing access token', { error });return null;} finally {// Clear the ongoing refresh promise so future calls can start a new refresh@@ -336,7 +337,7 @@ export const authServiceClient = {async macroApiToken() {const accessToken = await getAccessToken();if (!accessToken) {- logger.warn('No access token found, fetching with cookies');+ Telemetry.warn('No access token found, fetching with cookies');return authApiFetch<MacroApiTokenResponse>('/jwt/macro_api_token');}diff --git a/apps/web/src/lib/service-clients/service-storage/client.ts b/apps/web/src/lib/service-clients/service-storage/client.tsindex 032d41440d..dd9318acc0 100644--- a/apps/web/src/lib/service-clients/service-storage/client.ts+++ b/apps/web/src/lib/service-clients/service-storage/client.ts@@ -1488,7 +1488,10 @@ export const storageServiceClient = {async fetchCachedSnapshot(documentId: string): Promise<Result<Uint8Array, ResultError<FetchWithTokenErrorCode>[]>> {- return dssFetch<Uint8Array>(`/documents/${documentId}/cached_snapshot_url`);+ return dssFetch<Uint8Array>(+ `/documents/${documentId}/cached_snapshot_url`,+ { trace: { expectedStatusCodes: [404] } }+ );},async getDocumentShortId({diff --git a/apps/web/src/lib/service-clients/service-sync/source/helpers.ts b/apps/web/src/lib/service-clients/service-sync/source/helpers.tsindex 0c38fa6b85..3be192f2d4 100644--- a/apps/web/src/lib/service-clients/service-sync/source/helpers.ts+++ b/apps/web/src/lib/service-clients/service-sync/source/helpers.ts@@ -1,3 +1,4 @@+import { resumeDocumentSpan } from '@block-md/observability';import { SYNC_SERVICE_HOSTS } from '@core/constant/servers';import type {InitialSync,@@ -27,8 +28,16 @@ export function createSyncServiceSocket(documentId: string,initialToken: string): SyncWebsocket {- const connectUrl = (token: string) =>- `${SYNC_SERVICE_WS_URL}/${documentId}/connect?token=${token}`;+ const connectUrl = (token: string) => {+ const url = `${SYNC_SERVICE_WS_URL}/${documentId}/connect?token=${token}`;+ // Browsers can't set websocket headers, so the trace context rides a+ // query param; the sync service joins its spans to the active+ // transaction's trace.+ const traceparent = resumeDocumentSpan(documentId)?.traceparent();+ return traceparent+ ? `${url}&traceparent=${encodeURIComponent(traceparent)}`+ : url;+ };let initialUrl: string | undefined = connectUrl(initialToken);let fallbackUrl = initialUrl;diff --git a/apps/web/src/observability/browser.ts b/apps/web/src/observability/browser.tsnew file mode 100644index 0000000000..c48bd531e4--- /dev/null+++ b/apps/web/src/observability/browser.ts@@ -0,0 +1,77 @@+import { Telemetry } from '@macro-inc/observability';+import { createWebTracingProvider } from '@macro-inc/observability/web';+// This static import loads the zone.js Promise patch before application modules run.+import { ZoneContextManager } from '@macro-inc/observability/zone';++async function browserTelemetryEnabled(hasExporter: boolean): Promise<boolean> {+ const override = import.meta.env.VITE_ENABLE_BROWSER_OTEL;++ if (override === 'false') return false;+ if (override === 'true') return true;++ if (import.meta.hot) return hasExporter;++ if (!import.meta.env.VITE_POSTHOG_API_KEY) return false;++ const { analytics } = await import('@app/lib/analytics');+ const flag = 'enable-browser-otel';+ const current = analytics.posthog.isFeatureEnabled(flag);+ if (current !== undefined) return current;++ return new Promise((resolve) => {+ let unsubscribe: (() => void) | undefined;+ let settled = false;+ const timeout = window.setTimeout(() => finish(false), 3_000);++ const finish = (enabled: boolean) => {+ settled = true;+ window.clearTimeout(timeout);+ unsubscribe?.();+ resolve(enabled);+ };++ unsubscribe = analytics.posthog.onFeatureFlags((_flags, _variants, ctx) => {+ finish(+ !ctx?.errorsLoading &&+ (analytics.posthog.isFeatureEnabled(flag) ?? false)+ );+ });++ if (settled) unsubscribe();+ });+}++/** Initialize browser telemetry and its application-level lifecycle hooks. */+export async function initializeBrowserObservability(): Promise<void> {+ const tracesUrl =+ import.meta.env.VITE_OTEL_EXPORTER_URL ??+ (import.meta.hot ? 'http://localhost:8098/i/otlp/v1/traces' : undefined);+ const telemetryConfig = {+ serviceName: 'web-app',+ environment:+ import.meta.env.VITE_OTEL_ENV ??+ (import.meta.env.MODE === 'production' ? 'prod' : 'dev'),+ tracesUrl,+ logsUrl: tracesUrl?.replace(/\/v1\/traces\/?$/, '/v1/logs'),+ contextManager: new ZoneContextManager(),+ enabled: () => browserTelemetryEnabled(Boolean(tracesUrl)),+ };++ await Telemetry.init({+ ...telemetryConfig,+ tracingProvider: (resource, getUserId) =>+ createWebTracingProvider(telemetryConfig, resource, getUserId),+ });++ window.addEventListener('pagehide', () => void Telemetry.flush());+ window.addEventListener('error', (event) => {+ Telemetry.error(event.error ?? event.message, {+ 'error.source': 'window',+ });+ });+ window.addEventListener('unhandledrejection', (event) => {+ Telemetry.error(event.reason, {+ 'error.source': 'unhandledrejection',+ });+ });+}diff --git a/apps/web/src/routes/Root.tsx b/apps/web/src/routes/Root.tsxindex a7a3efcf55..0380aa9d99 100644--- a/apps/web/src/routes/Root.tsx+++ b/apps/web/src/routes/Root.tsx@@ -66,6 +66,7 @@ import { transformShortIdInUrlPathname } from '@core/util/url';import { EntityProvider } from '@entity';import { MaybeTauriProvider } from '@macro/tauri';import { TauriRouteListener } from '@macro/tauri/TauriProvider';+import { Telemetry } from '@macro-inc/observability';import {BrowserNotificationModal,createNotificationSource,@@ -74,10 +75,6 @@ import {usePlatformNotificationState,} from '@notifications';import { maybeHandlePlatformNotification } from '@notifications/notification-platform';-import {- clearUser as clearDatadogUser,- setUser as setDatadogUser,-} from '@observability';import {invalidateUserInfo,prefetchUserInfo,@@ -546,16 +543,12 @@ function UserInfoSideEffects() {let syncedPlanKey: string | undefined;createEffect(on(userInfo, (user) => {- // Keep Datadog log user context in sync with auth state: set on every- // authenticated load (the logs SDK doesn't persist across reloads), and- // clear on logout so logs aren't attributed to a signed-out user. Logout- // flips userInfo client-side, and on native mobile it's an SPA navigation- // with no page reload, so this effect is what clears it there.- if (user?.authenticated) {- setDatadogUser({ id: user.id, email: user.email });- } else {- clearDatadogUser();- }+ // Keep telemetry user context in sync with auth state: set on every+ // authenticated load, and clear on logout so spans and logs aren't+ // attributed to a signed-out user. Logout flips userInfo client-side,+ // and on native mobile it's an SPA navigation with no page reload, so+ // this effect is what clears it there.+ Telemetry.config.setUser(user?.authenticated ? user.id : undefined);if (!user || !user.authenticated) {syncedPlanKey = undefined;diff --git a/apps/web/src/vite-env.d.ts b/apps/web/src/vite-env.d.tsindex b8e8f303cd..98c7e129bc 100644--- a/apps/web/src/vite-env.d.ts+++ b/apps/web/src/vite-env.d.ts@@ -4,9 +4,11 @@ interface ImportMetaEnv {readonly __GIT_BRANCH__: string;readonly VITE_SEGMENT_WRITE_KEY: string;- readonly VITE_DD_WEB_APP_TOKEN: string;- readonly VITE_DD_HASH: string;readonly VITE_POSTHOG_API_KEY: string;++ readonly VITE_OTEL_EXPORTER_URL?: string;+ readonly VITE_OTEL_ENV?: string;+ readonly VITE_ENABLE_BROWSER_OTEL?: string;}interface ImportMeta {diff --git a/apps/web/tsconfig.json b/apps/web/tsconfig.jsonindex d77ddbe14c..e8870b678a 100644--- a/apps/web/tsconfig.json+++ b/apps/web/tsconfig.json@@ -182,12 +182,6 @@"@design/*": ["./src/lib/design/*"],- "@observability": [- "./src/lib/observability/"- ],- "@observability/*": [- "./src/lib/observability/*"- ],"@service-sync": ["./src/lib/service-clients/service-sync"],diff --git a/bun.lock b/bun.lockindex d968c01012..e00f612e5b 100644--- a/bun.lock+++ b/bun.lock@@ -18,7 +18,6 @@"@codemirror/language": "^6.11.3","@corvu/dialog": "^0.2.4","@corvu/drawer": "^0.2.4",- "@datadog/browser-logs": "^6.22.0","@date-fns/tz": "^1.4.1","@dinero.js/currencies": "2.0.0-alpha.8","@floating-ui/dom": "^1.7.4",@@ -45,7 +44,9 @@"@lukemorales/query-key-factory": "^1.3.4","@macro-inc/collaboration": "workspace:*","@macro-inc/lexical-core": "workspace:*",+ "@macro-inc/observability": "workspace:*","@normy/query-core": "^0.21.0",+ "@opentelemetry/api": "^1.9.1","@phosphor-icons/core": "^2.1.1","@solid-primitives/broadcast-channel": "^0.1.1","@solid-primitives/context": "^0.3.2",@@ -168,6 +169,7 @@"version": "0.1.0","dependencies": {"@loro-mirror/core": "workspace:*",+ "@macro-inc/observability": "workspace:*","@solid-primitives/event-bus": "^1.1.2","@tauri-apps/api": "^2.8.0","async-mutex": "^0.5.0",@@ -248,6 +250,30 @@"loro-crdt": "^1.4.4",},},+ "packages/observability": {+ "name": "@macro-inc/observability",+ "version": "0.0.1",+ "dependencies": {+ "@opentelemetry/api": "^1.9.1",+ "@opentelemetry/api-logs": "^0.220.0",+ "@opentelemetry/context-zone": "^2.9.0",+ "@opentelemetry/core": "^2.9.0",+ "@opentelemetry/exporter-logs-otlp-http": "^0.220.0",+ "@opentelemetry/exporter-trace-otlp-http": "^0.220.0",+ "@opentelemetry/resources": "^2.9.0",+ "@opentelemetry/sdk-logs": "^0.220.0",+ "@opentelemetry/sdk-trace-base": "^2.9.0",+ "@opentelemetry/sdk-trace-web": "^2.9.0",+ "@opentelemetry/semantic-conventions": "^1.43.0",+ },+ "devDependencies": {+ "@types/bun": "^1.3.14",+ "@types/node": "^24.8.0",+ "jsdom": "^26.1.0",+ "typescript": "^5.9.3",+ "vitest": "^3.2.4",+ },+ },"services/ai-editing-worker": {"name": "ai-editing-worker","version": "0.0.1",@@ -615,10 +641,6 @@"@csstools/css-tokenizer": ["@csstools/[email protected]", "", {}, "sha512-Vd/9EVDiu6PPJt9yAh6roZP6El1xHrdvIVGjyBsHR0RYwNHgL7FJPyIIW4fANJNG6FtyZfvlRPpFI4ZM/lubvw=="],- "@datadog/browser-core": ["@datadog/[email protected]", "", {}, "sha512-h/xuBuY4Xi32CbBHkChMMWlOQ4al/mFF08x3o0hqrderfJvX7VO5tD3OVNwF+pfMXOGlDtBCIhwazr9sn24h3g=="],-- "@datadog/browser-logs": ["@datadog/[email protected]", "", { "dependencies": { "@datadog/browser-core": "6.33.0" }, "peerDependencies": { "@datadog/browser-rum": "6.33.0" }, "optionalPeers": ["@datadog/browser-rum"] }, "sha512-WZ1XEIIYK2sijR0hUG3hTPu2kQs8hE/PpoJcMDE7p0/NPXX3TR3dGVrwag+gvIVyBaszRkITiBsHQfnRvxV1DA=="],-"@datadog/datadog-ci": ["@datadog/[email protected]", "", { "dependencies": { "@datadog/datadog-ci-base": "3.22.2", "@datadog/datadog-ci-plugin-aas": "3.22.2", "@datadog/datadog-ci-plugin-cloud-run": "3.22.2", "@datadog/datadog-ci-plugin-deployment": "3.22.2", "@datadog/datadog-ci-plugin-dora": "3.22.2", "@datadog/datadog-ci-plugin-gate": "3.22.2", "@datadog/datadog-ci-plugin-lambda": "3.22.2", "@datadog/datadog-ci-plugin-sarif": "3.22.2", "@datadog/datadog-ci-plugin-sbom": "3.22.2", "@datadog/datadog-ci-plugin-stepfunctions": "3.22.2", "@datadog/datadog-ci-plugin-synthetics": "3.22.2", "axios": "^1.12.1", "chalk": "3.0.0", "clipanion": "^3.2.1", "fast-xml-parser": "^4.4.1", "form-data": "^4.0.4", "js-yaml": "3.13.1", "semver": "^7.5.3", "simple-git": "3.16.0", "typanion": "^3.14.0", "upath": "^2.0.1", "uuid": "^9.0.0" }, "bin": "dist/cli.js" }, "sha512-OejGdY0g+S0/9oJNCueBSU3ZXjNCn+JWhxvx+jroB/FolkOEDESmugludf3/uOjM3mCeuUlGoDhwIAmezF0AvA=="],"@datadog/datadog-ci-base": ["@datadog/[email protected]", "", { "dependencies": { "@antfu/install-pkg": "^1.1.0", "@types/datadog-metrics": "0.6.1", "async-retry": "1.3.1", "axios": "^1.12.1", "chalk": "3.0.0", "clipanion": "^3.2.1", "datadog-metrics": "0.9.3", "debug": "^4.4.1", "deep-extend": "0.6.0", "fast-xml-parser": "^4.4.1", "form-data": "^4.0.4", "glob": "^10.4.5", "inquirer": "^8.2.5", "jszip": "^3.10.1", "proxy-agent": "^6.4.0", "semver": "^7.5.3", "simple-git": "3.16.0", "terminal-link": "2.1.1", "tiny-async-pool": "^2.1.0", "typanion": "^3.14.0", "upath": "^2.0.1" }, "peerDependencies": { "@datadog/datadog-ci-plugin-aas": "3.22.2", "@datadog/datadog-ci-plugin-cloud-run": "3.22.2", "@datadog/datadog-ci-plugin-deployment": "3.22.2", "@datadog/datadog-ci-plugin-dora": "3.22.2", "@datadog/datadog-ci-plugin-gate": "3.22.2", "@datadog/datadog-ci-plugin-lambda": "3.22.2", "@datadog/datadog-ci-plugin-sarif": "3.22.2", "@datadog/datadog-ci-plugin-sbom": "3.22.2", "@datadog/datadog-ci-plugin-stepfunctions": "3.22.2", "@datadog/datadog-ci-plugin-synthetics": "3.22.2" }, "optionalPeers": ["@datadog/datadog-ci-plugin-aas", "@datadog/datadog-ci-plugin-cloud-run", "@datadog/datadog-ci-plugin-deployment", "@datadog/datadog-ci-plugin-dora", "@datadog/datadog-ci-plugin-gate", "@datadog/datadog-ci-plugin-lambda", "@datadog/datadog-ci-plugin-sarif", "@datadog/datadog-ci-plugin-sbom", "@datadog/datadog-ci-plugin-stepfunctions", "@datadog/datadog-ci-plugin-synthetics"] }, "sha512-87Q1dHcOczRhddo9ugB6JZYOep70rQ9rHMByOF81RKFjufQWtbU+ZPl0pKge3DvDiR3oXo01PrzUsS+Ss38QOw=="],@@ -1041,6 +1063,8 @@"@macro-inc/lexical-core": ["@macro-inc/lexical-core@workspace:packages/lexical-core"],+ "@macro-inc/observability": ["@macro-inc/observability@workspace:packages/observability"],+"@marijn/find-cluster-break": ["@marijn/[email protected]", "", {}, "sha512-FY+MKLBoTsLNJF/eLWaOsXGdz6uh3Iu1axjPf6TUq92IYumcTcXWHoS747JARLkcdlJ/Waiaxc5wQfFO8jC6NA=="],"@mediapipe/tasks-vision": ["@mediapipe/[email protected]", "", {}, "sha512-vOifgZhkndgybdvoRITzRkIueWWSiCKuEUXXK6Q4FaJsFvRJuwgg++vqFUMlL0Uox62U5aEXFhHxlhV7Ja5e3Q=="],@@ -1061,6 +1085,36 @@"@opentelemetry/api": ["@opentelemetry/[email protected]", "", {}, "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q=="],+ "@opentelemetry/api-logs": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-CmVa4ImJ+ynfrPMNaAXHET6Bhb44SwzmfyVJFq9ni2jgXJR/l7C6gfVFddNmHP+ZOkP9cf4f9DBe68qVLTHc9w=="],++ "@opentelemetry/context-zone": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/context-zone-peer-dep": "2.9.0", "zone.js": "^0.11.0 || ^0.12.0 || ^0.13.0 || ^0.14.0 || ^0.15.0 || ^0.16.0" } }, "sha512-dUH2j6/jh5hgsAFvEqx+9MIjdi8OzBK1u8K6qsb2HE2R1AiR+u00GWCxbDts30JggTX4rwA8YNLO2xF5LC6EFg=="],++ "@opentelemetry/context-zone-peer-dep": ["@opentelemetry/[email protected]", "", { "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0", "zone.js": "^0.10.2 || ^0.11.0 || ^0.12.0 || ^0.13.0 || ^0.14.0 || ^0.15.0 || ^0.16.0" } }, "sha512-xECINyl1UHLNbaF1my3K5AqJGyox1ysLidplInAb/XTfBxc2CLDRuS4rUxiJs8irW/PkEDujdc/IPUqkJFnrzg=="],++ "@opentelemetry/core": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-m2nckMT80NnmjTYSPjJQObBJ+8dgkoajEOUbznL8AHZ3T3yHRk2P7gI1PhEBc1+lOnrYE9UWrWHqJDsmqjmNbw=="],++ "@opentelemetry/exporter-logs-otlp-http": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/api-logs": "0.220.0", "@opentelemetry/core": "2.9.0", "@opentelemetry/otlp-exporter-base": "0.220.0", "@opentelemetry/otlp-transformer": "0.220.0", "@opentelemetry/sdk-logs": "0.220.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-8186thl+pTw64iz/qEEen5oJZoZ/gO73XruChdaGlYdWOdBIQ42r+vHLf6a7vIDqTD4b8ZOoMlyxptanECaI9A=="],++ "@opentelemetry/exporter-trace-otlp-http": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/core": "2.9.0", "@opentelemetry/otlp-exporter-base": "0.220.0", "@opentelemetry/otlp-transformer": "0.220.0", "@opentelemetry/resources": "2.9.0", "@opentelemetry/sdk-trace": "2.9.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-/+ExB3lRkf+erv4PnoywyL7RHKITidxtUpUTS55k7OQ0dB42S7gEF1gry7swb9MSm1hYLUhJg4QQh9W8SpwwqA=="],++ "@opentelemetry/otlp-exporter-base": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/core": "2.9.0", "@opentelemetry/otlp-transformer": "0.220.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-CXYo8UD5Mn9YbgebO2EL4wejtA+gxLmLiu6HCk2KH2BR7XhFN6/6p1UlCb23DYCjeYkndevLHuejCCN1yx4+OQ=="],++ "@opentelemetry/otlp-transformer": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/api-logs": "0.220.0", "@opentelemetry/core": "2.9.0", "@opentelemetry/resources": "2.9.0", "@opentelemetry/sdk-logs": "0.220.0", "@opentelemetry/sdk-metrics": "2.9.0", "@opentelemetry/sdk-trace": "2.9.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-lXGrv7KXZ0gNH9SVNUaa6vv6phVYGvJxfXAlMbzbakiXru75f5MZl8Z7oqiMMQD77riVHJCFlQvbZs/VVN2/4A=="],++ "@opentelemetry/resources": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/core": "2.9.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-jyA5MBLQ+Dkl3+JsZkUoUvL7yHvU64kLsvpXKarWm6347Sl1t1bXFTFykUePNpT5WH5pm9a2Qtt03iIYQhZ1Fg=="],++ "@opentelemetry/sdk-logs": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/api-logs": "0.220.0", "@opentelemetry/core": "2.9.0", "@opentelemetry/resources": "2.9.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.4.0 <1.10.0" } }, "sha512-WywcTkQtv2iNmt+6y5Kcd4rzvx9bLVsBa2Nwcmg01IUaBTkTow3W4d9KE5vNBpEDtb9tp21WcRBY/lANRrApYA=="],++ "@opentelemetry/sdk-metrics": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/core": "2.9.0", "@opentelemetry/resources": "2.9.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.9.0 <1.10.0" } }, "sha512-Xx8RGS4H5XEBl01WuCreMIpiah9cCXMbSkeuIePPdD2cUpq/vUzYmj8E/MK1OsbOc93FuAD4jfn2WOacKwLn7Q=="],++ "@opentelemetry/sdk-trace": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/core": "2.9.0", "@opentelemetry/resources": "2.9.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-sGA19HvtrrSKYsseHphluH6j3p6Xa3fqc7c7y8f/7mYWejc1lyDFcpSdD1kYa50HCLUeEo4zA5bW0pniaPszuw=="],++ "@opentelemetry/sdk-trace-base": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/core": "2.9.0", "@opentelemetry/resources": "2.9.0", "@opentelemetry/sdk-trace": "2.9.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-cp9zmTl62R8PJrpvFcmc8N2JQU/xfa0S+61q511Nji+QxCfZ8Ifvg7H27G8cANe4crg4RTrWsVvanHiXjSp6ag=="],++ "@opentelemetry/sdk-trace-web": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/core": "2.9.0", "@opentelemetry/sdk-trace-base": "2.9.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-LS4XlzOK3e6YYdt84m15AmRR04121rKipmifi4XFZToH1h75f7F2bzHLbB1NMgIEYJ0jSKYm4VsK5mws/5kfTQ=="],++ "@opentelemetry/semantic-conventions": ["@opentelemetry/[email protected]", "", {}, "sha512-eSYWTm620tTk45EKSedaUL8MFYI8hW164hIXsgIHyxu3VobUB3fFCu5t0hQby6OoWRPsG1KkKUG2M5UadiLiVg=="],+"@orval/angular": ["@orval/[email protected]", "", { "dependencies": { "@orval/core": "7.21.0" } }, "sha512-AGelR1FfuimtIBBVccUI9MyjNOalLEyJFog8a94thFiqRGtz0JFIPnd8+IqRcmw3wE370PKQQMCqZl1WkjZi8w=="],"@orval/axios": ["@orval/[email protected]", "", { "dependencies": { "@orval/core": "7.21.0" } }, "sha512-0t2PqMSdjJ7No6Ng5gTEpqyILmngz1mDw7OQ2TPwOiAEepNfeDgE6NtRh3zbZvrH/H5I+W8dgn+wGYueZQCNjw=="],@@ -3173,6 +3227,8 @@"zod-validation-error": ["[email protected]", "", { "peerDependencies": { "zod": "^3.25.0 || ^4.0.0" } }, "sha512-hmk+pkyKq7Q71PiWVSDUc3VfpzpvcRHZ3QPw9yEMVvmtCekaMeOHnbr3WbxfrgEnQTv6haGP4cmv0Ojmihzsxw=="],+ "zone.js": ["[email protected]", "", {}, "sha512-Eky7p2Z1Ig3NnbfodSPoARCjKBSTFMnE/ACsP1L/XJEfY4SdOFce19BsUCWVwL6K5ABZFy5J3bjcMWffX+YM3Q=="],+"@antfu/install-pkg/tinyexec": ["[email protected]", "", {}, "sha512-SHf/r48b7vOrjve9PxJo3MN5v5yuyjHvdUcrQffT3WXMUfnGmHDVbC4k3sHJaJTgZCwpUplIaAo5ANtMyp3YHg=="],"@apidevtools/swagger-parser/@apidevtools/json-schema-ref-parser": ["@apidevtools/[email protected]", "", { "dependencies": { "@types/json-schema": "^7.0.15", "js-yaml": "^4.1.0" } }, "sha512-Oc96zvmxx1fqoSEdUmfmvvb59/KDOnUoJ7s2t7bISyAn0XEz57LCCw8k2Y4Pf3mwKaZLMciESALORLgfe2frCw=="],diff --git a/crates/macro_entrypoint/Cargo.toml b/crates/macro_entrypoint/Cargo.tomlindex db4c09fbc3..0d683aa466 100644--- a/crates/macro_entrypoint/Cargo.toml+++ b/crates/macro_entrypoint/Cargo.toml@@ -7,8 +7,8 @@ version = "0.1.0"[dependencies]dotenvy = { workspace = true }macro_env = { path = "../macro_env" }-opentelemetry = { workspace = true }-opentelemetry_sdk = { workspace = true }+opentelemetry = { workspace = true, features = ["trace"] }+opentelemetry_sdk = { workspace = true, features = ["rt-tokio", "trace"] }opentelemetry-otlp = { workspace = true }rootcause = { workspace = true }rootcause-tracing = { workspace = true }diff --git a/crates/macro_entrypoint/src/lib.rs b/crates/macro_entrypoint/src/lib.rsindex 7922a41e63..57e2300e80 100644--- a/crates/macro_entrypoint/src/lib.rs+++ b/crates/macro_entrypoint/src/lib.rs@@ -32,6 +32,7 @@ env_vars! {maybe_env_vars! {pub struct RustLog;+ pub struct OtelExporterOtlpEndpoint;pub struct OtelTraceFilter;}@@ -125,23 +126,45 @@ impl MacroEntrypoint {match (self.env, self.local) {(Environment::Local, LocalOptions { tree_tracing: None }) => {- // Keep rootcause from enabling callsites that RUST_LOG previously disabled.let rust_log_filter = rust_log_env_filter();- let fmt_layer = tracing_subscriber::fmt::layer()- .with_ansi(true)- .with_file(true)- .with_line_number(true)- .pretty()- .with_filter(rust_log_filter.clone());-- Registry::default()- .with(RootcauseLayer.with_filter(rust_log_filter))- .with(fmt_layer)- .init();-- InitializedEntrypoint {- tracer_provider: None,+ // Local OTLP export is opt-in. xtask injects this only for+ // trace-enabled local runs.+ let tracer_provider = OtelExporterOtlpEndpoint::new()+ .map(|_| init_opentelemetry("local".to_string()));++ if let Some(provider) = tracer_provider.as_ref() {+ let otel_filter = otel_env_filter();+ let rootcause_filter = rust_log_filter.clone().or(otel_filter.clone());+ let otel_layer = otel_layer_with_error_mapping(provider.tracer(service_name()))+ .with_filter(otel_filter);++ Registry::default()+ .with(RootcauseLayer.with_filter(rootcause_filter))+ .with(+ tracing_subscriber::fmt::layer()+ .with_ansi(true)+ .with_file(true)+ .with_line_number(true)+ .pretty()+ .with_filter(rust_log_filter),+ )+ .with(otel_layer)+ .init();+ } else {+ Registry::default()+ .with(RootcauseLayer.with_filter(rust_log_filter.clone()))+ .with(+ tracing_subscriber::fmt::layer()+ .with_ansi(true)+ .with_file(true)+ .with_line_number(true)+ .pretty()+ .with_filter(rust_log_filter),+ )+ .init();}++ InitializedEntrypoint { tracer_provider }}(Environment::Local,@@ -159,21 +182,18 @@ impl MacroEntrypoint {}}(Environment::Production | Environment::Develop, _) => {- let tracer_provider = init_opentelemetry();-- // Get service name for the tracer- let service_name = DdService::new()+ // Get environment from DD_ENV+ let env = DdEnv::new().map(|e| e.to_string())- .unwrap_or_else(|_| "unknown-service".to_string());+ .unwrap_or_else(|_| "unknown".to_string());+ let tracer_provider = init_opentelemetry(env);- let tracer = tracer_provider.tracer(service_name);+ let tracer = tracer_provider.tracer(service_name());let rust_log_filter = rust_log_env_filter();let otel_filter = otel_env_filter();// Capture anything already enabled for logs or OTEL without enabling new callsites.let rootcause_filter = rust_log_filter.clone().or(otel_filter.clone());- let otel_layer = tracing_opentelemetry::layer()- .with_tracer(tracer)- .with_filter(otel_filter);+ let otel_layer = otel_layer_with_error_mapping(tracer).with_filter(otel_filter);// Build the JSON event format, then wrap it with DatadogFormat// to inject dd.trace_id / dd.span_id for trace-log correlation.@@ -213,31 +233,70 @@ impl MacroEntrypoint {}}-/// Opentelemetry export endpoint to talk with datadog sidecar-const OTEL_EXPORTER_OTLP_ENDPOINT: &str = "http://127.0.0.1:4317";+/// Default OpenTelemetry export endpoint: the Datadog agent sidecar in prod/develop.+const DEFAULT_OTLP_ENDPOINT: &str = "http://127.0.0.1:4317";++/// Wraps the OTel layer so `tracing::error!` events on a span (e.g.+/// `macro_tower_layers::CustomOnFailure`'s "response failed", or an+/// `#[instrument(err)]` failure) become an OTLP exception with a message and+/// stack trace, and set the span's error status message. Without these, a+/// span can be marked errored (from the HTTP status) with no `error.message`+/// / `error.stack` for Datadog to show — "Missing error message and stack+/// trace" in the UI.+fn otel_layer_with_error_mapping<S>(+ tracer: opentelemetry_sdk::trace::SdkTracer,+) -> tracing_opentelemetry::OpenTelemetryLayer<S, opentelemetry_sdk::trace::SdkTracer>+where+ S: tracing::Subscriber + for<'a> tracing_subscriber::registry::LookupSpan<'a>,+{+ tracing_opentelemetry::layer()+ .with_tracer(tracer)+ .with_error_events_to_exceptions(true)+ .with_error_events_to_status(true)+ .with_error_fields_to_exceptions(true)+ .with_error_records_to_exceptions(true)+ .with_location(true)+}++/// The service name reported on traces: `DD_SERVICE` when set, otherwise the+/// current executable's file name so each service is distinguishable locally.+fn service_name() -> String {+ if let Ok(service) = DdService::new() {+ return service.to_string();+ }+ std::env::current_exe()+ .ok()+ .and_then(|exe| {+ exe.file_name()+ .map(|name| name.to_string_lossy().into_owned())+ })+ .unwrap_or_else(|| "unknown-service".to_string())+}++fn init_opentelemetry(deployment_environment: String) -> SdkTracerProvider {+ // W3C trace-context propagation: lets macro_tower_layers parent request+ // spans under an incoming `traceparent` (e.g. from the web app), and+ // service clients propagate context onward.+ opentelemetry::global::set_text_map_propagator(+ opentelemetry_sdk::propagation::TraceContextPropagator::new(),+ );++ let endpoint = OtelExporterOtlpEndpoint::new()+ .map(|e| e.to_string())+ .unwrap_or_else(|| DEFAULT_OTLP_ENDPOINT.to_string());-/// Initialize OpenTelemetry with OTLP exporter to the Datadog agent.-/// The Datadog agent sidecar listens on localhost:4317 for OTLP gRPC.-fn init_opentelemetry() -> SdkTracerProvider {let exporter = opentelemetry_otlp::SpanExporter::builder().with_tonic()- .with_endpoint(OTEL_EXPORTER_OTLP_ENDPOINT)+ .with_endpoint(endpoint).build().expect("failed to create OTLP span exporter");- // Get service name from DD_SERVICE or OTEL_SERVICE_NAME- let service_name = DdService::new()- .map(|e| e.to_string())- .unwrap_or_else(|_| "unknown-service".to_string());-- // Get environment from DD_ENV- let env = DdEnv::new()- .map(|e| e.to_string())- .unwrap_or_else(|_| "unknown".to_string());-let resource = opentelemetry_sdk::Resource::builder()- .with_service_name(service_name)- .with_attribute(opentelemetry::KeyValue::new("deployment.environment", env))+ .with_service_name(service_name())+ .with_attribute(opentelemetry::KeyValue::new(+ "deployment.environment",+ deployment_environment,+ )).build();SdkTracerProvider::builder()diff --git a/crates/macro_tower_layers/Cargo.toml b/crates/macro_tower_layers/Cargo.tomlindex 74d5f45541..9009e24a8b 100644--- a/crates/macro_tower_layers/Cargo.toml+++ b/crates/macro_tower_layers/Cargo.toml@@ -6,11 +6,16 @@ version = "0.1.0"[dependencies]http = { workspace = true }+opentelemetry = { workspace = true, features = ["trace"] }+opentelemetry-http = { workspace = true }tokio = { workspace = true, features = ["rt", "time"] }tower = { workspace = true }tower-http = { workspace = true, features = ["request-id", "util"] }tracing = { workspace = true }+tracing-opentelemetry = { workspace = true }workspace-hack = { version = "0.1", path = "../workspace-hack" }[dev-dependencies]+opentelemetry_sdk = { workspace = true, features = ["testing"] }tokio = { workspace = true, features = ["macros", "rt", "test-util", "time"] }+tracing-subscriber = { workspace = true }diff --git a/crates/macro_tower_layers/src/lib.rs b/crates/macro_tower_layers/src/lib.rsindex d10242f3ae..cd228577cf 100644--- a/crates/macro_tower_layers/src/lib.rs+++ b/crates/macro_tower_layers/src/lib.rs@@ -12,6 +12,7 @@ use std::{};use http::{HeaderValue, Request, Response};+use opentelemetry::trace::TraceContextExt;use tokio::time::MissedTickBehavior;use tower::{ServiceBuilder,@@ -24,6 +25,27 @@ use tower_http::{trace::{MakeSpan, OnFailure, OnResponse, TraceLayer},};use tracing::Span;+use tracing_opentelemetry::OpenTelemetrySpanExt;++/// Creates a safe HTTP request span and adopts an incoming W3C trace context.+///+/// A valid `traceparent` makes the request span a child of that remote span.+/// The global text-map propagator is registered by `macro_entrypoint`.+#[derive(Debug, Clone, Default)]+pub struct MakeSpanWithRemoteParent;++impl<B> MakeSpan<B> for MakeSpanWithRemoteParent {+ fn make_span(&mut self, request: &Request<B>) -> Span {+ let span = MakeHttpRequestSpan.make_span(request);+ let parent = opentelemetry::global::get_text_map_propagator(|propagator| {+ propagator.extract(&opentelemetry_http::HeaderExtractor(request.headers()))+ });+ if parent.span().span_context().is_valid() {+ let _ = span.set_parent(parent);+ }+ span+ }+}/// A very simple builder for x-request-ids#[derive(Default, Clone)]@@ -143,7 +165,7 @@ type ServiceBuilderAlias = ServiceBuilder<Stack<TraceLayer<SharedClassifier<ServerErrorsAsFailures>,- MakeHttpRequestSpan,+ MakeSpanWithRemoteParent,(),CustomOnResponse,tower_http::trace::DefaultOnBodyChunk,@@ -200,7 +222,7 @@ impl MacroRequestIdAndTracingLayer {.set_x_request_id(RequestIdBuilder::default()).layer(TraceLayer::new_for_http()- .make_span_with(MakeHttpRequestSpan)+ .make_span_with(MakeSpanWithRemoteParent).on_request(()).on_response(CustomOnResponse::new_with_threshold(warning_threshold)).on_failure(CustomOnFailure),diff --git a/crates/macro_tower_layers/src/test.rs b/crates/macro_tower_layers/src/test.rsindex bd51f37fc1..b1cfb78cea 100644--- a/crates/macro_tower_layers/src/test.rs+++ b/crates/macro_tower_layers/src/test.rs@@ -219,6 +219,87 @@ async fn starvation_detector_warns_when_runtime_blocked() {assert_eq!(captured.event_count(Level::WARN), 1);}+#[test]+fn header_extractor_extracts_w3c_traceparent() {+ let propagator = opentelemetry_sdk::propagation::TraceContextPropagator::new();+ let mut headers = http::HeaderMap::new();+ headers.insert(+ "traceparent",+ "00-5b8aa5a2d2c872e8321cf37308d69df2-051581bf3cb55c13-01"+ .parse()+ .unwrap(),+ );++ let cx = opentelemetry::propagation::TextMapPropagator::extract(+ &propagator,+ &opentelemetry_http::HeaderExtractor(&headers),+ );++ let span_context = cx.span().span_context().clone();+ assert!(span_context.is_valid());+ assert!(span_context.is_remote());+ assert_eq!(+ span_context.trace_id().to_string(),+ "5b8aa5a2d2c872e8321cf37308d69df2"+ );+ assert_eq!(span_context.span_id().to_string(), "051581bf3cb55c13");+}++#[test]+fn request_span_adopts_remote_traceparent() {+ use opentelemetry::trace::TracerProvider as _;+ use tracing_subscriber::layer::SubscriberExt;++ opentelemetry::global::set_text_map_propagator(+ opentelemetry_sdk::propagation::TraceContextPropagator::new(),+ );+ // Provider with no exporter: spans are created (so parentage is real) but+ // never shipped anywhere.+ let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder().build();+ let subscriber = tracing_subscriber::registry()+ .with(tracing_opentelemetry::layer().with_tracer(provider.tracer("test")));++ with_default(subscriber, || {+ let request = http::Request::builder()+ .uri("/test")+ .header(+ "traceparent",+ "00-5b8aa5a2d2c872e8321cf37308d69df2-051581bf3cb55c13-01",+ )+ .body(())+ .unwrap();++ let span = MakeSpanWithRemoteParent.make_span(&request);+ let trace_id = span.context().span().span_context().trace_id();+ assert_eq!(+ trace_id.to_string(),+ "5b8aa5a2d2c872e8321cf37308d69df2",+ "request span should continue the remote trace"+ );+ });+}++#[test]+fn request_span_without_traceparent_stays_root() {+ use opentelemetry::trace::TracerProvider as _;+ use tracing_subscriber::layer::SubscriberExt;++ let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder().build();+ let subscriber = tracing_subscriber::registry()+ .with(tracing_opentelemetry::layer().with_tracer(provider.tracer("test")));++ with_default(subscriber, || {+ let request = http::Request::builder().uri("/test").body(()).unwrap();+ let span = MakeSpanWithRemoteParent.make_span(&request);+ let trace_id = span.context().span().span_context().trace_id();+ assert_ne!(+ trace_id.to_string(),+ "5b8aa5a2d2c872e8321cf37308d69df2",+ "no traceparent should mean a fresh trace"+ );+ });+}+#[tokio::test]async fn starvation_detector_does_not_warn_within_grace_period() {tokio::time::pause();diff --git a/crates/worker-rs-otel/Cargo.toml b/crates/worker-rs-otel/Cargo.tomlnew file mode 100644index 0000000000..e261b56572--- /dev/null+++ b/crates/worker-rs-otel/Cargo.toml@@ -0,0 +1,23 @@+[package]+name = "worker-rs-otel"+version = "0.1.0"+edition = "2024"+publish = false++[dependencies]+getrandom = { workspace = true, features = ["wasm_js"] }+http.workspace = true+opentelemetry = { workspace = true, features = ["trace"] }+opentelemetry-proto = { version = "0.31.0", default-features = false, features = [+ "gen-tonic-messages",+ "logs",+ "with-serde",+ "trace",+] }+opentelemetry_sdk = { workspace = true, features = ["trace"] }+serde_json.workspace = true+tracing.workspace = true+tracing-subscriber.workspace = true+uuid = { workspace = true, features = ["js"] }+web-time.workspace = true+worker.workspace = truediff --git a/crates/worker-rs-otel/src/exporter/mod.rs b/crates/worker-rs-otel/src/exporter/mod.rsnew file mode 100644index 0000000000..524ae1de73--- /dev/null+++ b/crates/worker-rs-otel/src/exporter/mod.rs@@ -0,0 +1,136 @@+use std::{cell::RefCell, future::Future};++use super::WaitUntil;+use super::model::{ClosedLog, ClosedSpan};+use super::proto::{export_logs_request, export_traces_request};++const ENDPOINT_VAR: &str = "OTEL_EXPORTER_OTLP_ENDPOINT";+const ENVIRONMENT_VAR: &str = "ENVIRONMENT";+const MAX_BUFFERED_SPANS: usize = 512;+const MAX_BUFFERED_LOGS: usize = 512;++struct ExporterConfig {+ endpoint: Option<String>,+ environment: Option<String>,+}++thread_local! {+ static SPAN_BUFFER: RefCell<Vec<ClosedSpan>> = const { RefCell::new(Vec::new()) };+ static LOG_BUFFER: RefCell<Vec<ClosedLog>> = const { RefCell::new(Vec::new()) };+ static CONFIG: RefCell<Option<ExporterConfig>> = const { RefCell::new(None) };+}++pub(super) fn configure(env: &worker::Env) {+ CONFIG.with(|config| {+ let mut config = config.borrow_mut();+ if config.is_some() {+ return;+ }+ let get = |name: &str| {+ env.var(name)+ .ok()+ .map(|value| value.to_string())+ .filter(|value| !value.is_empty())+ };+ *config = Some(ExporterConfig {+ endpoint: get(ENDPOINT_VAR),+ environment: get(ENVIRONMENT_VAR),+ });+ });+}++fn signal_endpoint(endpoint: &str, signal: &str) -> String {+ format!("{}/v1/{signal}", endpoint.trim_end_matches('/'))+}++pub(super) fn buffer_span(span: ClosedSpan) {+ SPAN_BUFFER.with(|buffer| {+ let mut buffer = buffer.borrow_mut();+ if buffer.len() < MAX_BUFFERED_SPANS {+ buffer.push(span);+ }+ });+}++pub(super) fn buffer_log(log: ClosedLog) {+ LOG_BUFFER.with(|buffer| {+ let mut buffer = buffer.borrow_mut();+ if buffer.len() < MAX_BUFFERED_LOGS {+ buffer.push(log);+ }+ });+}++fn flush() -> Option<impl Future<Output = ()> + 'static> {+ let spans = SPAN_BUFFER.with(|buffer| std::mem::take(&mut *buffer.borrow_mut()));+ let logs = LOG_BUFFER.with(|buffer| std::mem::take(&mut *buffer.borrow_mut()));+ if spans.is_empty() && logs.is_empty() {+ return None;+ }+ let (endpoint, environment) = CONFIG.with(|config| {+ let config = config.borrow();+ let config = config.as_ref()?;+ Some((config.endpoint.clone()?, config.environment.clone()))+ })?;++ let traces = if spans.is_empty() {+ None+ } else {+ Some((+ signal_endpoint(&endpoint, "traces"),+ serde_json::to_string(&export_traces_request(spans, environment.as_deref())).ok()?,+ ))+ };+ let logs = if logs.is_empty() {+ None+ } else {+ Some((+ signal_endpoint(&endpoint, "logs"),+ serde_json::to_string(&export_logs_request(logs, environment.as_deref())).ok()?,+ ))+ };++ Some(async move {+ if let Some((endpoint, body)) = traces {+ _ = post_signal(&endpoint, body)+ .await+ .inspect_err(|error| tracing::debug!(error = ?error, "otel trace export failed"));+ }+ if let Some((endpoint, body)) = logs {+ _ = post_signal(&endpoint, body)+ .await+ .inspect_err(|error| tracing::debug!(error = ?error, "otel log export failed"));+ }+ })+}++pub(super) fn flush_into(wait_until: &impl WaitUntil) {+ if let Some(export) = flush() {+ wait_until.wait_until(Box::pin(export));+ }+}++async fn post_signal(endpoint: &str, body: String) -> worker::Result<()> {+ use worker::{Fetch, Method, Request, RequestInit};++ let mut request = Request::new_with_init(+ endpoint,+ RequestInit::new()+ .with_method(Method::Post)+ .with_body(Some(body.into())),+ )?;+ request+ .headers_mut()?+ .set("Content-Type", "application/json")?;+ let response = Fetch::Request(request).send().await?;+ if response.status_code() >= 300 {+ return Err(worker::Error::from(format!(+ "otel exporter returned {}",+ response.status_code()+ )));+ }+ Ok(())+}++#[cfg(test)]+mod test;diff --git a/crates/worker-rs-otel/src/exporter/test.rs b/crates/worker-rs-otel/src/exporter/test.rsnew file mode 100644index 0000000000..e375afcc29--- /dev/null+++ b/crates/worker-rs-otel/src/exporter/test.rs@@ -0,0 +1,13 @@+use super::signal_endpoint;++#[test]+fn appends_signal_path_to_collector_endpoint() {+ assert_eq!(+ signal_endpoint("http://otel-collector:4318/", "traces"),+ "http://otel-collector:4318/v1/traces"+ );+ assert_eq!(+ signal_endpoint("http://otel-collector:4318/", "logs"),+ "http://otel-collector:4318/v1/logs"+ );+}diff --git a/crates/worker-rs-otel/src/layer/mod.rs b/crates/worker-rs-otel/src/layer/mod.rsnew file mode 100644index 0000000000..65ec2aa01f--- /dev/null+++ b/crates/worker-rs-otel/src/layer/mod.rs@@ -0,0 +1,206 @@+use tracing::field::{Field, Visit};+use tracing_subscriber::{+ Layer,+ layer::Context as LayerContext,+ registry::{LookupSpan, Registry},+};++use super::exporter::{buffer_log, buffer_span};+use super::model::{+ ClosedLog, ClosedSpan, ClosedSpanEvent, LiveSpan, now_unix_nanos, random_span_id,+ random_trace_id,+};+use super::trace_context::{parse_span_id, parse_trace_id, sampled_traceparent};+use super::{REMOTE_PARENT_FIELD, REMOTE_TRACE_ID_FIELD};++#[derive(Default)]+struct FieldVisitor {+ remote_trace_id: Option<[u8; 16]>,+ remote_parent: Option<[u8; 8]>,+ fields: Vec<(String, String)>,+}++impl FieldVisitor {+ fn record(&mut self, field: &Field, value: String) {+ match field.name() {+ REMOTE_TRACE_ID_FIELD => self.remote_trace_id = parse_trace_id(&value),+ REMOTE_PARENT_FIELD => self.remote_parent = parse_span_id(&value),+ name => self.fields.push((name.to_string(), value)),+ }+ }+}++macro_rules! string_visit {+ ($ty:ty) => {+ impl Visit for $ty {+ fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) {+ self.record(field, format!("{value:?}"));+ }++ fn record_str(&mut self, field: &Field, value: &str) {+ self.record(field, value.to_string());+ }+ }+ };+}++string_visit!(FieldVisitor);++#[derive(Default)]+struct EventVisitor {+ message: Option<String>,+ fields: Vec<(String, String)>,+}++impl EventVisitor {+ fn record(&mut self, field: &Field, value: String) {+ if field.name() == "message" {+ self.message = Some(value);+ } else {+ self.fields.push((field.name().to_string(), value));+ }+ }+}++string_visit!(EventVisitor);++/// A tracing-subscriber layer that exports closed spans through OTLP/HTTP.+pub struct OtelLayer {+ service_name: &'static str,+}++impl OtelLayer {+ /// Create a layer that identifies exported resources as `service_name`.+ pub const fn new(service_name: &'static str) -> Self {+ Self { service_name }+ }+}++/// Build a sampled W3C `traceparent` value for a span managed by this layer.+pub fn traceparent_for_span(span: &tracing::Span) -> Option<String> {+ span.with_subscriber(|(id, dispatch)| {+ let registry = dispatch.downcast_ref::<Registry>()?;+ let span = registry.span(id)?;+ let extensions = span.extensions();+ let data = extensions.get::<LiveSpan>()?;+ sampled_traceparent(data.trace_id, data.span_id)+ })+ .flatten()+}++impl<S> Layer<S> for OtelLayer+where+ S: tracing::Subscriber + for<'a> LookupSpan<'a>,+{+ fn on_new_span(+ &self,+ attrs: &tracing::span::Attributes<'_>,+ id: &tracing::span::Id,+ ctx: LayerContext<'_, S>,+ ) {+ let Some(span) = ctx.span(id) else { return };++ let mut visitor = FieldVisitor::default();+ attrs.record(&mut visitor);++ let parent = span.parent().and_then(|p| {+ let ext = p.extensions();+ ext.get::<LiveSpan>().map(|d| (d.trace_id, d.span_id))+ });++ let (trace_id, parent_span_id, local_root) = match parent {+ Some((trace_id, parent_span_id)) => (trace_id, Some(parent_span_id), false),+ None => match visitor.remote_trace_id {+ Some(remote_trace_id) => (remote_trace_id, visitor.remote_parent, true),+ None => (random_trace_id(), None, true),+ },+ };++ span.extensions_mut().insert(LiveSpan {+ trace_id,+ span_id: random_span_id(),+ parent_span_id,+ local_root,+ start_ns: now_unix_nanos(),+ attrs: visitor.fields,+ events: Vec::new(),+ error_message: None,+ });+ }++ fn on_event(&self, event: &tracing::Event<'_>, ctx: LayerContext<'_, S>) {+ if *event.metadata().level() > tracing::Level::INFO {+ return;+ }+ let mut visitor = EventVisitor::default();+ event.record(&mut visitor);+ let meta = event.metadata();+ let level = *meta.level();+ let name = visitor.message.unwrap_or_else(|| meta.name().to_string());+ let mut attrs = visitor.fields;+ attrs.push(("level".to_string(), level.as_str().to_string()));+ let time_ns = now_unix_nanos();+ let trace_context = ctx.event_span(event).and_then(|span| {+ let mut extensions = span.extensions_mut();+ let data = extensions.get_mut::<LiveSpan>()?;+ if level == tracing::Level::ERROR && data.error_message.is_none() {+ data.error_message = Some(name.clone());+ }+ data.events.push(ClosedSpanEvent {+ name: name.clone(),+ time_ns,+ attrs: attrs.clone(),+ });+ Some((data.trace_id, data.span_id))+ });+ let (trace_id, span_id) = trace_context+ .map(|(trace_id, span_id)| (Some(trace_id), Some(span_id)))+ .unwrap_or_default();+ buffer_log(ClosedLog {+ service_name: self.service_name,+ time_ns,+ level,+ body: name,+ attrs,+ target: meta.target(),+ file: meta.file(),+ line: meta.line(),+ trace_id,+ span_id,+ });+ }++ fn on_record(+ &self,+ id: &tracing::span::Id,+ values: &tracing::span::Record<'_>,+ ctx: LayerContext<'_, S>,+ ) {+ let Some(span) = ctx.span(id) else { return };+ let mut visitor = FieldVisitor::default();+ values.record(&mut visitor);+ if let Some(data) = span.extensions_mut().get_mut::<LiveSpan>() {+ data.attrs.append(&mut visitor.fields);+ }+ }++ fn on_close(&self, id: tracing::span::Id, ctx: LayerContext<'_, S>) {+ let Some(span) = ctx.span(&id) else { return };+ let Some(data) = span.extensions_mut().remove::<LiveSpan>() else {+ return;+ };+ let meta = span.metadata();+ buffer_span(ClosedSpan {+ data,+ service_name: self.service_name,+ name: meta.name(),+ level: meta.level().as_str(),+ file: meta.file(),+ line: meta.line(),+ end_ns: now_unix_nanos(),+ });+ }+}++#[cfg(test)]+mod test;diff --git a/crates/worker-rs-otel/src/layer/test.rs b/crates/worker-rs-otel/src/layer/test.rsnew file mode 100644index 0000000000..ac18768024--- /dev/null+++ b/crates/worker-rs-otel/src/layer/test.rs@@ -0,0 +1,17 @@+use tracing_subscriber::layer::SubscriberExt;++use super::{OtelLayer, traceparent_for_span};+use crate::parse_traceparent;++#[test]+fn creates_traceparent_for_managed_span() {+ let subscriber = tracing_subscriber::registry().with(OtelLayer::new("test-service"));+ let dispatch = tracing::Dispatch::new(subscriber);++ tracing::dispatcher::with_default(&dispatch, || {+ let span = tracing::info_span!("outbound.request");+ let traceparent = traceparent_for_span(&span).expect("managed span context");+ assert!(parse_traceparent(&traceparent).is_some());+ assert!(traceparent.ends_with("-01"));+ });+}diff --git a/crates/worker-rs-otel/src/lib.rs b/crates/worker-rs-otel/src/lib.rsnew file mode 100644index 0000000000..3aae8fe6c6--- /dev/null+++ b/crates/worker-rs-otel/src/lib.rs@@ -0,0 +1,52 @@+//! OpenTelemetry tracing support for Cloudflare Workers built with worker-rs.++mod exporter;+mod layer;+mod model;+mod proto;+mod trace_context;++use std::{future::Future, pin::Pin};++pub use layer::{OtelLayer, traceparent_for_span};+pub use trace_context::{+ HeadersLike, TRACEPARENT, parse_traceparent, remote_fields, traceparent_from_headers,+ traceparent_from_request,+};++/// Reserved span field containing a 32-character hexadecimal remote trace ID.+pub const REMOTE_TRACE_ID_FIELD: &str = "trace.remote_id";+/// Reserved span field containing a 16-character hexadecimal remote parent span ID.+pub const REMOTE_PARENT_FIELD: &str = "trace.remote_parent";++type ExportFuture = Pin<Box<dyn Future<Output = ()> + 'static>>;++/// Registers background work against the current Cloudflare invocation.+pub trait WaitUntil {+ /// Keep the invocation alive until `future` completes.+ fn wait_until(&self, future: ExportFuture);+}++impl WaitUntil for worker::Context {+ fn wait_until(&self, future: ExportFuture) {+ worker::Context::wait_until(self, future);+ }+}++impl WaitUntil for worker::State {+ fn wait_until(&self, future: ExportFuture) {+ worker::State::wait_until(self, future);+ }+}++/// Run one Worker invocation and schedule its buffered telemetry for export.+pub async fn scope<T>(+ env: &worker::Env,+ wait_until: &impl WaitUntil,+ future: impl Future<Output = T>,+) -> T {+ exporter::configure(env);+ let output = future.await;+ exporter::flush_into(wait_until);+ output+}diff --git a/crates/worker-rs-otel/src/model.rs b/crates/worker-rs-otel/src/model.rsnew file mode 100644index 0000000000..e88dae5d79--- /dev/null+++ b/crates/worker-rs-otel/src/model.rs@@ -0,0 +1,57 @@+use web_time::{SystemTime, UNIX_EPOCH};++pub(super) struct ClosedSpanEvent {+ pub(super) name: String,+ pub(super) time_ns: u64,+ pub(super) attrs: Vec<(String, String)>,+}++pub(super) struct ClosedSpan {+ pub(super) data: LiveSpan,+ pub(super) service_name: &'static str,+ pub(super) name: &'static str,+ pub(super) level: &'static str,+ pub(super) file: Option<&'static str>,+ pub(super) line: Option<u32>,+ pub(super) end_ns: u64,+}++pub(super) struct ClosedLog {+ pub(super) service_name: &'static str,+ pub(super) time_ns: u64,+ pub(super) level: tracing::Level,+ pub(super) body: String,+ pub(super) attrs: Vec<(String, String)>,+ pub(super) target: &'static str,+ pub(super) file: Option<&'static str>,+ pub(super) line: Option<u32>,+ pub(super) trace_id: Option<[u8; 16]>,+ pub(super) span_id: Option<[u8; 8]>,+}++pub(super) struct LiveSpan {+ pub(super) trace_id: [u8; 16],+ pub(super) span_id: [u8; 8],+ pub(super) parent_span_id: Option<[u8; 8]>,+ pub(super) local_root: bool,+ pub(super) start_ns: u64,+ pub(super) attrs: Vec<(String, String)>,+ pub(super) events: Vec<ClosedSpanEvent>,+ pub(super) error_message: Option<String>,+}++pub(super) fn now_unix_nanos() -> u64 {+ SystemTime::now()+ .duration_since(UNIX_EPOCH)+ .map(|d| d.as_nanos() as u64)+ .unwrap_or_default()+}++pub(super) fn random_trace_id() -> [u8; 16] {+ uuid::Uuid::new_v4().into_bytes()+}++pub(super) fn random_span_id() -> [u8; 8] {+ let bytes = uuid::Uuid::new_v4().into_bytes();+ bytes[8..16].try_into().expect("8-byte slice")+}diff --git a/crates/worker-rs-otel/src/proto/mod.rs b/crates/worker-rs-otel/src/proto/mod.rsnew file mode 100644index 0000000000..4549b7b1f2--- /dev/null+++ b/crates/worker-rs-otel/src/proto/mod.rs@@ -0,0 +1,188 @@+use opentelemetry_proto::tonic::{+ collector::{logs::v1::ExportLogsServiceRequest, trace::v1::ExportTraceServiceRequest},+ common::v1::{AnyValue, InstrumentationScope, KeyValue, any_value},+ logs::v1::{LogRecord, ResourceLogs, ScopeLogs, SeverityNumber},+ resource::v1::Resource,+ trace::v1::{+ ResourceSpans, ScopeSpans, Span, Status, span::Event, span::SpanKind, status::StatusCode,+ },+};++use super::model::{ClosedLog, ClosedSpan};++fn string_value(value: &str) -> AnyValue {+ AnyValue {+ value: Some(any_value::Value::StringValue(value.to_string())),+ }+}++fn string_kv(key: &str, value: &str) -> KeyValue {+ KeyValue {+ key: key.to_string(),+ value: Some(string_value(value)),+ }+}++fn span_proto(span: ClosedSpan) -> Span {+ let data = span.data;+ let mut attributes: Vec<KeyValue> = data+ .attrs+ .into_iter()+ .map(|(key, value)| string_kv(&key, &value))+ .collect();+ attributes.push(string_kv("level", span.level));+ if let Some(file) = span.file {+ attributes.push(string_kv("code.filepath", file));+ }+ if let Some(line) = span.line {+ attributes.push(KeyValue {+ key: "code.lineno".to_string(),+ value: Some(AnyValue {+ value: Some(any_value::Value::IntValue(i64::from(line))),+ }),+ });+ }+ let events = data+ .events+ .into_iter()+ .map(|event| Event {+ time_unix_nano: event.time_ns,+ name: event.name,+ attributes: event+ .attrs+ .into_iter()+ .map(|(key, value)| string_kv(&key, &value))+ .collect(),+ ..Default::default()+ })+ .collect();+ Span {+ trace_id: data.trace_id.to_vec(),+ span_id: data.span_id.to_vec(),+ parent_span_id: data.parent_span_id.map(|p| p.to_vec()).unwrap_or_default(),+ name: span.name.to_string(),+ kind: if data.local_root {+ SpanKind::Server+ } else {+ SpanKind::Internal+ } as i32,+ start_time_unix_nano: data.start_ns,+ end_time_unix_nano: span.end_ns,+ flags: 1,+ attributes,+ events,+ status: data.error_message.map(|message| Status {+ code: StatusCode::Error as i32,+ message,+ }),+ ..Default::default()+ }+}++pub(super) fn export_traces_request(+ spans: Vec<ClosedSpan>,+ environment: Option<&str>,+) -> ExportTraceServiceRequest {+ let service_name = spans+ .first()+ .map(|span| span.service_name)+ .unwrap_or("unknown-service");+ let mut resource_attrs = vec![string_kv("service.name", service_name)];+ if let Some(environment) = environment {+ resource_attrs.push(string_kv("deployment.environment", environment));+ }+ ExportTraceServiceRequest {+ resource_spans: vec![ResourceSpans {+ resource: Some(Resource {+ attributes: resource_attrs,+ ..Default::default()+ }),+ scope_spans: vec![ScopeSpans {+ scope: Some(InstrumentationScope {+ name: service_name.to_string(),+ ..Default::default()+ }),+ spans: spans.into_iter().map(span_proto).collect(),+ ..Default::default()+ }],+ ..Default::default()+ }],+ }+}++fn severity_number(level: tracing::Level) -> SeverityNumber {+ match level {+ tracing::Level::ERROR => SeverityNumber::Error,+ tracing::Level::WARN => SeverityNumber::Warn,+ tracing::Level::INFO => SeverityNumber::Info,+ tracing::Level::DEBUG => SeverityNumber::Debug,+ tracing::Level::TRACE => SeverityNumber::Trace,+ }+}++fn log_proto(log: ClosedLog) -> LogRecord {+ let correlated = log.trace_id.is_some();+ let mut attributes: Vec<KeyValue> = log+ .attrs+ .into_iter()+ .map(|(key, value)| string_kv(&key, &value))+ .collect();+ attributes.push(string_kv("target", log.target));+ if let Some(file) = log.file {+ attributes.push(string_kv("code.filepath", file));+ }+ if let Some(line) = log.line {+ attributes.push(KeyValue {+ key: "code.lineno".to_string(),+ value: Some(AnyValue {+ value: Some(any_value::Value::IntValue(i64::from(line))),+ }),+ });+ }+ LogRecord {+ time_unix_nano: log.time_ns,+ observed_time_unix_nano: log.time_ns,+ severity_number: severity_number(log.level) as i32,+ severity_text: log.level.as_str().to_string(),+ body: Some(string_value(&log.body)),+ attributes,+ flags: u32::from(correlated),+ trace_id: log.trace_id.map(|id| id.to_vec()).unwrap_or_default(),+ span_id: log.span_id.map(|id| id.to_vec()).unwrap_or_default(),+ ..Default::default()+ }+}++pub(super) fn export_logs_request(+ logs: Vec<ClosedLog>,+ environment: Option<&str>,+) -> ExportLogsServiceRequest {+ let service_name = logs+ .first()+ .map(|log| log.service_name)+ .unwrap_or("unknown-service");+ let mut resource_attrs = vec![string_kv("service.name", service_name)];+ if let Some(environment) = environment {+ resource_attrs.push(string_kv("deployment.environment", environment));+ }+ ExportLogsServiceRequest {+ resource_logs: vec![ResourceLogs {+ resource: Some(Resource {+ attributes: resource_attrs,+ ..Default::default()+ }),+ scope_logs: vec![ScopeLogs {+ scope: Some(InstrumentationScope {+ name: service_name.to_string(),+ ..Default::default()+ }),+ log_records: logs.into_iter().map(log_proto).collect(),+ ..Default::default()+ }],+ ..Default::default()+ }],+ }+}++#[cfg(test)]+mod test;diff --git a/crates/worker-rs-otel/src/proto/test.rs b/crates/worker-rs-otel/src/proto/test.rsnew file mode 100644index 0000000000..1f6815aaa8--- /dev/null+++ b/crates/worker-rs-otel/src/proto/test.rs@@ -0,0 +1,32 @@+use opentelemetry_proto::tonic::{common::v1::any_value, logs::v1::SeverityNumber};++use super::export_logs_request;+use crate::model::ClosedLog;++#[test]+fn builds_correlated_log_record() {+ let request = export_logs_request(+ vec![ClosedLog {+ service_name: "sync-service",+ time_ns: 42,+ level: tracing::Level::WARN,+ body: "peer send failed".to_string(),+ attrs: vec![("document.id".to_string(), "doc-1".to_string())],+ target: "sync_service::websocket",+ file: Some("src/websocket.rs"),+ line: Some(10),+ trace_id: Some([1; 16]),+ span_id: Some([2; 8]),+ }],+ Some("test"),+ );++ let record = &request.resource_logs[0].scope_logs[0].log_records[0];+ assert_eq!(record.severity_number, SeverityNumber::Warn as i32);+ assert_eq!(record.trace_id, vec![1; 16]);+ assert_eq!(record.span_id, vec![2; 8]);+ assert!(matches!(+ record.body.as_ref().and_then(|body| body.value.as_ref()),+ Some(any_value::Value::StringValue(value)) if value == "peer send failed"+ ));+}diff --git a/crates/worker-rs-otel/src/trace_context/mod.rs b/crates/worker-rs-otel/src/trace_context/mod.rsnew file mode 100644index 0000000000..6be0e06ee8--- /dev/null+++ b/crates/worker-rs-otel/src/trace_context/mod.rs@@ -0,0 +1,98 @@+use std::{collections::HashMap, hash::BuildHasher};++use opentelemetry::Context;+use opentelemetry::propagation::TextMapPropagator;+use opentelemetry::trace::{SpanContext, SpanId, TraceContextExt, TraceFlags, TraceId, TraceState};+use opentelemetry_sdk::propagation::TraceContextPropagator;++/// W3C trace-context header and WebSocket query parameter.+pub const TRACEPARENT: &str = "traceparent";++/// Header collection that can provide values for trace-context propagation.+pub trait HeadersLike {+ /// Return a header value as text.+ fn header(&self, name: &str) -> Option<String>;+}++impl<S: BuildHasher> HeadersLike for HashMap<String, String, S> {+ fn header(&self, name: &str) -> Option<String> {+ self.get(name).cloned()+ }+}++impl HeadersLike for http::HeaderMap {+ fn header(&self, name: &str) -> Option<String> {+ self.get(name)?.to_str().ok().map(str::to_owned)+ }+}++impl HeadersLike for worker::Headers {+ fn header(&self, name: &str) -> Option<String> {+ self.get(name).ok().flatten()+ }+}++/// Parse a valid W3C `traceparent` value into OpenTelemetry's span context.+pub fn parse_traceparent(value: &str) -> Option<SpanContext> {+ let carrier = HashMap::from([(TRACEPARENT.to_string(), value.to_string())]);+ let context = TraceContextPropagator::new().extract(&carrier);+ let span_context = context.span().span_context().clone();+ span_context.is_valid().then_some(span_context)+}++pub(super) fn sampled_traceparent(trace_id: [u8; 16], span_id: [u8; 8]) -> Option<String> {+ let span_context = SpanContext::new(+ TraceId::from_bytes(trace_id),+ SpanId::from_bytes(span_id),+ TraceFlags::SAMPLED,+ false,+ TraceState::default(),+ );+ let context = Context::new().with_remote_span_context(span_context);+ let mut carrier = HashMap::new();+ TraceContextPropagator::new().inject_context(&context, &mut carrier);+ carrier.remove(TRACEPARENT)+}++/// Extract remote trace context from a supported header collection.+pub fn traceparent_from_headers(headers: &impl HeadersLike) -> Option<SpanContext> {+ headers+ .header(TRACEPARENT)+ .and_then(|value| parse_traceparent(&value))+}++/// Extract parsed remote trace context from a Worker request.+pub fn traceparent_from_request(request: &worker::Request) -> Option<SpanContext> {+ traceparent_from_headers(request.headers()).or_else(|| {+ request+ .url()+ .ok()?+ .query_pairs()+ .find(|(key, _)| key == TRACEPARENT)+ .and_then(|(_, value)| parse_traceparent(&value))+ })+}++/// Convert optional remote context into reserved tracing span field values.+pub fn remote_fields(traceparent: Option<&SpanContext>) -> (String, String) {+ traceparent+ .map(|value| (value.trace_id().to_string(), value.span_id().to_string()))+ .unwrap_or_default()+}++pub(super) fn parse_trace_id(value: &str) -> Option<[u8; 16]> {+ TraceId::from_hex(value)+ .ok()+ .filter(|id| *id != TraceId::INVALID)+ .map(|id| id.to_bytes())+}++pub(super) fn parse_span_id(value: &str) -> Option<[u8; 8]> {+ SpanId::from_hex(value)+ .ok()+ .filter(|id| *id != SpanId::INVALID)+ .map(|id| id.to_bytes())+}++#[cfg(test)]+mod test;diff --git a/crates/worker-rs-otel/src/trace_context/test.rs b/crates/worker-rs-otel/src/trace_context/test.rsnew file mode 100644index 0000000000..9fdc6d5b77--- /dev/null+++ b/crates/worker-rs-otel/src/trace_context/test.rs@@ -0,0 +1,46 @@+use opentelemetry::trace::TraceFlags;++use std::collections::HashMap;++use super::{TRACEPARENT, parse_traceparent, traceparent_from_headers};++#[test]+fn parses_valid_traceparent() {+ let traceparent = parse_traceparent("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01")+ .expect("valid traceparent");++ assert_eq!(+ traceparent.trace_id().to_string(),+ "4bf92f3577b34da6a3ce929d0e0e4736"+ );+ assert_eq!(traceparent.span_id().to_string(), "00f067aa0ba902b7");+ assert_eq!(traceparent.trace_flags(), TraceFlags::SAMPLED);+}++#[test]+fn rejects_invalid_traceparent() {+ assert!(parse_traceparent("invalid").is_none());+}++#[test]+fn extracts_traceparent_from_hash_map() {+ let headers = HashMap::from([(+ TRACEPARENT.to_string(),+ "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01".to_string(),+ )]);++ assert!(traceparent_from_headers(&headers).is_some());+}++#[test]+fn extracts_traceparent_from_http_headers() {+ let mut headers = http::HeaderMap::new();+ headers.insert(+ TRACEPARENT,+ "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"+ .parse()+ .expect("valid header value"),+ );++ assert!(traceparent_from_headers(&headers).is_some());+}diff --git a/crates/workspace-hack/Cargo.toml b/crates/workspace-hack/Cargo.tomlindex 370cd52490..c7a434b5d2 100644--- a/crates/workspace-hack/Cargo.toml+++ b/crates/workspace-hack/Cargo.toml@@ -80,8 +80,9 @@ num-bigint = { version = "0.4" }num-integer = { version = "0.1", features = ["i128"] }num-traits = { version = "0.2", features = ["i128", "libm"] }once_cell = { version = "1" }-opentelemetry = { version = "0.31", features = ["testing"] }-opentelemetry_sdk = { version = "0.31", features = ["testing"] }+opentelemetry = { version = "0.31", default-features = false, features = ["internal-logs", "logs", "metrics", "testing"] }+opentelemetry-http = { version = "0.31", default-features = false, features = ["internal-logs"] }+opentelemetry_sdk = { version = "0.31", default-features = false, features = ["internal-logs", "testing"] }percent-encoding = { version = "2" }phf = { version = "0.13", features = ["macros"] }phf_shared-594e8ee84c453af0 = { package = "phf_shared", version = "0.13" }@@ -118,6 +119,7 @@ sqlx-postgres = { version = "0.8", default-features = false, features = ["any",stable_deref_trait = { version = "1" }subtle = { version = "2" }sync_wrapper = { version = "1", default-features = false, features = ["futures"] }+thiserror = { version = "2" }time = { version = "0.3", features = ["formatting", "macros", "parsing"] }tokio = { version = "1", features = ["fs", "io-util", "macros", "net", "parking_lot", "process", "rt-multi-thread", "signal", "test-util"] }tokio-rustls = { version = "0.26", default-features = false, features = ["logging", "tls12"] }@@ -127,6 +129,8 @@ tower = { version = "0.5", default-features = false, features = ["balance", "buftower-http = { version = "0.6", features = ["compression-gzip", "cors", "fs", "limit", "request-id", "timeout", "trace", "util"] }tracing = { version = "0.1", features = ["log"] }tracing-core = { version = "0.1" }+tracing-log = { version = "0.2", default-features = false, features = ["log-tracer", "std"] }+tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }triomphe = { version = "0.1" }tungstenite = { version = "0.28", features = ["__rustls-tls", "url"] }typenum = { version = "1", default-features = false, features = ["const-generics"] }@@ -202,8 +206,9 @@ num-bigint = { version = "0.4" }num-integer = { version = "0.1", features = ["i128"] }num-traits = { version = "0.2", features = ["i128", "libm"] }once_cell = { version = "1" }-opentelemetry = { version = "0.31", features = ["testing"] }-opentelemetry_sdk = { version = "0.31", features = ["testing"] }+opentelemetry = { version = "0.31", default-features = false, features = ["internal-logs", "logs", "metrics", "testing"] }+opentelemetry-http = { version = "0.31", default-features = false, features = ["internal-logs"] }+opentelemetry_sdk = { version = "0.31", default-features = false, features = ["internal-logs", "testing"] }percent-encoding = { version = "2" }phf = { version = "0.13", features = ["macros"] }phf_shared-594e8ee84c453af0 = { package = "phf_shared", version = "0.13" }@@ -245,6 +250,7 @@ subtle = { version = "2" }syn-dff4ba8e3ae991db = { package = "syn", version = "1", features = ["extra-traits", "fold", "full"] }syn-f595c2ba2a3f28df = { package = "syn", version = "2", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] }sync_wrapper = { version = "1", default-features = false, features = ["futures"] }+thiserror = { version = "2" }time = { version = "0.3", features = ["formatting", "macros", "parsing"] }tokio = { version = "1", features = ["fs", "io-util", "macros", "net", "parking_lot", "process", "rt-multi-thread", "signal", "test-util"] }tokio-rustls = { version = "0.26", default-features = false, features = ["logging", "tls12"] }@@ -254,6 +260,8 @@ tower = { version = "0.5", default-features = false, features = ["balance", "buftower-http = { version = "0.6", features = ["compression-gzip", "cors", "fs", "limit", "request-id", "timeout", "trace", "util"] }tracing = { version = "0.1", features = ["log"] }tracing-core = { version = "0.1" }+tracing-log = { version = "0.2", default-features = false, features = ["log-tracer", "std"] }+tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }triomphe = { version = "0.1" }tungstenite = { version = "0.28", features = ["__rustls-tls", "url"] }typenum = { version = "1", default-features = false, features = ["const-generics"] }@@ -274,7 +282,6 @@ linux-raw-sys = { version = "0.12", default-features = false, features = ["auxveonce_cell = { version = "1", default-features = false, features = ["critical-section"] }rand-93f6ce9d446188ac = { package = "rand", version = "0.10" }rustix = { version = "1", features = ["event", "fs", "net", "pipe", "process", "system", "time"] }-thiserror = { version = "2" }tokio-rustls = { version = "0.26", default-features = false, features = ["aws-lc-rs", "ring"] }tokio-util = { version = "0.7", default-features = false, features = ["io-util"] }tower = { version = "0.5", default-features = false, features = ["retry", "timeout"] }@@ -292,7 +299,6 @@ linux-raw-sys = { version = "0.12", default-features = false, features = ["auxveonce_cell = { version = "1", default-features = false, features = ["critical-section"] }rand-93f6ce9d446188ac = { package = "rand", version = "0.10" }rustix = { version = "1", features = ["event", "fs", "net", "pipe", "process", "system", "time"] }-thiserror = { version = "2" }tokio-rustls = { version = "0.26", default-features = false, features = ["aws-lc-rs", "ring"] }tokio-util = { version = "0.7", default-features = false, features = ["io-util"] }tower = { version = "0.5", default-features = false, features = ["retry", "timeout"] }@@ -311,7 +317,6 @@ once_cell = { version = "1", default-features = false, features = ["critical-secrand-93f6ce9d446188ac = { package = "rand", version = "0.10" }rustix = { version = "1", features = ["event", "fs", "net", "pipe", "process", "system", "time"] }security-framework-sys = { version = "2" }-thiserror = { version = "2" }tokio-rustls = { version = "0.26", default-features = false, features = ["aws-lc-rs", "ring"] }tokio-util = { version = "0.7", default-features = false, features = ["io-util"] }tower = { version = "0.5", default-features = false, features = ["retry", "timeout"] }@@ -330,7 +335,6 @@ once_cell = { version = "1", default-features = false, features = ["critical-secrand-93f6ce9d446188ac = { package = "rand", version = "0.10" }rustix = { version = "1", features = ["event", "fs", "net", "pipe", "process", "system", "time"] }security-framework-sys = { version = "2" }-thiserror = { version = "2" }tokio-rustls = { version = "0.26", default-features = false, features = ["aws-lc-rs", "ring"] }tokio-util = { version = "0.7", default-features = false, features = ["io-util"] }tower = { version = "0.5", default-features = false, features = ["retry", "timeout"] }diff --git a/deny.toml b/deny.tomlindex b9d53a22e5..3a999ecc37 100644--- a/deny.toml+++ b/deny.toml@@ -201,6 +201,7 @@ deny = ["lambda_runtime_api_client", # External crate allowed to use tracing-subscriber"loom", # External concurrency testing dependency"macro_entrypoint",+ "macro_tower_layers", # OpenTelemetry layer tests"native_app_server", # Dev tooling only"notification_sandbox", # This is not production code its just tooling"projects", # Dev logging only@@ -210,6 +211,8 @@ deny = ["tracing-panic", # External crate allowed to use tracing-subscriber"tracing-tree","tracing-web", # External WASM tracing layer+ "worker-rs-otel", # Cloudflare/WASM OpenTelemetry integration+ "workspace-hack", # cargo-hakari generated crate], reason = "Use macro_entrypoint for tracing setup in workspace crates" },]@@ -334,7 +337,7 @@ allow-git = ["git+https://github.com/whutchinson98/jsonwebtoken"][sources.allow-org]# github.com organizations to allow git sources for-github = []+github = ["macro-inc"]# gitlab.com organizations to allow git sources forgitlab = []# bitbucket.org organizations to allow git sources fordiff --git a/docker/analytics-proxy.Dockerfile b/docker/analytics-proxy.Dockerfilenew file mode 100644index 0000000000..7457e1b096--- /dev/null+++ b/docker/analytics-proxy.Dockerfile@@ -0,0 +1,22 @@+FROM node:22-slim+RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates \+ && rm -rf /var/lib/apt/lists/*+RUN npm install -g bun++ENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt++WORKDIR /app/services/analytics-proxy+EXPOSE 8098++# The repo is bind-mounted at /app (see docker-compose.yml), so wrangler and+# node_modules come from the host install. Forward OTLP (both signals) to the+# local collector (--var overrides the wrangler.jsonc host defaults); no+# DD_API_KEY locally, so the worker skips key injection.+CMD ["sh", "-c", "\+ npx wrangler dev \+ --env local \+ --ip 0.0.0.0 \+ --port 8098 \+ --var OTLP_TRACES_INTAKE_URL:http://otel-collector:4318 \+ --var OTLP_LOGS_INTAKE_URL:http://otel-collector:4318\+"]diff --git a/docker/datadog-agent/datadog.yaml b/docker/datadog-agent/datadog.yamlnew file mode 100644index 0000000000..998b914aac--- /dev/null+++ b/docker/datadog-agent/datadog.yaml@@ -0,0 +1,20 @@+# Config for the local dev Datadog agent (docker-compose.tracing.yml).+# api_key and site come from the container environment, which overrides+# this file. This file exists because the OTLP receiver's CORS settings+# (required for browser -> agent requests) can't be set via env vars.+# Required for the agent to forward OTLP log records to Datadog Logs.+logs_enabled: true+otlp_config:+ receiver:+ protocols:+ grpc:+ endpoint: 0.0.0.0:4317+ http:+ endpoint: 0.0.0.0:4318+ cors:+ allowed_origins:+ - "*"+ logs:+ enabled: true+apm_config:+ enabled: truediff --git a/docker/docker-compose.yml b/docker/docker-compose.ymlindex 3d88dca340..24f7eb05d3 100644--- a/docker/docker-compose.yml+++ b/docker/docker-compose.yml@@ -406,6 +406,11 @@ services:- INTERNAL_API_SECRET_KEY=${INTERNAL_API_SECRET_KEY}- DOCUMENT_PERMISSIONS_SECRET=${DOCUMENT_PERMISSION_JWT}- DSS_INTERNAL_AUTH_KEY=${DOCUMENT_STORAGE_SERVICE_AUTH_KEY}+ # This worker exports OTLP/JSON over HTTP, so it needs the+ # HTTP intake (:4318) — NOT the gRPC :4317 endpoint the+ # shared env (env_layer) injects for the .with_tonic() Rust services.+ # Pinning it here overrides the shared env_file value for this service.+ - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318networks:services:aliases:@@ -441,6 +446,28 @@ services:aliases:- ai-editing-worker+ # ============================================================================+ # analytics_proxy (Port 8098)+ # First-party analytics/telemetry proxy (Cloudflare Worker, local wrangler+ # dev). Fronts PostHog and OTLP telemetry so ad blockers don't drop them and+ # the browser never holds the Datadog API key. Locally the+ # /i/otlp route forwards to the otel-collector alias (jaeger/datadog profile);+ # if neither profile is up, trace exports just fail harmlessly.+ # ============================================================================+ analytics_proxy:+ <<: *common-env+ build:+ context: .+ dockerfile: docker/analytics-proxy.Dockerfile+ ports:+ - "8098:8098"+ volumes:+ - .:/app+ networks:+ services:+ aliases:+ - analytics-proxy+# ============================================================================# lexical_service (Port 8096)# Lexical document conversion service - converts Lexical JSON to various formats@@ -482,6 +509,65 @@ services:retries: 3start_period: 10s+ # ============================================================================+ # jaeger (UI port 16686, OTLP 4317/4318) — profile: jaeger+ # Fully local trace viewer, no account needed. UI at http://localhost:16686.+ # Start: docker compose --project-directory . -f docker/docker-compose.yml up -d jaeger+ # ============================================================================+ jaeger:+ profiles:+ - jaeger+ image: jaegertracing/all-in-one:latest+ environment:+ COLLECTOR_OTLP_ENABLED: "true"+ # Browser -> collector requests need CORS on the OTLP HTTP receiver.+ # allowed-headers must include Content-Type: jaeger's default rejects the+ # preflight the browser OTLP exporter sends, even with origins=*.+ command:+ [+ "--collector.otlp.http.cors.allowed-origins=*",+ "--collector.otlp.http.cors.allowed-headers=*",+ ]+ ports:+ - "16686:16686"+ - "4317:4317"+ - "4318:4318"+ networks:+ services:+ aliases:+ - otel-collector++ # ============================================================================+ # datadog-agent (OTLP 4317/4318) — profile: datadog+ # Forwards OTLP traces to Datadog (us5); no local UI. View at+ # us5.datadoghq.com -> APM -> Traces, filtered by env:local (DD_ENV).+ # Binds the same OTLP ports as jaeger — run one or the other, not both.+ # Start: DD_API_KEY=<key> docker compose --project-directory . -f docker/docker-compose.yml up -d datadog-agent+ # ============================================================================+ datadog-agent:+ profiles:+ - datadog+ image: gcr.io/datadoghq/agent:7+ environment:+ # Default-empty (not :?) so the var is only required when this profile+ # actually runs; the agent itself fails fast without a key.+ DD_API_KEY: ${DD_API_KEY:-}+ DD_SITE: us5.datadoghq.com+ DD_ENV: ${DD_ENV:-local}+ DD_HOSTNAME: local-dev+ DD_APM_ENABLED: "true"+ volumes:+ # Mounted config exists to enable CORS on the OTLP HTTP receiver+ # (browser -> agent); that setting has no env-var form.+ - ./docker/datadog-agent/datadog.yaml:/etc/datadog-agent/datadog.yaml:ro+ ports:+ - "4317:4317"+ - "4318:4318"+ networks:+ services:+ aliases:+ - otel-collector+networks:services:driver: bridgediff --git a/opencode.json b/opencode.jsonnew file mode 100644index 0000000000..4dc34d5840--- /dev/null+++ b/opencode.json@@ -0,0 +1,13 @@+{+ "$schema": "https://opencode.ai/config.json",+ "plugin": [+ "@datadog/opencode-plugin"+ ],+ "mcp": {+ "datadog": {+ "type": "remote",+ "url": "https://mcp.us5.datadoghq.com/api/unstable/mcp-server/mcp?referrer_ide=opencode-plugin&plugin_version=0.7.13",+ "oauth": {}+ }+ }+}diff --git a/package.json b/package.jsonindex a5e2444a7c..4dc01acaba 100644--- a/package.json+++ b/package.json@@ -63,6 +63,7 @@"workspaces": {"packages": ["apps/web",+ "packages/observability","packages/collaboration","packages/loro-mirror","packages/lexical-core",diff --git a/packages/collaboration/package.json b/packages/collaboration/package.jsonindex a7c6a9f714..0e03a07018 100644--- a/packages/collaboration/package.json+++ b/packages/collaboration/package.json@@ -18,6 +18,7 @@},"dependencies": {"@loro-mirror/core": "workspace:*",+ "@macro-inc/observability": "workspace:*","@solid-primitives/event-bus": "^1.1.2","@tauri-apps/api": "^2.8.0","async-mutex": "^0.5.0",diff --git a/packages/collaboration/src/collab/engine.ts b/packages/collaboration/src/collab/engine.tsindex bba110415f..85c02921aa 100644--- a/packages/collaboration/src/collab/engine.ts+++ b/packages/collaboration/src/collab/engine.ts@@ -1,4 +1,5 @@import { type InferType, SyncDirection } from '@loro-mirror/core';+import type { Attributes } from '@macro-inc/observability';import { Mutex } from 'async-mutex';import type { VersionVector } from 'loro-crdt';import type { ResultAsync } from 'neverthrow';@@ -21,6 +22,7 @@ import {} from './manager';import type { GenericRootSchema, LoroRawUpdate, RawUpdate } from './shared';import type { SnapshotStore } from './snapshot-store';+import { peerCounterAttr, telemetrySpan } from './telemetry';// SnapshotStore in the engine is always Loro updates — RawUpdate.type LoroSnapshotStore = SnapshotStore<RawUpdate>;@@ -33,6 +35,15 @@ const SNAPSHOT_INTERVAL_MS = 5_000;const REQUEST_UPDATES_MAX_ATTEMPTS = 3;const REQUEST_UPDATES_RETRY_DELAY_MS = 2_000;+/** A version vector as a compact `peer:counter` span attribute. */+function vvAttr(vv: VersionVector): string {+ try {+ return peerCounterAttr(vv.toJSON().entries());+ } catch {+ return 'unavailable';+ }+}+export type EngineBindings<S extends GenericRootSchema> = {onRemoteState: (state: InferType<S>) => void;};@@ -108,7 +119,7 @@ export class SyncEngine<S extends GenericRootSchema, D> {private log(level: Parameters<typeof logSyncService>[0]['level'],message: string,- extra?: Record<string, unknown>+ extra?: Attributes) {logSyncService({documentId: this.syncs.live.documentId,@@ -197,7 +208,7 @@ export class SyncEngine<S extends GenericRootSchema, D> {if (syncResult.isErr()) {this.log('error', 'syncStateToLoro: failed, resetting engine', {- err: syncResult,+ err: JSON.stringify(syncResult.error),});this.reset();}@@ -223,7 +234,7 @@ export class SyncEngine<S extends GenericRootSchema, D> {const snapshot = await (snapshotThunk ?? this.defaultSnapshotThunk)();if (snapshot.isErr()) {this.log('error', 'engine.reset: failed to get snapshot', {- err: snapshot.error,+ err: String(snapshot.error),});return;}@@ -231,7 +242,7 @@ export class SyncEngine<S extends GenericRootSchema, D> {const resetResult = await this.loroManager.reset(snapshot.value);if (resetResult.isErr()) {this.log('error', 'engine.reset: loro manager reset failed', {- err: resetResult,+ err: JSON.stringify(resetResult.error),});return;}@@ -293,40 +304,49 @@ export class SyncEngine<S extends GenericRootSchema, D> {}}- private async handleRemoteUpdate(update: RawUpdate) {- await this.syncLock.runExclusive(async () => {- this.log('debug', 'engine: handling remote update');- const importResult = this.loroManager.importUpdate(update);- await Promise.resolve();- if (importResult.isErr()) {- const pendingOnly = importResult.error.every(- (e) => e.code === LoroManagerError.ImportPending- );- if (pendingOnly) {- // The update arrived ahead of its causal deps; Loro holds it and- // applies it once the gap fills. Pull the gap from the server- // instead of resetting. No rerun: if the gap's ops haven't reached- // the server yet (e.g. another tab's unflushed WAL), rerunning- // would poll in a tight loop; the held ops apply whenever the deps- // arrive through any channel.- this.log('info', 'engine: remote update pending on missing ops');- void this.convergeFromServer();- return;+ private async handleRemoteUpdate(+ update: RawUpdate+ ): Promise<'applied' | 'pending' | 'reset'> {+ return this.syncLock.runExclusive(() =>+ telemetrySpan(this.syncs.live.documentId, 'edit.apply', async (span) => {+ span.setAttr('update.bytes', update.length);+ const importResult = this.loroManager.importUpdate(update);+ await Promise.resolve();+ if (importResult.isErr()) {+ const pendingOnly = importResult.error.every(+ (e) => e.code === LoroManagerError.ImportPending+ );+ if (pendingOnly) {+ // Loro retains causally-ahead updates. Pull the missing operations+ // rather than resetting the document.+ this.log('debug', 'engine: remote update pending on missing ops');+ span.setAttr('outcome', 'pending');+ void this.convergeFromServer();+ return 'pending';+ }++ this.log(+ 'error',+ 'engine: failed to import remote update, resetting',+ {+ err: JSON.stringify(importResult.error),+ }+ );+ span.error(importResult.error);+ span.setAttr('outcome', 'reset');+ this.reset();+ return 'reset';}- this.log('error', 'engine: failed to import remote update, resetting', {- err: importResult,- });- this.reset();- return;- }- this.log('debug', 'engine: remote update imported ok');- });+ span.setAttr('outcome', 'applied');+ span.setAttr('did_change', importResult.value);+ return 'applied';+ })+ );}private handleSourceEvent(event: SyncSourceEvent) {switch (event.type) {case 'update':- this.log('debug', 'engine: source event: update');this.handleRemoteUpdate(event.update);break;case 'awareness':@@ -394,15 +414,30 @@ export class SyncEngine<S extends GenericRootSchema, D> {) {if (!this.isActiveGeneration(generation)) return;+ // The catch-up flow: "give me everything after the version I hold".+ // The span shows the anchor version, what came back, and whether it+ // applied — the trail for "client was stale and (never) reconciled".+ const span = telemetrySpan(this.syncs.live.documentId, 'doc.sync.catchup');+ span.setAttr('since.version', vvAttr(since));+ span.setAttr('attempt', attempt);this.log('debug', `engine: requestUpdatesSince (attempt ${attempt})`);const updates = await this.syncs.live.requestUpdatesSince(since);- if (!this.isActiveGeneration(generation)) return;+ if (!this.isActiveGeneration(generation)) {+ span.setAttr('outcome', 'cancelled');+ span.end();+ return;+ }if (updates.isErr() || !updates.value) {this.log('error', 'engine: requestUpdatesSince failed', {- err: 'error' in updates ? updates.error : 'update is undefined',+ err: updates.isErr() ? String(updates.error) : 'update is undefined',});- if (updates.isErr() && attempt < REQUEST_UPDATES_MAX_ATTEMPTS) {+ const retrying =+ updates.isErr() && attempt < REQUEST_UPDATES_MAX_ATTEMPTS;+ span.error(updates.isErr() ? updates.error : 'update is undefined');+ span.setAttr('outcome', retrying ? 'retrying' : 'failed');+ span.end();+ if (retrying) {await new Promise((resolve) =>setTimeout(resolve, REQUEST_UPDATES_RETRY_DELAY_MS));@@ -417,11 +452,17 @@ export class SyncEngine<S extends GenericRootSchema, D> {// would throw and trigger a reset — the noop live source used by// non-propagating AI edit sessions answers with exactly this.this.log('debug', 'engine: requestUpdatesSince ok, empty update');+ span.setAttr('outcome', 'noop');+ span.setAttr('update.bytes', 0);+ span.end();return;}this.log('debug', 'engine: requestUpdatesSince ok, applying update');- await this.handleRemoteUpdate(updates.value);+ const outcome = await this.handleRemoteUpdate(updates.value);+ span.setAttr('outcome', outcome);+ span.setAttr('update.bytes', updates.value.length);+ span.end();}}diff --git a/packages/collaboration/src/collab/logger.ts b/packages/collaboration/src/collab/logger.tsindex bcd1679891..1cd24ec3c3 100644--- a/packages/collaboration/src/collab/logger.ts+++ b/packages/collaboration/src/collab/logger.ts@@ -1,3 +1,6 @@+import type { Attributes } from '@macro-inc/observability';+import { type LogLevel, logTelemetry } from './telemetry';+export type WalContext = {count: number;dirty: number;@@ -6,9 +9,31 @@ export type WalContext = {export type SyncLogContext = {wal?: WalContext;- misc?: Record<string, unknown>;+ misc?: Attributes;};+/** `document.id` and `session.id` come from the sink's base attrs, not here. */+function contextAttrs(context: SyncLogContext): Attributes {+ const attributes: Attributes = {+ ...context.misc,+ ...(context.wal && {+ 'wal.count': context.wal.count,+ 'wal.dirty': context.wal.dirty,+ 'wal.most_recent_edit': context.wal.mostRecentEdit,+ }),+ };+ return attributes;+}++/**+ * Log a sync-engine message through the document's telemetry sink — the+ * observability library's logging pipeline, which exports the record (when+ * the host initialized an exporter) and always mirrors it to the console.+ *+ * `debug` lines are suppressed unless `window.debugSyncServiceLog` is set,+ * keeping the high-frequency narration out of both the console and the+ * export pipeline by default.+ */export function logSyncService({documentId,level,@@ -16,15 +41,14 @@ export function logSyncService({message,}: {documentId: string;- level: 'debug' | 'info' | 'warn' | 'error';+ level: LogLevel;context: SyncLogContext;message: string;}): void {const debugEnabled =- typeof window !== 'undefined' && (window as any).debugSyncServiceLog;+ typeof window !== 'undefined' &&+ (window as { debugSyncServiceLog?: boolean }).debugSyncServiceLog;if (level === 'debug' && !debugEnabled) return;- console[level === 'info' ? 'log' : level](- { documentId, t: Date.now(), ...context },- message- );++ logTelemetry(documentId, level, message, contextAttrs(context));}diff --git a/packages/collaboration/src/collab/manager.ts b/packages/collaboration/src/collab/manager.tsindex 34f81f4240..04eb894636 100644--- a/packages/collaboration/src/collab/manager.ts+++ b/packages/collaboration/src/collab/manager.ts@@ -19,6 +19,12 @@ import { onCleanup } from 'solid-js';import type { ResultError } from '../internal/result';import { logSyncService } from './logger';import type { GenericRootSchema, LoroRawUpdate, RawUpdate } from './shared';+import {+ disposeTelemetryFor,+ frontiersAttr,+ setTelemetryAttr,+ telemetrySpan,+} from './telemetry';export enum LoroManagerError {ImportFailed = 'IMPORT_FAILED',@@ -139,6 +145,9 @@ export class LoroManager<S extends GenericRootSchema = GenericRootSchema>constructor(schema: S, options: LoroManagerOptions) {this.schema = schema;this.options = options;+ // Stamp this doc's telemetry with the peer identity as soon as it+ // exists, so every span and log record carries who we are.+ setTelemetryAttr(options.documentId, 'loro.peer_id', this._doc.peerIdStr);}/** The inner LoroDoc. Only touch this if you know what you're doing. */@@ -532,32 +541,43 @@ export class LoroManager<S extends GenericRootSchema = GenericRootSchema>async ingest(input: SnapshotIngest): Promise<boolean> {if (this._initialized) return false;- const initResult = await this.initializeFromSnapshot(input.snapshot);- if (initResult.isErr()) {- logSyncService({- documentId: this.options.documentId,- level: 'error',- context: {},- message: `ingest(${input.kind}): seed failed`,- });- return false;- }-- // WAL entries are causally anchored to the local snapshot, so they're only- // safe to replay on top of the local seed we just applied.- if (input.kind === 'local' && input.walUpdates?.length) {- const replayResult = this.importBatchUpdates(input.walUpdates);- if (replayResult.isErr()) {+ // The winner of the snapshot race seeds the doc; its kind, size, and+ // resulting version are the anchor every later sync question ("since+ // what?") is asked against.+ return telemetrySpan(this.options.documentId, 'doc.seed', async (span) => {+ span.setAttr('seed.kind', input.kind);+ span.setAttr('snapshot.bytes', input.snapshot.length);+ const initResult = await this.initializeFromSnapshot(input.snapshot);+ if (initResult.isErr()) {logSyncService({documentId: this.options.documentId,level: 'error',context: {},- message: `ingest(${input.kind}): WAL replay failed`,+ message: `ingest(${input.kind}): seed failed`,});+ span.error(`seed from ${input.kind} snapshot failed`);+ return false;}- }- return true;+ // WAL entries are causally anchored to the local snapshot, so they're only+ // safe to replay on top of the local seed we just applied.+ if (input.kind === 'local' && input.walUpdates?.length) {+ span.setAttr('wal.replayed', input.walUpdates.length);+ const replayResult = this.importBatchUpdates(input.walUpdates);+ if (replayResult.isErr()) {+ logSyncService({+ documentId: this.options.documentId,+ level: 'error',+ context: {},+ message: `ingest(${input.kind}): WAL replay failed`,+ });+ span.error('WAL replay onto local seed failed');+ }+ }++ span.setAttr('version.after', frontiersAttr(this._doc));+ return true;+ });}/** Tear down subscriptions and free the underlying doc. */@@ -568,6 +588,7 @@ export class LoroManager<S extends GenericRootSchema = GenericRootSchema>this.initListeners.clear();this._mirror?.dispose();this._doc.free();+ disposeTelemetryFor(this.options.documentId);}}diff --git a/packages/collaboration/src/collab/snapshot-store.ts b/packages/collaboration/src/collab/snapshot-store.tsindex b315a39081..36772ca48a 100644--- a/packages/collaboration/src/collab/snapshot-store.ts+++ b/packages/collaboration/src/collab/snapshot-store.ts@@ -97,6 +97,7 @@ export async function loadCachedState<S extends GenericRootSchema>(}const pending = await walStore.getAll();+ let pendingCount = 0;for (const entry of pending) {const importResult = loroManager.importUpdate(entry.update);if (importResult.isErr()) {@@ -106,12 +107,7 @@ export async function loadCachedState<S extends GenericRootSchema>(if (pendingOnly) {// Loro holds the entry until its causal gap fills, so keep replaying:// a later WAL entry may fill the gap, and held ops apply on their own.- logSyncService({- documentId: 'unknown',- level: 'warn',- context: { misc: { entryId: entry.id } },- message: 'snapshot-store: WAL entry pending during cold load',- });+ pendingCount += 1;continue;}// Stop replaying. Skipped entries are safe: delivered ones are on the@@ -126,5 +122,13 @@ export async function loadCachedState<S extends GenericRootSchema>(break;}}+ if (pendingCount > 0) {+ logSyncService({+ documentId: 'unknown',+ level: 'warn',+ context: { misc: { pendingCount } },+ message: 'snapshot-store: WAL entries pending during cold load',+ });+ }return true;}diff --git a/packages/collaboration/src/collab/telemetry.test.ts b/packages/collaboration/src/collab/telemetry.test.tsnew file mode 100644index 0000000000..d98e605c04--- /dev/null+++ b/packages/collaboration/src/collab/telemetry.test.ts@@ -0,0 +1,49 @@+import { type Span, Telemetry } from '@macro-inc/observability';+import { afterEach, describe, expect, test, vi } from 'vitest';+import { disposeTelemetryFor, telemetrySpan } from './telemetry';++const documentId = 'telemetry-test-document';++function mockSpan() {+ const error = vi.fn();+ const span: Span = {+ span: (() => span) as Span['span'],+ run: (operation) => operation(),+ setAttr: vi.fn(),+ event: vi.fn(),+ error,+ traceparent: () => undefined,+ injectTraceHeaders: vi.fn(),+ end: vi.fn(),+ };+ vi.spyOn(Telemetry, 'span').mockImplementation(((+ _: string,+ operation?: (span: Span) => Promise<unknown>+ ) => (operation ? operation(span) : span)) as typeof Telemetry.span);+ return { span, error };+}++afterEach(() => {+ disposeTelemetryFor(documentId);+ vi.restoreAllMocks();+});++describe('telemetrySpan', () => {+ test('enriches and delegates the async operation', async () => {+ const { span, error } = mockSpan();++ const result = await telemetrySpan(+ documentId,+ 'test',+ async (activeSpan) => {+ expect(activeSpan).toBe(span);+ return 42;+ }+ );++ expect(result).toBe(42);+ expect(error).not.toHaveBeenCalled();+ expect(span.setAttr).toHaveBeenCalledWith('document.id', documentId);+ expect(span.setAttr).toHaveBeenCalledWith('session.id', expect.any(String));+ });+});diff --git a/packages/collaboration/src/collab/telemetry.ts b/packages/collaboration/src/collab/telemetry.tsnew file mode 100644index 0000000000..22305cf063--- /dev/null+++ b/packages/collaboration/src/collab/telemetry.ts@@ -0,0 +1,105 @@+import {+ type Attribute,+ type Attributes,+ type Span,+ Telemetry,+} from '@macro-inc/observability';++export type LogLevel = 'debug' | 'info' | 'warn' | 'error';++type DocumentTelemetry = {+ attributes: Attributes;+};++const documents = new Map<string, DocumentTelemetry>();++function documentTelemetry(documentId: string): DocumentTelemetry {+ let value = documents.get(documentId);+ if (!value) {+ value = {+ attributes: {+ 'document.id': documentId,+ 'session.id': crypto.randomUUID(),+ },+ };+ documents.set(documentId, value);+ }+ return value;+}++type SpanOperation<T> = (span: Span) => Promise<T>;++function setDocumentSpanAttributes(documentId: string, span: Span): void {+ for (const [key, value] of Object.entries(+ documentTelemetry(documentId).attributes+ )) {+ if (value !== undefined) span.setAttr(key, value);+ }+}++export function telemetrySpan<T>(+ documentId: string,+ name: string,+ operation: SpanOperation<T>+): Promise<T>;+export function telemetrySpan(documentId: string, name: string): Span;+export function telemetrySpan<T>(+ documentId: string,+ name: string,+ operation?: SpanOperation<T>+): Span | Promise<T> {+ if (operation) {+ return Telemetry.span(name, async (span) => {+ setDocumentSpanAttributes(documentId, span);+ return operation(span);+ });+ }++ const span = Telemetry.span(name);+ setDocumentSpanAttributes(documentId, span);+ return span;+}++export function setTelemetryAttr(+ documentId: string,+ name: string,+ value: Attribute+): void {+ documentTelemetry(documentId).attributes[name] = value;+}++export function logTelemetry(+ documentId: string,+ level: LogLevel,+ message: string,+ attributes?: Attributes+): void {+ Telemetry[level](message, {+ ...documentTelemetry(documentId).attributes,+ ...attributes,+ });+}++export function disposeTelemetryFor(documentId: string): void {+ documents.delete(documentId);+}++function cappedAttr(value: string, max = 256): string {+ return value.length > max ? `${value.slice(0, max)}...` : value;+}++/** `peer:counter` pairs as one compact span attribute. */+export function peerCounterAttr(+ pairs: Iterable<[peer: string, counter: number]>+): string {+ return cappedAttr(+ [...pairs].map(([peer, counter]) => `${peer}:${counter}`).join(',')+ );+}++/** A doc version as a compact span attribute (its frontiers). */+export function frontiersAttr(doc: {+ frontiers(): { peer: string; counter: number }[];+}): string {+ return peerCounterAttr(doc.frontiers().map((f) => [f.peer, f.counter]));+}diff --git a/packages/collaboration/src/collab/wal.test.ts b/packages/collaboration/src/collab/wal.test.tsindex 14e9402a12..a272890590 100644--- a/packages/collaboration/src/collab/wal.test.ts+++ b/packages/collaboration/src/collab/wal.test.ts@@ -1,8 +1,15 @@-import { describe, expect, it, vi } from 'vitest';+import { type Span, Telemetry } from '@macro-inc/observability';+import { afterEach, describe, expect, it, vi } from 'vitest';import type { RawUpdate } from './shared';+import { disposeTelemetryFor } from './telemetry';import { MockLiveSyncSource, MockWALStore, makeTestWAL } from './testing';import { WAL_TTL_MS, WALSyncer } from './wal';+afterEach(() => {+ disposeTelemetryFor('wal-telemetry-test');+ vi.restoreAllMocks();+});+function makeWAL(live: MockLiveSyncSource) {return makeTestWAL(live);}@@ -117,6 +124,43 @@ describe('WALSyncer', () => {expect(await undeliveredCount(walStore)).toBe(0);});+ it('records one delivery outage across repeated failures and ends it on recovery', async () => {+ const span: Span = {+ span: (() => span) as Span['span'],+ run: (operation) => operation(),+ setAttr: vi.fn(),+ event: vi.fn(),+ error: vi.fn(),+ traceparent: () => undefined,+ injectTraceHeaders: vi.fn(),+ end: vi.fn(),+ };+ const spanSpy = vi.spyOn(Telemetry, 'span').mockReturnValue(span as never);+ const store = new MockWALStore<RawUpdate>();+ let delivered = false;+ const push = vi.fn(async () => delivered);+ const wal = new WALSyncer(store, push, 'wal-telemetry-test');++ await wal.append(new Uint8Array([1]));+ await wal.pendingFlush;+ await wal.append(new Uint8Array([2]));+ await wal.pendingFlush;++ expect(spanSpy).toHaveBeenCalledTimes(1);+ expect(spanSpy).toHaveBeenCalledWith('wal.delivery_outage');+ expect(span.error).toHaveBeenCalledOnce();+ expect(span.end).not.toHaveBeenCalled();++ delivered = true;+ await wal.flush();++ expect(span.event).toHaveBeenCalledWith('wal.ack.confirmed', {+ 'wal.delivered': 2,+ });+ expect(span.setAttr).toHaveBeenCalledWith('outcome', 'recovered');+ expect(span.end).toHaveBeenCalledOnce();+ });+it('flushes updates that arrived during an in-flight flush', async () => {const live = new MockLiveSyncSource();const { wal, walStore } = makeWAL(live);diff --git a/packages/collaboration/src/collab/wal.ts b/packages/collaboration/src/collab/wal.tsindex a70408d8d4..49a8c25d7c 100644--- a/packages/collaboration/src/collab/wal.ts+++ b/packages/collaboration/src/collab/wal.ts@@ -1,7 +1,9 @@+import type { Span } from '@macro-inc/observability';import { type DBSchema, type IDBPDatabase, openDB as idbOpen } from 'idb';import { logSyncService, type WalContext } from './logger';import type { RawUpdate } from './shared';import type { LiveSyncSource } from './source';+import { telemetrySpan } from './telemetry';export type WALEntry<T> = {id: number;@@ -229,6 +231,15 @@ export class WALSyncer<T> {private hasNewPending = false;public pendingFlush: Promise<void> = Promise.resolve();private cleanupFns: Array<() => void> = [];+ private deliveryOutage:+ | {+ id: string;+ span: Span;+ startedAt: number;+ failedAttempts: number;+ maxUndelivered: number;+ }+ | undefined;/*** Exists so that we can not "do stuff" until we have pruned expired entries@@ -303,6 +314,10 @@ export class WALSyncer<T> {}public destroy(): void {+ if (this.deliveryOutage) {+ this.deliveryOutage.span.setAttr('outcome', 'abandoned');+ this.finishDeliveryOutage();+ }for (const fn of this.cleanupFns) fn();this.cleanupFns = [];}@@ -331,10 +346,17 @@ export class WALSyncer<T> {message: `WAL flush: pushing ${undelivered.length} entries`,});- const delivered = await this.push(undelivered.map((e) => e.update));+ let delivered: boolean;+ try {+ delivered = await this.push(undelivered.map((e) => e.update));+ } catch (e) {+ this.recordDeliveryFailure('push_error', undelivered.length, e);+ throw e;+ }if (delivered) {await this.store.markDelivered(undelivered.map((e) => e.id));+ this.recordDeliveryRecovery(undelivered.length);if (this.label)logSyncService({documentId: this.label,@@ -343,13 +365,7 @@ export class WALSyncer<T> {message: `WAL flush: delivered ${undelivered.length} entries`,});} else {- if (this.label)- logSyncService({- documentId: this.label,- level: 'warn',- context: { wal: await this.summary() },- message: `WAL flush: push not acked (${undelivered.length} entries)`,- });+ this.recordDeliveryFailure('not_acked', undelivered.length);succeeded = false;}} finally {@@ -361,6 +377,98 @@ export class WALSyncer<T> {return this.pendingFlush;}}++ private recordDeliveryFailure(+ kind: 'not_acked' | 'push_error',+ undelivered: number,+ error?: unknown+ ): void {+ if (!this.label) return;++ if (this.deliveryOutage) {+ this.deliveryOutage.failedAttempts++;+ this.deliveryOutage.maxUndelivered = Math.max(+ this.deliveryOutage.maxUndelivered,+ undelivered+ );+ this.deliveryOutage.span.setAttr(+ 'wal.failed_attempts',+ this.deliveryOutage.failedAttempts+ );+ this.deliveryOutage.span.setAttr(+ 'wal.max_undelivered',+ this.deliveryOutage.maxUndelivered+ );+ return;+ }++ const id = crypto.randomUUID();+ const span = telemetrySpan(this.label, 'wal.delivery_outage');+ span.setAttr('wal.outage.id', id);+ span.setAttr('wal.first_failure', kind);+ span.setAttr('wal.failed_attempts', 1);+ span.setAttr('wal.max_undelivered', undelivered);+ span.error(error ?? 'push not acked; entries remain undelivered');+ this.deliveryOutage = {+ id,+ span,+ startedAt: Date.now(),+ failedAttempts: 1,+ maxUndelivered: undelivered,+ };+ logSyncService({+ documentId: this.label,+ level: 'warn',+ context: {+ misc: {+ 'wal.outage.id': id,+ 'wal.first_failure': kind,+ 'wal.undelivered': undelivered,+ },+ },+ message: `WAL delivery outage started (${undelivered} entries)`,+ });+ }++ private recordDeliveryRecovery(delivered: number): void {+ if (!this.deliveryOutage || !this.label) return;++ const { id, span, startedAt, failedAttempts, maxUndelivered } =+ this.deliveryOutage;+ const outageMs = Date.now() - startedAt;+ span.event('wal.ack.confirmed', { 'wal.delivered': delivered });+ span.setAttr('outcome', 'recovered');+ span.setAttr('wal.outage_ms', outageMs);+ span.setAttr('wal.failed_attempts', failedAttempts);+ span.setAttr('wal.max_undelivered', maxUndelivered);+ span.setAttr('wal.delivered', delivered);+ span.end();+ this.deliveryOutage = undefined;+ logSyncService({+ documentId: this.label,+ level: 'info',+ context: {+ misc: {+ 'wal.outage.id': id,+ 'wal.outage_ms': outageMs,+ 'wal.failed_attempts': failedAttempts,+ 'wal.max_undelivered': maxUndelivered,+ 'wal.delivered': delivered,+ },+ },+ message: 'WAL delivery recovered',+ });+ }++ private finishDeliveryOutage(): void {+ if (!this.deliveryOutage) return;+ this.deliveryOutage.span.setAttr(+ 'wal.outage_ms',+ Date.now() - this.deliveryOutage.startedAt+ );+ this.deliveryOutage.span.end();+ this.deliveryOutage = undefined;+ }}/** Build a WAL syncer wired to a Loro live sync source: BrowserWALStore fordiff --git a/packages/collaboration/src/sync-service/source.test.ts b/packages/collaboration/src/sync-service/source.test.tsindex f77b02a1ed..d4cf39c2c4 100644--- a/packages/collaboration/src/sync-service/source.test.ts+++ b/packages/collaboration/src/sync-service/source.test.ts@@ -23,7 +23,11 @@ class FakeSocket implements SyncSocket {reconnects = 0;closed = false;private msg = new Set<AnyListener>();+ private openListeners = new Set<AnyListener>();private recon = new Set<AnyListener>();+ private closeListeners = new Set<AnyListener>();+ private retryListeners = new Set<AnyListener>();+ private heartbeatMissedListeners = new Set<AnyListener>();send(message: IFromPeer) {this.sent.push(message);@@ -43,15 +47,27 @@ class FakeSocket implements SyncSocket {listener: WebsocketEventListener<K, IFromPeer, FromRemote>) {if (type === WebsocketEvent.Message) this.msg.add(listener as AnyListener);+ else if (type === WebsocketEvent.Open)+ this.openListeners.add(listener as AnyListener);else if (type === WebsocketEvent.Reconnect)this.recon.add(listener as AnyListener);+ else if (type === WebsocketEvent.Close)+ this.closeListeners.add(listener as AnyListener);+ else if (type === WebsocketEvent.retry)+ this.retryListeners.add(listener as AnyListener);+ else if (type === WebsocketEvent.HeartbeatMissed)+ this.heartbeatMissedListeners.add(listener as AnyListener);}removeEventListener<K extends WebsocketEvent>(_type: K,listener: WebsocketEventListener<K, IFromPeer, FromRemote>) {this.msg.delete(listener as AnyListener);+ this.openListeners.delete(listener as AnyListener);this.recon.delete(listener as AnyListener);+ this.closeListeners.delete(listener as AnyListener);+ this.retryListeners.delete(listener as AnyListener);+ this.heartbeatMissedListeners.delete(listener as AnyListener);}// test drivers@@ -64,6 +80,28 @@ class FakeSocket implements SyncSocket {const event = {} as Parameters<AnyListener>[1];for (const listener of [...this.recon]) listener(this as never, event);}+ fireOpen() {+ this.connectionState = WebsocketConnectionState.Open;+ const event = {} as Parameters<AnyListener>[1];+ for (const listener of [...this.openListeners])+ listener(this as never, event);+ }+ fireClose(code: number, reason: string) {+ const event = {+ code,+ reason,+ wasClean: false,+ } as Parameters<AnyListener>[1];+ for (const listener of [...this.closeListeners])+ listener(this as never, event);+ }+ fireRetry(retries: number, backoff: number) {+ const event = {+ detail: { retries, backoff },+ } as Parameters<AnyListener>[1];+ for (const listener of [...this.retryListeners])+ listener(this as never, event);+ }}const GUARDS = {diff --git a/packages/collaboration/src/sync-service/source.ts b/packages/collaboration/src/sync-service/source.tsindex 209e21f8a5..502ebe80d1 100644--- a/packages/collaboration/src/sync-service/source.ts+++ b/packages/collaboration/src/sync-service/source.ts@@ -133,6 +133,11 @@ export class SyncServiceSource implements LiveSyncSource {});this.ws.addEventListener(WebsocketEvent.Message, this.onMessage);+ this.ws.addEventListener(WebsocketEvent.Close, this.onClose);+ this.ws.addEventListener(+ WebsocketEvent.HeartbeatMissed,+ this.onHeartbeatMissed+ );this.ws.addEventListener(WebsocketEvent.Reconnect, this.onReconnect);this.registerEnvironmentListeners();@@ -243,6 +248,11 @@ export class SyncServiceSource implements LiveSyncSource {document.removeEventListener('visibilitychange', this.onVisibilityChange);}this.ws.removeEventListener(WebsocketEvent.Message, this.onMessage);+ this.ws.removeEventListener(WebsocketEvent.Close, this.onClose);+ this.ws.removeEventListener(+ WebsocketEvent.HeartbeatMissed,+ this.onHeartbeatMissed+ );this.ws.removeEventListener(WebsocketEvent.Reconnect, this.onReconnect);this.ws.close();};@@ -312,6 +322,45 @@ export class SyncServiceSource implements LiveSyncSource {}};+ private onClose: WebsocketEventListener<+ WebsocketEvent.Close,+ IFromPeer,+ FromRemote+ > = (_ws, event) => {+ logSyncService({+ documentId: this.documentId,+ level: 'warn',+ context: {+ misc: {+ 'ws.close.code': event.code,+ 'ws.close.clean': event.wasClean,+ ...(event.reason && {+ 'ws.close.reason': event.reason.slice(0, 256),+ }),+ },+ },+ message: 'sync source: WebSocket disconnected',+ });+ };++ private onHeartbeatMissed: WebsocketEventListener<+ WebsocketEvent.HeartbeatMissed,+ IFromPeer,+ FromRemote+ > = (_ws, event) => {+ if (!event.detail.willReconnect) return;+ logSyncService({+ documentId: this.documentId,+ level: 'warn',+ context: {+ misc: {+ 'ws.heartbeat.missed': event.detail.missedHeartbeats,+ },+ },+ message: 'sync source: WebSocket heartbeat missed; reconnecting',+ });+ };+private onReconnect: WebsocketEventListener<WebsocketEvent.Reconnect,IFromPeer,@@ -347,7 +396,7 @@ export class SyncServiceSource implements LiveSyncSource {logSyncService({documentId: this.documentId,level: 'warn',- context: { misc: { error } },+ context: { misc: { error: String(error) } },message: 'sync source: reconnect initial sync timed out',});// Heartbeat is already running, so the connection remains monitoreddiff --git a/packages/observability/package.json b/packages/observability/package.jsonnew file mode 100644index 0000000000..5f8e20465d--- /dev/null+++ b/packages/observability/package.json@@ -0,0 +1,37 @@+{+ "name": "@macro-inc/observability",+ "version": "0.0.1",+ "private": true,+ "type": "module",+ "exports": {+ ".": "./src/index.ts",+ "./core": "./src/core.ts",+ "./web": "./src/web.ts",+ "./zone": "./src/zone.ts"+ },+ "scripts": {+ "lint": "biome check .",+ "test": "vitest run --config vitest.config.ts",+ "type-check": "tsc --noEmit --project tsconfig.json"+ },+ "dependencies": {+ "@opentelemetry/api": "^1.9.1",+ "@opentelemetry/api-logs": "^0.220.0",+ "@opentelemetry/context-zone": "^2.9.0",+ "@opentelemetry/core": "^2.9.0",+ "@opentelemetry/exporter-logs-otlp-http": "^0.220.0",+ "@opentelemetry/exporter-trace-otlp-http": "^0.220.0",+ "@opentelemetry/resources": "^2.9.0",+ "@opentelemetry/sdk-logs": "^0.220.0",+ "@opentelemetry/sdk-trace-base": "^2.9.0",+ "@opentelemetry/sdk-trace-web": "^2.9.0",+ "@opentelemetry/semantic-conventions": "^1.43.0"+ },+ "devDependencies": {+ "@types/bun": "^1.3.14",+ "@types/node": "^24.8.0",+ "jsdom": "^26.1.0",+ "typescript": "^5.9.3",+ "vitest": "^3.2.4"+ }+}diff --git a/packages/observability/src/config.ts b/packages/observability/src/config.tsnew file mode 100644index 0000000000..d74e418644--- /dev/null+++ b/packages/observability/src/config.ts@@ -0,0 +1,32 @@+import type {+ ContextManager,+ AttributeValue as OtelAttributeValue,+} from "@opentelemetry/api";+import type { TelemetryTracingProviderFactory } from "./provider";++export type Attribute = OtelAttributeValue;+export type Attributes = Record<string, Attribute | undefined>;+export type LogAttributes = Attributes;++export interface TelemetryInitConfig {+ serviceName: string;+ environment: string;+ tracesUrl?: string;+ logsUrl?: string;+ contextManager?: ContextManager;+ tracingProvider?: TelemetryTracingProviderFactory;+ enabled: () => Promise<boolean>;+}++/** Mutable attributes applied to telemetry created after they change. */+export class TelemetryRuntimeConfig {+ #userId: string | undefined;++ get userId(): string | undefined {+ return this.#userId;+ }++ setUser(userId: string | undefined): void {+ this.#userId = userId;+ }+}diff --git a/packages/observability/src/constants.ts b/packages/observability/src/constants.tsnew file mode 100644index 0000000000..85010dd888--- /dev/null+++ b/packages/observability/src/constants.ts@@ -0,0 +1,4 @@+export const INSTRUMENTATION_SCOPE_NAME = "@macro-inc/observability";+export const ATTR_USER_ID = "usr.id";+export const ATTR_DEPLOYMENT_ENVIRONMENT = "deployment.environment";+export const CONSOLE_PREFIX = "[observability]";diff --git a/packages/observability/src/core.ts b/packages/observability/src/core.tsnew file mode 100644index 0000000000..8a43a80583--- /dev/null+++ b/packages/observability/src/core.ts@@ -0,0 +1,13 @@+export type {+ Attribute,+ Attributes,+ LogAttributes,+ TelemetryInitConfig,+} from "./config";+export { TelemetryRuntimeConfig } from "./config";+export type {+ TelemetryTracingProvider,+ TelemetryTracingProviderFactory,+} from "./provider";+export type { Span } from "./span";+export { Telemetry } from "./telemetry";diff --git a/packages/observability/src/index.ts b/packages/observability/src/index.tsnew file mode 100644index 0000000000..cf304f99e2--- /dev/null+++ b/packages/observability/src/index.ts@@ -0,0 +1,9 @@+export type {+ Attribute,+ Attributes,+ LogAttributes,+ TelemetryInitConfig,+} from "./config";+export { TelemetryRuntimeConfig } from "./config";+export type { Span } from "./span";+export { Telemetry } from "./telemetry";diff --git a/packages/observability/src/logging.ts b/packages/observability/src/logging.tsnew file mode 100644index 0000000000..969b581133--- /dev/null+++ b/packages/observability/src/logging.ts@@ -0,0 +1,115 @@+import type { Attributes as OtelAttributes } from "@opentelemetry/api";+import { type Logger, logs, SeverityNumber } from "@opentelemetry/api-logs";+import { OTLPLogExporter } from "@opentelemetry/exporter-logs-otlp-http";+import type { Resource } from "@opentelemetry/resources";+import {+ BatchLogRecordProcessor,+ LoggerProvider,+} from "@opentelemetry/sdk-logs";+import {+ ATTR_EXCEPTION_MESSAGE,+ ATTR_EXCEPTION_STACKTRACE,+ ATTR_EXCEPTION_TYPE,+} from "@opentelemetry/semantic-conventions";+import type { LogAttributes, TelemetryInitConfig } from "./config";+import {+ ATTR_USER_ID,+ CONSOLE_PREFIX,+ INSTRUMENTATION_SCOPE_NAME,+} from "./constants";++type LogLevel = "debug" | "info" | "warn" | "error";++export class Logging {+ #provider: LoggerProvider | undefined;+ readonly #logger: Logger = logs.getLogger(INSTRUMENTATION_SCOPE_NAME);++ init(+ config: TelemetryInitConfig,+ resource: Resource,+ getUserId: () => string | undefined,+ ): void {+ if (!config.logsUrl) return;+ this.#provider = new LoggerProvider({+ resource,+ processors: [+ {+ onEmit: (record) => {+ const userId = getUserId();+ if (userId !== undefined) record.setAttribute(ATTR_USER_ID, userId);+ },+ forceFlush: () => Promise.resolve(),+ shutdown: () => Promise.resolve(),+ },+ new BatchLogRecordProcessor({+ exporter: new OTLPLogExporter({ url: config.logsUrl }),+ }),+ ],+ });+ logs.setGlobalLoggerProvider(this.#provider);+ }++ debug(message: string, attributes?: LogAttributes): void {+ this.#emit("debug", message, attributes);+ }++ info(message: string, attributes?: LogAttributes): void {+ this.#emit("info", message, attributes);+ }++ warn(message: string, attributes?: LogAttributes): void {+ this.#emit("warn", message, attributes);+ }++ error(error: unknown, attributes?: LogAttributes): void {+ const normalized = Logging.#normalizeError(error);+ this.#emit(+ "error",+ typeof normalized === "string"+ ? normalized+ : normalized.message || normalized.name,+ {+ ...attributes,+ ...(normalized instanceof Error && {+ [ATTR_EXCEPTION_TYPE]: normalized.name,+ [ATTR_EXCEPTION_MESSAGE]: normalized.message,+ [ATTR_EXCEPTION_STACKTRACE]: normalized.stack,+ }),+ },+ );+ }++ async flush(): Promise<void> {+ await this.#provider?.forceFlush();+ }++ async shutdown(): Promise<void> {+ await this.#provider?.shutdown();+ this.#provider = undefined;+ }++ #emit(level: LogLevel, message: string, attributes?: LogAttributes): void {+ const severity = {+ debug: SeverityNumber.DEBUG,+ info: SeverityNumber.INFO,+ warn: SeverityNumber.WARN,+ error: SeverityNumber.ERROR,+ }[level];+ this.#logger.emit({+ severityNumber: severity,+ severityText: level,+ body: message,+ attributes: attributes as OtelAttributes,+ });+ const consoleMethod = level === "info" ? "log" : level;+ console[consoleMethod](CONSOLE_PREFIX, message, attributes ?? {});+ }++ static #normalizeError(error: unknown): Error | string {+ return error instanceof Error ||+ typeof error === "string" ||+ (typeof error === "object" && error !== null && "message" in error)+ ? (error as Error | string)+ : String(error);+ }+}diff --git a/packages/observability/src/provider.ts b/packages/observability/src/provider.tsnew file mode 100644index 0000000000..2893cfa0f3--- /dev/null+++ b/packages/observability/src/provider.ts@@ -0,0 +1,14 @@+import type { TracerProvider } from "@opentelemetry/api";+import type { Resource } from "@opentelemetry/resources";++/** A runtime-specific OpenTelemetry tracer provider. */+export type TelemetryTracingProvider = TracerProvider & {+ forceFlush(): Promise<void>;+ shutdown(): Promise<void>;+};++/** Creates a tracer provider after telemetry has been enabled. */+export type TelemetryTracingProviderFactory = (+ resource: Resource,+ getUserId: () => string | undefined,+) => TelemetryTracingProvider;diff --git a/packages/observability/src/span.ts b/packages/observability/src/span.tsnew file mode 100644index 0000000000..178e9be3e5--- /dev/null+++ b/packages/observability/src/span.ts@@ -0,0 +1,108 @@+import {+ type Context,+ context,+ isSpanContextValid,+ type Attributes as OtelAttributes,+ propagation,+ type Span as OtelSpan,+ SpanStatusCode,+} from "@opentelemetry/api";+import type { Attribute, Attributes } from "./config";++/** A manually managed span. Calling {@link Span.end} ends it. */+export interface Span {+ /** Start a child span, or run and end one around an async operation. */+ span<T>(name: string, operation: (span: Span) => Promise<T>): Promise<T>;+ span(name: string): Span;+ /** Run with this span as the active OpenTelemetry context. */+ run<T>(operation: () => T): T;+ /** Set one span attribute. */+ setAttr(name: string, value: Attribute): void;+ /** Add a point-in-time event. */+ event(name: string, attributes?: Attributes): void;+ /** Record an exception and mark the span as failed. */+ error(error: unknown): void;+ /** Return this span's W3C trace context for transports without headers. */+ traceparent(): string | undefined;+ /** Inject this span's W3C trace context into an HTTP header carrier. */+ injectTraceHeaders(headers: Record<string, string>): void;+ /** End the span. Safe to call more than once. */+ end(): void;+}++export class SpanImpl implements Span {+ #ended = false;++ constructor(+ private readonly otelSpan: OtelSpan,+ private readonly ctx: Context,+ private readonly startChild: (name: string, parent: Context) => Span,+ ) {}++ span<T>(name: string, operation: (span: Span) => Promise<T>): Promise<T>;+ span(name: string): Span;+ span<T>(+ name: string,+ operation?: (span: Span) => Promise<T>,+ ): Span | Promise<T> {+ const span = this.startChild(name, this.ctx);+ if (!operation) return span;++ return span.run(async () => {+ try {+ return await operation(span);+ } finally {+ span.end();+ }+ });+ }++ run<T>(operation: () => T): T {+ return context.with(this.ctx, operation);+ }++ setAttr(name: string, value: Attribute): void {+ if (!this.#ended) this.otelSpan.setAttribute(name, value);+ }++ event(name: string, attributes?: Attributes): void {+ if (!this.#ended)+ this.otelSpan.addEvent(name, attributes as OtelAttributes);+ }++ error(error: unknown): void {+ if (this.#ended) return;+ const exception = SpanImpl.normalizeError(error);+ this.otelSpan.recordException(exception);+ this.otelSpan.setStatus({+ code: SpanStatusCode.ERROR,+ message:+ typeof exception === "string" ? exception : (exception.message ?? ""),+ });+ }++ traceparent(): string | undefined {+ const spanContext = this.otelSpan.spanContext();+ if (!isSpanContextValid(spanContext)) return undefined;+ const { traceId, spanId, traceFlags } = spanContext;+ return `00-${traceId}-${spanId}-${traceFlags.toString(16).padStart(2, "0")}`;+ }++ injectTraceHeaders(headers: Record<string, string>): void {+ if (!this.#ended) propagation.inject(this.ctx, headers);+ }++ end(): void {+ if (this.#ended) return;+ this.#ended = true;+ this.otelSpan.end();+ }++ private static normalizeError(error: unknown): Error | string {+ return error instanceof Error ||+ typeof error === "string" ||+ (typeof error === "object" && error !== null && "message" in error)+ ? (error as Error | string)+ : String(error);+ }+}diff --git a/packages/observability/src/telemetry.test.ts b/packages/observability/src/telemetry.test.tsnew file mode 100644index 0000000000..65820b54f3--- /dev/null+++ b/packages/observability/src/telemetry.test.ts@@ -0,0 +1,189 @@+// @vitest-environment jsdom+import { context, trace } from "@opentelemetry/api";+import { logs } from "@opentelemetry/api-logs";+import {+ InMemoryLogRecordExporter,+ LoggerProvider,+ SimpleLogRecordProcessor,+} from "@opentelemetry/sdk-logs";+import {+ BasicTracerProvider,+ InMemorySpanExporter,+ SimpleSpanProcessor,+} from "@opentelemetry/sdk-trace-base";+import { beforeEach, describe, expect, test } from "vitest";+import { Telemetry } from "./index";+import { ZoneContextManager } from "./zone";++context.setGlobalContextManager(new ZoneContextManager().enable());++const spanExporter = new InMemorySpanExporter();+trace.setGlobalTracerProvider(+ new BasicTracerProvider({+ spanProcessors: [new SimpleSpanProcessor(spanExporter)],+ }),+);++const logExporter = new InMemoryLogRecordExporter();+logs.setGlobalLoggerProvider(+ new LoggerProvider({+ processors: [new SimpleLogRecordProcessor({ exporter: logExporter })],+ }),+);++describe("Telemetry", () => {+ beforeEach(() => {+ spanExporter.reset();+ logExporter.reset();+ });++ test("awaits the async enablement decision", async () => {+ let decide!: (enabled: boolean) => void;+ let markDecisionRequested!: () => void;+ const decisionRequested = new Promise<void>((resolve) => {+ markDecisionRequested = resolve;+ });+ let initialized = false;+ const initialization = Telemetry.init({+ serviceName: "web-app",+ environment: "test",+ enabled: () =>+ new Promise<boolean>((resolve) => {+ decide = resolve;+ markDecisionRequested();+ }),+ }).then(() => {+ initialized = true;+ });++ await decisionRequested;+ expect(initialized).toBe(false);++ decide(false);+ await initialization;+ expect(initialized).toBe(true);+ });++ test("a span exports only when ended", () => {+ const span = Telemetry.span("doc.open");+ span.setAttr("document.id", "abc");+ expect(spanExporter.getFinishedSpans()).toHaveLength(0);++ span.end();+ span.end();++ expect(spanExporter.getFinishedSpans()).toHaveLength(1);+ expect(spanExporter.getFinishedSpans()[0]?.attributes).toMatchObject({+ "document.id": "abc",+ });+ });++ test("ends a callback span after the async operation", async () => {+ let complete: () => void = () => {};+ const operation = Telemetry.span("doc.save", async (span) => {+ span.setAttr("document.id", "abc");+ await new Promise<void>((resolve) => {+ complete = resolve;+ });+ return 42;+ });++ await Promise.resolve();+ expect(spanExporter.getFinishedSpans()).toHaveLength(0);+ complete();+ expect(await operation).toBe(42);+ expect(spanExporter.getFinishedSpans()).toHaveLength(1);+ });++ test("ends a callback span when the async operation throws", async () => {+ const failure = new Error("failed");++ await expect(+ Telemetry.span("doc.save", async () => {+ throw failure;+ }),+ ).rejects.toBe(failure);+ expect(spanExporter.getFinishedSpans()).toHaveLength(1);+ });++ test("starts children from the parent span context", () => {+ const root = Telemetry.span("doc.open");+ const child = root.span("doc.load");+ child.end();+ root.end();++ const spans = new Map(+ spanExporter.getFinishedSpans().map((span) => [span.name, span]),+ );+ expect(spans.get("doc.load")?.parentSpanContext?.spanId).toBe(+ spans.get("doc.open")?.spanContext().spanId,+ );+ });++ test("ends a callback child span after the async operation", async () => {+ const root = Telemetry.span("doc.open");+ let complete: () => void = () => {};+ const operation = root.span("doc.snapshot.dss-cache", async (span) => {+ span.setAttr("snapshot.source", "dss");+ await new Promise<void>((resolve) => {+ complete = resolve;+ });+ return "unavailable";+ });++ await Promise.resolve();+ expect(spanExporter.getFinishedSpans()).toHaveLength(0);+ complete();+ expect(await operation).toBe("unavailable");+ root.end();++ const spans = new Map(+ spanExporter.getFinishedSpans().map((span) => [span.name, span]),+ );+ expect(spans.get("doc.snapshot.dss-cache")?.attributes).toMatchObject({+ "snapshot.source": "dss",+ });+ expect(spans.get("doc.snapshot.dss-cache")?.parentSpanContext?.spanId).toBe(+ spans.get("doc.open")?.spanContext().spanId,+ );+ });++ test("run makes the span active", () => {+ const root = Telemetry.span("doc.open");+ const child = root.run(() => Telemetry.span("http request"));+ child.end();+ root.end();++ const spans = new Map(+ spanExporter.getFinishedSpans().map((span) => [span.name, span]),+ );+ expect(spans.get("http request")?.parentSpanContext?.spanId).toBe(+ spans.get("doc.open")?.spanContext().spanId,+ );+ });++ test("records span errors", () => {+ const span = Telemetry.span("doc.open");+ span.error(new Error("boom"));+ span.end();++ const [exported] = spanExporter.getFinishedSpans();+ expect(exported?.status.code).not.toBe(0);+ expect(exported?.events[0]?.name).toBe("exception");+ });++ test("emits structured logs", () => {+ Telemetry.warn("WAL flush not acked", {+ "document.id": "doc-1",+ dirty: 3,+ });++ const [record] = logExporter.getFinishedLogRecords();+ expect(record?.severityText).toBe("warn");+ expect(record?.body).toBe("WAL flush not acked");+ expect(record?.attributes).toMatchObject({+ "document.id": "doc-1",+ dirty: 3,+ });+ });+});diff --git a/packages/observability/src/telemetry.ts b/packages/observability/src/telemetry.tsnew file mode 100644index 0000000000..c872a5472d--- /dev/null+++ b/packages/observability/src/telemetry.ts@@ -0,0 +1,96 @@+import { resourceFromAttributes } from "@opentelemetry/resources";+import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";+import {+ type LogAttributes,+ type TelemetryInitConfig,+ TelemetryRuntimeConfig,+} from "./config";+import { ATTR_DEPLOYMENT_ENVIRONMENT, CONSOLE_PREFIX } from "./constants";+import { Logging } from "./logging";+import type { Span } from "./span";+import { Tracing } from "./tracing";++type SpanOperation<T> = (span: Span) => Promise<T>;++export class Telemetry {+ static readonly #tracing = new Tracing();+ static readonly #logging = new Logging();+ static #initialization: Promise<void> | undefined;++ private constructor() {}++ static readonly config = new TelemetryRuntimeConfig();++ /** Initialize providers once, after the optional async enablement decision. */+ static init(config: TelemetryInitConfig): Promise<void> {+ Telemetry.#initialization ??= Telemetry.#initialize(config);+ return Telemetry.#initialization;+ }++ static span<T>(name: string, operation: SpanOperation<T>): Promise<T>;+ static span(name: string): Span;+ static span<T>(+ name: string,+ operation?: SpanOperation<T>,+ ): Span | Promise<T> {+ const span = Telemetry.#tracing.span(name);+ if (!operation) return span;++ return span.run(async () => {+ try {+ return await operation(span);+ } finally {+ span.end();+ }+ });+ }++ /** Start an HTTP client span. Callers must end the returned span. */+ static clientSpan(name: string): Span {+ return Telemetry.#tracing.clientSpan(name);+ }++ static debug(message: string, attributes?: LogAttributes): void {+ Telemetry.#logging.debug(message, attributes);+ }++ static info(message: string, attributes?: LogAttributes): void {+ Telemetry.#logging.info(message, attributes);+ }++ static warn(message: string, attributes?: LogAttributes): void {+ Telemetry.#logging.warn(message, attributes);+ }++ static error(error: unknown, attributes?: LogAttributes): void {+ Telemetry.#logging.error(error, attributes);+ }++ static async flush(): Promise<void> {+ await Promise.all([Telemetry.#tracing.flush(), Telemetry.#logging.flush()]);+ }++ static async shutdown(): Promise<void> {+ await Promise.all([+ Telemetry.#tracing.shutdown(),+ Telemetry.#logging.shutdown(),+ ]);+ }++ static async #initialize(config: TelemetryInitConfig): Promise<void> {+ try {+ if (!(await config.enabled())) return;+ } catch (error) {+ console.warn(`${CONSOLE_PREFIX} Failed to resolve enablement`, error);+ return;+ }++ const resource = resourceFromAttributes({+ [ATTR_SERVICE_NAME]: config.serviceName,+ [ATTR_DEPLOYMENT_ENVIRONMENT]: config.environment,+ });+ const getUserId = () => Telemetry.config.userId;+ Telemetry.#tracing.init(config, resource, getUserId);+ Telemetry.#logging.init(config, resource, getUserId);+ }+}diff --git a/packages/observability/src/tracing.ts b/packages/observability/src/tracing.tsnew file mode 100644index 0000000000..37ad20ffb4--- /dev/null+++ b/packages/observability/src/tracing.ts@@ -0,0 +1,46 @@+import { type Context, context, SpanKind, trace } from "@opentelemetry/api";+import type { Resource } from "@opentelemetry/resources";+import type { TelemetryInitConfig } from "./config";+import { INSTRUMENTATION_SCOPE_NAME } from "./constants";+import type { TelemetryTracingProvider } from "./provider";+import { type Span, SpanImpl } from "./span";++export class Tracing {+ #provider: TelemetryTracingProvider | undefined;++ init(+ config: TelemetryInitConfig,+ resource: Resource,+ getUserId: () => string | undefined,+ ): void {+ this.#provider = config.tracingProvider?.(resource, getUserId);+ }++ span(name: string): Span {+ return this.#startSpan(name, context.active());+ }++ clientSpan(name: string): Span {+ return this.#startSpan(name, context.active(), SpanKind.CLIENT);+ }++ async flush(): Promise<void> {+ await this.#provider?.forceFlush();+ }++ async shutdown(): Promise<void> {+ await this.#provider?.shutdown();+ this.#provider = undefined;+ }++ #startSpan(name: string, parent: Context, kind?: SpanKind): Span {+ const otelSpan = trace+ .getTracer(INSTRUMENTATION_SCOPE_NAME)+ .startSpan(name, kind === undefined ? undefined : { kind }, parent);+ return new SpanImpl(+ otelSpan,+ trace.setSpan(parent, otelSpan),+ (childName, childParent) => this.#startSpan(childName, childParent),+ );+ }+}diff --git a/packages/observability/src/web.ts b/packages/observability/src/web.tsnew file mode 100644index 0000000000..3bff964281--- /dev/null+++ b/packages/observability/src/web.ts@@ -0,0 +1,42 @@+import { W3CTraceContextPropagator } from "@opentelemetry/core";+import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";+import type { Resource } from "@opentelemetry/resources";+import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-base";+import { WebTracerProvider } from "@opentelemetry/sdk-trace-web";+import type { TelemetryInitConfig } from "./config";+import { ATTR_USER_ID } from "./constants";+import type { TelemetryTracingProvider } from "./provider";++/** Creates the browser OpenTelemetry provider used by the web application. */+export function createWebTracingProvider(+ config: TelemetryInitConfig,+ resource: Resource,+ getUserId: () => string | undefined,+): TelemetryTracingProvider {+ const provider = new WebTracerProvider({+ resource,+ spanProcessors: [+ {+ onStart: (span) => {+ const userId = getUserId();+ if (userId !== undefined) span.setAttribute(ATTR_USER_ID, userId);+ },+ onEnd: () => {},+ forceFlush: () => Promise.resolve(),+ shutdown: () => Promise.resolve(),+ },+ ...(config.tracesUrl+ ? [+ new BatchSpanProcessor(+ new OTLPTraceExporter({ url: config.tracesUrl }),+ ),+ ]+ : []),+ ],+ });+ provider.register({+ ...(config.contextManager && { contextManager: config.contextManager }),+ propagator: new W3CTraceContextPropagator(),+ });+ return provider;+}diff --git a/packages/observability/src/zone.ts b/packages/observability/src/zone.tsnew file mode 100644index 0000000000..2340ab0544--- /dev/null+++ b/packages/observability/src/zone.ts@@ -0,0 +1,6 @@+// Importing context-zone loads zone.js, which patches Promise & friends as a+// module side effect, quarantined in its own entry so it never rides into+// non-browser consumers. Hosts must import this statically at startup: patching+// lazily is unreliable, since context would not flow through Promises created+// before the patch.+export { ZoneContextManager } from "@opentelemetry/context-zone";diff --git a/packages/observability/tsconfig.json b/packages/observability/tsconfig.jsonnew file mode 100644index 0000000000..b231a634a4--- /dev/null+++ b/packages/observability/tsconfig.json@@ -0,0 +1,18 @@+{+ "compilerOptions": {+ "allowImportingTsExtensions": true,+ "esModuleInterop": true,+ "forceConsistentCasingInFileNames": true,+ "isolatedModules": true,+ "lib": ["DOM", "DOM.Iterable", "ESNext"],+ "module": "ESNext",+ "moduleResolution": "Bundler",+ "noEmit": true,+ "skipLibCheck": true,+ "strict": true,+ "target": "ESNext",+ "types": ["@types/bun", "@types/node"]+ },+ "include": ["src/**/*"],+ "exclude": ["node_modules"]+}diff --git a/packages/observability/vitest.config.ts b/packages/observability/vitest.config.tsnew file mode 100644index 0000000000..36acedb5d1--- /dev/null+++ b/packages/observability/vitest.config.ts@@ -0,0 +1,9 @@+import { defineConfig } from "vitest/config";++export default defineConfig({+ test: {+ name: "observability",+ environment: "jsdom",+ include: ["src/**/*.test.ts"],+ },+});diff --git a/services/analytics-proxy/bun.lock b/services/analytics-proxy/bun.lockindex ef140d2865..4585fc38b8 100644--- a/services/analytics-proxy/bun.lock+++ b/services/analytics-proxy/bun.lock@@ -4,6 +4,9 @@"workspaces": {"": {"name": "analytics-proxy",+ "dependencies": {+ "hono": "^4.6.20",+ },"devDependencies": {"typescript": "^5.7.3","wrangler": "^4.25.1",@@ -159,6 +162,8 @@"fsevents": ["[email protected]", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="],+ "hono": ["[email protected]", "", {}, "sha512-XcuyW9qE2kJn07PkecMOBd5Vq/hMy7mmGw+idz1yblbg9N17ijJODrvPkn7/dwL3Kulj8LcRJ69DLOWf91dRUg=="],+"kleur": ["[email protected]", "", {}, "sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ=="],"miniflare": ["[email protected]", "", { "dependencies": { "@cspotcode/source-map-support": "0.8.1", "sharp": "^0.34.5", "undici": "7.18.2", "workerd": "1.20260312.1", "ws": "8.18.0", "youch": "4.1.0-beta.10" }, "bin": { "miniflare": "bootstrap.js" } }, "sha512-YSWxec9ssisqkQgaCgcIQxZlB41E9hMiq1nxUgxXHRrE9NsfyC6ptSt8yfgBobsKIseAVKLTB/iEDpMumBv8oA=="],diff --git a/services/analytics-proxy/package.json b/services/analytics-proxy/package.jsonindex c1f4d7a347..98dc10cbd4 100644--- a/services/analytics-proxy/package.json+++ b/services/analytics-proxy/package.json@@ -14,5 +14,8 @@"devDependencies": {"typescript": "^5.7.3","wrangler": "^4.25.1"+ },+ "dependencies": {+ "hono": "^4.6.20"}}diff --git a/services/analytics-proxy/src/index.ts b/services/analytics-proxy/src/index.tsindex 94756f6230..1ea15a3408 100644--- a/services/analytics-proxy/src/index.ts+++ b/services/analytics-proxy/src/index.ts@@ -1,59 +1,132 @@-const POSTHOG_PROVIDER_PREFIX = '/i/ph';-const POSTHOG_HOST = 'us.i.posthog.com';-// Privacy filter lists block the upstream filename; keep the browser-facing alias opaque.+/// <reference path="../worker-configuration.d.ts" />++import { Hono } from 'hono';+import { cors } from 'hono/cors';++// The OTLP traces intake needs a server-side Datadog API key the browser must+// never hold, so it lives as a Worker secret (set via+// `wrangler secret put DD_API_KEY`) and is injected here. The generated Env+// only knows vars/bindings declared in wrangler.jsonc, so augment it with the+// secret. It is absent locally (the dev OTLP collector trusts the container+// network), so the injection is conditional. Other providers rely on the+// client-side token already present on the request, so they need no+// augmentation.+declare global {+ interface Env {+ /** Datadog API key for the OTLP traces intake. Unset in local dev. */+ DD_API_KEY?: string;+ }+}++const POSTHOG_PREFIX = '/i/ph';+const POSTHOG_ORIGIN = 'https://us.i.posthog.com';+// Privacy filter lists block the upstream filename; keep the browser-facing+// alias opaque and restore the real filename only for the upstream request.const POSTHOG_RECORDER_SCRIPT_NAME = 'posthog-recorder.js';const POSTHOG_RECORDER_PROXY_SCRIPT_NAME = 'runtime.js';-const PROVIDERS: Record<string, string> = {- [POSTHOG_PROVIDER_PREFIX]: POSTHOG_HOST,- // Datadog browser logs intake for the us5 site (site: us5.datadoghq.com).- // The observability SDK's `proxy` option targets the `/i/dd` prefix.- '/i/dd': 'browser-intake-us5-datadoghq.com',-};--function getProvider(pathname: string): { apiHost: string; path: string } | null {- if (- pathname ===- `${POSTHOG_PROVIDER_PREFIX}/static/${POSTHOG_RECORDER_PROXY_SCRIPT_NAME}`- ) {- return {- apiHost: POSTHOG_HOST,- path: `/static/${POSTHOG_RECORDER_SCRIPT_NAME}`,- };- }+// OTLP telemetry is proxied to a per-signal intake origin (full origin,+// scheme included, so it can be Datadog's https intakes in deployed envs or+// the plaintext otel-collector in the local pool — where one collector+// serves both signals). Split per signal because Datadog has no single OTLP+// origin: traces and logs use different intake hosts. Unlike the static+// providers the browser can't authenticate, so the Worker injects dd-api-key+// when present.+const OTLP_PREFIX = '/i/otlp';- for (const [prefix, apiHost] of Object.entries(PROVIDERS)) {- if (pathname.startsWith(prefix)) {- return { apiHost, path: pathname.slice(prefix.length) || '/' };- }- }+/** Intake origin for an OTLP subpath, or null for an unknown signal. */+function otlpIntakeUrl(env: Env, path: string): string | null {+ if (path.startsWith('/v1/traces')) return env.OTLP_TRACES_INTAKE_URL;+ if (path.startsWith('/v1/logs')) return env.OTLP_LOGS_INTAKE_URL;return null;}-async function handleProxy(request: Request, apiHost: string, pathWithSearch: string): Promise<Response> {+async function handleProxy(+ request: Request,+ origin: string,+ pathWithSearch: string,+ extraHeaders?: Record<string, string>+): Promise<Response> {const originHeaders = new Headers(request.headers);originHeaders.delete('cookie');- originHeaders.set('X-Forwarded-For', request.headers.get('CF-Connecting-IP') || '');+ originHeaders.set(+ 'X-Forwarded-For',+ request.headers.get('CF-Connecting-IP') || ''+ );+ for (const [key, value] of Object.entries(extraHeaders ?? {})) {+ originHeaders.set(key, value);+ }- const originRequest = new Request(`https://${apiHost}${pathWithSearch}`, {+ const originRequest = new Request(`${origin}${pathWithSearch}`, {method: request.method,headers: originHeaders,- body: request.method !== 'GET' && request.method !== 'HEAD' ? await request.arrayBuffer() : null,+ body:+ request.method !== 'GET' && request.method !== 'HEAD'+ ? await request.arrayBuffer()+ : null,redirect: request.redirect,});return await fetch(originRequest);}-export default {- async fetch(request: Request): Promise<Response> {- const url = new URL(request.url);- const provider = getProvider(url.pathname);+const app = new Hono<{ Bindings: Env }>();++// OTLP uses protobuf cross-origin. Cookies are stripped by handleProxy, so+// wildcard origins and headers are safe and preflights need no credentials.+app.use(+ `${OTLP_PREFIX}/*`,+ cors({+ origin: '*',+ allowMethods: ['POST', 'OPTIONS'],+ maxAge: 86400,+ })+);++// DD is expensive and I don't want someone hammering it. not a perfect solution+// but better than nothing.+app.use('*', async (c, next) => {+ const url = new URL(c.req.url);++ // Cloudflare's edge sets CF-Connecting-IP on every real request; the local+ // pool's Caddy route forwards it too.+ const isLocalhost =+ url.hostname === 'localhost' || url.hostname === '127.0.0.1';+ const clientIp =+ c.req.header('CF-Connecting-IP') ||+ (isLocalhost ? 'local-development' : null);+ if (!clientIp) return c.text('Missing CF-Connecting-IP', 400);++ const { success } = await c.env.RATE_LIMITER.limit({ key: clientIp });+ if (!success) return c.text('Too Many Requests', 429);++ await next();+});++app.all(`${OTLP_PREFIX}/*`, async (c) => {+ const url = new URL(c.req.url);+ const path = url.pathname.slice(OTLP_PREFIX.length) || '/';+ const intake = otlpIntakeUrl(c.env, path);+ if (!intake) return c.text('Not found', 404);++ return handleProxy(+ c.req.raw,+ intake,+ path + url.search,+ c.env.DD_API_KEY ? { 'dd-api-key': c.env.DD_API_KEY } : undefined+ );+});++app.all(`${POSTHOG_PREFIX}/*`, (c) => {+ const url = new URL(c.req.url);+ const proxyPath = url.pathname.slice(POSTHOG_PREFIX.length) || '/';+ const path =+ proxyPath === `/static/${POSTHOG_RECORDER_PROXY_SCRIPT_NAME}`+ ? `/static/${POSTHOG_RECORDER_SCRIPT_NAME}`+ : proxyPath;+ return handleProxy(c.req.raw, POSTHOG_ORIGIN, path + url.search);+});- if (!provider) {- return new Response('Not found', { status: 404 });- }+app.notFound((c) => c.text('Not found', 404));- return handleProxy(request, provider.apiHost, provider.path + url.search);- },-};+export default app;diff --git a/services/analytics-proxy/wrangler.jsonc b/services/analytics-proxy/wrangler.jsoncindex 89e288f74b..b279468698 100644--- a/services/analytics-proxy/wrangler.jsonc+++ b/services/analytics-proxy/wrangler.jsonc@@ -6,23 +6,70 @@"observability": {"enabled": true},+ // Cap each client to 30 req/s. The binding only allows a 10s or 60s window,+ // so 300 per 10s expresses 30/s. Bindings are not inherited by named+ // environments, so this is repeated under each env below.+ "ratelimits": [+ {+ "name": "RATE_LIMITER",+ "namespace_id": "1001",+ "simple": { "limit": 300, "period": 10 }+ }+ ],"vars": {- "POSTHOG_HOST": "https://us.i.posthog.com"+ "POSTHOG_HOST": "https://us.i.posthog.com",+ // Full origins (scheme included) the `/i/otlp` route forwards to, split+ // per signal because Datadog has no single OTLP origin: traces and logs+ // use different intake hosts. In deployed envs the server-side dd-api-key+ // is injected (a secret set via `wrangler secret put DD_API_KEY`, never+ // committed here). In the local pool the docker command overrides both to+ // the otel-collector alias via --var (one collector serves both paths).+ "OTLP_TRACES_INTAKE_URL": "https://trace.agent.us5.datadoghq.com",+ "OTLP_LOGS_INTAKE_URL": "https://otlp-http-intake.logs.us5.datadoghq.com"},"env": {"local": {+ "ratelimits": [+ {+ "name": "RATE_LIMITER",+ "namespace_id": "1001",+ "simple": { "limit": 300, "period": 10 }+ }+ ],"vars": {- "POSTHOG_HOST": "https://us.i.posthog.com"+ "POSTHOG_HOST": "https://us.i.posthog.com",+ // Host defaults for a bare `bun run dev`; the docker pool overrides+ // both to http://otel-collector:4318 via --var (see docker-compose.yml).+ "OTLP_TRACES_INTAKE_URL": "http://localhost:4318",+ "OTLP_LOGS_INTAKE_URL": "http://localhost:4318"}},"dev": {+ "ratelimits": [+ {+ "name": "RATE_LIMITER",+ "namespace_id": "1001",+ "simple": { "limit": 300, "period": 10 }+ }+ ],"vars": {- "POSTHOG_HOST": "https://us.i.posthog.com"+ "POSTHOG_HOST": "https://us.i.posthog.com",+ "OTLP_TRACES_INTAKE_URL": "https://trace.agent.us5.datadoghq.com",+ "OTLP_LOGS_INTAKE_URL": "https://otlp-http-intake.logs.us5.datadoghq.com"}},"prod": {+ "ratelimits": [+ {+ "name": "RATE_LIMITER",+ "namespace_id": "1001",+ "simple": { "limit": 300, "period": 10 }+ }+ ],"vars": {- "POSTHOG_HOST": "https://us.i.posthog.com"+ "POSTHOG_HOST": "https://us.i.posthog.com",+ "OTLP_TRACES_INTAKE_URL": "https://trace.agent.us5.datadoghq.com",+ "OTLP_LOGS_INTAKE_URL": "https://otlp-http-intake.logs.us5.datadoghq.com"}}},diff --git a/services/sync-service/Cargo.toml b/services/sync-service/Cargo.tomlindex 262ff6767b..46687da287 100644--- a/services/sync-service/Cargo.toml+++ b/services/sync-service/Cargo.toml@@ -28,20 +28,22 @@ release = falsewasm-opt = true[dependencies]-worker = { version = "0.8.1", features = ['d1'] }+worker = { workspace = true, features = ['d1'] }tracing.workspace = truetracing-subscriber = { workspace = true, features = ['time',] }tracing-web = "0.1"-web-time = "1.1.0"+web-time.workspace = truetime = { workspace = true, features = ['wasm-bindgen'] }bebop.workspace = truesync_service_bebop_schema = { path = "bebop-schema" }serde_json.workspace = trueserde.workspace = true+getrandom = { workspace = true, features = ["wasm_js"] }wasm-bindgen.workspace = trueuuid = { workspace = true, features = ["js"] }loro = "1.13"macro_sync_service_jwt = { path = "../../crates/macro_sync_service_jwt" }matchit = "0.8.4"+worker-rs-otel = { path = "../../crates/worker-rs-otel" }diff --git a/services/sync-service/src/cf_worker.rs b/services/sync-service/src/cf_worker.rsindex e2137c89fa..f4f9778ad8 100644--- a/services/sync-service/src/cf_worker.rs+++ b/services/sync-service/src/cf_worker.rs@@ -1,5 +1,5 @@use bebop::{Record, SubRecord};-use tracing::error;+use tracing::{Instrument, error};use wasm_bindgen::JsValue;use worker::{Env, Error, Headers, Method, Request, RequestInit, Response, Result, Stub};@@ -76,6 +76,7 @@ pub async fn copy_handler(env: Env, mut req: Request, document_id: &str) -> Resu&headers,MACRO_INTERNAL_AUTH_KEY_HEADER_KEY,)?;+ mv_header(og_req.headers(), &headers, worker_rs_otel::TRACEPARENT)?;ss_req.with_headers(headers);let mut url = og_req.url()?;@@ -140,8 +141,13 @@ pub async fn pass_to_durable_object(document_id: &str,) -> Result<Response> {let stub = get_durable_object(env, document_id)?;+ let span = tracing::info_span!("do.fetch", document.id = %document_id);+ if let Some(traceparent) = worker_rs_otel::traceparent_for_span(&span) {+ req.headers()+ .set(worker_rs_otel::TRACEPARENT, &traceparent)?;+ }- let fut = timeout(stub.fetch_with_request(req), DEFAULT_TIMEOUT_MS);+ let fut = timeout(stub.fetch_with_request(req), DEFAULT_TIMEOUT_MS).instrument(span);let res = timeit_log!("worker -> do_fetch", fut.await);Ok(match res {crate::timeout::TimeoutResult::Ok(x) => x?,diff --git a/services/sync-service/src/durable_object.rs b/services/sync-service/src/durable_object.rsindex fd206e0f0b..1b04830f7a 100644--- a/services/sync-service/src/durable_object.rs+++ b/services/sync-service/src/durable_object.rs@@ -9,7 +9,7 @@ use bebop::Record;use loro::{ExportMode, awareness::EphemeralStore};use matchit::Router;use serde::{Deserialize, Serialize};-use tracing::{debug, error, info, instrument, trace, warn};+use tracing::{Instrument, debug, error, info, instrument, trace, warn};use worker::{Cors, Date, DurableObject, Env, Error, Method, Request, Response, ResponseBody,ResponseBuilder, Result, ScheduledTime, State, WebSocket, WebSocketIncomingMessage,@@ -354,6 +354,7 @@ impl DocumentSyncSession {match (*matched.value,matched.params.get("document_id").inspect(|document_id| {+ tracing::Span::current().record("document.id", *document_id);trace!(route =? matched.value, document_id = document_id, "matched route");}),) {@@ -682,6 +683,9 @@ impl DocumentSyncSession {.and_then(|state| state.export_shallow_snapshot());if let Ok(snapshot) = snapshot {+ // Size of the initial sync this connect sent — the server end+ // of the client's `doc.sync.initial-sync` span.+ tracing::Span::current().record("snapshot.bytes", snapshot.len());websocket::send_initial_sync(&pair.server,snapshot.as_slice(),@@ -936,11 +940,26 @@ impl DurableObject for DocumentSyncSession {.with_cors(&cors(set_allow_origin.as_deref()))?.empty());}- let res = self- .inner_fetch(req)- .await- .context("DurableObject::fetch error")?;- res.with_cors(&cors(set_allow_origin.as_deref()))+ let traceparent = worker_rs_otel::traceparent_from_request(&req);+ let (remote_id, remote_parent) = worker_rs_otel::remote_fields(traceparent.as_ref());+ let path = req.path();+ let span = tracing::info_span!(+ "do.request",+ http.path = %path,+ document.id = tracing::field::Empty,+ snapshot.bytes = tracing::field::Empty,+ trace.remote_id = %remote_id,+ trace.remote_parent = %remote_parent,+ );++ let res = worker_rs_otel::scope(+ &self.env,+ &self.state,+ self.inner_fetch(req).instrument(span),+ )+ .await;+ res.context("DurableObject::fetch error")?+ .with_cors(&cors(set_allow_origin.as_deref()))}async fn websocket_message(&self, ws: WebSocket, msg: WebSocketIncomingMessage) -> Result<()> {@@ -948,41 +967,82 @@ impl DurableObject for DocumentSyncSession {const PING: &str = "ping";let binary_message = match msg {WebSocketIncomingMessage::String(message) => {- // TODO do keepalive?+ // Heartbeat: deliberately unspanned to keep it out of traces.if message == PING {ws.send_with_str(PONG).ok();} else {- warn!("Received unknown 'String' message: {message:?}");+ return worker_rs_otel::scope(&self.env, &self.state, async {+ warn!("Received unknown 'String' message: {message:?}");+ Ok(())+ })+ .await;}return Ok(());}WebSocketIncomingMessage::Binary(bm) => bm,};+ worker_rs_otel::scope(&self.env, &self.state, async {+ let mut telemetry = websocket::InboundMessageTelemetry::new(binary_message.len());++ let res: Result<()> = async {+ let document_id = self.document_id().await.inspect_err(|_| {+ telemetry.record_error_stage("document_context");+ })?;+ let ws_id = get_ws_id_from_tags(&self.state.get_tags(&ws)).ok();+ telemetry.record_context(&document_id, ws_id);++ let message =+ websocket::deserialize_message(&binary_message).inspect_err(|_| {+ telemetry.record_message_type("invalid");+ telemetry.record_error_stage("deserialize");+ })?;+ telemetry.record_message_type(websocket::message_type(&message));++ websocket::process_message(+ &ws,+ &document_id,+ &*self.document_state().await?,+ &*self.session_storage().await?,+ &self.awareness,+ message,+ self.msg_buffer.clone(),+ self,+ &mut telemetry,+ )+ .await+ .inspect_err(|_| {+ telemetry.record_error_stage("process");+ })+ .context("failed to process websocket message")?;++ bump_alarm(&self.state)+ .await+ .inspect_err(|_| {+ telemetry.record_error_stage("alarm");+ })+ .context("failed to keep document alive")?;++ Ok(())+ }+ .await;- websocket::process_message(- &ws,- &self.document_id().await?,- &*self.document_state().await?,- &*self.session_storage().await?,- &self.awareness,- binary_message,- self.msg_buffer.clone(),- self,- )+ if let Err(error) = &res {+ let span = telemetry.error_span();+ {+ let _entered = span.enter();+ tracing::error!(error = ?error, "failed to handle websocket message");+ }+ }+ res+ }).await- .context("failed to process websocket message")?;-- bump_alarm(&self.state)- .await- .context("failed to keep document alive")?;-- Ok(())}/// Save document if needed- #[instrument(skip_all, err)]async fn alarm(&self) -> Result<Response> {- let state = match self+ let span = tracing::info_span!("do.alarm");+ worker_rs_otel::scope(&self.env, &self.state, async {+ let state = match self.document_state().await.context("failed to get document_state")@@ -1046,7 +1106,10 @@ impl DurableObject for DocumentSyncSession {info!("durable object has reached 0 connections")}- Response::ok("ok")+ Response::ok("ok")+ }+ .instrument(span))+ .await}async fn websocket_close(@@ -1056,40 +1119,46 @@ impl DurableObject for DocumentSyncSession {_reason: String,_was_clean: bool,) -> Result<()> {- let peer_ids = Wsm::new(self, &ws).get_peer_ids().await?;- for peer_id in peer_ids {- self.awareness.delete(&peer_id.to_string());- let update = self.awareness.encode(&peer_id.to_string());-- // Don't silently discard the error- websocket::broadcast_awareness(- &ws,- self.state.get_websockets().as_slice(),- update.as_slice(),- self.msg_buffer.clone(),- )- .context("failed to broadcast awareness")?;- }+ worker_rs_otel::scope(&self.env, &self.state, async {+ let peer_ids = Wsm::new(self, &ws).get_peer_ids().await?;+ for peer_id in peer_ids {+ self.awareness.delete(&peer_id.to_string());+ let update = self.awareness.encode(&peer_id.to_string());++ // Don't silently discard the error+ websocket::broadcast_awareness(+ &ws,+ self.state.get_websockets().as_slice(),+ update.as_slice(),+ self.msg_buffer.clone(),+ )+ .context("failed to broadcast awareness")?;+ }- if self.state.get_websockets().len() == 1- && let Ok(document_id) = self.document_id().await- && let Ok(state) = self.document_state().await- && let Ok(snapshot) = state.export_shallow_snapshot()- {- let env = self.env.clone();- self.state.wait_until(async move {- report_new_doc_state(&document_id, &snapshot, &env).await;- report_interaction(&document_id, &env, InteractionReason::LastLeave).await;- });- }- Ok(())+ if self.state.get_websockets().len() == 1+ && let Ok(document_id) = self.document_id().await+ && let Ok(state) = self.document_state().await+ && let Ok(snapshot) = state.export_shallow_snapshot()+ {+ let env = self.env.clone();+ self.state.wait_until(async move {+ report_new_doc_state(&document_id, &snapshot, &env).await;+ report_interaction(&document_id, &env, InteractionReason::LastLeave).await;+ });+ }+ Ok(())+ })+ .await}async fn websocket_error(&self, ws: WebSocket, error: Error) -> Result<()> {- let ws_id = get_ws_id(&self.state, &ws)?;- error!(ws_id = ws_id, error = ?error, "websocket error");- // TODO update awareness stuff- Ok(())+ worker_rs_otel::scope(&self.env, &self.state, async {+ let ws_id = get_ws_id(&self.state, &ws)?;+ error!(ws_id = ws_id, error = ?error, "websocket error");+ // TODO update awareness stuff+ Ok(())+ })+ .await}}diff --git a/services/sync-service/src/lib.rs b/services/sync-service/src/lib.rsindex 6285506a80..c473781bf0 100644--- a/services/sync-service/src/lib.rs+++ b/services/sync-service/src/lib.rs@@ -49,6 +49,7 @@ fn inner_start() {performance_layer().with_details_from_fields(tracing_subscriber::fmt::format::Pretty::default()),)+ .with(worker_rs_otel::OtelLayer::new("sync-service")).init();}#[cfg(not(target_arch = "wasm32"))]@@ -64,7 +65,21 @@ fn start() {}#[event(fetch)]-async fn fetch(req: worker::Request, env: Env, _ctx: Context) -> Result<worker::Response> {+async fn fetch(req: worker::Request, env: Env, ctx: Context) -> Result<worker::Response> {use crate::cf_worker::router;- router(env, req).await+ use tracing::Instrument;++ let traceparent = worker_rs_otel::traceparent_from_request(&req);+ let (remote_id, remote_parent) = worker_rs_otel::remote_fields(traceparent.as_ref());+ let method = req.method().to_string();+ let path = req.path();+ let span = tracing::info_span!(+ "request",+ http.method = %method,+ http.path = %path,+ trace.remote_id = %remote_id,+ trace.remote_parent = %remote_parent,+ );++ worker_rs_otel::scope(&env, &ctx, router(env.clone(), req).instrument(span)).await}diff --git a/services/sync-service/src/websocket.rs b/services/sync-service/src/websocket.rsindex 45ddf378cb..b73c5ad7e6 100644--- a/services/sync-service/src/websocket.rs+++ b/services/sync-service/src/websocket.rs@@ -23,6 +23,118 @@ fn serialize<'a, T: bebop::Record<'a>>(Ok(msg_buf)}+/// Decodes an inbound WebSocket payload once, before its trace span is created.+pub fn deserialize_message(message: &[u8]) -> Result<FromPeer<'_>> {+ if message.len() > MAX_MESSAGE_SIZE {+ tracing::warn!("received message might be too large {}", message.len());+ }+ FromPeer::deserialize(message).context(format!(+ "failed to deserialize message, message length {}",+ message.len()+ ))+}++/// Returns the protocol-level type of an inbound WebSocket message.+pub fn message_type(message: &FromPeer<'_>) -> &'static str {+ match message {+ FromPeer::PeerRegisterId { .. } => "register_id",+ FromPeer::PeerUpdate { .. } => "update",+ FromPeer::PeerAwareness { .. } => "awareness",+ FromPeer::PeerRequestSince { .. } => "request_since",+ FromPeer::PeerRequestSnapshot { .. } => "request_snapshot",+ FromPeer::Unknown => "unknown",+ }+}++/// Diagnostic fields collected while handling an inbound WebSocket message.+pub struct InboundMessageTelemetry {+ document_id: Option<String>,+ ws_id: Option<String>,+ message_type: &'static str,+ message_bytes: usize,+ error_stage: &'static str,+ op_id: Option<String>,+ peer_id: Option<u64>,+ update_bytes: Option<usize>,+ response_bytes: Option<usize>,+ broadcast_targets: Option<usize>,+}++impl InboundMessageTelemetry {+ /// Start collecting diagnostics without creating a span.+ pub fn new(message_bytes: usize) -> Self {+ Self {+ document_id: None,+ ws_id: None,+ message_type: "unknown",+ message_bytes,+ error_stage: "unknown",+ op_id: None,+ peer_id: None,+ update_bytes: None,+ response_bytes: None,+ broadcast_targets: None,+ }+ }++ /// Record identifiers available from the Durable Object callback.+ pub fn record_context(&mut self, document_id: &str, ws_id: Option<String>) {+ self.document_id = Some(document_id.to_string());+ self.ws_id = ws_id;+ }++ /// Record the decoded protocol message type.+ pub fn record_message_type(&mut self, message_type: &'static str) {+ self.message_type = message_type;+ }++ /// Record which stage of message handling failed.+ pub fn record_error_stage(&mut self, error_stage: &'static str) {+ self.error_stage = error_stage;+ }++ /// Create the only span emitted for this message, after handling fails.+ pub fn error_span(&self) -> tracing::Span {+ let span = tracing::error_span!(+ "ws.message.error",+ document.id = tracing::field::Empty,+ ws.id = tracing::field::Empty,+ message.type = self.message_type,+ message.bytes = self.message_bytes,+ error.stage = self.error_stage,+ op.id = tracing::field::Empty,+ peer.id = tracing::field::Empty,+ update.bytes = tracing::field::Empty,+ response.bytes = tracing::field::Empty,+ broadcast.targets = tracing::field::Empty,+ );++ if let Some(document_id) = &self.document_id {+ span.record("document.id", document_id.as_str());+ }+ if let Some(ws_id) = &self.ws_id {+ span.record("ws.id", ws_id.as_str());+ }+ if let Some(op_id) = &self.op_id {+ span.record("op.id", op_id.as_str());+ }+ if let Some(peer_id) = self.peer_id {+ span.record("peer.id", peer_id);+ }+ if let Some(update_bytes) = self.update_bytes {+ span.record("update.bytes", update_bytes);+ }+ if let Some(response_bytes) = self.response_bytes {+ span.record("response.bytes", response_bytes);+ }+ if let Some(broadcast_targets) = self.broadcast_targets {+ span.record("broadcast.targets", broadcast_targets);+ }++ span+ }+}+/// Sends the initial sync message to the client over the websocket/// The initial sync message contains the snapshot of the current state of the documentpub fn send_initial_sync(@@ -79,19 +191,11 @@ pub async fn process_message(document_state: &DocumentState,session_storage: &SessionStorage,awareness: &EphemeralStore,- message: Vec<u8>,+ message: FromPeer<'_>,buf: Arc<Mutex<Vec<u8>>>,dss: &DocumentSyncSession,+ telemetry: &mut InboundMessageTelemetry,) -> Result<()> {- if message.len() > MAX_MESSAGE_SIZE {- tracing::warn!("received message might be too large {}", message.len());- }-- let message: FromPeer = FromPeer::deserialize(message.as_slice()).context(format!(- "failed to deserialize message, message length {}",- message.len()- ))?;-trace!(message = tracing::field::display(&message),"process websocket message"@@ -100,6 +204,7 @@ pub async fn process_message(// Handle peer id registration// This registers a peer id to the owner of the current websocketFromPeer::PeerRegisterId { peerid } => {+ telemetry.peer_id = Some(peerid);Wsm::new(dss, ws).add_new_peerid(peerid, document_id).await?;@@ -108,6 +213,8 @@ pub async fn process_message(// Should extract binary update and broadcast it to all other connected peers// Should also store the update in the operation log to be applied to the remote docFromPeer::PeerUpdate { updates, id } => {+ telemetry.op_id = Some(id.to_string());+ telemetry.update_bytes = Some(updates.iter().map(|u| u.len()).sum());if !Wsm::new(dss, ws).can_edit().await? {tracing::warn!("received update from peer without edit permission");return Ok(());@@ -115,6 +222,7 @@ pub async fn process_message(let peer_ids = Wsm::new(dss, ws).get_peer_ids().await.unwrap_or_default();let peer_id = peer_ids.first().copied();+ telemetry.peer_id = peer_id;let now_ms = web_time::SystemTime::now().duration_since(web_time::UNIX_EPOCH).map(|d| d.as_millis() as i64)@@ -148,9 +256,19 @@ pub async fn process_message(let mut buf = buf.lock("serialize RemoteUpdateAck in PeerUpdate handler");let serialized =serialize(message, &mut buf).context("Failed serializing update")?;- ws.send_with_bytes(serialized).context("failed to send")?;+ ws.send_with_bytes(serialized)+ .inspect_err(|error| {+ tracing::error!(error = ?error, op.id = %id, "failed to send update ack");+ })+ .context("failed to send update ack")?;}+ // Peers receiving the rebroadcast (everyone but the sender). The+ // count is the key Teo/Hutch diagnostic: if it excludes a peer the+ // DO thinks isn't connected, that's the dropped-update bug.+ let sockets = dss.get_websockets();+ let targets: Vec<&WebSocket> = sockets.iter().filter(|w| w != &ws).collect();+ telemetry.broadcast_targets = Some(targets.len());for update in &updates {// broadcast each update to other peerslet message = FromRemote::RemoteUpdate {@@ -159,7 +277,7 @@ pub async fn process_message(let mut buf = buf.lock("serialize RemoteUpdate in PeerUpdate handler");let serialized =serialize(message, &mut buf).context("Failed serializing update")?;- for w in dss.get_websockets().iter().filter(|w| w != &ws) {+ for w in &targets {// A dead peer socket must not abort delivery to the// remaining peers.if let Err(e) = w.send_with_bytes(serialized) {@@ -178,7 +296,10 @@ pub async fn process_message(return Ok(());}let encodede = awareness.encode_all();- broadcast_awareness(ws, &dss.get_websockets(), &encodede, buf)+ let sockets = dss.get_websockets();+ telemetry.broadcast_targets =+ Some(sockets.iter().filter(|socket| socket != &ws).count());+ broadcast_awareness(ws, &sockets, &encodede, buf).context("failed to broadcast awareness")?;}// Handle a peer requesting a specific set of updates from the document.@@ -191,6 +312,8 @@ pub async fn process_message(let update = document_state.export_updates_since(&decoded).context("failed to export updates")?;+ // Server end of the client's `doc.sync.catchup` span.+ telemetry.response_bytes = Some(update.len());// Echo the client's *original* vv bytes back, not a re-encoded copy.// The client correlates the response by byte-exact match on the vv it@@ -211,6 +334,7 @@ pub async fn process_message(// Peer is requesting a snapshot from the remoteFromPeer::PeerRequestSnapshot {} => {let snapshot = document_state.export_shallow_snapshot()?;+ telemetry.response_bytes = Some(snapshot.len());let message = FromRemote::RemoteSnapshot {snapshot: SliceWrapper::Raw(&snapshot),diff --git a/services/sync-service/wrangler.docker.toml b/services/sync-service/wrangler.docker.tomlindex 2a653af9f9..68c1342960 100644--- a/services/sync-service/wrangler.docker.toml+++ b/services/sync-service/wrangler.docker.toml@@ -33,6 +33,12 @@ INTERNAL_API_SECRET_KEY = "LOCAL_API_KEY"SPS_URL = "http://search-processing-service:8080"DSS_URL = "http://document_storage_service:8080"DSS_INTERNAL_AUTH_KEY = "local"+# Tags exported spans with deployment.environment=local, matching the+# Datadog agent's DD_ENV so the env:local trace/log views include them.+ENVIRONMENT = "local"+# jaeger/datadog-agent alias on the compose `services` network.+# Unset in deployed envs = trace and log export disabled.+OTEL_EXPORTER_OTLP_ENDPOINT = "http://otel-collector:4318"[[kv_namespaces]]binding = "DOCUMENT_VERSIONING_KV"diff --git a/services/sync-service/wrangler.toml b/services/sync-service/wrangler.tomlindex 8b49e8f9ef..9252dfcd7d 100644--- a/services/sync-service/wrangler.toml+++ b/services/sync-service/wrangler.toml@@ -34,6 +34,12 @@ INTERNAL_API_SECRET_KEY = "LOCAL_API_KEY"SPS_URL = "https://search-processing-dev.macro.com"DSS_URL = "http://localhost:8086"DSS_INTERNAL_AUTH_KEY = "local"+# Tags exported spans with deployment.environment=local, matching the+# Datadog agent's DD_ENV so the env:local trace/log views include them.+ENVIRONMENT = "local"+# Local jaeger/datadog-agent OTLP receiver (docker port-mapped to the host).+# Unset in deployed envs = trace and log export disabled.+OTEL_EXPORTER_OTLP_ENDPOINT = "http://localhost:4318"[[kv_namespaces]]binding = "DOCUMENT_VERSIONING_KV"diff --git a/tooling/xtask/crates/xtask_local/src/local/cli.rs b/tooling/xtask/crates/xtask_local/src/local/cli.rsindex 55d14e8c9d..4d207609e8 100644--- a/tooling/xtask/crates/xtask_local/src/local/cli.rs+++ b/tooling/xtask/crates/xtask_local/src/local/cli.rs@@ -118,6 +118,48 @@ pub struct RunArgs {/// Stream subprocess output and show per-step timings.#[arg(long, short)]pub verbose: bool,+ /// Start a local OTLP trace collector and wire services to export to it.+ /// Omit to leave tracing off (the default) — see `docker/docker-compose.yml`+ /// for what each backend does.+ #[arg(long)]+ pub traces: Option<TracesBackend>,+}++/// Which OTLP trace collector `--traces` should bring up.+#[derive(clap::ValueEnum, Clone, Copy, Debug, PartialEq, Eq)]+pub enum TracesBackend {+ /// Fully local trace viewer at http://localhost:16686, no account needed.+ Jaeger,+ /// Forwards to Datadog APM (us5); requires `DD_API_KEY`.+ Datadog,+}++impl TracesBackend {+ /// The compose service name gated by this backend's profile.+ pub fn compose_service(self) -> &'static str {+ match self {+ TracesBackend::Jaeger => "jaeger",+ TracesBackend::Datadog => "datadog-agent",+ }+ }++ /// The compose profile gating this backend's service.+ pub fn compose_profile(self) -> &'static str {+ match self {+ TracesBackend::Jaeger => "jaeger",+ TracesBackend::Datadog => "datadog",+ }+ }++ /// Env var the backend needs to forward telemetry, if any. Checked+ /// before starting the collector so a missing key fails loud instead of+ /// silently dropping everything at the intake.+ pub fn required_env(self) -> Option<&'static str> {+ match self {+ TracesBackend::Jaeger => None,+ TracesBackend::Datadog => Some("DD_API_KEY"),+ }+ }}#[derive(Args, Clone, Default)]diff --git a/tooling/xtask/crates/xtask_local/src/local/e2e.rs b/tooling/xtask/crates/xtask_local/src/local/e2e.rsindex 0f9118a0a4..2a67f8d86a 100644--- a/tooling/xtask/crates/xtask_local/src/local/e2e.rs+++ b/tooling/xtask/crates/xtask_local/src/local/e2e.rs@@ -122,6 +122,7 @@ pub fn run(args: &LocalE2eArgs) -> Result<()> {env: EnvArgs::default(),build: BuildArgs::default(),no_frontend: true,+ traces: None,verbose: false,},..stack::UpArgs::default()diff --git a/tooling/xtask/crates/xtask_local/src/local/env_layer.rs b/tooling/xtask/crates/xtask_local/src/local/env_layer.rsindex 0d35e668da..20227761a8 100644--- a/tooling/xtask/crates/xtask_local/src/local/env_layer.rs+++ b/tooling/xtask/crates/xtask_local/src/local/env_layer.rs@@ -60,6 +60,18 @@ pub fn resolve(for (k, v) in local_env::LocalEnv::for_instance(mode, instance).to_env() {env.insert(k, v);}+ // Opt-in local trace export: point services at the local OTLP collector+ // (docker-network alias `otel-collector`) only when one answers on the+ // OTLP HTTP port, so services don't spam export errors when none is+ // running. Both viewers bind 4318 — Jaeger (compose profile `jaeger`)+ // and the Datadog agent (profile `datadog`) — so (re)start the stack+ // after starting one to pick up tracing.+ if super::summary::port_open(4318) {+ env.insert(+ "OTEL_EXPORTER_OTLP_ENDPOINT".into(),+ "http://otel-collector:4317".into(),+ );+ }}// `--env-file` overlay (the only dotenv we still parse).diff --git a/tooling/xtask/crates/xtask_local/src/local/frontend.rs b/tooling/xtask/crates/xtask_local/src/local/frontend.rsindex 804e19dbbd..d8197eb8bf 100644--- a/tooling/xtask/crates/xtask_local/src/local/frontend.rs+++ b/tooling/xtask/crates/xtask_local/src/local/frontend.rs@@ -90,8 +90,15 @@ pub fn build_static(stage: &Stage, instance: &Instance, prebuilt: Option<&Path>)/// The env the dev server runs with. Both local and dev point the whole app at/// the local proxy origin (single backend origin); the modes differ only in/// what the *services* behind the proxy talk to (local infra vs dev resources).-fn dev_env(instance: &Instance) -> Vec<(String, String)> {- vec![+///+/// When a trace collector is up (`--traces`), point the OTel exporter at the+/// analytics-proxy through the same proxy origin (`/i/otlp`), so tracing works+/// for any instance (named instances have no host-port binding for the proxy)+/// and exercises the real browser -> proxy -> collector path. Left unset+/// otherwise. This overrides the bare-dev defaults in apps/web/.env.local+/// (Vite lets process env win).+fn dev_env(instance: &Instance, traces_enabled: bool) -> Vec<(String, String)> {+ let mut env = vec![("PORT".to_string(),instance.port(Port::Frontend).to_string(),@@ -101,7 +108,25 @@ fn dev_env(instance: &Instance) -> Vec<(String, String)> {"VITE_LOCAL_BACKEND_ORIGIN".to_string(),proxy::url(instance),),- ]+ ];+ if traces_enabled {+ env.push((+ "VITE_OTEL_EXPORTER_URL".to_string(),+ format!("{}/i/otlp/v1/traces", proxy::url(instance)),+ ));+ // Tag frontend telemetry with the same env the Datadog agent uses+ // (DD_ENV, default `local`), so the summary's traces/logs links —+ // filtered by that env — actually match the emitted spans/records.+ env.push((+ "VITE_OTEL_ENV".to_string(),+ macro_env_var::maybe_read_env("DD_ENV").unwrap_or_else(|| "local".into()),+ ));+ }+ env.push((+ "VITE_ENABLE_BROWSER_OTEL".to_string(),+ traces_enabled.to_string(),+ ));+ env}/// Poll the backend (auth health, through the proxy) until ready.@@ -163,7 +188,12 @@ fn tail_str(s: &str, n: usize) -> String {/// is captured (suppressed — the vite banner would duplicate the summary's URL)/// but retained so an unexpected exit can be diagnosed. Returns `None` in/// dry-run mode.-pub fn start(stage: &Stage, instance: &Instance, mode: Mode) -> Result<Option<Frontend>> {+pub fn start(+ stage: &Stage,+ instance: &Instance,+ mode: Mode,+ traces_enabled: bool,+) -> Result<Option<Frontend>> {if mode.spec().wait_backend_before_frontend {wait_backend_ready(stage, instance)?;}@@ -205,7 +235,7 @@ pub fn start(stage: &Stage, instance: &Instance, mode: Mode) -> Result<Option<Fr.stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::piped());- for (k, v) in dev_env(instance) {+ for (k, v) in dev_env(instance, traces_enabled) {cmd.env(k, v);}let mut child = cmd.spawn().context("launching `bun run dev`")?;diff --git a/tooling/xtask/crates/xtask_local/src/local/gen_compose.rs b/tooling/xtask/crates/xtask_local/src/local/gen_compose.rsindex b6c0e847cd..09d8f9d844 100644--- a/tooling/xtask/crates/xtask_local/src/local/gen_compose.rs+++ b/tooling/xtask/crates/xtask_local/src/local/gen_compose.rs@@ -33,12 +33,14 @@ pub const CADDY_IMAGE: &str = "caddy:2-alpine";/// Base-compose services that bind fixed host ports but aren't in our inventory/// (reached via the proxy / container network, not the host). For named/// instances we drop their host-port bindings so concurrent stacks don't collide-/// on 8787/6969/8096/8100.+/// on 8787/6969/8096/8100/8933.const AUX_HOST_PORT_SERVICES: &[&str] = &["sync_service","websocket_service","lexical_service","static_file_cdn",+ "ai_editing_worker",+ "analytics_proxy",];/// The host directory bind-mounted into FusionAuth atdiff --git a/tooling/xtask/crates/xtask_local/src/local/mod.rs b/tooling/xtask/crates/xtask_local/src/local/mod.rsindex b56e569520..d374b5ce62 100644--- a/tooling/xtask/crates/xtask_local/src/local/mod.rs+++ b/tooling/xtask/crates/xtask_local/src/local/mod.rs@@ -185,6 +185,13 @@ pub fn run_stack(mode: Mode, args: &cli::RunArgs) -> Result<()> {stack::clear_state(&instance)?;}+ // Before `prepare` (which resolves env and reads the OTLP port to decide+ // whether to wire `OTEL_EXPORTER_OTLP_ENDPOINT`), so a `--traces` run gets+ // the same auto-wiring as a collector started manually beforehand.+ if let Some(backend) = args.traces {+ ensure_tracing_backend(&stage, backend)?;+ }+// `run_local`/`run_dev` are full delete + full create: tear the previous// stack and ALL its stateful volumes down so the bring-up is always from a// clean slate. That makes the command unconditionally idempotent — no@@ -240,7 +247,7 @@ pub fn run_stack(mode: Mode, args: &cli::RunArgs) -> Result<()> {stage.note("frontend disabled (--no-frontend)");None} else {- frontend::start(&stage, &instance, mode)?+ frontend::start(&stage, &instance, mode, args.traces.is_some())?};summary::print(@@ -650,7 +657,93 @@ fn bring_up_app(}up.arg("proxy");}- stage.run("Starting services (docker compose up -d)", &mut up)+ stage.run("Starting services (docker compose up -d)", &mut up)?;+ connect_tracing_network(instance);+ Ok(())+}++/// Start the requested trace collector (`--traces`) under its own compose+/// project, the same one a developer would use manually — see+/// `docker/docker-compose.yml`'s `jaeger`/`datadog-agent` services. Global and+/// idempotent (like `start_localstack`): one collector per machine, shared+/// across instances, left running across `run_local` invocations.+fn ensure_tracing_backend(stage: &Stage, backend: cli::TracesBackend) -> Result<()> {+ // A keyed backend with a missing key accepts telemetry locally and drops+ // every payload at the vendor intake (403), which looks like "traces are+ // broken" rather than "key is missing" — so fail loud up front.+ if let Some(var) = backend.required_env()+ && macro_env_var::maybe_read_env(var).is_none_or(|v| v.is_empty())+ {+ anyhow::bail!(+ "--traces {} requires the {var} env var to be set (export it in \+ the shell you run this from)",+ backend.compose_profile()+ );+ }+ let compose = repo_root().join("docker/docker-compose.yml");+ let mut up = Command::new("docker");+ up.arg("compose")+ .arg("--project-directory")+ .arg(repo_root())+ .arg("-f")+ .arg(&compose)+ .arg("--profile")+ .arg(backend.compose_profile())+ .arg("up")+ .arg("-d")+ .arg("--remove-orphans")+ .arg(backend.compose_service());+ stage.run(+ &format!("Starting {} trace collector", backend.compose_profile()),+ &mut up,+ )?;++ // `up -d` returns once the container starts, not once it's accepting+ // connections; `env_layer::resolve` (which runs right after this, in+ // `prepare`) needs the OTLP port live NOW to decide whether to wire+ // `OTEL_EXPORTER_OTLP_ENDPOINT`, so wait for it here instead of racing.+ for _ in 0..50 {+ if summary::port_open(4318) {+ return Ok(());+ }+ std::thread::sleep(std::time::Duration::from_millis(100));+ }+ anyhow::bail!(+ "{} trace collector did not come up on port 4318 in time",+ backend.compose_profile()+ )+}++/// If the global trace collector (Jaeger or the Datadog agent) is running,+/// attach it to this instance's `services` network under the `otel-collector`+/// alias that the injected `OTEL_EXPORTER_OTLP_ENDPOINT` points at (see+/// `env_layer::resolve`). The collectors are started under their own compose+/// project (`docker/docker-compose.yml`, profiles `jaeger`/`datadog`), and+/// Compose prefixes networks per project — so without this, instance+/// containers can't resolve `otel-collector` and span exports fail with DNS+/// errors. Best-effort: "already connected" (rerun) and other failures are+/// ignored, they only mean traces don't flow.+fn connect_tracing_network(instance: &Instance) {+ if !summary::port_open(4318) {+ return;+ }+ // Find the collector container (Jaeger or Datadog agent — both publish the+ // OTLP HTTP port) by that port rather than assuming a container name, in+ // case it was started under a different project.+ let Some(collector) = Command::new("docker")+ .args(["ps", "--filter", "publish=4318", "--format", "{{.Names}}"])+ .output()+ .ok()+ .map(|out| String::from_utf8_lossy(&out.stdout).trim().to_string())+ .filter(|name| !name.is_empty())+ else {+ return;+ };+ let _ = Command::new("docker")+ .args(["network", "connect", "--alias", "otel-collector"])+ .arg(format!("{}_services", instance.project_name()))+ .arg(collector)+ .output();}/// Restart the given services' containers so they re-exec their freshly builtdiff --git a/tooling/xtask/crates/xtask_local/src/local/proxy.rs b/tooling/xtask/crates/xtask_local/src/local/proxy.rsindex b809517aa6..062689a36a 100644--- a/tooling/xtask/crates/xtask_local/src/local/proxy.rs+++ b/tooling/xtask/crates/xtask_local/src/local/proxy.rs@@ -121,6 +121,16 @@ const SPECIAL_ROUTES: &str = r#" @websocket path /websocket /websocket/*uri strip_prefix /syncreverse_proxy sync-service:8787}+ # Analytics/telemetry proxy worker (PostHog and OTLP traces/logs).+ # No prefix strip: the worker itself routes on the /i/{ph,dd,otlp} prefix,+ # and it listens on 8098, not the :8080 the generated routes assume.+ # Set CF-Connecting-IP (absent without Cloudflare's edge in front) so the+ # worker's rate-limit keying has a client IP instead of erroring.+ handle /i/* {+ reverse_proxy analytics-proxy:8098 {+ header_up CF-Connecting-IP {http.request.remote.host}+ }+ }handle_path /lexical/* {reverse_proxy lexical-service:8096}diff --git a/tooling/xtask/crates/xtask_local/src/local/proxy/test.rs b/tooling/xtask/crates/xtask_local/src/local/proxy/test.rsindex d7917ca37d..070ffce3e1 100644--- a/tooling/xtask/crates/xtask_local/src/local/proxy/test.rs+++ b/tooling/xtask/crates/xtask_local/src/local/proxy/test.rs@@ -35,6 +35,15 @@ fn websocket_services_use_a_matcher() {assert!(caddy.contains("handle_path /auth/* {"));}+/// The analytics-proxy worker is reached through the single origin at `/i/*`+/// (PostHog and OTLP traces/logs), un-stripped, on its own :8098 port.+#[test]+fn analytics_proxy_route_is_present() {+ let caddy = caddyfile(Mode::Local, false);+ assert!(caddy.contains("handle /i/* {"));+ assert!(caddy.contains("reverse_proxy analytics-proxy:8098"));+}+#[test]fn document_content_services_are_available_through_the_proxy() {let caddy = caddyfile(Mode::Local, false);diff --git a/tooling/xtask/crates/xtask_local/src/local/summary.rs b/tooling/xtask/crates/xtask_local/src/local/summary.rsindex bd35529afe..43e638ae8b 100644--- a/tooling/xtask/crates/xtask_local/src/local/summary.rs+++ b/tooling/xtask/crates/xtask_local/src/local/summary.rs@@ -146,7 +146,60 @@ pub fn print(};row(label, value);}+ if let Some(url) = traces_url() {+ row("traces", url);+ }+ if let Some(url) = dd_logs_url() {+ row("dd logs", url);+ }+row("logs", logs_command(instance, &env.generated_path));row("stop", stop_command(instance));println!();}++/// The trace viewer for the OTel spans the web app emits, if one is running.+///+/// The viewers are global (fixed ports, one per machine, started manually via+/// compose profiles — see `docker/docker-compose.yml`), so this probes rather+/// than consulting the instance: the Jaeger UI on 16686, else a Datadog agent+/// on the OTLP port 4318, whose traces land in the Datadog APM UI under the+/// `env:` its compose profile sets (`DD_ENV`, default `local`).+fn traces_url() -> Option<String> {+ if port_open(16686) {+ return Some("http://localhost:16686".into());+ }+ if port_open(4318) {+ return Some(format!(+ "https://us5.datadoghq.com/apm/traces?query=env%3A{}",+ dd_env()+ ));+ }+ None+}++/// The Datadog Logs Explorer for the OTel log records the web app emits —+/// only when the Datadog agent is the running collector (Jaeger has no log+/// UI; its OTLP logs are dropped).+fn dd_logs_url() -> Option<String> {+ if port_open(16686) || !port_open(4318) {+ return None;+ }+ Some(format!(+ "https://us5.datadoghq.com/logs?query=env%3A{}",+ dd_env()+ ))+}++fn dd_env() -> String {+ macro_env_var::maybe_read_env("DD_ENV").unwrap_or_else(|| "local".into())+}++/// Whether something is listening on `port` on localhost.+pub(super) fn port_open(port: u16) -> bool {+ std::net::TcpStream::connect_timeout(+ &([127, 0, 0, 1], port).into(),+ std::time::Duration::from_millis(150),+ )+ .is_ok()+}diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/deploy_preview.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/deploy_preview.rsindex eb8e7cfc1c..89fb7a2a5c 100644--- a/tooling/xtask/crates/xtask_workflows/src/workflows/deploy_preview.rs+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/deploy_preview.rs@@ -74,8 +74,6 @@ fn build() -> Step<Run> {Step::new("Build").run("just build-dev").working_directory(xtask_paths::repo_dir!("apps/web"))- .add_env(Env::new("VITE_DD_WEB_APP_TOKEN", vars::DD_WEB_APP_TOKEN))- .add_env(Env::new("VITE_DD_HASH", "${{ github.sha }}")).add_env(Env::new("VITE_SEGMENT_WRITE_KEY",vars::SEGMENT_WRITE_KEY_PRODUCTION,diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/deploy_web_app.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/deploy_web_app.rsindex 9a720382c3..07635b6ca3 100644--- a/tooling/xtask/crates/xtask_workflows/src/workflows/deploy_web_app.rs+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/deploy_web_app.rs@@ -81,8 +81,6 @@ pub fn patch(root: &mut serde_yaml::Value) -> Result<()> {required: trueDD_API_KEY:required: true- DD_WEB_APP_TOKEN:- required: trueSEGMENT_WRITE_KEY:required: truePOSTHOG_API_KEY:@@ -123,8 +121,6 @@ fn build() -> Step<Run> {Step::new("Build").run("just build-${{ inputs.environment }}").working_directory(xtask_paths::repo_dir!("apps/web"))- .add_env(Env::new("VITE_DD_WEB_APP_TOKEN", vars::DD_WEB_APP_TOKEN))- .add_env(Env::new("VITE_DD_HASH", "${{ github.sha }}")).add_env(Env::new("VITE_SEGMENT_WRITE_KEY", vars::SEGMENT_WRITE_KEY)).add_env(Env::new("VITE_POSTHOG_API_KEY", vars::POSTHOG_API_KEY))}diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/deploy_web_app_dev_push.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/deploy_web_app_dev_push.rsindex 9b136d81b2..1f34f88e63 100644--- a/tooling/xtask/crates/xtask_workflows/src/workflows/deploy_web_app_dev_push.rs+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/deploy_web_app_dev_push.rs@@ -42,7 +42,6 @@ pub fn patch(root: &mut serde_yaml::Value) -> Result<()> {PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }}DD_APP_KEY: ${{ secrets.DD_APP_KEY }}DD_API_KEY: ${{ secrets.DD_API_KEY }}- DD_WEB_APP_TOKEN: ${{ secrets.DD_WEB_APP_TOKEN }}SEGMENT_WRITE_KEY: ${{ secrets.SEGMENT_WRITE_KEY_PRODUCTION }}POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}"#})?,diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/vars.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/vars.rsindex cc59718369..db6e6759d3 100644--- a/tooling/xtask/crates/xtask_workflows/src/workflows/vars.rs+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/vars.rs@@ -17,7 +17,6 @@ secret!(CACHIX_AUTH_TOKEN);secret!(CLOUDFLARE_API_TOKEN);secret!(DD_API_KEY);secret!(DD_APP_KEY);-secret!(DD_WEB_APP_TOKEN);secret!(DOPPLER_PREVIEW_TOKEN);secret!(DOPPLER_TOKEN);secret!(FLY_API_TOKEN);