Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

wip: add frontend distributed otel (#5188)

Wolf Mermelstein2a29d67aec21
diff --git a/.cargo/config.toml b/.cargo/config.toml
index 8cfa9bf0f6..dce456f5bd 100644
--- a/.cargo/config.toml
+++ b/.cargo/config.toml
@@ -16,3 +16,9 @@
# (e.g. `cargo x cache-wasm` never builds guppy/hakari/aws-sdk/tokio).
[alias]
x = "run -p xtask --"
+
+# worker-rs-otel (wasm32-unknown-unknown) pulls getrandom 0.3 via OTLP types;
+# getrandom 0.3 needs its backend selected at
+# build time. Harmless for wasm crates that don't use getrandom 0.3.
+[target.wasm32-unknown-unknown]
+rustflags = ['--cfg', 'getrandom_backend="wasm_js"']
diff --git a/.config/hakari.toml b/.config/hakari.toml
index f0b54a5ff9..742fa1f783 100644
--- a/.config/hakari.toml
+++ b/.config/hakari.toml
@@ -59,6 +59,7 @@ workspace-members = [
"macro_sync_service_jwt",
"sync_service",
"sync_service_bebop_schema",
+ "worker-rs-otel",
# maybe_send: zero-dep marker traits used by cache-core; must stay
# wasm-safe (workspace-hack pulls native-only deps like mio into wasm32).
"maybe_send",
diff --git a/.github/workflows/deploy_preview.yml b/.github/workflows/deploy_preview.yml
index 711e919d23..636e2271b6 100644
--- a/.github/workflows/deploy_preview.yml
+++ b/.github/workflows/deploy_preview.yml
@@ -42,8 +42,6 @@ jobs:
- name: Build
run: just build-dev
env:
- VITE_DD_WEB_APP_TOKEN: ${{ secrets.DD_WEB_APP_TOKEN }}
- VITE_DD_HASH: ${{ github.sha }}
VITE_SEGMENT_WRITE_KEY: ${{ secrets.SEGMENT_WRITE_KEY_PRODUCTION }}
VITE_POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}
working-directory: apps/web
diff --git a/.github/workflows/deploy_web_app.yml b/.github/workflows/deploy_web_app.yml
index da462bae72..802c6b05da 100644
--- a/.github/workflows/deploy_web_app.yml
+++ b/.github/workflows/deploy_web_app.yml
@@ -22,8 +22,6 @@ on:
required: true
DD_API_KEY:
required: true
- DD_WEB_APP_TOKEN:
- required: true
SEGMENT_WRITE_KEY:
required: true
POSTHOG_API_KEY:
@@ -64,8 +62,6 @@ jobs:
- name: Build
run: just build-${{ inputs.environment }}
env:
- VITE_DD_WEB_APP_TOKEN: ${{ secrets.DD_WEB_APP_TOKEN }}
- VITE_DD_HASH: ${{ github.sha }}
VITE_SEGMENT_WRITE_KEY: ${{ secrets.SEGMENT_WRITE_KEY }}
VITE_POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}
working-directory: apps/web
diff --git a/.github/workflows/deploy_web_app_dev_push.yml b/.github/workflows/deploy_web_app_dev_push.yml
index 6e5b2a91c0..fd972eaf30 100644
--- a/.github/workflows/deploy_web_app_dev_push.yml
+++ b/.github/workflows/deploy_web_app_dev_push.yml
@@ -33,7 +33,6 @@ jobs:
PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }}
DD_APP_KEY: ${{ secrets.DD_APP_KEY }}
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- DD_WEB_APP_TOKEN: ${{ secrets.DD_WEB_APP_TOKEN }}
SEGMENT_WRITE_KEY: ${{ secrets.SEGMENT_WRITE_KEY_PRODUCTION }}
POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}
concurrency:
diff --git a/.github/workflows/release-production.yml b/.github/workflows/release-production.yml
index f5a6680c73..021ecfe604 100644
--- a/.github/workflows/release-production.yml
+++ b/.github/workflows/release-production.yml
@@ -59,8 +59,6 @@ jobs:
working-directory: apps/web
run: just build-prod
env:
- VITE_DD_WEB_APP_TOKEN: ${{ secrets.DD_WEB_APP_TOKEN }}
- VITE_DD_HASH: ${{ github.sha }}
VITE_SEGMENT_WRITE_KEY: ${{ secrets.SEGMENT_WRITE_KEY_PRODUCTION }}
VITE_POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}
- name: Upload Build Artifact
diff --git a/.github/workspace-dep-closures.json b/.github/workspace-dep-closures.json
index 898d4abc37..3a900a9559 100644
--- a/.github/workspace-dep-closures.json
+++ b/.github/workspace-dep-closures.json
@@ -5570,6 +5570,7 @@
],
"sync_service": [
"crates/macro_sync_service_jwt",
+ "crates/worker-rs-otel",
"services/sync-service",
"services/sync-service/bebop-schema"
],
@@ -6046,6 +6047,9 @@
"crates/webhook",
"crates/workspace-hack"
],
+ "worker-rs-otel": [
+ "crates/worker-rs-otel"
+ ],
"worker_trigger": [
"crates/macro_aws_config",
"crates/macro_entrypoint",
diff --git a/Cargo.lock b/Cargo.lock
index 7ea064e6ed..88abf3dfb6 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3238,6 +3238,18 @@ dependencies = [
"web-sys",
]
+[[package]]
+name = "const-hex"
+version = "1.19.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "33e2a781ebdf4467d1428dc4593067825fb646f6871475098d8577421af73558"
+dependencies = [
+ "cfg-if",
+ "cpufeatures 0.2.17",
+ "proptest",
+ "serde_core",
+]
+
[[package]]
name = "const-oid"
version = "0.9.6"
@@ -8785,10 +8797,15 @@ name = "macro_tower_layers"
version = "0.1.0"
dependencies = [
"http 1.4.0",
+ "opentelemetry",
+ "opentelemetry-http",
+ "opentelemetry_sdk",
"tokio",
"tower 0.5.3",
"tower-http",
"tracing",
+ "tracing-opentelemetry",
+ "tracing-subscriber",
"workspace-hack",
]
@@ -10511,9 +10528,13 @@ version = "0.31.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7175df06de5eaee9909d4805a3d07e28bb752c34cab57fa9cff549da596b30f"
dependencies = [
+ "base64 0.22.1",
+ "const-hex",
"opentelemetry",
"opentelemetry_sdk",
"prost 0.14.3",
+ "serde",
+ "serde_json",
"tonic",
"tonic-prost",
]
@@ -11485,6 +11506,21 @@ dependencies = [
"workspace-hack",
]
+[[package]]
+name = "proptest"
+version = "1.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744"
+dependencies = [
+ "bitflags 2.11.1",
+ "num-traits",
+ "rand 0.9.4",
+ "rand_chacha 0.9.0",
+ "rand_xorshift",
+ "regex-syntax",
+ "unarray",
+]
+
[[package]]
name = "prost"
version = "0.12.6"
@@ -11794,6 +11830,15 @@ dependencies = [
"rand_core 0.5.1",
]
+[[package]]
+name = "rand_xorshift"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a"
+dependencies = [
+ "rand_core 0.9.5",
+]
+
[[package]]
name = "rand_xoshiro"
version = "0.6.0"
@@ -14338,6 +14383,7 @@ name = "sync_service"
version = "0.1.0"
dependencies = [
"bebop",
+ "getrandom 0.3.4",
"loro",
"macro_sync_service_jwt",
"matchit 0.8.4",
@@ -14352,6 +14398,7 @@ dependencies = [
"wasm-bindgen",
"web-time",
"worker",
+ "worker-rs-otel",
]
[[package]]
@@ -15294,6 +15341,12 @@ version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971"
+[[package]]
+name = "unarray"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94"
+
[[package]]
name = "unfurl"
version = "0.1.0"
@@ -16550,6 +16603,23 @@ dependencies = [
"worker-sys",
]
+[[package]]
+name = "worker-rs-otel"
+version = "0.1.0"
+dependencies = [
+ "getrandom 0.3.4",
+ "http 1.4.0",
+ "opentelemetry",
+ "opentelemetry-proto",
+ "opentelemetry_sdk",
+ "serde_json",
+ "tracing",
+ "tracing-subscriber",
+ "uuid",
+ "web-time",
+ "worker",
+]
+
[[package]]
name = "worker-sys"
version = "0.8.1"
@@ -16660,6 +16730,7 @@ dependencies = [
"num-traits",
"once_cell",
"opentelemetry",
+ "opentelemetry-http",
"opentelemetry_sdk",
"percent-encoding",
"phf 0.13.1",
@@ -16715,6 +16786,8 @@ dependencies = [
"tower-http",
"tracing",
"tracing-core",
+ "tracing-log",
+ "tracing-subscriber",
"triomphe",
"tungstenite 0.28.0",
"typenum",
diff --git a/Cargo.toml b/Cargo.toml
index 6809e8f9b3..0117574552 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -48,6 +48,7 @@ members = [
"crates/soup_realtime",
"crates/task_dedup",
"crates/unfurl",
+ "crates/worker-rs-otel",
"crates/workspace-hack",
"services/ai_projections_refresh_handler",
"services/authentication_service",
@@ -141,6 +142,7 @@ dotenvy = "0.15.7"
ego-tree = "0.6.2"
either = "1"
futures = "0.3.31"
+getrandom = "0.3"
glob = "0.3.3"
# Pinned engine for the generated dependency artifacts (used by the xtask
# graph/deps commands). This version lives in Cargo.lock, so dev machines and
@@ -172,7 +174,8 @@ lol_html = "2.4.0"
lru = "0.16"
mockall = "0.13"
nom = "8"
-opentelemetry = "0.31.0"
+opentelemetry = { version = "0.31.0", default-features = false }
+opentelemetry-http = { version = "0.31.0", default-features = false }
opentelemetry-otlp = { version = "0.31.0", default-features = false, features = [
"grpc-tonic",
"internal-logs",
@@ -180,7 +183,7 @@ opentelemetry-otlp = { version = "0.31.0", default-features = false, features =
"metrics",
"trace",
] }
-opentelemetry_sdk = { version = "0.31.0", features = ["rt-tokio"] }
+opentelemetry_sdk = { version = "0.31.0", default-features = false }
ordered-float = "5"
paste = "1"
pdfium-render = { version = "0.8.37", default-features = false, features = [
@@ -263,6 +266,8 @@ uuid = { version = "1.15.1", features = ["fast-rng", "serde", "v4", "v5", "v7"]
# anything older than 0.2.123, so keep this floor above that.
wasm-bindgen = "0.2.123"
wasm-bindgen-futures = "0.4"
+web-time = "1.1.0"
+worker = "0.8.1"
# this is pinned to match axum 0.7.9 dependency requirements
async-stripe = { version = "0.41", default-features = false, features = [
"billing",
diff --git a/apps/web/package.json b/apps/web/package.json
index 1c8e360d30..cc648c3c43 100644
--- a/apps/web/package.json
+++ b/apps/web/package.json
@@ -29,8 +29,6 @@
"knip": "bunx --bun knip"
},
"dependencies": {
- "@datadog/browser-logs": "^6.22.0",
- "@dinero.js/currencies": "2.0.0-alpha.8",
"@aws-crypto/sha256-js": "^5.2.0",
"@codemirror/lang-cpp": "^6.0.3",
"@codemirror/lang-css": "^6.3.1",
@@ -43,6 +41,7 @@
"@corvu/dialog": "^0.2.4",
"@corvu/drawer": "^0.2.4",
"@date-fns/tz": "^1.4.1",
+ "@dinero.js/currencies": "2.0.0-alpha.8",
"@floating-ui/dom": "^1.7.4",
"@fontsource-variable/inter": "^5.2.8",
"@fontsource-variable/playfair-display": "^5.2.8",
@@ -67,7 +66,9 @@
"@lukemorales/query-key-factory": "^1.3.4",
"@macro-inc/collaboration": "workspace:*",
"@macro-inc/lexical-core": "workspace:*",
+ "@macro-inc/observability": "workspace:*",
"@normy/query-core": "^0.21.0",
+ "@opentelemetry/api": "^1.9.1",
"@phosphor-icons/core": "^2.1.1",
"@solid-primitives/broadcast-channel": "^0.1.1",
"@solid-primitives/context": "^0.3.2",
@@ -98,10 +99,10 @@
"async-mutex": "^0.5.0",
"codemirror": "^6.0.2",
"colorjs.io": "^0.5.2",
- "date-fns": "^4.1.0",
"d3-array": "^3.2.4",
"d3-scale": "^4.0.2",
"d3-shape": "^3.2.0",
+ "date-fns": "^4.1.0",
"detect-browser": "^5.3.0",
"dinero.js": "2.0.0-alpha.8",
"email-validator": "^2.0.4",
@@ -116,13 +117,13 @@
"loro-crdt": "catalog:",
"nanoid": "^3.3.7",
"neverthrow": "^8.2.0",
- "posthog-js": "^1.387.0",
"pdfjs-dist": "github:macro-inc/pdf.js#v2.16.52-web",
+ "posthog-js": "^1.387.0",
"quill": "^2.0.3",
"short-uuid": "^5.2.0",
"signature_pad": "^5.0.10",
- "solid-gesture": "^0.0.3",
"solid-custom-scrollbars": "^0.1.7",
+ "solid-gesture": "^0.0.3",
"solid-js": "^1.9.7",
"solid-styled-components": "^0.28.5",
"solid-transition-group": "^0.2.3",
@@ -130,8 +131,8 @@
"tabbable": "^6.2.0",
"tailwind-merge": "^3.4.0",
"tailwindcss": "^4.2.4",
- "tauri-plugin-safe-area-insets": "^0.1.0",
"tauri-plugin-device-info-api": "^1.0.1",
+ "tauri-plugin-safe-area-insets": "^0.1.0",
"ts-pattern": "^5.9.0",
"unicode-emoji-json": "^0.8.0",
"uuid": "^10.0.0",
diff --git a/apps/web/src/components/app/FatalError.tsx b/apps/web/src/components/app/FatalError.tsx
index 40dbfef8a8..4d6d51367a 100644
--- a/apps/web/src/components/app/FatalError.tsx
+++ b/apps/web/src/components/app/FatalError.tsx
@@ -1,4 +1,5 @@
-import { logger } from '@observability';
+import { Telemetry } from '@macro-inc/observability';
+
import ResetIcon from '@phosphor/arrow-clockwise.svg';
import HomeIcon from '@phosphor/house.svg';
import { Button, Dialog, Surface } from '@ui';
@@ -10,7 +11,7 @@ interface FatalErrorProps {
}
export function FatalError(props: FatalErrorProps) {
- logger.error(props.error || 'Unknown error', {
+ Telemetry.error(props.error || 'Unknown error', {
url: window.location.href,
});
diff --git a/apps/web/src/features/block-email/component/BaseInput.tsx b/apps/web/src/features/block-email/component/BaseInput.tsx
index 7ca5b64abd..2d082353db 100644
--- a/apps/web/src/features/block-email/component/BaseInput.tsx
+++ b/apps/web/src/features/block-email/component/BaseInput.tsx
@@ -44,7 +44,7 @@ import {
$appendWatermarkNodeToLast,
$removeAllWatermarkNodes,
} from '@macro-inc/lexical-core';
-import { logger } from '@observability';
+import { Telemetry } from '@macro-inc/observability';
import ChevronDown from '@phosphor/caret-down.svg';
import CaretRight from '@phosphor/caret-right.svg';
import DotsThree from '@phosphor/dots-three.svg';
@@ -691,7 +691,7 @@ export function BaseInput(props: {
$removeAllWatermarkNodes(editor());
const prepared = prepareEmailBody(editor());
if (!prepared) {
- logger.error(
+ Telemetry.error(
new Error('Unable to prepare email body for draft collection.')
);
return null;
@@ -740,12 +740,12 @@ export function BaseInput(props: {
const newMessage = props.newMessage ?? false;
if (!currentThread && !newMessage) {
- logger.error(new Error('Failed to save draft: thread not found'));
+ Telemetry.error(new Error('Failed to save draft: thread not found'));
return;
}
if (newMessage && currentThread) {
- logger.error(
+ Telemetry.error(
new Error(
'Failed to save draft: new message and current thread cannot be provided together'
)
@@ -986,14 +986,14 @@ export function BaseInput(props: {
const newMessage = props.newMessage ?? false;
if (!currentThread && !newMessage) {
- logger.error(new Error("Can't send email, no email thread found"));
+ Telemetry.error(new Error("Can't send email, no email thread found"));
toast.failure('Email failed to send');
return;
}
if (newMessage && currentThread) {
toast.failure('Email failed to send');
- logger.error('New message and thread cannot be provided together');
+ Telemetry.error('New message and thread cannot be provided together');
return;
}
@@ -1006,14 +1006,14 @@ export function BaseInput(props: {
if (emailLinksQuery.isError) {
toast.failure('Email failed to send: Could not load email accounts');
- logger.error('Failed to load email links');
+ Telemetry.error('Failed to load email links');
return;
}
const linksData = emailLinksQuery.data;
if (!linksData || linksData.links.length < 1) {
toast.failure('Email failed to send: No email account connected');
- logger.error('No links found');
+ Telemetry.error('No links found');
return;
}
linkId = primaryLinkId() ?? linksData.links[0].id;
diff --git a/apps/web/src/features/block-email/component/MessageContainer.tsx b/apps/web/src/features/block-email/component/MessageContainer.tsx
index eb8a201696..601cbbf99a 100644
--- a/apps/web/src/features/block-email/component/MessageContainer.tsx
+++ b/apps/web/src/features/block-email/component/MessageContainer.tsx
@@ -13,8 +13,7 @@ import { UserIcon, type UserIconProps } from '@core/component/UserIcon';
import { VideoPreview } from '@core/component/VideoPreview';
import { fileTypeToBlockName } from '@core/constant/allBlocks';
import { isMobile } from '@core/mobile/isMobile';
-
-import { logger } from '@observability';
+import { Telemetry } from '@macro-inc/observability';
import { refetchSoupEntity } from '@queries/soup/cache';
import { emailClient } from '@service-email/client';
import type { ApiMessage, Attachment } from '@service-email/generated/schemas';
@@ -168,11 +167,11 @@ export function MessageContainer(props: MessageContainerProps) {
});
if (response.isErr()) {
toast.failure('Failed to get attachment. Please try again.');
- return logger.error('Failed to get or create attachment document id', {
- error: new Error(
+ return Telemetry.error(
+ new Error(
'Failed to get or create attachment document id: ' + response.error
- ),
- });
+ )
+ );
}
const { document_id } = response.value;
@@ -182,14 +181,11 @@ export function MessageContainer(props: MessageContainerProps) {
});
if (maybeDocumentMetadata.isErr()) {
toast.failure('Failed to get attachment. Please try again.');
- return logger.error(
- 'Failed to get or create attachment document metadata',
- {
- error: new Error(
- 'Failed to get or create attachment document metadata: ' +
- maybeDocumentMetadata.error
- ),
- }
+ return Telemetry.error(
+ new Error(
+ 'Failed to get or create attachment document metadata: ' +
+ maybeDocumentMetadata.error
+ )
);
}
diff --git a/apps/web/src/features/block-email/component/compose/Compose.tsx b/apps/web/src/features/block-email/component/compose/Compose.tsx
index bb797af6d8..31ae28015b 100644
--- a/apps/web/src/features/block-email/component/compose/Compose.tsx
+++ b/apps/web/src/features/block-email/component/compose/Compose.tsx
@@ -49,7 +49,8 @@ import {
$appendWatermarkNodeToLast,
$removeAllWatermarkNodes,
} from '@macro-inc/lexical-core';
-import { logger } from '@observability/logger';
+import { Telemetry } from '@macro-inc/observability';
+
import ArrowCounterClockwise from '@phosphor-icons/core/regular/arrow-counter-clockwise.svg?component-solid';
import { queryClient } from '@queries/client';
import {
@@ -235,7 +236,7 @@ export function EmailCompose(props: EmailComposeProps) {
$removeAllWatermarkNodes(editor());
const prepared = prepareEmailBody(editor());
if (!prepared) {
- logger.error(
+ Telemetry.error(
new Error('Unable to prepare email body for draft collection.')
);
return null;
diff --git a/apps/web/src/features/block-email/component/compose/ComposeBody.tsx b/apps/web/src/features/block-email/component/compose/ComposeBody.tsx
index d3122a219d..3c0faa0543 100644
--- a/apps/web/src/features/block-email/component/compose/ComposeBody.tsx
+++ b/apps/web/src/features/block-email/component/compose/ComposeBody.tsx
@@ -8,7 +8,8 @@ import { MarkdownTextarea } from '@core/component/LexicalMarkdown/component/core
import { createFilesReadyHandler } from '@core/component/LexicalMarkdown/utils/fileUploadUtils';
import { fileFolderDrop } from '@core/directive/fileFolderDrop';
import { handleFileFolderDrop } from '@core/util/upload';
-import { logger } from '@observability/logger';
+import { Telemetry } from '@macro-inc/observability';
+
import { cn, Scroll } from '@ui';
import type { LexicalEditor } from 'lexical';
import {
@@ -88,7 +89,7 @@ export function ComposeBody(props: {
const logComposeBody = (event: string, details?: Record<string, unknown>) => {
if (!props.debugName) return;
- logger.log(`[ComposeBody] ${event}`, {
+ Telemetry.info(`[ComposeBody] ${event}`, {
debugName: props.debugName,
...details,
});
diff --git a/apps/web/src/features/block-email/util/makeAttachmentPublic.ts b/apps/web/src/features/block-email/util/makeAttachmentPublic.ts
index 811d340ef4..2aaae06638 100644
--- a/apps/web/src/features/block-email/util/makeAttachmentPublic.ts
+++ b/apps/web/src/features/block-email/util/makeAttachmentPublic.ts
@@ -1,7 +1,8 @@
import { analytics } from '@app/lib/analytics';
import { toast } from '@core/component/Toast/Toast';
-import { logger } from '@observability';
+import { Telemetry } from '@macro-inc/observability';
+
import { storageServiceClient } from '@service-storage/client';
export const makeAttachmentPublic = async (attachmentId: string) => {
@@ -35,6 +36,8 @@ export const makeAttachmentPublic = async (attachmentId: string) => {
toast.alert('Recipients may not be able to view this file', {
subtext: 'Please consult the document owner to change share permissions',
});
- logger.error('Failed to make attachment public', { errors: result.error });
+ Telemetry.error('Failed to make attachment public', {
+ errors: JSON.stringify(result.error),
+ });
}
};
diff --git a/apps/web/src/features/block-md/component/Block.tsx b/apps/web/src/features/block-md/component/Block.tsx
index a09bf215e9..fdec9f6570 100644
--- a/apps/web/src/features/block-md/component/Block.tsx
+++ b/apps/web/src/features/block-md/component/Block.tsx
@@ -20,6 +20,7 @@ import {
LORO_WAL_DB_NAME,
} from '@macro-inc/collaboration/collab/wal';
import { MARKDOWN_LORO_SCHEMA } from '@macro-inc/lexical-core/markdown-loro-schema';
+import type { Span } from '@macro-inc/observability';
import { DocumentDebouncedNotificationReadMarker } from '@notifications';
import { useInstructionsMdIdQuery } from '@queries/storage/instructions-md';
import { storageServiceClient } from '@service-storage/client';
@@ -35,6 +36,7 @@ import {
import type { MarkdownData } from '../definition';
import { HistoryProvider } from '../history/HistoryContext';
import { OldOverlay } from '../history/OldOverlay';
+import { resumeDocumentSpan, stampLoroSnapshotState } from '../observability';
import { blockDataSignal, mdStore } from '../signal/markdownBlockData';
import { FindAndReplace } from './FindAndReplace';
import { MarkdownNameProvider, useMarkdownName } from './MarkdownNameProvider';
@@ -51,16 +53,78 @@ export interface BlockMarkdownProps {
}
type MarkdownLoroManager = LoroManager<typeof MARKDOWN_LORO_SCHEMA>;
+type SnapshotResult = {
+ outcome: 'seeded' | 'discarded' | 'unavailable' | 'error';
+ bytes?: number;
+};
+
+const snapshotStepNames = {
+ optimistic: 'doc.snapshot.optimistic',
+ local: 'doc.snapshot.local-cache',
+ s3: 'doc.snapshot.s3-cache',
+ remote: 'doc.snapshot.remote-sync',
+} as const;
+
+type SnapshotSource = keyof typeof snapshotStepNames;
+
+function startSnapshotIngest(
+ parentSpan: Span | undefined,
+ source: SnapshotSource,
+ loroManager: MarkdownLoroManager,
+ ingest: () => Promise<SnapshotResult>
+): void {
+ parentSpan?.event('doc.snapshot.attempt', {
+ 'snapshot.source': source,
+ });
+
+ const operation = parentSpan
+ ? parentSpan.span(snapshotStepNames[source], async (snapshotSpan) => {
+ snapshotSpan.setAttr('snapshot.source', source);
+ try {
+ const result = await ingest();
+ snapshotSpan.setAttr('outcome', result.outcome);
+ if (result.bytes !== undefined) {
+ snapshotSpan.setAttr('snapshot.bytes', result.bytes);
+ }
+ if (result.outcome === 'seeded') {
+ stampLoroSnapshotState(snapshotSpan, loroManager.doc);
+ } else if (result.outcome === 'error') {
+ snapshotSpan.error('snapshot ingestion failed');
+ }
+ return result;
+ } catch (error) {
+ snapshotSpan.error(error);
+ snapshotSpan.setAttr('outcome', 'error');
+ throw error;
+ }
+ })
+ : ingest();
+
+ void operation
+ .then(({ outcome, bytes }) => {
+ parentSpan?.event('doc.snapshot.result', {
+ 'snapshot.source': source,
+ ...(bytes !== undefined && { 'snapshot.bytes': bytes }),
+ outcome,
+ });
+ })
+ .catch(() => {
+ parentSpan?.event('doc.snapshot.result', {
+ 'snapshot.source': source,
+ outcome: 'error',
+ });
+ });
+}
async function ingestLocalSnapshot(
loroManager: MarkdownLoroManager,
snapshotStore: IDBSnapshotStore<RawUpdate>,
walStore: BrowserWALStore<RawUpdate>
-) {
+): Promise<SnapshotResult> {
const localSnapshot = await snapshotStore.load();
- if (!localSnapshot) return;
+ if (!localSnapshot) return { outcome: 'unavailable' };
const walEntries = await walStore.getAll();
- await loroManager.ingest({
+ const seeded = await loroManager.ingest({
kind: 'local',
snapshot: localSnapshot,
walUpdates: walEntries.map((entry) => entry.update),
@@ -79,32 +143,47 @@ async function ingestLocalSnapshot(
});
await snapshotStore.save(snapshot);
}
+ return {
+ outcome: seeded ? 'seeded' : 'discarded',
+ bytes: localSnapshot.length,
+ };
}
+/**
+ * Ingest the sync-service initial sync. `discarded` means the snapshot
+ * arrived fine but lost the seed race.
+ */
async function ingestRemoteSnapshot(
loroManager: MarkdownLoroManager,
doInitialSync: MarkdownData['doInitialSync']
-): Promise<boolean> {
+): Promise<SnapshotResult> {
const sync = await doInitialSync();
if (sync.isErr()) {
console.error('Failed to receive initial sync', sync.error);
- return loroManager.initialized;
+ return { outcome: 'error' };
}
- await loroManager.ingest({
+ const bytes = sync.value.snapshot.length;
+ const seeded = await loroManager.ingest({
kind: 'dss',
snapshot: sync.value.snapshot,
});
- return true;
+ return { outcome: seeded ? 'seeded' : 'discarded', bytes };
}
async function ingestS3Snapshot(
loroManager: MarkdownLoroManager,
blockId: string
-) {
+): Promise<SnapshotResult> {
const result = await storageServiceClient.fetchCachedSnapshot(blockId);
- if (result.isOk()) {
- await loroManager.ingest({ kind: 's3', snapshot: result.value });
- }
+ if (result.isErr()) return { outcome: 'unavailable' };
+ const seeded = await loroManager.ingest({
+ kind: 's3',
+ snapshot: result.value,
+ });
+ return {
+ outcome: seeded ? 'seeded' : 'discarded',
+ bytes: result.value.length,
+ };
}
export default function BlockMarkdown(props: BlockMarkdownProps) {
@@ -142,17 +221,32 @@ function BlockMarkdownContent({ optimisticSnapshot }: BlockMarkdownProps) {
}
setBlockError(null);
+ // optimistic is the "hard coded" snapshot like the golden one when you create a
+ // new document
+ const span = resumeDocumentSpan(blockId);
if (optimisticSnapshot) {
- loroManager.ingest({
- kind: 'optimistic',
- snapshot: optimisticSnapshot,
+ startSnapshotIngest(span, 'optimistic', loroManager, async () => {
+ const seeded = await loroManager.ingest({
+ kind: 'optimistic',
+ snapshot: optimisticSnapshot,
+ });
+ return {
+ outcome: seeded ? 'seeded' : 'discarded',
+ bytes: optimisticSnapshot.length,
+ };
});
}
// First one wins automatically (loro manager takes care of ignoring the rest)
- ingestLocalSnapshot(loroManager, snapshotStore, walStore);
- ingestS3Snapshot(loroManager, blockId);
- ingestRemoteSnapshot(loroManager, data.doInitialSync);
+ startSnapshotIngest(span, 'local', loroManager, () =>
+ ingestLocalSnapshot(loroManager, snapshotStore, walStore)
+ );
+ startSnapshotIngest(span, 's3', loroManager, () =>
+ ingestS3Snapshot(loroManager, blockId)
+ );
+ startSnapshotIngest(span, 'remote', loroManager, () =>
+ ingestRemoteSnapshot(loroManager, data.doInitialSync)
+ );
})
);
diff --git a/apps/web/src/features/block-md/component/MarkdownCollabProvider.tsx b/apps/web/src/features/block-md/component/MarkdownCollabProvider.tsx
index cab5166639..5c7adefea5 100644
--- a/apps/web/src/features/block-md/component/MarkdownCollabProvider.tsx
+++ b/apps/web/src/features/block-md/component/MarkdownCollabProvider.tsx
@@ -68,6 +68,7 @@ import {
onCleanup,
type Setter,
} from 'solid-js';
+import { endDocumentSpan, resumeDocumentSpan } from '../observability';
import { CollabStatus } from './CollabStatus';
type MutatedNodes = UpdateListenerPayload['mutatedNodes'];
@@ -459,7 +460,7 @@ export function MarkdownCollabProvider(props: MarkdownCollabProviderProps) {
const state = loroManager.state;
const empty = state
? isStateEmpty(state.state as unknown as SerializedEditorState)
- : null;
+ : undefined;
logSyncService({
documentId: syncSource()!.documentId,
@@ -478,6 +479,7 @@ export function MarkdownCollabProvider(props: MarkdownCollabProviderProps) {
message:
'editor init: no state from loroManager — editor will NOT become ready (skeleton stays)',
});
+ endDocumentSpan(syncSource()!.documentId);
return;
}
@@ -514,6 +516,7 @@ export function MarkdownCollabProvider(props: MarkdownCollabProviderProps) {
'editor init: initializeEditorWithVersionedState failed',
});
props.setEditorError(initError);
+ endDocumentSpan(syncSource()!.documentId);
return;
}
}
@@ -521,6 +524,9 @@ export function MarkdownCollabProvider(props: MarkdownCollabProviderProps) {
// Start the sync engine
startSync();
props.setEditorReady(true);
+ const documentId = syncSource()!.documentId;
+ resumeDocumentSpan(documentId)?.event('editor.ready');
+ endDocumentSpan(documentId);
logSyncService({
documentId: syncSource()!.documentId,
level: 'info',
@@ -551,6 +557,8 @@ export function MarkdownCollabProvider(props: MarkdownCollabProviderProps) {
onCleanup(() => {
syncEngine.stop();
walSyncer.destroy();
+ const documentId = syncSource()?.documentId;
+ if (documentId) endDocumentSpan(documentId);
});
return (
diff --git a/apps/web/src/features/block-md/definition.ts b/apps/web/src/features/block-md/definition.ts
index dd54763baf..5656d60edc 100644
--- a/apps/web/src/features/block-md/definition.ts
+++ b/apps/web/src/features/block-md/definition.ts
@@ -5,6 +5,7 @@ import {
loadResult,
} from '@core/block';
import { ENABLE_MARKDOWN_LIVE_COLLABORATION } from '@core/constant/featureFlags';
+import { ThrownResultError } from '@core/util/result';
import {
fetchDocumentLocation,
waitForDocumentSyncServiceReady,
@@ -14,6 +15,12 @@ import { makeFileFromBlob } from '@service-storage/util/makeFileFromBlob';
import { createSyncServiceSource } from '@service-sync/source';
import { err, ok } from 'neverthrow';
import MarkdownBlock from './component/Block';
+import {
+ endDocumentSpan,
+ registerDocumentSpan,
+ resumeDocumentSpan,
+ startDocumentSpan,
+} from './observability';
import type { MarkdownRewriteOutput } from './signal/rewriteSignal';
export const definition = defineBlock({
@@ -38,72 +45,120 @@ export const definition = defineBlock({
});
}
- const [maybeBundle, maybeLocation] = await Promise.all([
- fetchDocumentLoadBundle(documentId),
- loadResult(fetchDocumentLocation({ documentId })),
- ]);
- if (maybeBundle.isErr()) return err(maybeBundle.error);
- const { token, documentMetadata, userAccessLevel } = maybeBundle.value;
-
- if (maybeLocation.isErr()) return err(maybeLocation.error);
+ let rootSpan = resumeDocumentSpan(documentId);
+ if (!rootSpan) {
+ rootSpan = startDocumentSpan('doc.open');
+ rootSpan.setAttr('doc.type', 'md');
+ rootSpan.setAttr('document.id', documentId);
+ registerDocumentSpan(documentId, rootSpan);
+ }
+ return rootSpan.span('doc.load', async (loadSpan) => {
+ const loadBundle = () =>
+ loadSpan.span('doc.load.bundle', async (bundleSpan) => {
+ const result = await fetchDocumentLoadBundle(documentId);
+ if (result.isErr()) {
+ bundleSpan.error(new ThrownResultError(result.error));
+ }
+ return result;
+ });
- let location = maybeLocation.value;
- if (
- location.type === 'presignedUrl' &&
- location.content.state === 'pending'
- ) {
- location = await waitForDocumentSyncServiceReady({ documentId }).catch(
- (error) => {
- console.error(
- 'Failed waiting for markdown sync-service location',
- error
+ // The location span covers the sync-service readiness wait too, so it
+ // reflects when the document actually became loadable.
+ const loadLocation = () =>
+ loadSpan.span('doc.load.location', async (locationSpan) => {
+ const result = await loadResult(
+ fetchDocumentLocation({ documentId })
);
- return location;
- }
- );
- }
+ if (result.isErr()) {
+ locationSpan.error(new ThrownResultError(result.error));
+ return result;
+ }
+ let location = result.value;
+ if (
+ location.type === 'presignedUrl' &&
+ location.content.state === 'pending'
+ ) {
+ location = await waitForDocumentSyncServiceReady({
+ documentId,
+ }).catch((error) => {
+ console.error(
+ 'Failed waiting for markdown sync-service location',
+ error
+ );
+ return location;
+ });
+ locationSpan.setAttr('location.pending', true);
+ return ok(location);
+ }
+ return ok(location);
+ });
- // Markdown initialization and lifecycle persistence are backend-owned.
- // If a markdown document still resolves to object storage here, opening
- // it would require a backend repair/backfill path rather than a frontend
- // sync-service mutation that leaves DB content metadata inconsistent.
- if (location.type !== 'syncServiceContent') {
- console.error(
- 'Markdown document is not available in sync-service',
- documentId,
- location.content
- );
- return LoadErrors.INVALID;
- }
+ const [maybeBundle, maybeLocation] = await Promise.all([
+ loadBundle(),
+ loadLocation(),
+ ]);
+ if (maybeBundle.isErr()) {
+ loadSpan.error('load bundle failed');
+ rootSpan.error('load bundle failed');
+ endDocumentSpan(documentId);
+ return err(maybeBundle.error);
+ }
+ const { token, documentMetadata, userAccessLevel } = maybeBundle.value;
+
+ if (maybeLocation.isErr()) {
+ loadSpan.error('load location failed');
+ rootSpan.error('load location failed');
+ endDocumentSpan(documentId);
+ return err(maybeLocation.error);
+ }
- const { source: syncSource, doInitialSync } = createSyncServiceSource(
- source.id,
- token
- );
+ const location = maybeLocation.value;
- // HACK: unfortunately, most blocks still rely on a dssFile for things like
- // metadata and fileName. so I'm creating an empty blob file to get around that.
- const fileWithoutBlob = await makeFileFromBlob({
- blob: new Blob([]),
- documentKeyParts: {
- owner: documentMetadata.owner,
- documentId: documentMetadata.documentId,
- documentVersionId: documentMetadata.documentVersionId.toString(),
+ // Markdown initialization and lifecycle persistence are backend-owned.
+ // If a markdown document still resolves to object storage here, opening
+ // it would require a backend repair/backfill path rather than a frontend
+ // sync-service mutation that leaves DB content metadata inconsistent.
+ if (location.type !== 'syncServiceContent') {
+ console.error(
+ 'Markdown document is not available in sync-service',
+ documentId,
+ location.content
+ );
+ loadSpan.error('markdown document not in sync-service');
+ rootSpan.error('markdown document not in sync-service');
+ endDocumentSpan(documentId);
+ return LoadErrors.INVALID;
+ }
+
+ const { source: syncSource, doInitialSync } = createSyncServiceSource(
+ source.id,
+ token
+ );
+
+ // HACK: unfortunately, most blocks still rely on a dssFile for things like
+ // metadata and fileName. so I'm creating an empty blob file to get around that.
+ const fileWithoutBlob = await makeFileFromBlob({
+ blob: new Blob([]),
+ documentKeyParts: {
+ owner: documentMetadata.owner,
+ documentId: documentMetadata.documentId,
+ documentVersionId: documentMetadata.documentVersionId.toString(),
+ // @ts-ignore: TODO: fix / replace @macro-inc/document-processing-job-types
+ fileType: 'md',
+ },
+ fileName: documentMetadata.documentName,
+ mimeType: definition.accepted['md']!,
// @ts-ignore: TODO: fix / replace @macro-inc/document-processing-job-types
- fileType: 'md',
- },
- fileName: documentMetadata.documentName,
- mimeType: definition.accepted['md']!,
- // @ts-ignore: TODO: fix / replace @macro-inc/document-processing-job-types
- metadata: documentMetadata,
- });
+ metadata: documentMetadata,
+ });
- return ok({
- dssFile: fileWithoutBlob,
- userAccessLevel,
- syncSource,
- doInitialSync,
- documentMetadata,
+ return ok({
+ dssFile: fileWithoutBlob,
+ userAccessLevel,
+ syncSource,
+ doInitialSync,
+ documentMetadata,
+ });
});
}
return LoadErrors.INVALID;
diff --git a/apps/web/src/features/block-md/history/HistoryContext.tsx b/apps/web/src/features/block-md/history/HistoryContext.tsx
index 3180206b3a..75f1b6ff78 100644
--- a/apps/web/src/features/block-md/history/HistoryContext.tsx
+++ b/apps/web/src/features/block-md/history/HistoryContext.tsx
@@ -1,5 +1,6 @@
import { MACRO_AGENT_BOT_ID } from '@core/constant/macroAgent';
import { tryMacroId, useDisplayName } from '@core/user';
+import { ThrownResultError } from '@core/util/result';
import { isAiPeer } from '@macro-inc/collaboration/collab/ai-peer';
import {
buildDiffState,
@@ -101,7 +102,7 @@ export function HistoryProvider(props: {
() => (isOpen() || shouldLoad() ? props.documentId() : undefined),
async (documentId) => {
const result = await syncServiceClient.getSnapshot({ documentId });
- if (result.isErr()) throw new Error(String(result.error));
+ if (result.isErr()) throw new ThrownResultError(result.error);
const doc = new LoroDoc();
doc.import(result.value);
return doc;
diff --git a/apps/web/src/features/block-md/observability.ts b/apps/web/src/features/block-md/observability.ts
new file mode 100644
index 0000000000..21412f4696
--- /dev/null
+++ b/apps/web/src/features/block-md/observability.ts
@@ -0,0 +1,41 @@
+import { type Span, Telemetry } from '@macro-inc/observability';
+import type { LoroDoc } from 'loro-crdt';
+
+const documentSpans = new Map<string, Span>();
+
+export function startDocumentSpan(name: string): Span {
+ return Telemetry.span(name);
+}
+
+export function registerDocumentSpan(documentId: string, span: Span): void {
+ const previous = documentSpans.get(documentId);
+ if (previous !== span) previous?.end();
+ documentSpans.set(documentId, span);
+}
+
+export function resumeDocumentSpan(documentId: string): Span | undefined {
+ return documentSpans.get(documentId);
+}
+
+export function endDocumentSpan(documentId: string): void {
+ const span = documentSpans.get(documentId);
+ if (!span) return;
+ documentSpans.delete(documentId);
+ span.end();
+}
+
+export function endTrackedDocumentSpan(span: Span): void {
+ for (const [documentId, candidate] of documentSpans) {
+ if (candidate === span) documentSpans.delete(documentId);
+ }
+ span.end();
+}
+
+export function stampLoroSnapshotState(span: Span, doc: LoroDoc): void {
+ const version = doc.oplogVersion();
+
+ span.setAttr('snapshot.op_count', doc.opCount());
+ span.setAttr('snapshot.peer_count', version.length());
+ span.setAttr('snapshot.frontier_count', doc.oplogFrontiers().length);
+ span.setAttr('snapshot.is_shallow', doc.isShallow());
+}
diff --git a/apps/web/src/features/command/Launcher.tsx b/apps/web/src/features/command/Launcher.tsx
index 232b5e5d84..c788c7a922 100644
--- a/apps/web/src/features/command/Launcher.tsx
+++ b/apps/web/src/features/command/Launcher.tsx
@@ -1,6 +1,11 @@
import { useFeatureFlag } from '@app/lib/analytics/posthog';
import { setAutomationComposerOpen } from '@block-automation/component';
import { EMAIL_COMPOSE_TO_INPUT_ID } from '@block-email/constants';
+import {
+ endTrackedDocumentSpan,
+ registerDocumentSpan,
+ startDocumentSpan,
+} from '@block-md/observability';
import { openNewChannelModal } from '@channel/CreateChannelModal';
import { useSplitLayout } from '@components/app/split-layout/layout';
import type { BlockAlias, BlockName } from '@core/block';
@@ -51,6 +56,7 @@ import { AnimatedTaskIcon } from '@icon/wide-task';
import WideTask from '@icon/wide-task.svg';
import { Dialog } from '@kobalte/core/dialog';
import { getMarkdownGoldenBytes } from '@macro-inc/lexical-core/markdown-golden';
+import type { Span } from '@macro-inc/observability';
import ArrowRight from '@phosphor/arrow-right.svg';
import { createProject } from '@queries/storage/projects';
import { CommandMenuShell, cn, Hotkey, Layer } from '@ui';
@@ -72,9 +78,11 @@ const createBlock = async (spec: {
createFn: () => Promise<string | undefined>;
loading?: boolean;
shouldInsert?: boolean;
+ /** Active creation span; registered by document id after creation. */
+ span?: Span;
}) => {
const { openWithSplit } = useSplitLayout();
- const { blockName, createFn, loading } = spec;
+ const { blockName, createFn, loading, span } = spec;
setCreateMenuOpen(false, false);
@@ -96,10 +104,15 @@ const createBlock = async (spec: {
const id = await createFn();
if (!id) {
+ span?.setAttr('error', true);
+ span?.end();
split?.goBack();
return;
}
+ span?.setAttr('document.id', id);
+ if (span) registerDocumentSpan(id, span);
+
// If we are creating a new markdown document "from scratch" then we can let
// them instantly start editing
const createMdParams =
@@ -130,6 +143,8 @@ const createBlock = async (spec: {
}
);
}
+
+ span?.event('doc.navigated');
};
const createComponent = async (spec: {
@@ -168,20 +183,33 @@ export function runCreateAction(
const source = options.source ?? 'create_menu';
switch (blockName) {
- case 'md':
- createBlock({
- blockName: 'md',
- loading: true,
- createFn: () =>
- createMarkdownFile({
- title: '',
- content: '',
- projectId: undefined,
- source,
- }),
- shouldInsert,
- });
+ case 'md': {
+ const span = startDocumentSpan('doc.create');
+ span.setAttr('doc.type', 'md');
+ span.setAttr('doc.source', source);
+ void span
+ .run(() =>
+ createBlock({
+ blockName: 'md',
+ loading: true,
+ span,
+ createFn: () =>
+ createMarkdownFile({
+ title: '',
+ content: '',
+ projectId: undefined,
+ source,
+ }),
+ shouldInsert,
+ })
+ )
+ .catch((error) => {
+ span.error(error);
+ endTrackedDocumentSpan(span);
+ throw error;
+ });
return;
+ }
case 'canvas':
createBlock({
blockName: 'canvas',
diff --git a/apps/web/src/index.tsx b/apps/web/src/index.tsx
index f21a381854..4c9e8dccad 100644
--- a/apps/web/src/index.tsx
+++ b/apps/web/src/index.tsx
@@ -1,4 +1,6 @@
+import { initializeBrowserObservability } from './observability/browser';
import './index.css';
+
import '@fontsource-variable/inter';
import '@fontsource-variable/roboto-mono';
import '@fontsource-variable/playfair-display';
@@ -85,21 +87,13 @@ const renderApp = () => {
render(() => <Root />, root);
};
-function main() {
+async function main() {
+ await initializeBrowserObservability();
+
console.log('App Version ', import.meta.env.__APP_VERSION__);
// during `vite dev` (but not dev builds), don't inject analytics/observability
if (!import.meta.hot) {
- const scheduleIdleTask =
- window.requestIdleCallback ?? ((cb: () => void) => setTimeout(cb, 1));
-
- // Lazy load and init observability (Datadog) to reduce initial bundle
- scheduleIdleTask(() => {
- import('@observability').then((Observability) => {
- Observability.init(import.meta.env.__APP_VERSION__);
- });
- });
-
// this event is emitted when dynamically loading a module fails
// for example when you're using the app and a new version is deployed
window.addEventListener('vite:preloadError', () =>
@@ -110,4 +104,5 @@ function main() {
renderApp();
}
+// unawaited
main();
diff --git a/apps/web/src/lib/core/component/LexicalMarkdown/utils/fileUploadUtils.ts b/apps/web/src/lib/core/component/LexicalMarkdown/utils/fileUploadUtils.ts
index 4e2ea2541b..6a91bb15e2 100644
--- a/apps/web/src/lib/core/component/LexicalMarkdown/utils/fileUploadUtils.ts
+++ b/apps/web/src/lib/core/component/LexicalMarkdown/utils/fileUploadUtils.ts
@@ -11,7 +11,8 @@ import {
type UploadInput,
uploadFiles,
} from '@core/util/upload';
-import { logger } from '@observability';
+import { Telemetry } from '@macro-inc/observability';
+
import { fileExtension } from '@service-storage/util/filename';
import type { LexicalEditor } from 'lexical';
import {
@@ -131,9 +132,7 @@ async function onFilesReady(
const item = await documentUploadToItem(result);
if (!item) {
toast.failure('Document upload failed or timed out');
- logger.error('Document upload failed or timed out', {
- cause: new Error(),
- });
+ Telemetry.error(new Error('Document upload failed or timed out'));
continue;
}
uploadedItemIds.push(item.id);
@@ -149,9 +148,7 @@ async function onFilesReady(
const item = await documentUploadToItem(result);
if (!item) {
toast.failure('Folder upload failed or timed out');
- logger.error('Folder upload failed or timed out', {
- cause: new Error(),
- });
+ Telemetry.error(new Error('Folder upload failed or timed out'));
continue;
}
uploadedItemIds.push(item.id);
diff --git a/apps/web/src/lib/core/hotkey/hotkeys.ts b/apps/web/src/lib/core/hotkey/hotkeys.ts
index 92d6ece56e..60b63ac6fc 100644
--- a/apps/web/src/lib/core/hotkey/hotkeys.ts
+++ b/apps/web/src/lib/core/hotkey/hotkeys.ts
@@ -1,7 +1,8 @@
import { IS_MAC } from '@core/constant/isMac';
import { isTouchDevice } from '@core/mobile/isTouchDevice';
import { isEditableInput } from '@core/util/isEditableInput';
-import { logger } from '@observability';
+import { Telemetry } from '@macro-inc/observability';
+
import { onCleanup, onMount, untrack } from 'solid-js';
import {
EVENT_MODIFIER_KEYS,
@@ -145,16 +146,17 @@ export function registerHotkey(
};
if (!scopeId) {
- logger.error('Scope ID is required for hotkey registration.', {
- error: new Error('No scope ID provided'),
- scopeId,
- });
+ Telemetry.error(
+ new Error('Scope ID is required for hotkey registration.'),
+ {
+ scopeId,
+ }
+ );
return noopDisposer;
}
const scopeNode = hotkeyScopeTree.get(scopeId);
if (!scopeNode) {
- logger.error('Scope ID not found.', {
- error: new Error('Scope ID not found'),
+ Telemetry.error(new Error('Scope ID not found.'), {
scopeId,
});
return noopDisposer;
@@ -175,15 +177,12 @@ export function registerHotkey(
existingHotkeys.length !== hotkeys?.length ||
!existingHotkeys.every((el, i) => el === hotkeys[i])
) {
- logger.log(
+ Telemetry.warn(
`Hotkey token "${hotkeyToken}" is already registered with a different command. ` +
`Existing: ${existingCommand.description}, New: ${description}. ` +
`This is likely a bug, please fix it. `,
{
- level: 'warn',
- error: new Error(
- 'Hotkey token already registered with a different command'
- ),
+ error: 'Hotkey token already registered with a different command',
}
);
}
@@ -194,15 +193,13 @@ export function registerHotkey(
if (providedCommandScopeId) {
const existingScope = hotkeyScopeTree.get(providedCommandScopeId);
if (!existingScope) {
- logger.error('Provided command scope ID not found.', {
- error: new Error('Provided command scope ID not found'),
+ Telemetry.error(new Error('Provided command scope ID not found.'), {
providedCommandScopeId,
});
return noopDisposer;
}
if (existingScope.type !== 'command') {
- logger.error('Provided scope is not a command scope.', {
- error: new Error('Provided scope is not a command scope'),
+ Telemetry.error(new Error('Provided scope is not a command scope.'), {
providedCommandScopeId,
scopeType: existingScope.type,
});
@@ -250,11 +247,10 @@ export function registerHotkey(
hotkeys?.forEach((h) => {
const existingHandlers = scopeNode.hotkeyCommands.get(h);
if (existingHandlers && existingHandlers.length > 0) {
- logger.log(
+ Telemetry.warn(
`Hotkey ${h} already registered in scope ${scopeId}. Previous hotkey is being overwritten.`,
{
- level: 'warn',
- error: new Error('Hotkey already registered in scope'),
+ error: 'Hotkey already registered in scope',
}
);
}
@@ -516,7 +512,7 @@ export function useHotkeyDOMScope(
const currentDataAttribute = el.getAttribute('data-hotkey-scope');
if (currentDataAttribute) {
- logger.error(
+ Telemetry.error(
`Attempting to attach ${scopeId} hotkey scope, but the specified element already has a hotkey scope attached to it. This is an error and will break the hotkey scope tree. Please use that scope, ${currentDataAttribute}, instead of inventing your own.`
);
}
@@ -553,7 +549,7 @@ export function attachGlobalDOMScope(el: Element) {
const currentDataAttribute = el.getAttribute('data-hotkey-scope');
if (currentDataAttribute) {
- logger.error(
+ Telemetry.error(
`Attempting to attach global hotkey scope, but the specified element already has a hotkey scope attached to it. This is an error and will break the hotkey scope tree. Please use that scope, ${currentDataAttribute}, instead of inventing your own.`
);
}
@@ -660,10 +656,12 @@ export function useHotKeyRoot() {
}
const isEditableFocused = isEditableInput(document.activeElement);
if (!currentScopeId) {
- return logger.error(`Could not find current hotkey scope.`, {
- error: new Error('Could not find current hotkey scope'),
- currentScopeId,
- });
+ return Telemetry.error(
+ new Error('Could not find current hotkey scope.'),
+ {
+ currentScopeId,
+ }
+ );
}
const pressedKeysString = getKeyString(currentPressedKeys);
diff --git a/apps/web/src/lib/core/hotkey/utils.ts b/apps/web/src/lib/core/hotkey/utils.ts
index b57d213c89..975cdd7c7d 100644
--- a/apps/web/src/lib/core/hotkey/utils.ts
+++ b/apps/web/src/lib/core/hotkey/utils.ts
@@ -1,5 +1,6 @@
import { IS_MAC } from '@core/constant/isMac';
-import { logger } from '@observability/logger';
+import { Telemetry } from '@macro-inc/observability';
+
import { createEffect, createMemo } from 'solid-js';
import {
macOptionReverse,
@@ -347,12 +348,10 @@ export function getActiveCommandByToken(
commandScope.originalParentScopeId
);
if (!originalParentScope) {
- logger.error('Original parent scope not found for command scope:', {
- error: new Error(
- 'Original parent scope not found for command scope'
- ),
- commandScopeId: commandScope?.scopeId,
- });
+ Telemetry.error(
+ new Error('Original parent scope not found for command scope'),
+ { commandScopeId: commandScope?.scopeId }
+ );
return [];
}
diff --git a/apps/web/src/lib/core/util/fetchWithToken.ts b/apps/web/src/lib/core/util/fetchWithToken.ts
index e6db7c2fa0..cf177a9f01 100644
--- a/apps/web/src/lib/core/util/fetchWithToken.ts
+++ b/apps/web/src/lib/core/util/fetchWithToken.ts
@@ -1,6 +1,7 @@
import { ENABLE_BEARER_TOKEN_AUTH } from '@core/constant/featureFlags';
import { SERVER_HOSTS } from '@core/constant/servers';
-import { logger } from '@observability';
+import { Telemetry } from '@macro-inc/observability';
+
import { fetchWithAuth } from '@service-auth/fetch';
import { err, ok, type Result } from 'neverthrow';
import type { ObjectLike, ResultError } from './result';
@@ -143,10 +144,10 @@ export async function fetchWithToken<
if (ENABLE_BEARER_TOKEN_AUTH) {
const result = await fetchWithAuth<T, CustomErrorCode>(input, init);
if (result.isErr()) {
- logger.error('fetchWithToken: fetchWithAuth failed', {
- input,
- init,
- errors: result.error,
+ Telemetry.error('fetchWithToken: fetchWithAuth failed', {
+ input: typeof input === 'string' ? input : input.url,
+ method: init?.method,
+ errors: JSON.stringify(result.error),
});
}
diff --git a/apps/web/src/lib/core/util/result.spec.ts b/apps/web/src/lib/core/util/result.spec.ts
index 4223adb878..451bd394f2 100644
--- a/apps/web/src/lib/core/util/result.spec.ts
+++ b/apps/web/src/lib/core/util/result.spec.ts
@@ -10,11 +10,14 @@ describe('throwOnErr', () => {
});
it('throws a ThrownResultError for err values', async () => {
- const error = [{ code: 'ERROR', message: 'Test error' }];
+ const error = [
+ { code: 'ERROR', message: 'First error' },
+ { code: 'ERROR', message: 'Second error' },
+ ];
await expect(throwOnErr(async () => err(error))).rejects.toMatchObject({
errors: error,
- message: 'Test error',
+ message: 'First error, Second error',
});
});
});
diff --git a/apps/web/src/lib/core/util/safeFetch.spec.ts b/apps/web/src/lib/core/util/safeFetch.spec.ts
index 3a2b6ef648..ad23224484 100644
--- a/apps/web/src/lib/core/util/safeFetch.spec.ts
+++ b/apps/web/src/lib/core/util/safeFetch.spec.ts
@@ -129,6 +129,25 @@ describe('safeFetch', () => {
}
});
+ test('keeps an expected 404 as a NOT_FOUND result', async () => {
+ mockFetch.mockImplementationOnce(() =>
+ Promise.resolve({
+ ok: false,
+ status: 404,
+ } as Response)
+ );
+
+ const result = await safeFetch<{ data: string }>(
+ 'https://localhost/data',
+ { trace: { expectedStatusCodes: [404] } }
+ );
+
+ expect(result).toMatchObject({
+ error: [{ code: 'NOT_FOUND' }],
+ });
+ expect(mockFetch.mock.calls[0]?.[1]).not.toHaveProperty('trace');
+ });
+
test('handle 401 Unauthorized', async () => {
mockFetch.mockImplementationOnce(() =>
Promise.resolve({
diff --git a/apps/web/src/lib/core/util/safeFetch.ts b/apps/web/src/lib/core/util/safeFetch.ts
index dddb10dcd6..6aac644423 100644
--- a/apps/web/src/lib/core/util/safeFetch.ts
+++ b/apps/web/src/lib/core/util/safeFetch.ts
@@ -1,8 +1,84 @@
+import { SERVER_HOSTS, SYNC_SERVICE_HOSTS } from '@core/constant/servers';
+import { Telemetry } from '@macro-inc/observability';
import { err, ok, type Result } from 'neverthrow';
import { platformFetch } from './platformFetch';
import type { ObjectLike, ResultError } from './result';
import { sleep } from './sleep';
+const tracedOrigins: ReadonlySet<string> = (() => {
+ const origins = new Set<string>();
+ for (const host of [
+ ...Object.values(SERVER_HOSTS),
+ SYNC_SERVICE_HOSTS.worker,
+ ]) {
+ try {
+ origins.add(new URL(host).origin);
+ } catch {}
+ }
+ if (typeof window !== 'undefined') origins.add(window.location.origin);
+ return origins;
+})();
+
+function isTracedOrigin(url: URL): boolean {
+ return tracedOrigins.has(url.origin);
+}
+
+function requestUrl(input: RequestInfo): URL | undefined {
+ const raw = typeof input === 'string' ? input : input.url;
+ try {
+ return new URL(
+ raw,
+ typeof window === 'undefined' ? undefined : window.location.origin
+ );
+ } catch {
+ return undefined;
+ }
+}
+
+function tracedFetch(
+ input: RequestInfo,
+ init: RequestInit & { headers: Record<string, string> },
+ traceOptions?: SafeFetchTraceOptions
+): Promise<Response> {
+ const url = requestUrl(input);
+ if (!url) return platformFetch(input, init);
+ const method = (init.method ?? 'GET').toUpperCase();
+ const span = Telemetry.clientSpan(`http ${method} ${url.pathname}`);
+ span.setAttr('http.method', method);
+ // Path only: query strings can carry tokens.
+ span.setAttr('http.url', `${url.origin}${url.pathname}`);
+ return span.run(async () => {
+ try {
+ if (isTracedOrigin(url)) {
+ span.injectTraceHeaders(init.headers);
+ }
+ const response = await platformFetch(input, init);
+ span.setAttr('http.status_code', response.status);
+ if (!response.ok) {
+ const expected = traceOptions?.expectedStatusCodes?.includes(
+ response.status
+ );
+ if (expected) {
+ span.setAttr('http.expected_status', true);
+ return response;
+ }
+ const message = `HTTP ${response.status} for ${method} ${url.pathname}`;
+ span.error({
+ name: 'HttpError',
+ message,
+ stack: new Error(message).stack,
+ });
+ }
+ return response;
+ } catch (error) {
+ span.error(error);
+ throw error;
+ } finally {
+ span.end();
+ }
+ });
+}
+
/**
* Base error codes for fetch operations.
*/
@@ -61,11 +137,18 @@ export interface RetryConfig {
delay?: 'exponential' | number;
}
+/** Trace-only classification for responses that are expected by the caller. */
+export interface SafeFetchTraceOptions {
+ /** Non-OK responses that should not mark the HTTP span as failed. */
+ expectedStatusCodes?: readonly number[];
+}
+
/**
* Extended RequestInit interface that includes retry configuration.
*/
export interface SafeFetchInit extends RequestInit {
retry?: RetryConfig;
+ trace?: SafeFetchTraceOptions;
}
export type TextResponse = { contentType: 'text/plain'; body: string };
@@ -160,7 +243,7 @@ export async function safeFetch<
init?: SafeFetchInit,
errorResponseHandler?: ErrorResponseHandler<CustomErrorCode>
): Promise<Result<T, ResultError<BaseFetchErrorCode | CustomErrorCode>[]>> {
- const { retry, ...fetchInit } = init || {};
+ const { retry, trace, ...fetchInit } = init || {};
const maxTries = retry?.maxTries ?? 1;
const delay = retry?.delay ?? 0;
type ErrorCode = BaseFetchErrorCode | CustomErrorCode;
@@ -170,20 +253,24 @@ export async function safeFetch<
for (let attempt = 1; attempt <= maxTries; attempt++) {
try {
- const response = await platformFetch(input, {
- ...fetchInit,
- headers: {
- ...(fetchInit?.method !== 'GET' &&
- fetchInit?.method !== 'HEAD' &&
- !(fetchInit?.body instanceof FormData) && {
- 'Content-Type':
- (fetchInit?.headers as Record<string, string> | undefined)?.[
- 'Content-Type'
- ] ?? 'application/json',
- }),
- ...fetchInit?.headers,
- } as Record<string, string>,
- });
+ const response = await tracedFetch(
+ input,
+ {
+ ...fetchInit,
+ headers: {
+ ...(fetchInit?.method !== 'GET' &&
+ fetchInit?.method !== 'HEAD' &&
+ !(fetchInit?.body instanceof FormData) && {
+ 'Content-Type':
+ (fetchInit?.headers as Record<string, string> | undefined)?.[
+ 'Content-Type'
+ ] ?? 'application/json',
+ }),
+ ...fetchInit?.headers,
+ } as Record<string, string>,
+ },
+ trace
+ );
if (!response.ok) {
if (errorResponseHandler) {
diff --git a/apps/web/src/lib/observability/index.ts b/apps/web/src/lib/observability/index.ts
deleted file mode 100644
index 1d41e57efc..0000000000
--- a/apps/web/src/lib/observability/index.ts
+++ /dev/null
@@ -1,80 +0,0 @@
-/// <reference types="../../vite-env.d.ts" />
-
-import { datadogLogs } from '@datadog/browser-logs';
-import { isInitialized, setInitialized } from './shared';
-
-const clientToken = import.meta.env.VITE_DD_WEB_APP_TOKEN;
-const env = import.meta.env.MODE === 'production' ? 'prod' : 'dev';
-const service = 'web-app';
-const site = 'us5.datadoghq.com';
-
-// Route intake through the first-party analytics proxy (Cloudflare Worker) so
-// ad blockers / tracking protection don't drop logs the way they block
-// requests sent straight to *.datadoghq.com. The worker maps the `/i/dd`
-// prefix to the us5 browser intake origin; see services/analytics-proxy.
-const proxy = (options: { path: string; parameters: string }) =>
- `https://macro-prox.macroverse.workers.dev/i/dd${options.path}?${options.parameters}`;
-
-interface User {
- id: string;
- email: string;
- [key: string]: any;
-}
-
-// init() is deferred to an idle callback, so setUser() can run before the SDK
-// is ready. Buffer the latest user and apply it once initialized.
-let pendingUser: User | undefined;
-
-export function init(version = import.meta.env.__APP_VERSION__) {
- if (import.meta.hot || isInitialized()) return;
-
- datadogLogs.init({
- clientToken,
- env,
- version,
- service,
- site,
- proxy,
- // Catch exceptions without RUM: forwards uncaught exceptions, unhandled
- // promise rejections, and failed network requests (XHR/fetch) to Datadog
- // as error-level logs (with stack traces).
- forwardErrorsToLogs: true,
- // Also forward explicit console.error() calls.
- forwardConsoleLogs: ['error'],
- // Forward browser Reporting API entries (CSP violations, deprecations,
- // interventions).
- forwardReports: 'all',
- telemetrySampleRate: 0,
- beforeSend: (event, _context) => {
- if (event.message.includes('unfurl-service')) return false;
-
- // these are from VList and can be ignored: https://github.com/inokawa/virtua?tab=readme-ov-file#what-is-resizeobserver-loop-completed-with-undelivered-notifications-error
- if (
- event.message.includes(
- 'ResizeObserver loop completed with undelivered notifications'
- )
- )
- return false;
-
- return true;
- },
- });
-
- setInitialized(true);
-
- if (pendingUser) datadogLogs.setUser(pendingUser);
-}
-
-export function setUser(user: User) {
- pendingUser = user;
- if (isInitialized()) datadogLogs.setUser(user);
-}
-
-// Drop the user from log context on logout so logs aren't attributed to a
-// signed-out user. Mirrors the analytics.reset() in the logout flow.
-export function clearUser() {
- pendingUser = undefined;
- if (isInitialized()) datadogLogs.clearUser();
-}
-
-export { error, log, logger } from './logger';
diff --git a/apps/web/src/lib/observability/logger.ts b/apps/web/src/lib/observability/logger.ts
deleted file mode 100644
index 0eed3eea8f..0000000000
--- a/apps/web/src/lib/observability/logger.ts
+++ /dev/null
@@ -1,79 +0,0 @@
-import { datadogLogs } from '@datadog/browser-logs';
-import { isInitialized } from './shared';
-
-interface Context {
- error?: Error;
- level?: 'debug' | 'info' | 'warn' | 'error';
- [key: string]: any;
-}
-
-/**
- * Log a message to Datadog.
- * @param messsage - The message to log.
- * @param context - The context of the log such as error, level, etc.
- */
-export function log(messsage: string, context?: Context) {
- if (import.meta.hot || !isInitialized())
- return console.log(messsage, context);
-
- const { error, level, ...messageContext } = {
- error: undefined,
- level: 'info',
- ...context,
- } as const;
-
- datadogLogs.logger.log(messsage, messageContext, level, error);
-}
-
-interface ErrorContext extends Context {
- cause?: Error;
-}
-
-/**
- * Log a warning to Datadog.
- * @param message - The message to log.
- * @param context - The context of the log such as error, level, etc.
- */
-function warn(message: string, context?: Context) {
- if (import.meta.hot || !isInitialized())
- return console.warn(message, context);
-
- const { error, level, ...messageContext } = {
- error: undefined,
- level: 'warn',
- ...context,
- } as const;
-
- datadogLogs.logger.log(message, messageContext, level, error);
-}
-
-/**
- * Log an error to Datadog.
- * @param errorMessage - The Error object or message to log.
- * @param context - The context of the log such as cause, level, etc.
- *
- * If used in a catch block, set the context.cause to the error thrown.
- */
-export function error(errorMessage: Error | string, context?: ErrorContext) {
- if (import.meta.hot || !isInitialized())
- return console.error(errorMessage, context);
-
- const { error, ...errorContext } = {
- error:
- typeof errorMessage === 'string' ? new Error(errorMessage) : errorMessage,
- level: 'error',
- ...context,
- } as const;
-
- if (context?.cause) {
- error.cause = context.cause;
- }
-
- datadogLogs.logger.error(error.message || error.name, errorContext, error);
-}
-
-export const logger = {
- log,
- warn,
- error,
-};
diff --git a/apps/web/src/lib/observability/shared.ts b/apps/web/src/lib/observability/shared.ts
deleted file mode 100644
index 50e670a034..0000000000
--- a/apps/web/src/lib/observability/shared.ts
+++ /dev/null
@@ -1,8 +0,0 @@
-let initialized = false;
-export function isInitialized() {
- return initialized;
-}
-
-export function setInitialized(value: boolean) {
- initialized = value;
-}
diff --git a/apps/web/src/lib/queries/soup/reactive-items.ts b/apps/web/src/lib/queries/soup/reactive-items.ts
index 3826b835c1..13d536a8f0 100644
--- a/apps/web/src/lib/queries/soup/reactive-items.ts
+++ b/apps/web/src/lib/queries/soup/reactive-items.ts
@@ -11,7 +11,7 @@
*/
import { createQuerySignal } from '@graphql-cache/solid/create-query-signal';
-import { logger } from '@observability/logger';
+import { Telemetry } from '@macro-inc/observability';
import { useInstructionsMdIdQuery } from '@queries/storage/instructions-md';
import {
SoupDocument,
@@ -124,7 +124,7 @@ export function useReactiveSoupAstItemsQuery(
createComputed(
on(error, (queryError) => {
if (queryError) {
- logger.error(queryError, { graphqlOperation: 'Soup' });
+ Telemetry.error(queryError, { graphqlOperation: 'Soup' });
}
})
);
diff --git a/apps/web/src/lib/queries/sync/document-peers.ts b/apps/web/src/lib/queries/sync/document-peers.ts
index e45b75f87d..156b2cafb3 100644
--- a/apps/web/src/lib/queries/sync/document-peers.ts
+++ b/apps/web/src/lib/queries/sync/document-peers.ts
@@ -1,3 +1,4 @@
+import { ThrownResultError } from '@core/util/result';
import { syncServiceClient } from '@service-sync/client';
import { useQuery } from '@tanstack/solid-query';
import type { Accessor } from 'solid-js';
@@ -7,7 +8,7 @@ async function fetchDocumentPeers(
documentId: string
): Promise<Map<string, string>> {
const result = await syncServiceClient.getDocumentMetadata({ documentId });
- if (result.isErr()) throw new Error(String(result.error));
+ if (result.isErr()) throw new ThrownResultError(result.error);
return new Map(result.value.peers.map((p) => [String(p.peer_id), p.user_id]));
}
diff --git a/apps/web/src/lib/service-clients/service-auth/client.ts b/apps/web/src/lib/service-clients/service-auth/client.ts
index d41a911940..341e9508d3 100644
--- a/apps/web/src/lib/service-clients/service-auth/client.ts
+++ b/apps/web/src/lib/service-clients/service-auth/client.ts
@@ -8,7 +8,8 @@ import {
type SafeFetchInit,
safeFetch,
} from '@core/util/safeFetch';
-import { logger } from '@observability';
+import { Telemetry } from '@macro-inc/observability';
+
import { makePersisted } from '@solid-primitives/storage';
import { err, ok } from 'neverthrow';
import { createSignal } from 'solid-js';
@@ -124,7 +125,7 @@ async function getAccessToken(): Promise<string | null> {
return null;
}
} catch (error) {
- logger.error('Error refreshing access token', { error });
+ Telemetry.error('Error refreshing access token', { error });
return null;
} finally {
// Clear the ongoing refresh promise so future calls can start a new refresh
@@ -336,7 +337,7 @@ export const authServiceClient = {
async macroApiToken() {
const accessToken = await getAccessToken();
if (!accessToken) {
- logger.warn('No access token found, fetching with cookies');
+ Telemetry.warn('No access token found, fetching with cookies');
return authApiFetch<MacroApiTokenResponse>('/jwt/macro_api_token');
}
diff --git a/apps/web/src/lib/service-clients/service-storage/client.ts b/apps/web/src/lib/service-clients/service-storage/client.ts
index 032d41440d..dd9318acc0 100644
--- a/apps/web/src/lib/service-clients/service-storage/client.ts
+++ b/apps/web/src/lib/service-clients/service-storage/client.ts
@@ -1488,7 +1488,10 @@ export const storageServiceClient = {
async fetchCachedSnapshot(
documentId: string
): Promise<Result<Uint8Array, ResultError<FetchWithTokenErrorCode>[]>> {
- return dssFetch<Uint8Array>(`/documents/${documentId}/cached_snapshot_url`);
+ return dssFetch<Uint8Array>(
+ `/documents/${documentId}/cached_snapshot_url`,
+ { trace: { expectedStatusCodes: [404] } }
+ );
},
async getDocumentShortId({
diff --git a/apps/web/src/lib/service-clients/service-sync/source/helpers.ts b/apps/web/src/lib/service-clients/service-sync/source/helpers.ts
index 0c38fa6b85..3be192f2d4 100644
--- a/apps/web/src/lib/service-clients/service-sync/source/helpers.ts
+++ b/apps/web/src/lib/service-clients/service-sync/source/helpers.ts
@@ -1,3 +1,4 @@
+import { resumeDocumentSpan } from '@block-md/observability';
import { SYNC_SERVICE_HOSTS } from '@core/constant/servers';
import type {
InitialSync,
@@ -27,8 +28,16 @@ export function createSyncServiceSocket(
documentId: string,
initialToken: string
): SyncWebsocket {
- const connectUrl = (token: string) =>
- `${SYNC_SERVICE_WS_URL}/${documentId}/connect?token=${token}`;
+ const connectUrl = (token: string) => {
+ const url = `${SYNC_SERVICE_WS_URL}/${documentId}/connect?token=${token}`;
+ // Browsers can't set websocket headers, so the trace context rides a
+ // query param; the sync service joins its spans to the active
+ // transaction's trace.
+ const traceparent = resumeDocumentSpan(documentId)?.traceparent();
+ return traceparent
+ ? `${url}&traceparent=${encodeURIComponent(traceparent)}`
+ : url;
+ };
let initialUrl: string | undefined = connectUrl(initialToken);
let fallbackUrl = initialUrl;
diff --git a/apps/web/src/observability/browser.ts b/apps/web/src/observability/browser.ts
new file mode 100644
index 0000000000..c48bd531e4
--- /dev/null
+++ b/apps/web/src/observability/browser.ts
@@ -0,0 +1,77 @@
+import { Telemetry } from '@macro-inc/observability';
+import { createWebTracingProvider } from '@macro-inc/observability/web';
+// This static import loads the zone.js Promise patch before application modules run.
+import { ZoneContextManager } from '@macro-inc/observability/zone';
+
+async function browserTelemetryEnabled(hasExporter: boolean): Promise<boolean> {
+ const override = import.meta.env.VITE_ENABLE_BROWSER_OTEL;
+
+ if (override === 'false') return false;
+ if (override === 'true') return true;
+
+ if (import.meta.hot) return hasExporter;
+
+ if (!import.meta.env.VITE_POSTHOG_API_KEY) return false;
+
+ const { analytics } = await import('@app/lib/analytics');
+ const flag = 'enable-browser-otel';
+ const current = analytics.posthog.isFeatureEnabled(flag);
+ if (current !== undefined) return current;
+
+ return new Promise((resolve) => {
+ let unsubscribe: (() => void) | undefined;
+ let settled = false;
+ const timeout = window.setTimeout(() => finish(false), 3_000);
+
+ const finish = (enabled: boolean) => {
+ settled = true;
+ window.clearTimeout(timeout);
+ unsubscribe?.();
+ resolve(enabled);
+ };
+
+ unsubscribe = analytics.posthog.onFeatureFlags((_flags, _variants, ctx) => {
+ finish(
+ !ctx?.errorsLoading &&
+ (analytics.posthog.isFeatureEnabled(flag) ?? false)
+ );
+ });
+
+ if (settled) unsubscribe();
+ });
+}
+
+/** Initialize browser telemetry and its application-level lifecycle hooks. */
+export async function initializeBrowserObservability(): Promise<void> {
+ const tracesUrl =
+ import.meta.env.VITE_OTEL_EXPORTER_URL ??
+ (import.meta.hot ? 'http://localhost:8098/i/otlp/v1/traces' : undefined);
+ const telemetryConfig = {
+ serviceName: 'web-app',
+ environment:
+ import.meta.env.VITE_OTEL_ENV ??
+ (import.meta.env.MODE === 'production' ? 'prod' : 'dev'),
+ tracesUrl,
+ logsUrl: tracesUrl?.replace(/\/v1\/traces\/?$/, '/v1/logs'),
+ contextManager: new ZoneContextManager(),
+ enabled: () => browserTelemetryEnabled(Boolean(tracesUrl)),
+ };
+
+ await Telemetry.init({
+ ...telemetryConfig,
+ tracingProvider: (resource, getUserId) =>
+ createWebTracingProvider(telemetryConfig, resource, getUserId),
+ });
+
+ window.addEventListener('pagehide', () => void Telemetry.flush());
+ window.addEventListener('error', (event) => {
+ Telemetry.error(event.error ?? event.message, {
+ 'error.source': 'window',
+ });
+ });
+ window.addEventListener('unhandledrejection', (event) => {
+ Telemetry.error(event.reason, {
+ 'error.source': 'unhandledrejection',
+ });
+ });
+}
diff --git a/apps/web/src/routes/Root.tsx b/apps/web/src/routes/Root.tsx
index a7a3efcf55..0380aa9d99 100644
--- a/apps/web/src/routes/Root.tsx
+++ b/apps/web/src/routes/Root.tsx
@@ -66,6 +66,7 @@ import { transformShortIdInUrlPathname } from '@core/util/url';
import { EntityProvider } from '@entity';
import { MaybeTauriProvider } from '@macro/tauri';
import { TauriRouteListener } from '@macro/tauri/TauriProvider';
+import { Telemetry } from '@macro-inc/observability';
import {
BrowserNotificationModal,
createNotificationSource,
@@ -74,10 +75,6 @@ import {
usePlatformNotificationState,
} from '@notifications';
import { maybeHandlePlatformNotification } from '@notifications/notification-platform';
-import {
- clearUser as clearDatadogUser,
- setUser as setDatadogUser,
-} from '@observability';
import {
invalidateUserInfo,
prefetchUserInfo,
@@ -546,16 +543,12 @@ function UserInfoSideEffects() {
let syncedPlanKey: string | undefined;
createEffect(
on(userInfo, (user) => {
- // Keep Datadog log user context in sync with auth state: set on every
- // authenticated load (the logs SDK doesn't persist across reloads), and
- // clear on logout so logs aren't attributed to a signed-out user. Logout
- // flips userInfo client-side, and on native mobile it's an SPA navigation
- // with no page reload, so this effect is what clears it there.
- if (user?.authenticated) {
- setDatadogUser({ id: user.id, email: user.email });
- } else {
- clearDatadogUser();
- }
+ // Keep telemetry user context in sync with auth state: set on every
+ // authenticated load, and clear on logout so spans and logs aren't
+ // attributed to a signed-out user. Logout flips userInfo client-side,
+ // and on native mobile it's an SPA navigation with no page reload, so
+ // this effect is what clears it there.
+ Telemetry.config.setUser(user?.authenticated ? user.id : undefined);
if (!user || !user.authenticated) {
syncedPlanKey = undefined;
diff --git a/apps/web/src/vite-env.d.ts b/apps/web/src/vite-env.d.ts
index b8e8f303cd..98c7e129bc 100644
--- a/apps/web/src/vite-env.d.ts
+++ b/apps/web/src/vite-env.d.ts
@@ -4,9 +4,11 @@ interface ImportMetaEnv {
readonly __GIT_BRANCH__: string;
readonly VITE_SEGMENT_WRITE_KEY: string;
- readonly VITE_DD_WEB_APP_TOKEN: string;
- readonly VITE_DD_HASH: string;
readonly VITE_POSTHOG_API_KEY: string;
+
+ readonly VITE_OTEL_EXPORTER_URL?: string;
+ readonly VITE_OTEL_ENV?: string;
+ readonly VITE_ENABLE_BROWSER_OTEL?: string;
}
interface ImportMeta {
diff --git a/apps/web/tsconfig.json b/apps/web/tsconfig.json
index d77ddbe14c..e8870b678a 100644
--- a/apps/web/tsconfig.json
+++ b/apps/web/tsconfig.json
@@ -182,12 +182,6 @@
"@design/*": [
"./src/lib/design/*"
],
- "@observability": [
- "./src/lib/observability/"
- ],
- "@observability/*": [
- "./src/lib/observability/*"
- ],
"@service-sync": [
"./src/lib/service-clients/service-sync"
],
diff --git a/bun.lock b/bun.lock
index d968c01012..e00f612e5b 100644
--- a/bun.lock
+++ b/bun.lock
@@ -18,7 +18,6 @@
"@codemirror/language": "^6.11.3",
"@corvu/dialog": "^0.2.4",
"@corvu/drawer": "^0.2.4",
- "@datadog/browser-logs": "^6.22.0",
"@date-fns/tz": "^1.4.1",
"@dinero.js/currencies": "2.0.0-alpha.8",
"@floating-ui/dom": "^1.7.4",
@@ -45,7 +44,9 @@
"@lukemorales/query-key-factory": "^1.3.4",
"@macro-inc/collaboration": "workspace:*",
"@macro-inc/lexical-core": "workspace:*",
+ "@macro-inc/observability": "workspace:*",
"@normy/query-core": "^0.21.0",
+ "@opentelemetry/api": "^1.9.1",
"@phosphor-icons/core": "^2.1.1",
"@solid-primitives/broadcast-channel": "^0.1.1",
"@solid-primitives/context": "^0.3.2",
@@ -168,6 +169,7 @@
"version": "0.1.0",
"dependencies": {
"@loro-mirror/core": "workspace:*",
+ "@macro-inc/observability": "workspace:*",
"@solid-primitives/event-bus": "^1.1.2",
"@tauri-apps/api": "^2.8.0",
"async-mutex": "^0.5.0",
@@ -248,6 +250,30 @@
"loro-crdt": "^1.4.4",
},
},
+ "packages/observability": {
+ "name": "@macro-inc/observability",
+ "version": "0.0.1",
+ "dependencies": {
+ "@opentelemetry/api": "^1.9.1",
+ "@opentelemetry/api-logs": "^0.220.0",
+ "@opentelemetry/context-zone": "^2.9.0",
+ "@opentelemetry/core": "^2.9.0",
+ "@opentelemetry/exporter-logs-otlp-http": "^0.220.0",
+ "@opentelemetry/exporter-trace-otlp-http": "^0.220.0",
+ "@opentelemetry/resources": "^2.9.0",
+ "@opentelemetry/sdk-logs": "^0.220.0",
+ "@opentelemetry/sdk-trace-base": "^2.9.0",
+ "@opentelemetry/sdk-trace-web": "^2.9.0",
+ "@opentelemetry/semantic-conventions": "^1.43.0",
+ },
+ "devDependencies": {
+ "@types/bun": "^1.3.14",
+ "@types/node": "^24.8.0",
+ "jsdom": "^26.1.0",
+ "typescript": "^5.9.3",
+ "vitest": "^3.2.4",
+ },
+ },
"services/ai-editing-worker": {
"name": "ai-editing-worker",
"version": "0.0.1",
@@ -615,10 +641,6 @@
"@csstools/css-tokenizer": ["@csstools/[email protected]", "", {}, "sha512-Vd/9EVDiu6PPJt9yAh6roZP6El1xHrdvIVGjyBsHR0RYwNHgL7FJPyIIW4fANJNG6FtyZfvlRPpFI4ZM/lubvw=="],
- "@datadog/browser-core": ["@datadog/[email protected]", "", {}, "sha512-h/xuBuY4Xi32CbBHkChMMWlOQ4al/mFF08x3o0hqrderfJvX7VO5tD3OVNwF+pfMXOGlDtBCIhwazr9sn24h3g=="],
-
- "@datadog/browser-logs": ["@datadog/[email protected]", "", { "dependencies": { "@datadog/browser-core": "6.33.0" }, "peerDependencies": { "@datadog/browser-rum": "6.33.0" }, "optionalPeers": ["@datadog/browser-rum"] }, "sha512-WZ1XEIIYK2sijR0hUG3hTPu2kQs8hE/PpoJcMDE7p0/NPXX3TR3dGVrwag+gvIVyBaszRkITiBsHQfnRvxV1DA=="],
-
"@datadog/datadog-ci": ["@datadog/[email protected]", "", { "dependencies": { "@datadog/datadog-ci-base": "3.22.2", "@datadog/datadog-ci-plugin-aas": "3.22.2", "@datadog/datadog-ci-plugin-cloud-run": "3.22.2", "@datadog/datadog-ci-plugin-deployment": "3.22.2", "@datadog/datadog-ci-plugin-dora": "3.22.2", "@datadog/datadog-ci-plugin-gate": "3.22.2", "@datadog/datadog-ci-plugin-lambda": "3.22.2", "@datadog/datadog-ci-plugin-sarif": "3.22.2", "@datadog/datadog-ci-plugin-sbom": "3.22.2", "@datadog/datadog-ci-plugin-stepfunctions": "3.22.2", "@datadog/datadog-ci-plugin-synthetics": "3.22.2", "axios": "^1.12.1", "chalk": "3.0.0", "clipanion": "^3.2.1", "fast-xml-parser": "^4.4.1", "form-data": "^4.0.4", "js-yaml": "3.13.1", "semver": "^7.5.3", "simple-git": "3.16.0", "typanion": "^3.14.0", "upath": "^2.0.1", "uuid": "^9.0.0" }, "bin": "dist/cli.js" }, "sha512-OejGdY0g+S0/9oJNCueBSU3ZXjNCn+JWhxvx+jroB/FolkOEDESmugludf3/uOjM3mCeuUlGoDhwIAmezF0AvA=="],
"@datadog/datadog-ci-base": ["@datadog/[email protected]", "", { "dependencies": { "@antfu/install-pkg": "^1.1.0", "@types/datadog-metrics": "0.6.1", "async-retry": "1.3.1", "axios": "^1.12.1", "chalk": "3.0.0", "clipanion": "^3.2.1", "datadog-metrics": "0.9.3", "debug": "^4.4.1", "deep-extend": "0.6.0", "fast-xml-parser": "^4.4.1", "form-data": "^4.0.4", "glob": "^10.4.5", "inquirer": "^8.2.5", "jszip": "^3.10.1", "proxy-agent": "^6.4.0", "semver": "^7.5.3", "simple-git": "3.16.0", "terminal-link": "2.1.1", "tiny-async-pool": "^2.1.0", "typanion": "^3.14.0", "upath": "^2.0.1" }, "peerDependencies": { "@datadog/datadog-ci-plugin-aas": "3.22.2", "@datadog/datadog-ci-plugin-cloud-run": "3.22.2", "@datadog/datadog-ci-plugin-deployment": "3.22.2", "@datadog/datadog-ci-plugin-dora": "3.22.2", "@datadog/datadog-ci-plugin-gate": "3.22.2", "@datadog/datadog-ci-plugin-lambda": "3.22.2", "@datadog/datadog-ci-plugin-sarif": "3.22.2", "@datadog/datadog-ci-plugin-sbom": "3.22.2", "@datadog/datadog-ci-plugin-stepfunctions": "3.22.2", "@datadog/datadog-ci-plugin-synthetics": "3.22.2" }, "optionalPeers": ["@datadog/datadog-ci-plugin-aas", "@datadog/datadog-ci-plugin-cloud-run", "@datadog/datadog-ci-plugin-deployment", "@datadog/datadog-ci-plugin-dora", "@datadog/datadog-ci-plugin-gate", "@datadog/datadog-ci-plugin-lambda", "@datadog/datadog-ci-plugin-sarif", "@datadog/datadog-ci-plugin-sbom", "@datadog/datadog-ci-plugin-stepfunctions", "@datadog/datadog-ci-plugin-synthetics"] }, "sha512-87Q1dHcOczRhddo9ugB6JZYOep70rQ9rHMByOF81RKFjufQWtbU+ZPl0pKge3DvDiR3oXo01PrzUsS+Ss38QOw=="],
@@ -1041,6 +1063,8 @@
"@macro-inc/lexical-core": ["@macro-inc/lexical-core@workspace:packages/lexical-core"],
+ "@macro-inc/observability": ["@macro-inc/observability@workspace:packages/observability"],
+
"@marijn/find-cluster-break": ["@marijn/[email protected]", "", {}, "sha512-FY+MKLBoTsLNJF/eLWaOsXGdz6uh3Iu1axjPf6TUq92IYumcTcXWHoS747JARLkcdlJ/Waiaxc5wQfFO8jC6NA=="],
"@mediapipe/tasks-vision": ["@mediapipe/[email protected]", "", {}, "sha512-vOifgZhkndgybdvoRITzRkIueWWSiCKuEUXXK6Q4FaJsFvRJuwgg++vqFUMlL0Uox62U5aEXFhHxlhV7Ja5e3Q=="],
@@ -1061,6 +1085,36 @@
"@opentelemetry/api": ["@opentelemetry/[email protected]", "", {}, "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q=="],
+ "@opentelemetry/api-logs": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-CmVa4ImJ+ynfrPMNaAXHET6Bhb44SwzmfyVJFq9ni2jgXJR/l7C6gfVFddNmHP+ZOkP9cf4f9DBe68qVLTHc9w=="],
+
+ "@opentelemetry/context-zone": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/context-zone-peer-dep": "2.9.0", "zone.js": "^0.11.0 || ^0.12.0 || ^0.13.0 || ^0.14.0 || ^0.15.0 || ^0.16.0" } }, "sha512-dUH2j6/jh5hgsAFvEqx+9MIjdi8OzBK1u8K6qsb2HE2R1AiR+u00GWCxbDts30JggTX4rwA8YNLO2xF5LC6EFg=="],
+
+ "@opentelemetry/context-zone-peer-dep": ["@opentelemetry/[email protected]", "", { "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0", "zone.js": "^0.10.2 || ^0.11.0 || ^0.12.0 || ^0.13.0 || ^0.14.0 || ^0.15.0 || ^0.16.0" } }, "sha512-xECINyl1UHLNbaF1my3K5AqJGyox1ysLidplInAb/XTfBxc2CLDRuS4rUxiJs8irW/PkEDujdc/IPUqkJFnrzg=="],
+
+ "@opentelemetry/core": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-m2nckMT80NnmjTYSPjJQObBJ+8dgkoajEOUbznL8AHZ3T3yHRk2P7gI1PhEBc1+lOnrYE9UWrWHqJDsmqjmNbw=="],
+
+ "@opentelemetry/exporter-logs-otlp-http": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/api-logs": "0.220.0", "@opentelemetry/core": "2.9.0", "@opentelemetry/otlp-exporter-base": "0.220.0", "@opentelemetry/otlp-transformer": "0.220.0", "@opentelemetry/sdk-logs": "0.220.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-8186thl+pTw64iz/qEEen5oJZoZ/gO73XruChdaGlYdWOdBIQ42r+vHLf6a7vIDqTD4b8ZOoMlyxptanECaI9A=="],
+
+ "@opentelemetry/exporter-trace-otlp-http": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/core": "2.9.0", "@opentelemetry/otlp-exporter-base": "0.220.0", "@opentelemetry/otlp-transformer": "0.220.0", "@opentelemetry/resources": "2.9.0", "@opentelemetry/sdk-trace": "2.9.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-/+ExB3lRkf+erv4PnoywyL7RHKITidxtUpUTS55k7OQ0dB42S7gEF1gry7swb9MSm1hYLUhJg4QQh9W8SpwwqA=="],
+
+ "@opentelemetry/otlp-exporter-base": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/core": "2.9.0", "@opentelemetry/otlp-transformer": "0.220.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-CXYo8UD5Mn9YbgebO2EL4wejtA+gxLmLiu6HCk2KH2BR7XhFN6/6p1UlCb23DYCjeYkndevLHuejCCN1yx4+OQ=="],
+
+ "@opentelemetry/otlp-transformer": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/api-logs": "0.220.0", "@opentelemetry/core": "2.9.0", "@opentelemetry/resources": "2.9.0", "@opentelemetry/sdk-logs": "0.220.0", "@opentelemetry/sdk-metrics": "2.9.0", "@opentelemetry/sdk-trace": "2.9.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-lXGrv7KXZ0gNH9SVNUaa6vv6phVYGvJxfXAlMbzbakiXru75f5MZl8Z7oqiMMQD77riVHJCFlQvbZs/VVN2/4A=="],
+
+ "@opentelemetry/resources": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/core": "2.9.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-jyA5MBLQ+Dkl3+JsZkUoUvL7yHvU64kLsvpXKarWm6347Sl1t1bXFTFykUePNpT5WH5pm9a2Qtt03iIYQhZ1Fg=="],
+
+ "@opentelemetry/sdk-logs": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/api-logs": "0.220.0", "@opentelemetry/core": "2.9.0", "@opentelemetry/resources": "2.9.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.4.0 <1.10.0" } }, "sha512-WywcTkQtv2iNmt+6y5Kcd4rzvx9bLVsBa2Nwcmg01IUaBTkTow3W4d9KE5vNBpEDtb9tp21WcRBY/lANRrApYA=="],
+
+ "@opentelemetry/sdk-metrics": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/core": "2.9.0", "@opentelemetry/resources": "2.9.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.9.0 <1.10.0" } }, "sha512-Xx8RGS4H5XEBl01WuCreMIpiah9cCXMbSkeuIePPdD2cUpq/vUzYmj8E/MK1OsbOc93FuAD4jfn2WOacKwLn7Q=="],
+
+ "@opentelemetry/sdk-trace": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/core": "2.9.0", "@opentelemetry/resources": "2.9.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-sGA19HvtrrSKYsseHphluH6j3p6Xa3fqc7c7y8f/7mYWejc1lyDFcpSdD1kYa50HCLUeEo4zA5bW0pniaPszuw=="],
+
+ "@opentelemetry/sdk-trace-base": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/core": "2.9.0", "@opentelemetry/resources": "2.9.0", "@opentelemetry/sdk-trace": "2.9.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-cp9zmTl62R8PJrpvFcmc8N2JQU/xfa0S+61q511Nji+QxCfZ8Ifvg7H27G8cANe4crg4RTrWsVvanHiXjSp6ag=="],
+
+ "@opentelemetry/sdk-trace-web": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/core": "2.9.0", "@opentelemetry/sdk-trace-base": "2.9.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-LS4XlzOK3e6YYdt84m15AmRR04121rKipmifi4XFZToH1h75f7F2bzHLbB1NMgIEYJ0jSKYm4VsK5mws/5kfTQ=="],
+
+ "@opentelemetry/semantic-conventions": ["@opentelemetry/[email protected]", "", {}, "sha512-eSYWTm620tTk45EKSedaUL8MFYI8hW164hIXsgIHyxu3VobUB3fFCu5t0hQby6OoWRPsG1KkKUG2M5UadiLiVg=="],
+
"@orval/angular": ["@orval/[email protected]", "", { "dependencies": { "@orval/core": "7.21.0" } }, "sha512-AGelR1FfuimtIBBVccUI9MyjNOalLEyJFog8a94thFiqRGtz0JFIPnd8+IqRcmw3wE370PKQQMCqZl1WkjZi8w=="],
"@orval/axios": ["@orval/[email protected]", "", { "dependencies": { "@orval/core": "7.21.0" } }, "sha512-0t2PqMSdjJ7No6Ng5gTEpqyILmngz1mDw7OQ2TPwOiAEepNfeDgE6NtRh3zbZvrH/H5I+W8dgn+wGYueZQCNjw=="],
@@ -3173,6 +3227,8 @@
"zod-validation-error": ["[email protected]", "", { "peerDependencies": { "zod": "^3.25.0 || ^4.0.0" } }, "sha512-hmk+pkyKq7Q71PiWVSDUc3VfpzpvcRHZ3QPw9yEMVvmtCekaMeOHnbr3WbxfrgEnQTv6haGP4cmv0Ojmihzsxw=="],
+ "zone.js": ["[email protected]", "", {}, "sha512-Eky7p2Z1Ig3NnbfodSPoARCjKBSTFMnE/ACsP1L/XJEfY4SdOFce19BsUCWVwL6K5ABZFy5J3bjcMWffX+YM3Q=="],
+
"@antfu/install-pkg/tinyexec": ["[email protected]", "", {}, "sha512-SHf/r48b7vOrjve9PxJo3MN5v5yuyjHvdUcrQffT3WXMUfnGmHDVbC4k3sHJaJTgZCwpUplIaAo5ANtMyp3YHg=="],
"@apidevtools/swagger-parser/@apidevtools/json-schema-ref-parser": ["@apidevtools/[email protected]", "", { "dependencies": { "@types/json-schema": "^7.0.15", "js-yaml": "^4.1.0" } }, "sha512-Oc96zvmxx1fqoSEdUmfmvvb59/KDOnUoJ7s2t7bISyAn0XEz57LCCw8k2Y4Pf3mwKaZLMciESALORLgfe2frCw=="],
diff --git a/crates/macro_entrypoint/Cargo.toml b/crates/macro_entrypoint/Cargo.toml
index db4c09fbc3..0d683aa466 100644
--- a/crates/macro_entrypoint/Cargo.toml
+++ b/crates/macro_entrypoint/Cargo.toml
@@ -7,8 +7,8 @@ version = "0.1.0"
[dependencies]
dotenvy = { workspace = true }
macro_env = { path = "../macro_env" }
-opentelemetry = { workspace = true }
-opentelemetry_sdk = { workspace = true }
+opentelemetry = { workspace = true, features = ["trace"] }
+opentelemetry_sdk = { workspace = true, features = ["rt-tokio", "trace"] }
opentelemetry-otlp = { workspace = true }
rootcause = { workspace = true }
rootcause-tracing = { workspace = true }
diff --git a/crates/macro_entrypoint/src/lib.rs b/crates/macro_entrypoint/src/lib.rs
index 7922a41e63..57e2300e80 100644
--- a/crates/macro_entrypoint/src/lib.rs
+++ b/crates/macro_entrypoint/src/lib.rs
@@ -32,6 +32,7 @@ env_vars! {
maybe_env_vars! {
pub struct RustLog;
+ pub struct OtelExporterOtlpEndpoint;
pub struct OtelTraceFilter;
}
@@ -125,23 +126,45 @@ impl MacroEntrypoint {
match (self.env, self.local) {
(Environment::Local, LocalOptions { tree_tracing: None }) => {
- // Keep rootcause from enabling callsites that RUST_LOG previously disabled.
let rust_log_filter = rust_log_env_filter();
- let fmt_layer = tracing_subscriber::fmt::layer()
- .with_ansi(true)
- .with_file(true)
- .with_line_number(true)
- .pretty()
- .with_filter(rust_log_filter.clone());
-
- Registry::default()
- .with(RootcauseLayer.with_filter(rust_log_filter))
- .with(fmt_layer)
- .init();
-
- InitializedEntrypoint {
- tracer_provider: None,
+ // Local OTLP export is opt-in. xtask injects this only for
+ // trace-enabled local runs.
+ let tracer_provider = OtelExporterOtlpEndpoint::new()
+ .map(|_| init_opentelemetry("local".to_string()));
+
+ if let Some(provider) = tracer_provider.as_ref() {
+ let otel_filter = otel_env_filter();
+ let rootcause_filter = rust_log_filter.clone().or(otel_filter.clone());
+ let otel_layer = otel_layer_with_error_mapping(provider.tracer(service_name()))
+ .with_filter(otel_filter);
+
+ Registry::default()
+ .with(RootcauseLayer.with_filter(rootcause_filter))
+ .with(
+ tracing_subscriber::fmt::layer()
+ .with_ansi(true)
+ .with_file(true)
+ .with_line_number(true)
+ .pretty()
+ .with_filter(rust_log_filter),
+ )
+ .with(otel_layer)
+ .init();
+ } else {
+ Registry::default()
+ .with(RootcauseLayer.with_filter(rust_log_filter.clone()))
+ .with(
+ tracing_subscriber::fmt::layer()
+ .with_ansi(true)
+ .with_file(true)
+ .with_line_number(true)
+ .pretty()
+ .with_filter(rust_log_filter),
+ )
+ .init();
}
+
+ InitializedEntrypoint { tracer_provider }
}
(
Environment::Local,
@@ -159,21 +182,18 @@ impl MacroEntrypoint {
}
}
(Environment::Production | Environment::Develop, _) => {
- let tracer_provider = init_opentelemetry();
-
- // Get service name for the tracer
- let service_name = DdService::new()
+ // Get environment from DD_ENV
+ let env = DdEnv::new()
.map(|e| e.to_string())
- .unwrap_or_else(|_| "unknown-service".to_string());
+ .unwrap_or_else(|_| "unknown".to_string());
+ let tracer_provider = init_opentelemetry(env);
- let tracer = tracer_provider.tracer(service_name);
+ let tracer = tracer_provider.tracer(service_name());
let rust_log_filter = rust_log_env_filter();
let otel_filter = otel_env_filter();
// Capture anything already enabled for logs or OTEL without enabling new callsites.
let rootcause_filter = rust_log_filter.clone().or(otel_filter.clone());
- let otel_layer = tracing_opentelemetry::layer()
- .with_tracer(tracer)
- .with_filter(otel_filter);
+ let otel_layer = otel_layer_with_error_mapping(tracer).with_filter(otel_filter);
// Build the JSON event format, then wrap it with DatadogFormat
// to inject dd.trace_id / dd.span_id for trace-log correlation.
@@ -213,31 +233,70 @@ impl MacroEntrypoint {
}
}
-/// Opentelemetry export endpoint to talk with datadog sidecar
-const OTEL_EXPORTER_OTLP_ENDPOINT: &str = "http://127.0.0.1:4317";
+/// Default OpenTelemetry export endpoint: the Datadog agent sidecar in prod/develop.
+const DEFAULT_OTLP_ENDPOINT: &str = "http://127.0.0.1:4317";
+
+/// Wraps the OTel layer so `tracing::error!` events on a span (e.g.
+/// `macro_tower_layers::CustomOnFailure`'s "response failed", or an
+/// `#[instrument(err)]` failure) become an OTLP exception with a message and
+/// stack trace, and set the span's error status message. Without these, a
+/// span can be marked errored (from the HTTP status) with no `error.message`
+/// / `error.stack` for Datadog to show — "Missing error message and stack
+/// trace" in the UI.
+fn otel_layer_with_error_mapping<S>(
+ tracer: opentelemetry_sdk::trace::SdkTracer,
+) -> tracing_opentelemetry::OpenTelemetryLayer<S, opentelemetry_sdk::trace::SdkTracer>
+where
+ S: tracing::Subscriber + for<'a> tracing_subscriber::registry::LookupSpan<'a>,
+{
+ tracing_opentelemetry::layer()
+ .with_tracer(tracer)
+ .with_error_events_to_exceptions(true)
+ .with_error_events_to_status(true)
+ .with_error_fields_to_exceptions(true)
+ .with_error_records_to_exceptions(true)
+ .with_location(true)
+}
+
+/// The service name reported on traces: `DD_SERVICE` when set, otherwise the
+/// current executable's file name so each service is distinguishable locally.
+fn service_name() -> String {
+ if let Ok(service) = DdService::new() {
+ return service.to_string();
+ }
+ std::env::current_exe()
+ .ok()
+ .and_then(|exe| {
+ exe.file_name()
+ .map(|name| name.to_string_lossy().into_owned())
+ })
+ .unwrap_or_else(|| "unknown-service".to_string())
+}
+
+fn init_opentelemetry(deployment_environment: String) -> SdkTracerProvider {
+ // W3C trace-context propagation: lets macro_tower_layers parent request
+ // spans under an incoming `traceparent` (e.g. from the web app), and
+ // service clients propagate context onward.
+ opentelemetry::global::set_text_map_propagator(
+ opentelemetry_sdk::propagation::TraceContextPropagator::new(),
+ );
+
+ let endpoint = OtelExporterOtlpEndpoint::new()
+ .map(|e| e.to_string())
+ .unwrap_or_else(|| DEFAULT_OTLP_ENDPOINT.to_string());
-/// Initialize OpenTelemetry with OTLP exporter to the Datadog agent.
-/// The Datadog agent sidecar listens on localhost:4317 for OTLP gRPC.
-fn init_opentelemetry() -> SdkTracerProvider {
let exporter = opentelemetry_otlp::SpanExporter::builder()
.with_tonic()
- .with_endpoint(OTEL_EXPORTER_OTLP_ENDPOINT)
+ .with_endpoint(endpoint)
.build()
.expect("failed to create OTLP span exporter");
- // Get service name from DD_SERVICE or OTEL_SERVICE_NAME
- let service_name = DdService::new()
- .map(|e| e.to_string())
- .unwrap_or_else(|_| "unknown-service".to_string());
-
- // Get environment from DD_ENV
- let env = DdEnv::new()
- .map(|e| e.to_string())
- .unwrap_or_else(|_| "unknown".to_string());
-
let resource = opentelemetry_sdk::Resource::builder()
- .with_service_name(service_name)
- .with_attribute(opentelemetry::KeyValue::new("deployment.environment", env))
+ .with_service_name(service_name())
+ .with_attribute(opentelemetry::KeyValue::new(
+ "deployment.environment",
+ deployment_environment,
+ ))
.build();
SdkTracerProvider::builder()
diff --git a/crates/macro_tower_layers/Cargo.toml b/crates/macro_tower_layers/Cargo.toml
index 74d5f45541..9009e24a8b 100644
--- a/crates/macro_tower_layers/Cargo.toml
+++ b/crates/macro_tower_layers/Cargo.toml
@@ -6,11 +6,16 @@ version = "0.1.0"
[dependencies]
http = { workspace = true }
+opentelemetry = { workspace = true, features = ["trace"] }
+opentelemetry-http = { workspace = true }
tokio = { workspace = true, features = ["rt", "time"] }
tower = { workspace = true }
tower-http = { workspace = true, features = ["request-id", "util"] }
tracing = { workspace = true }
+tracing-opentelemetry = { workspace = true }
workspace-hack = { version = "0.1", path = "../workspace-hack" }
[dev-dependencies]
+opentelemetry_sdk = { workspace = true, features = ["testing"] }
tokio = { workspace = true, features = ["macros", "rt", "test-util", "time"] }
+tracing-subscriber = { workspace = true }
diff --git a/crates/macro_tower_layers/src/lib.rs b/crates/macro_tower_layers/src/lib.rs
index d10242f3ae..cd228577cf 100644
--- a/crates/macro_tower_layers/src/lib.rs
+++ b/crates/macro_tower_layers/src/lib.rs
@@ -12,6 +12,7 @@ use std::{
};
use http::{HeaderValue, Request, Response};
+use opentelemetry::trace::TraceContextExt;
use tokio::time::MissedTickBehavior;
use tower::{
ServiceBuilder,
@@ -24,6 +25,27 @@ use tower_http::{
trace::{MakeSpan, OnFailure, OnResponse, TraceLayer},
};
use tracing::Span;
+use tracing_opentelemetry::OpenTelemetrySpanExt;
+
+/// Creates a safe HTTP request span and adopts an incoming W3C trace context.
+///
+/// A valid `traceparent` makes the request span a child of that remote span.
+/// The global text-map propagator is registered by `macro_entrypoint`.
+#[derive(Debug, Clone, Default)]
+pub struct MakeSpanWithRemoteParent;
+
+impl<B> MakeSpan<B> for MakeSpanWithRemoteParent {
+ fn make_span(&mut self, request: &Request<B>) -> Span {
+ let span = MakeHttpRequestSpan.make_span(request);
+ let parent = opentelemetry::global::get_text_map_propagator(|propagator| {
+ propagator.extract(&opentelemetry_http::HeaderExtractor(request.headers()))
+ });
+ if parent.span().span_context().is_valid() {
+ let _ = span.set_parent(parent);
+ }
+ span
+ }
+}
/// A very simple builder for x-request-ids
#[derive(Default, Clone)]
@@ -143,7 +165,7 @@ type ServiceBuilderAlias = ServiceBuilder<
Stack<
TraceLayer<
SharedClassifier<ServerErrorsAsFailures>,
- MakeHttpRequestSpan,
+ MakeSpanWithRemoteParent,
(),
CustomOnResponse,
tower_http::trace::DefaultOnBodyChunk,
@@ -200,7 +222,7 @@ impl MacroRequestIdAndTracingLayer {
.set_x_request_id(RequestIdBuilder::default())
.layer(
TraceLayer::new_for_http()
- .make_span_with(MakeHttpRequestSpan)
+ .make_span_with(MakeSpanWithRemoteParent)
.on_request(())
.on_response(CustomOnResponse::new_with_threshold(warning_threshold))
.on_failure(CustomOnFailure),
diff --git a/crates/macro_tower_layers/src/test.rs b/crates/macro_tower_layers/src/test.rs
index bd51f37fc1..b1cfb78cea 100644
--- a/crates/macro_tower_layers/src/test.rs
+++ b/crates/macro_tower_layers/src/test.rs
@@ -219,6 +219,87 @@ async fn starvation_detector_warns_when_runtime_blocked() {
assert_eq!(captured.event_count(Level::WARN), 1);
}
+#[test]
+fn header_extractor_extracts_w3c_traceparent() {
+ let propagator = opentelemetry_sdk::propagation::TraceContextPropagator::new();
+ let mut headers = http::HeaderMap::new();
+ headers.insert(
+ "traceparent",
+ "00-5b8aa5a2d2c872e8321cf37308d69df2-051581bf3cb55c13-01"
+ .parse()
+ .unwrap(),
+ );
+
+ let cx = opentelemetry::propagation::TextMapPropagator::extract(
+ &propagator,
+ &opentelemetry_http::HeaderExtractor(&headers),
+ );
+
+ let span_context = cx.span().span_context().clone();
+ assert!(span_context.is_valid());
+ assert!(span_context.is_remote());
+ assert_eq!(
+ span_context.trace_id().to_string(),
+ "5b8aa5a2d2c872e8321cf37308d69df2"
+ );
+ assert_eq!(span_context.span_id().to_string(), "051581bf3cb55c13");
+}
+
+#[test]
+fn request_span_adopts_remote_traceparent() {
+ use opentelemetry::trace::TracerProvider as _;
+ use tracing_subscriber::layer::SubscriberExt;
+
+ opentelemetry::global::set_text_map_propagator(
+ opentelemetry_sdk::propagation::TraceContextPropagator::new(),
+ );
+ // Provider with no exporter: spans are created (so parentage is real) but
+ // never shipped anywhere.
+ let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder().build();
+ let subscriber = tracing_subscriber::registry()
+ .with(tracing_opentelemetry::layer().with_tracer(provider.tracer("test")));
+
+ with_default(subscriber, || {
+ let request = http::Request::builder()
+ .uri("/test")
+ .header(
+ "traceparent",
+ "00-5b8aa5a2d2c872e8321cf37308d69df2-051581bf3cb55c13-01",
+ )
+ .body(())
+ .unwrap();
+
+ let span = MakeSpanWithRemoteParent.make_span(&request);
+ let trace_id = span.context().span().span_context().trace_id();
+ assert_eq!(
+ trace_id.to_string(),
+ "5b8aa5a2d2c872e8321cf37308d69df2",
+ "request span should continue the remote trace"
+ );
+ });
+}
+
+#[test]
+fn request_span_without_traceparent_stays_root() {
+ use opentelemetry::trace::TracerProvider as _;
+ use tracing_subscriber::layer::SubscriberExt;
+
+ let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder().build();
+ let subscriber = tracing_subscriber::registry()
+ .with(tracing_opentelemetry::layer().with_tracer(provider.tracer("test")));
+
+ with_default(subscriber, || {
+ let request = http::Request::builder().uri("/test").body(()).unwrap();
+ let span = MakeSpanWithRemoteParent.make_span(&request);
+ let trace_id = span.context().span().span_context().trace_id();
+ assert_ne!(
+ trace_id.to_string(),
+ "5b8aa5a2d2c872e8321cf37308d69df2",
+ "no traceparent should mean a fresh trace"
+ );
+ });
+}
+
#[tokio::test]
async fn starvation_detector_does_not_warn_within_grace_period() {
tokio::time::pause();
diff --git a/crates/worker-rs-otel/Cargo.toml b/crates/worker-rs-otel/Cargo.toml
new file mode 100644
index 0000000000..e261b56572
--- /dev/null
+++ b/crates/worker-rs-otel/Cargo.toml
@@ -0,0 +1,23 @@
+[package]
+name = "worker-rs-otel"
+version = "0.1.0"
+edition = "2024"
+publish = false
+
+[dependencies]
+getrandom = { workspace = true, features = ["wasm_js"] }
+http.workspace = true
+opentelemetry = { workspace = true, features = ["trace"] }
+opentelemetry-proto = { version = "0.31.0", default-features = false, features = [
+ "gen-tonic-messages",
+ "logs",
+ "with-serde",
+ "trace",
+] }
+opentelemetry_sdk = { workspace = true, features = ["trace"] }
+serde_json.workspace = true
+tracing.workspace = true
+tracing-subscriber.workspace = true
+uuid = { workspace = true, features = ["js"] }
+web-time.workspace = true
+worker.workspace = true
diff --git a/crates/worker-rs-otel/src/exporter/mod.rs b/crates/worker-rs-otel/src/exporter/mod.rs
new file mode 100644
index 0000000000..524ae1de73
--- /dev/null
+++ b/crates/worker-rs-otel/src/exporter/mod.rs
@@ -0,0 +1,136 @@
+use std::{cell::RefCell, future::Future};
+
+use super::WaitUntil;
+use super::model::{ClosedLog, ClosedSpan};
+use super::proto::{export_logs_request, export_traces_request};
+
+const ENDPOINT_VAR: &str = "OTEL_EXPORTER_OTLP_ENDPOINT";
+const ENVIRONMENT_VAR: &str = "ENVIRONMENT";
+const MAX_BUFFERED_SPANS: usize = 512;
+const MAX_BUFFERED_LOGS: usize = 512;
+
+struct ExporterConfig {
+ endpoint: Option<String>,
+ environment: Option<String>,
+}
+
+thread_local! {
+ static SPAN_BUFFER: RefCell<Vec<ClosedSpan>> = const { RefCell::new(Vec::new()) };
+ static LOG_BUFFER: RefCell<Vec<ClosedLog>> = const { RefCell::new(Vec::new()) };
+ static CONFIG: RefCell<Option<ExporterConfig>> = const { RefCell::new(None) };
+}
+
+pub(super) fn configure(env: &worker::Env) {
+ CONFIG.with(|config| {
+ let mut config = config.borrow_mut();
+ if config.is_some() {
+ return;
+ }
+ let get = |name: &str| {
+ env.var(name)
+ .ok()
+ .map(|value| value.to_string())
+ .filter(|value| !value.is_empty())
+ };
+ *config = Some(ExporterConfig {
+ endpoint: get(ENDPOINT_VAR),
+ environment: get(ENVIRONMENT_VAR),
+ });
+ });
+}
+
+fn signal_endpoint(endpoint: &str, signal: &str) -> String {
+ format!("{}/v1/{signal}", endpoint.trim_end_matches('/'))
+}
+
+pub(super) fn buffer_span(span: ClosedSpan) {
+ SPAN_BUFFER.with(|buffer| {
+ let mut buffer = buffer.borrow_mut();
+ if buffer.len() < MAX_BUFFERED_SPANS {
+ buffer.push(span);
+ }
+ });
+}
+
+pub(super) fn buffer_log(log: ClosedLog) {
+ LOG_BUFFER.with(|buffer| {
+ let mut buffer = buffer.borrow_mut();
+ if buffer.len() < MAX_BUFFERED_LOGS {
+ buffer.push(log);
+ }
+ });
+}
+
+fn flush() -> Option<impl Future<Output = ()> + 'static> {
+ let spans = SPAN_BUFFER.with(|buffer| std::mem::take(&mut *buffer.borrow_mut()));
+ let logs = LOG_BUFFER.with(|buffer| std::mem::take(&mut *buffer.borrow_mut()));
+ if spans.is_empty() && logs.is_empty() {
+ return None;
+ }
+ let (endpoint, environment) = CONFIG.with(|config| {
+ let config = config.borrow();
+ let config = config.as_ref()?;
+ Some((config.endpoint.clone()?, config.environment.clone()))
+ })?;
+
+ let traces = if spans.is_empty() {
+ None
+ } else {
+ Some((
+ signal_endpoint(&endpoint, "traces"),
+ serde_json::to_string(&export_traces_request(spans, environment.as_deref())).ok()?,
+ ))
+ };
+ let logs = if logs.is_empty() {
+ None
+ } else {
+ Some((
+ signal_endpoint(&endpoint, "logs"),
+ serde_json::to_string(&export_logs_request(logs, environment.as_deref())).ok()?,
+ ))
+ };
+
+ Some(async move {
+ if let Some((endpoint, body)) = traces {
+ _ = post_signal(&endpoint, body)
+ .await
+ .inspect_err(|error| tracing::debug!(error = ?error, "otel trace export failed"));
+ }
+ if let Some((endpoint, body)) = logs {
+ _ = post_signal(&endpoint, body)
+ .await
+ .inspect_err(|error| tracing::debug!(error = ?error, "otel log export failed"));
+ }
+ })
+}
+
+pub(super) fn flush_into(wait_until: &impl WaitUntil) {
+ if let Some(export) = flush() {
+ wait_until.wait_until(Box::pin(export));
+ }
+}
+
+async fn post_signal(endpoint: &str, body: String) -> worker::Result<()> {
+ use worker::{Fetch, Method, Request, RequestInit};
+
+ let mut request = Request::new_with_init(
+ endpoint,
+ RequestInit::new()
+ .with_method(Method::Post)
+ .with_body(Some(body.into())),
+ )?;
+ request
+ .headers_mut()?
+ .set("Content-Type", "application/json")?;
+ let response = Fetch::Request(request).send().await?;
+ if response.status_code() >= 300 {
+ return Err(worker::Error::from(format!(
+ "otel exporter returned {}",
+ response.status_code()
+ )));
+ }
+ Ok(())
+}
+
+#[cfg(test)]
+mod test;
diff --git a/crates/worker-rs-otel/src/exporter/test.rs b/crates/worker-rs-otel/src/exporter/test.rs
new file mode 100644
index 0000000000..e375afcc29
--- /dev/null
+++ b/crates/worker-rs-otel/src/exporter/test.rs
@@ -0,0 +1,13 @@
+use super::signal_endpoint;
+
+#[test]
+fn appends_signal_path_to_collector_endpoint() {
+ assert_eq!(
+ signal_endpoint("http://otel-collector:4318/", "traces"),
+ "http://otel-collector:4318/v1/traces"
+ );
+ assert_eq!(
+ signal_endpoint("http://otel-collector:4318/", "logs"),
+ "http://otel-collector:4318/v1/logs"
+ );
+}
diff --git a/crates/worker-rs-otel/src/layer/mod.rs b/crates/worker-rs-otel/src/layer/mod.rs
new file mode 100644
index 0000000000..65ec2aa01f
--- /dev/null
+++ b/crates/worker-rs-otel/src/layer/mod.rs
@@ -0,0 +1,206 @@
+use tracing::field::{Field, Visit};
+use tracing_subscriber::{
+ Layer,
+ layer::Context as LayerContext,
+ registry::{LookupSpan, Registry},
+};
+
+use super::exporter::{buffer_log, buffer_span};
+use super::model::{
+ ClosedLog, ClosedSpan, ClosedSpanEvent, LiveSpan, now_unix_nanos, random_span_id,
+ random_trace_id,
+};
+use super::trace_context::{parse_span_id, parse_trace_id, sampled_traceparent};
+use super::{REMOTE_PARENT_FIELD, REMOTE_TRACE_ID_FIELD};
+
+#[derive(Default)]
+struct FieldVisitor {
+ remote_trace_id: Option<[u8; 16]>,
+ remote_parent: Option<[u8; 8]>,
+ fields: Vec<(String, String)>,
+}
+
+impl FieldVisitor {
+ fn record(&mut self, field: &Field, value: String) {
+ match field.name() {
+ REMOTE_TRACE_ID_FIELD => self.remote_trace_id = parse_trace_id(&value),
+ REMOTE_PARENT_FIELD => self.remote_parent = parse_span_id(&value),
+ name => self.fields.push((name.to_string(), value)),
+ }
+ }
+}
+
+macro_rules! string_visit {
+ ($ty:ty) => {
+ impl Visit for $ty {
+ fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) {
+ self.record(field, format!("{value:?}"));
+ }
+
+ fn record_str(&mut self, field: &Field, value: &str) {
+ self.record(field, value.to_string());
+ }
+ }
+ };
+}
+
+string_visit!(FieldVisitor);
+
+#[derive(Default)]
+struct EventVisitor {
+ message: Option<String>,
+ fields: Vec<(String, String)>,
+}
+
+impl EventVisitor {
+ fn record(&mut self, field: &Field, value: String) {
+ if field.name() == "message" {
+ self.message = Some(value);
+ } else {
+ self.fields.push((field.name().to_string(), value));
+ }
+ }
+}
+
+string_visit!(EventVisitor);
+
+/// A tracing-subscriber layer that exports closed spans through OTLP/HTTP.
+pub struct OtelLayer {
+ service_name: &'static str,
+}
+
+impl OtelLayer {
+ /// Create a layer that identifies exported resources as `service_name`.
+ pub const fn new(service_name: &'static str) -> Self {
+ Self { service_name }
+ }
+}
+
+/// Build a sampled W3C `traceparent` value for a span managed by this layer.
+pub fn traceparent_for_span(span: &tracing::Span) -> Option<String> {
+ span.with_subscriber(|(id, dispatch)| {
+ let registry = dispatch.downcast_ref::<Registry>()?;
+ let span = registry.span(id)?;
+ let extensions = span.extensions();
+ let data = extensions.get::<LiveSpan>()?;
+ sampled_traceparent(data.trace_id, data.span_id)
+ })
+ .flatten()
+}
+
+impl<S> Layer<S> for OtelLayer
+where
+ S: tracing::Subscriber + for<'a> LookupSpan<'a>,
+{
+ fn on_new_span(
+ &self,
+ attrs: &tracing::span::Attributes<'_>,
+ id: &tracing::span::Id,
+ ctx: LayerContext<'_, S>,
+ ) {
+ let Some(span) = ctx.span(id) else { return };
+
+ let mut visitor = FieldVisitor::default();
+ attrs.record(&mut visitor);
+
+ let parent = span.parent().and_then(|p| {
+ let ext = p.extensions();
+ ext.get::<LiveSpan>().map(|d| (d.trace_id, d.span_id))
+ });
+
+ let (trace_id, parent_span_id, local_root) = match parent {
+ Some((trace_id, parent_span_id)) => (trace_id, Some(parent_span_id), false),
+ None => match visitor.remote_trace_id {
+ Some(remote_trace_id) => (remote_trace_id, visitor.remote_parent, true),
+ None => (random_trace_id(), None, true),
+ },
+ };
+
+ span.extensions_mut().insert(LiveSpan {
+ trace_id,
+ span_id: random_span_id(),
+ parent_span_id,
+ local_root,
+ start_ns: now_unix_nanos(),
+ attrs: visitor.fields,
+ events: Vec::new(),
+ error_message: None,
+ });
+ }
+
+ fn on_event(&self, event: &tracing::Event<'_>, ctx: LayerContext<'_, S>) {
+ if *event.metadata().level() > tracing::Level::INFO {
+ return;
+ }
+ let mut visitor = EventVisitor::default();
+ event.record(&mut visitor);
+ let meta = event.metadata();
+ let level = *meta.level();
+ let name = visitor.message.unwrap_or_else(|| meta.name().to_string());
+ let mut attrs = visitor.fields;
+ attrs.push(("level".to_string(), level.as_str().to_string()));
+ let time_ns = now_unix_nanos();
+ let trace_context = ctx.event_span(event).and_then(|span| {
+ let mut extensions = span.extensions_mut();
+ let data = extensions.get_mut::<LiveSpan>()?;
+ if level == tracing::Level::ERROR && data.error_message.is_none() {
+ data.error_message = Some(name.clone());
+ }
+ data.events.push(ClosedSpanEvent {
+ name: name.clone(),
+ time_ns,
+ attrs: attrs.clone(),
+ });
+ Some((data.trace_id, data.span_id))
+ });
+ let (trace_id, span_id) = trace_context
+ .map(|(trace_id, span_id)| (Some(trace_id), Some(span_id)))
+ .unwrap_or_default();
+ buffer_log(ClosedLog {
+ service_name: self.service_name,
+ time_ns,
+ level,
+ body: name,
+ attrs,
+ target: meta.target(),
+ file: meta.file(),
+ line: meta.line(),
+ trace_id,
+ span_id,
+ });
+ }
+
+ fn on_record(
+ &self,
+ id: &tracing::span::Id,
+ values: &tracing::span::Record<'_>,
+ ctx: LayerContext<'_, S>,
+ ) {
+ let Some(span) = ctx.span(id) else { return };
+ let mut visitor = FieldVisitor::default();
+ values.record(&mut visitor);
+ if let Some(data) = span.extensions_mut().get_mut::<LiveSpan>() {
+ data.attrs.append(&mut visitor.fields);
+ }
+ }
+
+ fn on_close(&self, id: tracing::span::Id, ctx: LayerContext<'_, S>) {
+ let Some(span) = ctx.span(&id) else { return };
+ let Some(data) = span.extensions_mut().remove::<LiveSpan>() else {
+ return;
+ };
+ let meta = span.metadata();
+ buffer_span(ClosedSpan {
+ data,
+ service_name: self.service_name,
+ name: meta.name(),
+ level: meta.level().as_str(),
+ file: meta.file(),
+ line: meta.line(),
+ end_ns: now_unix_nanos(),
+ });
+ }
+}
+
+#[cfg(test)]
+mod test;
diff --git a/crates/worker-rs-otel/src/layer/test.rs b/crates/worker-rs-otel/src/layer/test.rs
new file mode 100644
index 0000000000..ac18768024
--- /dev/null
+++ b/crates/worker-rs-otel/src/layer/test.rs
@@ -0,0 +1,17 @@
+use tracing_subscriber::layer::SubscriberExt;
+
+use super::{OtelLayer, traceparent_for_span};
+use crate::parse_traceparent;
+
+#[test]
+fn creates_traceparent_for_managed_span() {
+ let subscriber = tracing_subscriber::registry().with(OtelLayer::new("test-service"));
+ let dispatch = tracing::Dispatch::new(subscriber);
+
+ tracing::dispatcher::with_default(&dispatch, || {
+ let span = tracing::info_span!("outbound.request");
+ let traceparent = traceparent_for_span(&span).expect("managed span context");
+ assert!(parse_traceparent(&traceparent).is_some());
+ assert!(traceparent.ends_with("-01"));
+ });
+}
diff --git a/crates/worker-rs-otel/src/lib.rs b/crates/worker-rs-otel/src/lib.rs
new file mode 100644
index 0000000000..3aae8fe6c6
--- /dev/null
+++ b/crates/worker-rs-otel/src/lib.rs
@@ -0,0 +1,52 @@
+//! OpenTelemetry tracing support for Cloudflare Workers built with worker-rs.
+
+mod exporter;
+mod layer;
+mod model;
+mod proto;
+mod trace_context;
+
+use std::{future::Future, pin::Pin};
+
+pub use layer::{OtelLayer, traceparent_for_span};
+pub use trace_context::{
+ HeadersLike, TRACEPARENT, parse_traceparent, remote_fields, traceparent_from_headers,
+ traceparent_from_request,
+};
+
+/// Reserved span field containing a 32-character hexadecimal remote trace ID.
+pub const REMOTE_TRACE_ID_FIELD: &str = "trace.remote_id";
+/// Reserved span field containing a 16-character hexadecimal remote parent span ID.
+pub const REMOTE_PARENT_FIELD: &str = "trace.remote_parent";
+
+type ExportFuture = Pin<Box<dyn Future<Output = ()> + 'static>>;
+
+/// Registers background work against the current Cloudflare invocation.
+pub trait WaitUntil {
+ /// Keep the invocation alive until `future` completes.
+ fn wait_until(&self, future: ExportFuture);
+}
+
+impl WaitUntil for worker::Context {
+ fn wait_until(&self, future: ExportFuture) {
+ worker::Context::wait_until(self, future);
+ }
+}
+
+impl WaitUntil for worker::State {
+ fn wait_until(&self, future: ExportFuture) {
+ worker::State::wait_until(self, future);
+ }
+}
+
+/// Run one Worker invocation and schedule its buffered telemetry for export.
+pub async fn scope<T>(
+ env: &worker::Env,
+ wait_until: &impl WaitUntil,
+ future: impl Future<Output = T>,
+) -> T {
+ exporter::configure(env);
+ let output = future.await;
+ exporter::flush_into(wait_until);
+ output
+}
diff --git a/crates/worker-rs-otel/src/model.rs b/crates/worker-rs-otel/src/model.rs
new file mode 100644
index 0000000000..e88dae5d79
--- /dev/null
+++ b/crates/worker-rs-otel/src/model.rs
@@ -0,0 +1,57 @@
+use web_time::{SystemTime, UNIX_EPOCH};
+
+pub(super) struct ClosedSpanEvent {
+ pub(super) name: String,
+ pub(super) time_ns: u64,
+ pub(super) attrs: Vec<(String, String)>,
+}
+
+pub(super) struct ClosedSpan {
+ pub(super) data: LiveSpan,
+ pub(super) service_name: &'static str,
+ pub(super) name: &'static str,
+ pub(super) level: &'static str,
+ pub(super) file: Option<&'static str>,
+ pub(super) line: Option<u32>,
+ pub(super) end_ns: u64,
+}
+
+pub(super) struct ClosedLog {
+ pub(super) service_name: &'static str,
+ pub(super) time_ns: u64,
+ pub(super) level: tracing::Level,
+ pub(super) body: String,
+ pub(super) attrs: Vec<(String, String)>,
+ pub(super) target: &'static str,
+ pub(super) file: Option<&'static str>,
+ pub(super) line: Option<u32>,
+ pub(super) trace_id: Option<[u8; 16]>,
+ pub(super) span_id: Option<[u8; 8]>,
+}
+
+pub(super) struct LiveSpan {
+ pub(super) trace_id: [u8; 16],
+ pub(super) span_id: [u8; 8],
+ pub(super) parent_span_id: Option<[u8; 8]>,
+ pub(super) local_root: bool,
+ pub(super) start_ns: u64,
+ pub(super) attrs: Vec<(String, String)>,
+ pub(super) events: Vec<ClosedSpanEvent>,
+ pub(super) error_message: Option<String>,
+}
+
+pub(super) fn now_unix_nanos() -> u64 {
+ SystemTime::now()
+ .duration_since(UNIX_EPOCH)
+ .map(|d| d.as_nanos() as u64)
+ .unwrap_or_default()
+}
+
+pub(super) fn random_trace_id() -> [u8; 16] {
+ uuid::Uuid::new_v4().into_bytes()
+}
+
+pub(super) fn random_span_id() -> [u8; 8] {
+ let bytes = uuid::Uuid::new_v4().into_bytes();
+ bytes[8..16].try_into().expect("8-byte slice")
+}
diff --git a/crates/worker-rs-otel/src/proto/mod.rs b/crates/worker-rs-otel/src/proto/mod.rs
new file mode 100644
index 0000000000..4549b7b1f2
--- /dev/null
+++ b/crates/worker-rs-otel/src/proto/mod.rs
@@ -0,0 +1,188 @@
+use opentelemetry_proto::tonic::{
+ collector::{logs::v1::ExportLogsServiceRequest, trace::v1::ExportTraceServiceRequest},
+ common::v1::{AnyValue, InstrumentationScope, KeyValue, any_value},
+ logs::v1::{LogRecord, ResourceLogs, ScopeLogs, SeverityNumber},
+ resource::v1::Resource,
+ trace::v1::{
+ ResourceSpans, ScopeSpans, Span, Status, span::Event, span::SpanKind, status::StatusCode,
+ },
+};
+
+use super::model::{ClosedLog, ClosedSpan};
+
+fn string_value(value: &str) -> AnyValue {
+ AnyValue {
+ value: Some(any_value::Value::StringValue(value.to_string())),
+ }
+}
+
+fn string_kv(key: &str, value: &str) -> KeyValue {
+ KeyValue {
+ key: key.to_string(),
+ value: Some(string_value(value)),
+ }
+}
+
+fn span_proto(span: ClosedSpan) -> Span {
+ let data = span.data;
+ let mut attributes: Vec<KeyValue> = data
+ .attrs
+ .into_iter()
+ .map(|(key, value)| string_kv(&key, &value))
+ .collect();
+ attributes.push(string_kv("level", span.level));
+ if let Some(file) = span.file {
+ attributes.push(string_kv("code.filepath", file));
+ }
+ if let Some(line) = span.line {
+ attributes.push(KeyValue {
+ key: "code.lineno".to_string(),
+ value: Some(AnyValue {
+ value: Some(any_value::Value::IntValue(i64::from(line))),
+ }),
+ });
+ }
+ let events = data
+ .events
+ .into_iter()
+ .map(|event| Event {
+ time_unix_nano: event.time_ns,
+ name: event.name,
+ attributes: event
+ .attrs
+ .into_iter()
+ .map(|(key, value)| string_kv(&key, &value))
+ .collect(),
+ ..Default::default()
+ })
+ .collect();
+ Span {
+ trace_id: data.trace_id.to_vec(),
+ span_id: data.span_id.to_vec(),
+ parent_span_id: data.parent_span_id.map(|p| p.to_vec()).unwrap_or_default(),
+ name: span.name.to_string(),
+ kind: if data.local_root {
+ SpanKind::Server
+ } else {
+ SpanKind::Internal
+ } as i32,
+ start_time_unix_nano: data.start_ns,
+ end_time_unix_nano: span.end_ns,
+ flags: 1,
+ attributes,
+ events,
+ status: data.error_message.map(|message| Status {
+ code: StatusCode::Error as i32,
+ message,
+ }),
+ ..Default::default()
+ }
+}
+
+pub(super) fn export_traces_request(
+ spans: Vec<ClosedSpan>,
+ environment: Option<&str>,
+) -> ExportTraceServiceRequest {
+ let service_name = spans
+ .first()
+ .map(|span| span.service_name)
+ .unwrap_or("unknown-service");
+ let mut resource_attrs = vec![string_kv("service.name", service_name)];
+ if let Some(environment) = environment {
+ resource_attrs.push(string_kv("deployment.environment", environment));
+ }
+ ExportTraceServiceRequest {
+ resource_spans: vec![ResourceSpans {
+ resource: Some(Resource {
+ attributes: resource_attrs,
+ ..Default::default()
+ }),
+ scope_spans: vec![ScopeSpans {
+ scope: Some(InstrumentationScope {
+ name: service_name.to_string(),
+ ..Default::default()
+ }),
+ spans: spans.into_iter().map(span_proto).collect(),
+ ..Default::default()
+ }],
+ ..Default::default()
+ }],
+ }
+}
+
+fn severity_number(level: tracing::Level) -> SeverityNumber {
+ match level {
+ tracing::Level::ERROR => SeverityNumber::Error,
+ tracing::Level::WARN => SeverityNumber::Warn,
+ tracing::Level::INFO => SeverityNumber::Info,
+ tracing::Level::DEBUG => SeverityNumber::Debug,
+ tracing::Level::TRACE => SeverityNumber::Trace,
+ }
+}
+
+fn log_proto(log: ClosedLog) -> LogRecord {
+ let correlated = log.trace_id.is_some();
+ let mut attributes: Vec<KeyValue> = log
+ .attrs
+ .into_iter()
+ .map(|(key, value)| string_kv(&key, &value))
+ .collect();
+ attributes.push(string_kv("target", log.target));
+ if let Some(file) = log.file {
+ attributes.push(string_kv("code.filepath", file));
+ }
+ if let Some(line) = log.line {
+ attributes.push(KeyValue {
+ key: "code.lineno".to_string(),
+ value: Some(AnyValue {
+ value: Some(any_value::Value::IntValue(i64::from(line))),
+ }),
+ });
+ }
+ LogRecord {
+ time_unix_nano: log.time_ns,
+ observed_time_unix_nano: log.time_ns,
+ severity_number: severity_number(log.level) as i32,
+ severity_text: log.level.as_str().to_string(),
+ body: Some(string_value(&log.body)),
+ attributes,
+ flags: u32::from(correlated),
+ trace_id: log.trace_id.map(|id| id.to_vec()).unwrap_or_default(),
+ span_id: log.span_id.map(|id| id.to_vec()).unwrap_or_default(),
+ ..Default::default()
+ }
+}
+
+pub(super) fn export_logs_request(
+ logs: Vec<ClosedLog>,
+ environment: Option<&str>,
+) -> ExportLogsServiceRequest {
+ let service_name = logs
+ .first()
+ .map(|log| log.service_name)
+ .unwrap_or("unknown-service");
+ let mut resource_attrs = vec![string_kv("service.name", service_name)];
+ if let Some(environment) = environment {
+ resource_attrs.push(string_kv("deployment.environment", environment));
+ }
+ ExportLogsServiceRequest {
+ resource_logs: vec![ResourceLogs {
+ resource: Some(Resource {
+ attributes: resource_attrs,
+ ..Default::default()
+ }),
+ scope_logs: vec![ScopeLogs {
+ scope: Some(InstrumentationScope {
+ name: service_name.to_string(),
+ ..Default::default()
+ }),
+ log_records: logs.into_iter().map(log_proto).collect(),
+ ..Default::default()
+ }],
+ ..Default::default()
+ }],
+ }
+}
+
+#[cfg(test)]
+mod test;
diff --git a/crates/worker-rs-otel/src/proto/test.rs b/crates/worker-rs-otel/src/proto/test.rs
new file mode 100644
index 0000000000..1f6815aaa8
--- /dev/null
+++ b/crates/worker-rs-otel/src/proto/test.rs
@@ -0,0 +1,32 @@
+use opentelemetry_proto::tonic::{common::v1::any_value, logs::v1::SeverityNumber};
+
+use super::export_logs_request;
+use crate::model::ClosedLog;
+
+#[test]
+fn builds_correlated_log_record() {
+ let request = export_logs_request(
+ vec![ClosedLog {
+ service_name: "sync-service",
+ time_ns: 42,
+ level: tracing::Level::WARN,
+ body: "peer send failed".to_string(),
+ attrs: vec![("document.id".to_string(), "doc-1".to_string())],
+ target: "sync_service::websocket",
+ file: Some("src/websocket.rs"),
+ line: Some(10),
+ trace_id: Some([1; 16]),
+ span_id: Some([2; 8]),
+ }],
+ Some("test"),
+ );
+
+ let record = &request.resource_logs[0].scope_logs[0].log_records[0];
+ assert_eq!(record.severity_number, SeverityNumber::Warn as i32);
+ assert_eq!(record.trace_id, vec![1; 16]);
+ assert_eq!(record.span_id, vec![2; 8]);
+ assert!(matches!(
+ record.body.as_ref().and_then(|body| body.value.as_ref()),
+ Some(any_value::Value::StringValue(value)) if value == "peer send failed"
+ ));
+}
diff --git a/crates/worker-rs-otel/src/trace_context/mod.rs b/crates/worker-rs-otel/src/trace_context/mod.rs
new file mode 100644
index 0000000000..6be0e06ee8
--- /dev/null
+++ b/crates/worker-rs-otel/src/trace_context/mod.rs
@@ -0,0 +1,98 @@
+use std::{collections::HashMap, hash::BuildHasher};
+
+use opentelemetry::Context;
+use opentelemetry::propagation::TextMapPropagator;
+use opentelemetry::trace::{SpanContext, SpanId, TraceContextExt, TraceFlags, TraceId, TraceState};
+use opentelemetry_sdk::propagation::TraceContextPropagator;
+
+/// W3C trace-context header and WebSocket query parameter.
+pub const TRACEPARENT: &str = "traceparent";
+
+/// Header collection that can provide values for trace-context propagation.
+pub trait HeadersLike {
+ /// Return a header value as text.
+ fn header(&self, name: &str) -> Option<String>;
+}
+
+impl<S: BuildHasher> HeadersLike for HashMap<String, String, S> {
+ fn header(&self, name: &str) -> Option<String> {
+ self.get(name).cloned()
+ }
+}
+
+impl HeadersLike for http::HeaderMap {
+ fn header(&self, name: &str) -> Option<String> {
+ self.get(name)?.to_str().ok().map(str::to_owned)
+ }
+}
+
+impl HeadersLike for worker::Headers {
+ fn header(&self, name: &str) -> Option<String> {
+ self.get(name).ok().flatten()
+ }
+}
+
+/// Parse a valid W3C `traceparent` value into OpenTelemetry's span context.
+pub fn parse_traceparent(value: &str) -> Option<SpanContext> {
+ let carrier = HashMap::from([(TRACEPARENT.to_string(), value.to_string())]);
+ let context = TraceContextPropagator::new().extract(&carrier);
+ let span_context = context.span().span_context().clone();
+ span_context.is_valid().then_some(span_context)
+}
+
+pub(super) fn sampled_traceparent(trace_id: [u8; 16], span_id: [u8; 8]) -> Option<String> {
+ let span_context = SpanContext::new(
+ TraceId::from_bytes(trace_id),
+ SpanId::from_bytes(span_id),
+ TraceFlags::SAMPLED,
+ false,
+ TraceState::default(),
+ );
+ let context = Context::new().with_remote_span_context(span_context);
+ let mut carrier = HashMap::new();
+ TraceContextPropagator::new().inject_context(&context, &mut carrier);
+ carrier.remove(TRACEPARENT)
+}
+
+/// Extract remote trace context from a supported header collection.
+pub fn traceparent_from_headers(headers: &impl HeadersLike) -> Option<SpanContext> {
+ headers
+ .header(TRACEPARENT)
+ .and_then(|value| parse_traceparent(&value))
+}
+
+/// Extract parsed remote trace context from a Worker request.
+pub fn traceparent_from_request(request: &worker::Request) -> Option<SpanContext> {
+ traceparent_from_headers(request.headers()).or_else(|| {
+ request
+ .url()
+ .ok()?
+ .query_pairs()
+ .find(|(key, _)| key == TRACEPARENT)
+ .and_then(|(_, value)| parse_traceparent(&value))
+ })
+}
+
+/// Convert optional remote context into reserved tracing span field values.
+pub fn remote_fields(traceparent: Option<&SpanContext>) -> (String, String) {
+ traceparent
+ .map(|value| (value.trace_id().to_string(), value.span_id().to_string()))
+ .unwrap_or_default()
+}
+
+pub(super) fn parse_trace_id(value: &str) -> Option<[u8; 16]> {
+ TraceId::from_hex(value)
+ .ok()
+ .filter(|id| *id != TraceId::INVALID)
+ .map(|id| id.to_bytes())
+}
+
+pub(super) fn parse_span_id(value: &str) -> Option<[u8; 8]> {
+ SpanId::from_hex(value)
+ .ok()
+ .filter(|id| *id != SpanId::INVALID)
+ .map(|id| id.to_bytes())
+}
+
+#[cfg(test)]
+mod test;
diff --git a/crates/worker-rs-otel/src/trace_context/test.rs b/crates/worker-rs-otel/src/trace_context/test.rs
new file mode 100644
index 0000000000..9fdc6d5b77
--- /dev/null
+++ b/crates/worker-rs-otel/src/trace_context/test.rs
@@ -0,0 +1,46 @@
+use opentelemetry::trace::TraceFlags;
+
+use std::collections::HashMap;
+
+use super::{TRACEPARENT, parse_traceparent, traceparent_from_headers};
+
+#[test]
+fn parses_valid_traceparent() {
+ let traceparent = parse_traceparent("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01")
+ .expect("valid traceparent");
+
+ assert_eq!(
+ traceparent.trace_id().to_string(),
+ "4bf92f3577b34da6a3ce929d0e0e4736"
+ );
+ assert_eq!(traceparent.span_id().to_string(), "00f067aa0ba902b7");
+ assert_eq!(traceparent.trace_flags(), TraceFlags::SAMPLED);
+}
+
+#[test]
+fn rejects_invalid_traceparent() {
+ assert!(parse_traceparent("invalid").is_none());
+}
+
+#[test]
+fn extracts_traceparent_from_hash_map() {
+ let headers = HashMap::from([(
+ TRACEPARENT.to_string(),
+ "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01".to_string(),
+ )]);
+
+ assert!(traceparent_from_headers(&headers).is_some());
+}
+
+#[test]
+fn extracts_traceparent_from_http_headers() {
+ let mut headers = http::HeaderMap::new();
+ headers.insert(
+ TRACEPARENT,
+ "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"
+ .parse()
+ .expect("valid header value"),
+ );
+
+ assert!(traceparent_from_headers(&headers).is_some());
+}
diff --git a/crates/workspace-hack/Cargo.toml b/crates/workspace-hack/Cargo.toml
index 370cd52490..c7a434b5d2 100644
--- a/crates/workspace-hack/Cargo.toml
+++ b/crates/workspace-hack/Cargo.toml
@@ -80,8 +80,9 @@ num-bigint = { version = "0.4" }
num-integer = { version = "0.1", features = ["i128"] }
num-traits = { version = "0.2", features = ["i128", "libm"] }
once_cell = { version = "1" }
-opentelemetry = { version = "0.31", features = ["testing"] }
-opentelemetry_sdk = { version = "0.31", features = ["testing"] }
+opentelemetry = { version = "0.31", default-features = false, features = ["internal-logs", "logs", "metrics", "testing"] }
+opentelemetry-http = { version = "0.31", default-features = false, features = ["internal-logs"] }
+opentelemetry_sdk = { version = "0.31", default-features = false, features = ["internal-logs", "testing"] }
percent-encoding = { version = "2" }
phf = { version = "0.13", features = ["macros"] }
phf_shared-594e8ee84c453af0 = { package = "phf_shared", version = "0.13" }
@@ -118,6 +119,7 @@ sqlx-postgres = { version = "0.8", default-features = false, features = ["any",
stable_deref_trait = { version = "1" }
subtle = { version = "2" }
sync_wrapper = { version = "1", default-features = false, features = ["futures"] }
+thiserror = { version = "2" }
time = { version = "0.3", features = ["formatting", "macros", "parsing"] }
tokio = { version = "1", features = ["fs", "io-util", "macros", "net", "parking_lot", "process", "rt-multi-thread", "signal", "test-util"] }
tokio-rustls = { version = "0.26", default-features = false, features = ["logging", "tls12"] }
@@ -127,6 +129,8 @@ tower = { version = "0.5", default-features = false, features = ["balance", "buf
tower-http = { version = "0.6", features = ["compression-gzip", "cors", "fs", "limit", "request-id", "timeout", "trace", "util"] }
tracing = { version = "0.1", features = ["log"] }
tracing-core = { version = "0.1" }
+tracing-log = { version = "0.2", default-features = false, features = ["log-tracer", "std"] }
+tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }
triomphe = { version = "0.1" }
tungstenite = { version = "0.28", features = ["__rustls-tls", "url"] }
typenum = { version = "1", default-features = false, features = ["const-generics"] }
@@ -202,8 +206,9 @@ num-bigint = { version = "0.4" }
num-integer = { version = "0.1", features = ["i128"] }
num-traits = { version = "0.2", features = ["i128", "libm"] }
once_cell = { version = "1" }
-opentelemetry = { version = "0.31", features = ["testing"] }
-opentelemetry_sdk = { version = "0.31", features = ["testing"] }
+opentelemetry = { version = "0.31", default-features = false, features = ["internal-logs", "logs", "metrics", "testing"] }
+opentelemetry-http = { version = "0.31", default-features = false, features = ["internal-logs"] }
+opentelemetry_sdk = { version = "0.31", default-features = false, features = ["internal-logs", "testing"] }
percent-encoding = { version = "2" }
phf = { version = "0.13", features = ["macros"] }
phf_shared-594e8ee84c453af0 = { package = "phf_shared", version = "0.13" }
@@ -245,6 +250,7 @@ subtle = { version = "2" }
syn-dff4ba8e3ae991db = { package = "syn", version = "1", features = ["extra-traits", "fold", "full"] }
syn-f595c2ba2a3f28df = { package = "syn", version = "2", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] }
sync_wrapper = { version = "1", default-features = false, features = ["futures"] }
+thiserror = { version = "2" }
time = { version = "0.3", features = ["formatting", "macros", "parsing"] }
tokio = { version = "1", features = ["fs", "io-util", "macros", "net", "parking_lot", "process", "rt-multi-thread", "signal", "test-util"] }
tokio-rustls = { version = "0.26", default-features = false, features = ["logging", "tls12"] }
@@ -254,6 +260,8 @@ tower = { version = "0.5", default-features = false, features = ["balance", "buf
tower-http = { version = "0.6", features = ["compression-gzip", "cors", "fs", "limit", "request-id", "timeout", "trace", "util"] }
tracing = { version = "0.1", features = ["log"] }
tracing-core = { version = "0.1" }
+tracing-log = { version = "0.2", default-features = false, features = ["log-tracer", "std"] }
+tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }
triomphe = { version = "0.1" }
tungstenite = { version = "0.28", features = ["__rustls-tls", "url"] }
typenum = { version = "1", default-features = false, features = ["const-generics"] }
@@ -274,7 +282,6 @@ linux-raw-sys = { version = "0.12", default-features = false, features = ["auxve
once_cell = { version = "1", default-features = false, features = ["critical-section"] }
rand-93f6ce9d446188ac = { package = "rand", version = "0.10" }
rustix = { version = "1", features = ["event", "fs", "net", "pipe", "process", "system", "time"] }
-thiserror = { version = "2" }
tokio-rustls = { version = "0.26", default-features = false, features = ["aws-lc-rs", "ring"] }
tokio-util = { version = "0.7", default-features = false, features = ["io-util"] }
tower = { version = "0.5", default-features = false, features = ["retry", "timeout"] }
@@ -292,7 +299,6 @@ linux-raw-sys = { version = "0.12", default-features = false, features = ["auxve
once_cell = { version = "1", default-features = false, features = ["critical-section"] }
rand-93f6ce9d446188ac = { package = "rand", version = "0.10" }
rustix = { version = "1", features = ["event", "fs", "net", "pipe", "process", "system", "time"] }
-thiserror = { version = "2" }
tokio-rustls = { version = "0.26", default-features = false, features = ["aws-lc-rs", "ring"] }
tokio-util = { version = "0.7", default-features = false, features = ["io-util"] }
tower = { version = "0.5", default-features = false, features = ["retry", "timeout"] }
@@ -311,7 +317,6 @@ once_cell = { version = "1", default-features = false, features = ["critical-sec
rand-93f6ce9d446188ac = { package = "rand", version = "0.10" }
rustix = { version = "1", features = ["event", "fs", "net", "pipe", "process", "system", "time"] }
security-framework-sys = { version = "2" }
-thiserror = { version = "2" }
tokio-rustls = { version = "0.26", default-features = false, features = ["aws-lc-rs", "ring"] }
tokio-util = { version = "0.7", default-features = false, features = ["io-util"] }
tower = { version = "0.5", default-features = false, features = ["retry", "timeout"] }
@@ -330,7 +335,6 @@ once_cell = { version = "1", default-features = false, features = ["critical-sec
rand-93f6ce9d446188ac = { package = "rand", version = "0.10" }
rustix = { version = "1", features = ["event", "fs", "net", "pipe", "process", "system", "time"] }
security-framework-sys = { version = "2" }
-thiserror = { version = "2" }
tokio-rustls = { version = "0.26", default-features = false, features = ["aws-lc-rs", "ring"] }
tokio-util = { version = "0.7", default-features = false, features = ["io-util"] }
tower = { version = "0.5", default-features = false, features = ["retry", "timeout"] }
diff --git a/deny.toml b/deny.toml
index b9d53a22e5..3a999ecc37 100644
--- a/deny.toml
+++ b/deny.toml
@@ -201,6 +201,7 @@ deny = [
"lambda_runtime_api_client", # External crate allowed to use tracing-subscriber
"loom", # External concurrency testing dependency
"macro_entrypoint",
+ "macro_tower_layers", # OpenTelemetry layer tests
"native_app_server", # Dev tooling only
"notification_sandbox", # This is not production code its just tooling
"projects", # Dev logging only
@@ -210,6 +211,8 @@ deny = [
"tracing-panic", # External crate allowed to use tracing-subscriber
"tracing-tree",
"tracing-web", # External WASM tracing layer
+ "worker-rs-otel", # Cloudflare/WASM OpenTelemetry integration
+ "workspace-hack", # cargo-hakari generated crate
], reason = "Use macro_entrypoint for tracing setup in workspace crates" },
]
@@ -334,7 +337,7 @@ allow-git = ["git+https://github.com/whutchinson98/jsonwebtoken"]
[sources.allow-org]
# github.com organizations to allow git sources for
-github = []
+github = ["macro-inc"]
# gitlab.com organizations to allow git sources for
gitlab = []
# bitbucket.org organizations to allow git sources for
diff --git a/docker/analytics-proxy.Dockerfile b/docker/analytics-proxy.Dockerfile
new file mode 100644
index 0000000000..7457e1b096
--- /dev/null
+++ b/docker/analytics-proxy.Dockerfile
@@ -0,0 +1,22 @@
+FROM node:22-slim
+RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates \
+ && rm -rf /var/lib/apt/lists/*
+RUN npm install -g bun
+
+ENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt
+
+WORKDIR /app/services/analytics-proxy
+EXPOSE 8098
+
+# The repo is bind-mounted at /app (see docker-compose.yml), so wrangler and
+# node_modules come from the host install. Forward OTLP (both signals) to the
+# local collector (--var overrides the wrangler.jsonc host defaults); no
+# DD_API_KEY locally, so the worker skips key injection.
+CMD ["sh", "-c", "\
+ npx wrangler dev \
+ --env local \
+ --ip 0.0.0.0 \
+ --port 8098 \
+ --var OTLP_TRACES_INTAKE_URL:http://otel-collector:4318 \
+ --var OTLP_LOGS_INTAKE_URL:http://otel-collector:4318\
+"]
diff --git a/docker/datadog-agent/datadog.yaml b/docker/datadog-agent/datadog.yaml
new file mode 100644
index 0000000000..998b914aac
--- /dev/null
+++ b/docker/datadog-agent/datadog.yaml
@@ -0,0 +1,20 @@
+# Config for the local dev Datadog agent (docker-compose.tracing.yml).
+# api_key and site come from the container environment, which overrides
+# this file. This file exists because the OTLP receiver's CORS settings
+# (required for browser -> agent requests) can't be set via env vars.
+# Required for the agent to forward OTLP log records to Datadog Logs.
+logs_enabled: true
+otlp_config:
+ receiver:
+ protocols:
+ grpc:
+ endpoint: 0.0.0.0:4317
+ http:
+ endpoint: 0.0.0.0:4318
+ cors:
+ allowed_origins:
+ - "*"
+ logs:
+ enabled: true
+apm_config:
+ enabled: true
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
index 3d88dca340..24f7eb05d3 100644
--- a/docker/docker-compose.yml
+++ b/docker/docker-compose.yml
@@ -406,6 +406,11 @@ services:
- INTERNAL_API_SECRET_KEY=${INTERNAL_API_SECRET_KEY}
- DOCUMENT_PERMISSIONS_SECRET=${DOCUMENT_PERMISSION_JWT}
- DSS_INTERNAL_AUTH_KEY=${DOCUMENT_STORAGE_SERVICE_AUTH_KEY}
+ # This worker exports OTLP/JSON over HTTP, so it needs the
+ # HTTP intake (:4318) — NOT the gRPC :4317 endpoint the
+ # shared env (env_layer) injects for the .with_tonic() Rust services.
+ # Pinning it here overrides the shared env_file value for this service.
+ - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
networks:
services:
aliases:
@@ -441,6 +446,28 @@ services:
aliases:
- ai-editing-worker
+ # ============================================================================
+ # analytics_proxy (Port 8098)
+ # First-party analytics/telemetry proxy (Cloudflare Worker, local wrangler
+ # dev). Fronts PostHog and OTLP telemetry so ad blockers don't drop them and
+ # the browser never holds the Datadog API key. Locally the
+ # /i/otlp route forwards to the otel-collector alias (jaeger/datadog profile);
+ # if neither profile is up, trace exports just fail harmlessly.
+ # ============================================================================
+ analytics_proxy:
+ <<: *common-env
+ build:
+ context: .
+ dockerfile: docker/analytics-proxy.Dockerfile
+ ports:
+ - "8098:8098"
+ volumes:
+ - .:/app
+ networks:
+ services:
+ aliases:
+ - analytics-proxy
+
# ============================================================================
# lexical_service (Port 8096)
# Lexical document conversion service - converts Lexical JSON to various formats
@@ -482,6 +509,65 @@ services:
retries: 3
start_period: 10s
+ # ============================================================================
+ # jaeger (UI port 16686, OTLP 4317/4318) — profile: jaeger
+ # Fully local trace viewer, no account needed. UI at http://localhost:16686.
+ # Start: docker compose --project-directory . -f docker/docker-compose.yml up -d jaeger
+ # ============================================================================
+ jaeger:
+ profiles:
+ - jaeger
+ image: jaegertracing/all-in-one:latest
+ environment:
+ COLLECTOR_OTLP_ENABLED: "true"
+ # Browser -> collector requests need CORS on the OTLP HTTP receiver.
+ # allowed-headers must include Content-Type: jaeger's default rejects the
+ # preflight the browser OTLP exporter sends, even with origins=*.
+ command:
+ [
+ "--collector.otlp.http.cors.allowed-origins=*",
+ "--collector.otlp.http.cors.allowed-headers=*",
+ ]
+ ports:
+ - "16686:16686"
+ - "4317:4317"
+ - "4318:4318"
+ networks:
+ services:
+ aliases:
+ - otel-collector
+
+ # ============================================================================
+ # datadog-agent (OTLP 4317/4318) — profile: datadog
+ # Forwards OTLP traces to Datadog (us5); no local UI. View at
+ # us5.datadoghq.com -> APM -> Traces, filtered by env:local (DD_ENV).
+ # Binds the same OTLP ports as jaeger — run one or the other, not both.
+ # Start: DD_API_KEY=<key> docker compose --project-directory . -f docker/docker-compose.yml up -d datadog-agent
+ # ============================================================================
+ datadog-agent:
+ profiles:
+ - datadog
+ image: gcr.io/datadoghq/agent:7
+ environment:
+ # Default-empty (not :?) so the var is only required when this profile
+ # actually runs; the agent itself fails fast without a key.
+ DD_API_KEY: ${DD_API_KEY:-}
+ DD_SITE: us5.datadoghq.com
+ DD_ENV: ${DD_ENV:-local}
+ DD_HOSTNAME: local-dev
+ DD_APM_ENABLED: "true"
+ volumes:
+ # Mounted config exists to enable CORS on the OTLP HTTP receiver
+ # (browser -> agent); that setting has no env-var form.
+ - ./docker/datadog-agent/datadog.yaml:/etc/datadog-agent/datadog.yaml:ro
+ ports:
+ - "4317:4317"
+ - "4318:4318"
+ networks:
+ services:
+ aliases:
+ - otel-collector
+
networks:
services:
driver: bridge
diff --git a/opencode.json b/opencode.json
new file mode 100644
index 0000000000..4dc34d5840
--- /dev/null
+++ b/opencode.json
@@ -0,0 +1,13 @@
+{
+ "$schema": "https://opencode.ai/config.json",
+ "plugin": [
+ "@datadog/opencode-plugin"
+ ],
+ "mcp": {
+ "datadog": {
+ "type": "remote",
+ "url": "https://mcp.us5.datadoghq.com/api/unstable/mcp-server/mcp?referrer_ide=opencode-plugin&plugin_version=0.7.13",
+ "oauth": {}
+ }
+ }
+}
diff --git a/package.json b/package.json
index a5e2444a7c..4dc01acaba 100644
--- a/package.json
+++ b/package.json
@@ -63,6 +63,7 @@
"workspaces": {
"packages": [
"apps/web",
+ "packages/observability",
"packages/collaboration",
"packages/loro-mirror",
"packages/lexical-core",
diff --git a/packages/collaboration/package.json b/packages/collaboration/package.json
index a7c6a9f714..0e03a07018 100644
--- a/packages/collaboration/package.json
+++ b/packages/collaboration/package.json
@@ -18,6 +18,7 @@
},
"dependencies": {
"@loro-mirror/core": "workspace:*",
+ "@macro-inc/observability": "workspace:*",
"@solid-primitives/event-bus": "^1.1.2",
"@tauri-apps/api": "^2.8.0",
"async-mutex": "^0.5.0",
diff --git a/packages/collaboration/src/collab/engine.ts b/packages/collaboration/src/collab/engine.ts
index bba110415f..85c02921aa 100644
--- a/packages/collaboration/src/collab/engine.ts
+++ b/packages/collaboration/src/collab/engine.ts
@@ -1,4 +1,5 @@
import { type InferType, SyncDirection } from '@loro-mirror/core';
+import type { Attributes } from '@macro-inc/observability';
import { Mutex } from 'async-mutex';
import type { VersionVector } from 'loro-crdt';
import type { ResultAsync } from 'neverthrow';
@@ -21,6 +22,7 @@ import {
} from './manager';
import type { GenericRootSchema, LoroRawUpdate, RawUpdate } from './shared';
import type { SnapshotStore } from './snapshot-store';
+import { peerCounterAttr, telemetrySpan } from './telemetry';
// SnapshotStore in the engine is always Loro updates — RawUpdate.
type LoroSnapshotStore = SnapshotStore<RawUpdate>;
@@ -33,6 +35,15 @@ const SNAPSHOT_INTERVAL_MS = 5_000;
const REQUEST_UPDATES_MAX_ATTEMPTS = 3;
const REQUEST_UPDATES_RETRY_DELAY_MS = 2_000;
+/** A version vector as a compact `peer:counter` span attribute. */
+function vvAttr(vv: VersionVector): string {
+ try {
+ return peerCounterAttr(vv.toJSON().entries());
+ } catch {
+ return 'unavailable';
+ }
+}
+
export type EngineBindings<S extends GenericRootSchema> = {
onRemoteState: (state: InferType<S>) => void;
};
@@ -108,7 +119,7 @@ export class SyncEngine<S extends GenericRootSchema, D> {
private log(
level: Parameters<typeof logSyncService>[0]['level'],
message: string,
- extra?: Record<string, unknown>
+ extra?: Attributes
) {
logSyncService({
documentId: this.syncs.live.documentId,
@@ -197,7 +208,7 @@ export class SyncEngine<S extends GenericRootSchema, D> {
if (syncResult.isErr()) {
this.log('error', 'syncStateToLoro: failed, resetting engine', {
- err: syncResult,
+ err: JSON.stringify(syncResult.error),
});
this.reset();
}
@@ -223,7 +234,7 @@ export class SyncEngine<S extends GenericRootSchema, D> {
const snapshot = await (snapshotThunk ?? this.defaultSnapshotThunk)();
if (snapshot.isErr()) {
this.log('error', 'engine.reset: failed to get snapshot', {
- err: snapshot.error,
+ err: String(snapshot.error),
});
return;
}
@@ -231,7 +242,7 @@ export class SyncEngine<S extends GenericRootSchema, D> {
const resetResult = await this.loroManager.reset(snapshot.value);
if (resetResult.isErr()) {
this.log('error', 'engine.reset: loro manager reset failed', {
- err: resetResult,
+ err: JSON.stringify(resetResult.error),
});
return;
}
@@ -293,40 +304,49 @@ export class SyncEngine<S extends GenericRootSchema, D> {
}
}
- private async handleRemoteUpdate(update: RawUpdate) {
- await this.syncLock.runExclusive(async () => {
- this.log('debug', 'engine: handling remote update');
- const importResult = this.loroManager.importUpdate(update);
- await Promise.resolve();
- if (importResult.isErr()) {
- const pendingOnly = importResult.error.every(
- (e) => e.code === LoroManagerError.ImportPending
- );
- if (pendingOnly) {
- // The update arrived ahead of its causal deps; Loro holds it and
- // applies it once the gap fills. Pull the gap from the server
- // instead of resetting. No rerun: if the gap's ops haven't reached
- // the server yet (e.g. another tab's unflushed WAL), rerunning
- // would poll in a tight loop; the held ops apply whenever the deps
- // arrive through any channel.
- this.log('info', 'engine: remote update pending on missing ops');
- void this.convergeFromServer();
- return;
+ private async handleRemoteUpdate(
+ update: RawUpdate
+ ): Promise<'applied' | 'pending' | 'reset'> {
+ return this.syncLock.runExclusive(() =>
+ telemetrySpan(this.syncs.live.documentId, 'edit.apply', async (span) => {
+ span.setAttr('update.bytes', update.length);
+ const importResult = this.loroManager.importUpdate(update);
+ await Promise.resolve();
+ if (importResult.isErr()) {
+ const pendingOnly = importResult.error.every(
+ (e) => e.code === LoroManagerError.ImportPending
+ );
+ if (pendingOnly) {
+ // Loro retains causally-ahead updates. Pull the missing operations
+ // rather than resetting the document.
+ this.log('debug', 'engine: remote update pending on missing ops');
+ span.setAttr('outcome', 'pending');
+ void this.convergeFromServer();
+ return 'pending';
+ }
+
+ this.log(
+ 'error',
+ 'engine: failed to import remote update, resetting',
+ {
+ err: JSON.stringify(importResult.error),
+ }
+ );
+ span.error(importResult.error);
+ span.setAttr('outcome', 'reset');
+ this.reset();
+ return 'reset';
}
- this.log('error', 'engine: failed to import remote update, resetting', {
- err: importResult,
- });
- this.reset();
- return;
- }
- this.log('debug', 'engine: remote update imported ok');
- });
+ span.setAttr('outcome', 'applied');
+ span.setAttr('did_change', importResult.value);
+ return 'applied';
+ })
+ );
}
private handleSourceEvent(event: SyncSourceEvent) {
switch (event.type) {
case 'update':
- this.log('debug', 'engine: source event: update');
this.handleRemoteUpdate(event.update);
break;
case 'awareness':
@@ -394,15 +414,30 @@ export class SyncEngine<S extends GenericRootSchema, D> {
) {
if (!this.isActiveGeneration(generation)) return;
+ // The catch-up flow: "give me everything after the version I hold".
+ // The span shows the anchor version, what came back, and whether it
+ // applied — the trail for "client was stale and (never) reconciled".
+ const span = telemetrySpan(this.syncs.live.documentId, 'doc.sync.catchup');
+ span.setAttr('since.version', vvAttr(since));
+ span.setAttr('attempt', attempt);
this.log('debug', `engine: requestUpdatesSince (attempt ${attempt})`);
const updates = await this.syncs.live.requestUpdatesSince(since);
- if (!this.isActiveGeneration(generation)) return;
+ if (!this.isActiveGeneration(generation)) {
+ span.setAttr('outcome', 'cancelled');
+ span.end();
+ return;
+ }
if (updates.isErr() || !updates.value) {
this.log('error', 'engine: requestUpdatesSince failed', {
- err: 'error' in updates ? updates.error : 'update is undefined',
+ err: updates.isErr() ? String(updates.error) : 'update is undefined',
});
- if (updates.isErr() && attempt < REQUEST_UPDATES_MAX_ATTEMPTS) {
+ const retrying =
+ updates.isErr() && attempt < REQUEST_UPDATES_MAX_ATTEMPTS;
+ span.error(updates.isErr() ? updates.error : 'update is undefined');
+ span.setAttr('outcome', retrying ? 'retrying' : 'failed');
+ span.end();
+ if (retrying) {
await new Promise((resolve) =>
setTimeout(resolve, REQUEST_UPDATES_RETRY_DELAY_MS)
);
@@ -417,11 +452,17 @@ export class SyncEngine<S extends GenericRootSchema, D> {
// would throw and trigger a reset — the noop live source used by
// non-propagating AI edit sessions answers with exactly this.
this.log('debug', 'engine: requestUpdatesSince ok, empty update');
+ span.setAttr('outcome', 'noop');
+ span.setAttr('update.bytes', 0);
+ span.end();
return;
}
this.log('debug', 'engine: requestUpdatesSince ok, applying update');
- await this.handleRemoteUpdate(updates.value);
+ const outcome = await this.handleRemoteUpdate(updates.value);
+ span.setAttr('outcome', outcome);
+ span.setAttr('update.bytes', updates.value.length);
+ span.end();
}
}
diff --git a/packages/collaboration/src/collab/logger.ts b/packages/collaboration/src/collab/logger.ts
index bcd1679891..1cd24ec3c3 100644
--- a/packages/collaboration/src/collab/logger.ts
+++ b/packages/collaboration/src/collab/logger.ts
@@ -1,3 +1,6 @@
+import type { Attributes } from '@macro-inc/observability';
+import { type LogLevel, logTelemetry } from './telemetry';
+
export type WalContext = {
count: number;
dirty: number;
@@ -6,9 +9,31 @@ export type WalContext = {
export type SyncLogContext = {
wal?: WalContext;
- misc?: Record<string, unknown>;
+ misc?: Attributes;
};
+/** `document.id` and `session.id` come from the sink's base attrs, not here. */
+function contextAttrs(context: SyncLogContext): Attributes {
+ const attributes: Attributes = {
+ ...context.misc,
+ ...(context.wal && {
+ 'wal.count': context.wal.count,
+ 'wal.dirty': context.wal.dirty,
+ 'wal.most_recent_edit': context.wal.mostRecentEdit,
+ }),
+ };
+ return attributes;
+}
+
+/**
+ * Log a sync-engine message through the document's telemetry sink — the
+ * observability library's logging pipeline, which exports the record (when
+ * the host initialized an exporter) and always mirrors it to the console.
+ *
+ * `debug` lines are suppressed unless `window.debugSyncServiceLog` is set,
+ * keeping the high-frequency narration out of both the console and the
+ * export pipeline by default.
+ */
export function logSyncService({
documentId,
level,
@@ -16,15 +41,14 @@ export function logSyncService({
message,
}: {
documentId: string;
- level: 'debug' | 'info' | 'warn' | 'error';
+ level: LogLevel;
context: SyncLogContext;
message: string;
}): void {
const debugEnabled =
- typeof window !== 'undefined' && (window as any).debugSyncServiceLog;
+ typeof window !== 'undefined' &&
+ (window as { debugSyncServiceLog?: boolean }).debugSyncServiceLog;
if (level === 'debug' && !debugEnabled) return;
- console[level === 'info' ? 'log' : level](
- { documentId, t: Date.now(), ...context },
- message
- );
+
+ logTelemetry(documentId, level, message, contextAttrs(context));
}
diff --git a/packages/collaboration/src/collab/manager.ts b/packages/collaboration/src/collab/manager.ts
index 34f81f4240..04eb894636 100644
--- a/packages/collaboration/src/collab/manager.ts
+++ b/packages/collaboration/src/collab/manager.ts
@@ -19,6 +19,12 @@ import { onCleanup } from 'solid-js';
import type { ResultError } from '../internal/result';
import { logSyncService } from './logger';
import type { GenericRootSchema, LoroRawUpdate, RawUpdate } from './shared';
+import {
+ disposeTelemetryFor,
+ frontiersAttr,
+ setTelemetryAttr,
+ telemetrySpan,
+} from './telemetry';
export enum LoroManagerError {
ImportFailed = 'IMPORT_FAILED',
@@ -139,6 +145,9 @@ export class LoroManager<S extends GenericRootSchema = GenericRootSchema>
constructor(schema: S, options: LoroManagerOptions) {
this.schema = schema;
this.options = options;
+ // Stamp this doc's telemetry with the peer identity as soon as it
+ // exists, so every span and log record carries who we are.
+ setTelemetryAttr(options.documentId, 'loro.peer_id', this._doc.peerIdStr);
}
/** The inner LoroDoc. Only touch this if you know what you're doing. */
@@ -532,32 +541,43 @@ export class LoroManager<S extends GenericRootSchema = GenericRootSchema>
async ingest(input: SnapshotIngest): Promise<boolean> {
if (this._initialized) return false;
- const initResult = await this.initializeFromSnapshot(input.snapshot);
- if (initResult.isErr()) {
- logSyncService({
- documentId: this.options.documentId,
- level: 'error',
- context: {},
- message: `ingest(${input.kind}): seed failed`,
- });
- return false;
- }
-
- // WAL entries are causally anchored to the local snapshot, so they're only
- // safe to replay on top of the local seed we just applied.
- if (input.kind === 'local' && input.walUpdates?.length) {
- const replayResult = this.importBatchUpdates(input.walUpdates);
- if (replayResult.isErr()) {
+ // The winner of the snapshot race seeds the doc; its kind, size, and
+ // resulting version are the anchor every later sync question ("since
+ // what?") is asked against.
+ return telemetrySpan(this.options.documentId, 'doc.seed', async (span) => {
+ span.setAttr('seed.kind', input.kind);
+ span.setAttr('snapshot.bytes', input.snapshot.length);
+ const initResult = await this.initializeFromSnapshot(input.snapshot);
+ if (initResult.isErr()) {
logSyncService({
documentId: this.options.documentId,
level: 'error',
context: {},
- message: `ingest(${input.kind}): WAL replay failed`,
+ message: `ingest(${input.kind}): seed failed`,
});
+ span.error(`seed from ${input.kind} snapshot failed`);
+ return false;
}
- }
- return true;
+ // WAL entries are causally anchored to the local snapshot, so they're only
+ // safe to replay on top of the local seed we just applied.
+ if (input.kind === 'local' && input.walUpdates?.length) {
+ span.setAttr('wal.replayed', input.walUpdates.length);
+ const replayResult = this.importBatchUpdates(input.walUpdates);
+ if (replayResult.isErr()) {
+ logSyncService({
+ documentId: this.options.documentId,
+ level: 'error',
+ context: {},
+ message: `ingest(${input.kind}): WAL replay failed`,
+ });
+ span.error('WAL replay onto local seed failed');
+ }
+ }
+
+ span.setAttr('version.after', frontiersAttr(this._doc));
+ return true;
+ });
}
/** Tear down subscriptions and free the underlying doc. */
@@ -568,6 +588,7 @@ export class LoroManager<S extends GenericRootSchema = GenericRootSchema>
this.initListeners.clear();
this._mirror?.dispose();
this._doc.free();
+ disposeTelemetryFor(this.options.documentId);
}
}
diff --git a/packages/collaboration/src/collab/snapshot-store.ts b/packages/collaboration/src/collab/snapshot-store.ts
index b315a39081..36772ca48a 100644
--- a/packages/collaboration/src/collab/snapshot-store.ts
+++ b/packages/collaboration/src/collab/snapshot-store.ts
@@ -97,6 +97,7 @@ export async function loadCachedState<S extends GenericRootSchema>(
}
const pending = await walStore.getAll();
+ let pendingCount = 0;
for (const entry of pending) {
const importResult = loroManager.importUpdate(entry.update);
if (importResult.isErr()) {
@@ -106,12 +107,7 @@ export async function loadCachedState<S extends GenericRootSchema>(
if (pendingOnly) {
// Loro holds the entry until its causal gap fills, so keep replaying:
// a later WAL entry may fill the gap, and held ops apply on their own.
- logSyncService({
- documentId: 'unknown',
- level: 'warn',
- context: { misc: { entryId: entry.id } },
- message: 'snapshot-store: WAL entry pending during cold load',
- });
+ pendingCount += 1;
continue;
}
// Stop replaying. Skipped entries are safe: delivered ones are on the
@@ -126,5 +122,13 @@ export async function loadCachedState<S extends GenericRootSchema>(
break;
}
}
+ if (pendingCount > 0) {
+ logSyncService({
+ documentId: 'unknown',
+ level: 'warn',
+ context: { misc: { pendingCount } },
+ message: 'snapshot-store: WAL entries pending during cold load',
+ });
+ }
return true;
}
diff --git a/packages/collaboration/src/collab/telemetry.test.ts b/packages/collaboration/src/collab/telemetry.test.ts
new file mode 100644
index 0000000000..d98e605c04
--- /dev/null
+++ b/packages/collaboration/src/collab/telemetry.test.ts
@@ -0,0 +1,49 @@
+import { type Span, Telemetry } from '@macro-inc/observability';
+import { afterEach, describe, expect, test, vi } from 'vitest';
+import { disposeTelemetryFor, telemetrySpan } from './telemetry';
+
+const documentId = 'telemetry-test-document';
+
+function mockSpan() {
+ const error = vi.fn();
+ const span: Span = {
+ span: (() => span) as Span['span'],
+ run: (operation) => operation(),
+ setAttr: vi.fn(),
+ event: vi.fn(),
+ error,
+ traceparent: () => undefined,
+ injectTraceHeaders: vi.fn(),
+ end: vi.fn(),
+ };
+ vi.spyOn(Telemetry, 'span').mockImplementation(((
+ _: string,
+ operation?: (span: Span) => Promise<unknown>
+ ) => (operation ? operation(span) : span)) as typeof Telemetry.span);
+ return { span, error };
+}
+
+afterEach(() => {
+ disposeTelemetryFor(documentId);
+ vi.restoreAllMocks();
+});
+
+describe('telemetrySpan', () => {
+ test('enriches and delegates the async operation', async () => {
+ const { span, error } = mockSpan();
+
+ const result = await telemetrySpan(
+ documentId,
+ 'test',
+ async (activeSpan) => {
+ expect(activeSpan).toBe(span);
+ return 42;
+ }
+ );
+
+ expect(result).toBe(42);
+ expect(error).not.toHaveBeenCalled();
+ expect(span.setAttr).toHaveBeenCalledWith('document.id', documentId);
+ expect(span.setAttr).toHaveBeenCalledWith('session.id', expect.any(String));
+ });
+});
diff --git a/packages/collaboration/src/collab/telemetry.ts b/packages/collaboration/src/collab/telemetry.ts
new file mode 100644
index 0000000000..22305cf063
--- /dev/null
+++ b/packages/collaboration/src/collab/telemetry.ts
@@ -0,0 +1,105 @@
+import {
+ type Attribute,
+ type Attributes,
+ type Span,
+ Telemetry,
+} from '@macro-inc/observability';
+
+export type LogLevel = 'debug' | 'info' | 'warn' | 'error';
+
+type DocumentTelemetry = {
+ attributes: Attributes;
+};
+
+const documents = new Map<string, DocumentTelemetry>();
+
+function documentTelemetry(documentId: string): DocumentTelemetry {
+ let value = documents.get(documentId);
+ if (!value) {
+ value = {
+ attributes: {
+ 'document.id': documentId,
+ 'session.id': crypto.randomUUID(),
+ },
+ };
+ documents.set(documentId, value);
+ }
+ return value;
+}
+
+type SpanOperation<T> = (span: Span) => Promise<T>;
+
+function setDocumentSpanAttributes(documentId: string, span: Span): void {
+ for (const [key, value] of Object.entries(
+ documentTelemetry(documentId).attributes
+ )) {
+ if (value !== undefined) span.setAttr(key, value);
+ }
+}
+
+export function telemetrySpan<T>(
+ documentId: string,
+ name: string,
+ operation: SpanOperation<T>
+): Promise<T>;
+export function telemetrySpan(documentId: string, name: string): Span;
+export function telemetrySpan<T>(
+ documentId: string,
+ name: string,
+ operation?: SpanOperation<T>
+): Span | Promise<T> {
+ if (operation) {
+ return Telemetry.span(name, async (span) => {
+ setDocumentSpanAttributes(documentId, span);
+ return operation(span);
+ });
+ }
+
+ const span = Telemetry.span(name);
+ setDocumentSpanAttributes(documentId, span);
+ return span;
+}
+
+export function setTelemetryAttr(
+ documentId: string,
+ name: string,
+ value: Attribute
+): void {
+ documentTelemetry(documentId).attributes[name] = value;
+}
+
+export function logTelemetry(
+ documentId: string,
+ level: LogLevel,
+ message: string,
+ attributes?: Attributes
+): void {
+ Telemetry[level](message, {
+ ...documentTelemetry(documentId).attributes,
+ ...attributes,
+ });
+}
+
+export function disposeTelemetryFor(documentId: string): void {
+ documents.delete(documentId);
+}
+
+function cappedAttr(value: string, max = 256): string {
+ return value.length > max ? `${value.slice(0, max)}...` : value;
+}
+
+/** `peer:counter` pairs as one compact span attribute. */
+export function peerCounterAttr(
+ pairs: Iterable<[peer: string, counter: number]>
+): string {
+ return cappedAttr(
+ [...pairs].map(([peer, counter]) => `${peer}:${counter}`).join(',')
+ );
+}
+
+/** A doc version as a compact span attribute (its frontiers). */
+export function frontiersAttr(doc: {
+ frontiers(): { peer: string; counter: number }[];
+}): string {
+ return peerCounterAttr(doc.frontiers().map((f) => [f.peer, f.counter]));
+}
diff --git a/packages/collaboration/src/collab/wal.test.ts b/packages/collaboration/src/collab/wal.test.ts
index 14e9402a12..a272890590 100644
--- a/packages/collaboration/src/collab/wal.test.ts
+++ b/packages/collaboration/src/collab/wal.test.ts
@@ -1,8 +1,15 @@
-import { describe, expect, it, vi } from 'vitest';
+import { type Span, Telemetry } from '@macro-inc/observability';
+import { afterEach, describe, expect, it, vi } from 'vitest';
import type { RawUpdate } from './shared';
+import { disposeTelemetryFor } from './telemetry';
import { MockLiveSyncSource, MockWALStore, makeTestWAL } from './testing';
import { WAL_TTL_MS, WALSyncer } from './wal';
+afterEach(() => {
+ disposeTelemetryFor('wal-telemetry-test');
+ vi.restoreAllMocks();
+});
+
function makeWAL(live: MockLiveSyncSource) {
return makeTestWAL(live);
}
@@ -117,6 +124,43 @@ describe('WALSyncer', () => {
expect(await undeliveredCount(walStore)).toBe(0);
});
+ it('records one delivery outage across repeated failures and ends it on recovery', async () => {
+ const span: Span = {
+ span: (() => span) as Span['span'],
+ run: (operation) => operation(),
+ setAttr: vi.fn(),
+ event: vi.fn(),
+ error: vi.fn(),
+ traceparent: () => undefined,
+ injectTraceHeaders: vi.fn(),
+ end: vi.fn(),
+ };
+ const spanSpy = vi.spyOn(Telemetry, 'span').mockReturnValue(span as never);
+ const store = new MockWALStore<RawUpdate>();
+ let delivered = false;
+ const push = vi.fn(async () => delivered);
+ const wal = new WALSyncer(store, push, 'wal-telemetry-test');
+
+ await wal.append(new Uint8Array([1]));
+ await wal.pendingFlush;
+ await wal.append(new Uint8Array([2]));
+ await wal.pendingFlush;
+
+ expect(spanSpy).toHaveBeenCalledTimes(1);
+ expect(spanSpy).toHaveBeenCalledWith('wal.delivery_outage');
+ expect(span.error).toHaveBeenCalledOnce();
+ expect(span.end).not.toHaveBeenCalled();
+
+ delivered = true;
+ await wal.flush();
+
+ expect(span.event).toHaveBeenCalledWith('wal.ack.confirmed', {
+ 'wal.delivered': 2,
+ });
+ expect(span.setAttr).toHaveBeenCalledWith('outcome', 'recovered');
+ expect(span.end).toHaveBeenCalledOnce();
+ });
+
it('flushes updates that arrived during an in-flight flush', async () => {
const live = new MockLiveSyncSource();
const { wal, walStore } = makeWAL(live);
diff --git a/packages/collaboration/src/collab/wal.ts b/packages/collaboration/src/collab/wal.ts
index a70408d8d4..49a8c25d7c 100644
--- a/packages/collaboration/src/collab/wal.ts
+++ b/packages/collaboration/src/collab/wal.ts
@@ -1,7 +1,9 @@
+import type { Span } from '@macro-inc/observability';
import { type DBSchema, type IDBPDatabase, openDB as idbOpen } from 'idb';
import { logSyncService, type WalContext } from './logger';
import type { RawUpdate } from './shared';
import type { LiveSyncSource } from './source';
+import { telemetrySpan } from './telemetry';
export type WALEntry<T> = {
id: number;
@@ -229,6 +231,15 @@ export class WALSyncer<T> {
private hasNewPending = false;
public pendingFlush: Promise<void> = Promise.resolve();
private cleanupFns: Array<() => void> = [];
+ private deliveryOutage:
+ | {
+ id: string;
+ span: Span;
+ startedAt: number;
+ failedAttempts: number;
+ maxUndelivered: number;
+ }
+ | undefined;
/**
* Exists so that we can not "do stuff" until we have pruned expired entries
@@ -303,6 +314,10 @@ export class WALSyncer<T> {
}
public destroy(): void {
+ if (this.deliveryOutage) {
+ this.deliveryOutage.span.setAttr('outcome', 'abandoned');
+ this.finishDeliveryOutage();
+ }
for (const fn of this.cleanupFns) fn();
this.cleanupFns = [];
}
@@ -331,10 +346,17 @@ export class WALSyncer<T> {
message: `WAL flush: pushing ${undelivered.length} entries`,
});
- const delivered = await this.push(undelivered.map((e) => e.update));
+ let delivered: boolean;
+ try {
+ delivered = await this.push(undelivered.map((e) => e.update));
+ } catch (e) {
+ this.recordDeliveryFailure('push_error', undelivered.length, e);
+ throw e;
+ }
if (delivered) {
await this.store.markDelivered(undelivered.map((e) => e.id));
+ this.recordDeliveryRecovery(undelivered.length);
if (this.label)
logSyncService({
documentId: this.label,
@@ -343,13 +365,7 @@ export class WALSyncer<T> {
message: `WAL flush: delivered ${undelivered.length} entries`,
});
} else {
- if (this.label)
- logSyncService({
- documentId: this.label,
- level: 'warn',
- context: { wal: await this.summary() },
- message: `WAL flush: push not acked (${undelivered.length} entries)`,
- });
+ this.recordDeliveryFailure('not_acked', undelivered.length);
succeeded = false;
}
} finally {
@@ -361,6 +377,98 @@ export class WALSyncer<T> {
return this.pendingFlush;
}
}
+
+ private recordDeliveryFailure(
+ kind: 'not_acked' | 'push_error',
+ undelivered: number,
+ error?: unknown
+ ): void {
+ if (!this.label) return;
+
+ if (this.deliveryOutage) {
+ this.deliveryOutage.failedAttempts++;
+ this.deliveryOutage.maxUndelivered = Math.max(
+ this.deliveryOutage.maxUndelivered,
+ undelivered
+ );
+ this.deliveryOutage.span.setAttr(
+ 'wal.failed_attempts',
+ this.deliveryOutage.failedAttempts
+ );
+ this.deliveryOutage.span.setAttr(
+ 'wal.max_undelivered',
+ this.deliveryOutage.maxUndelivered
+ );
+ return;
+ }
+
+ const id = crypto.randomUUID();
+ const span = telemetrySpan(this.label, 'wal.delivery_outage');
+ span.setAttr('wal.outage.id', id);
+ span.setAttr('wal.first_failure', kind);
+ span.setAttr('wal.failed_attempts', 1);
+ span.setAttr('wal.max_undelivered', undelivered);
+ span.error(error ?? 'push not acked; entries remain undelivered');
+ this.deliveryOutage = {
+ id,
+ span,
+ startedAt: Date.now(),
+ failedAttempts: 1,
+ maxUndelivered: undelivered,
+ };
+ logSyncService({
+ documentId: this.label,
+ level: 'warn',
+ context: {
+ misc: {
+ 'wal.outage.id': id,
+ 'wal.first_failure': kind,
+ 'wal.undelivered': undelivered,
+ },
+ },
+ message: `WAL delivery outage started (${undelivered} entries)`,
+ });
+ }
+
+ private recordDeliveryRecovery(delivered: number): void {
+ if (!this.deliveryOutage || !this.label) return;
+
+ const { id, span, startedAt, failedAttempts, maxUndelivered } =
+ this.deliveryOutage;
+ const outageMs = Date.now() - startedAt;
+ span.event('wal.ack.confirmed', { 'wal.delivered': delivered });
+ span.setAttr('outcome', 'recovered');
+ span.setAttr('wal.outage_ms', outageMs);
+ span.setAttr('wal.failed_attempts', failedAttempts);
+ span.setAttr('wal.max_undelivered', maxUndelivered);
+ span.setAttr('wal.delivered', delivered);
+ span.end();
+ this.deliveryOutage = undefined;
+ logSyncService({
+ documentId: this.label,
+ level: 'info',
+ context: {
+ misc: {
+ 'wal.outage.id': id,
+ 'wal.outage_ms': outageMs,
+ 'wal.failed_attempts': failedAttempts,
+ 'wal.max_undelivered': maxUndelivered,
+ 'wal.delivered': delivered,
+ },
+ },
+ message: 'WAL delivery recovered',
+ });
+ }
+
+ private finishDeliveryOutage(): void {
+ if (!this.deliveryOutage) return;
+ this.deliveryOutage.span.setAttr(
+ 'wal.outage_ms',
+ Date.now() - this.deliveryOutage.startedAt
+ );
+ this.deliveryOutage.span.end();
+ this.deliveryOutage = undefined;
+ }
}
/** Build a WAL syncer wired to a Loro live sync source: BrowserWALStore for
diff --git a/packages/collaboration/src/sync-service/source.test.ts b/packages/collaboration/src/sync-service/source.test.ts
index f77b02a1ed..d4cf39c2c4 100644
--- a/packages/collaboration/src/sync-service/source.test.ts
+++ b/packages/collaboration/src/sync-service/source.test.ts
@@ -23,7 +23,11 @@ class FakeSocket implements SyncSocket {
reconnects = 0;
closed = false;
private msg = new Set<AnyListener>();
+ private openListeners = new Set<AnyListener>();
private recon = new Set<AnyListener>();
+ private closeListeners = new Set<AnyListener>();
+ private retryListeners = new Set<AnyListener>();
+ private heartbeatMissedListeners = new Set<AnyListener>();
send(message: IFromPeer) {
this.sent.push(message);
@@ -43,15 +47,27 @@ class FakeSocket implements SyncSocket {
listener: WebsocketEventListener<K, IFromPeer, FromRemote>
) {
if (type === WebsocketEvent.Message) this.msg.add(listener as AnyListener);
+ else if (type === WebsocketEvent.Open)
+ this.openListeners.add(listener as AnyListener);
else if (type === WebsocketEvent.Reconnect)
this.recon.add(listener as AnyListener);
+ else if (type === WebsocketEvent.Close)
+ this.closeListeners.add(listener as AnyListener);
+ else if (type === WebsocketEvent.retry)
+ this.retryListeners.add(listener as AnyListener);
+ else if (type === WebsocketEvent.HeartbeatMissed)
+ this.heartbeatMissedListeners.add(listener as AnyListener);
}
removeEventListener<K extends WebsocketEvent>(
_type: K,
listener: WebsocketEventListener<K, IFromPeer, FromRemote>
) {
this.msg.delete(listener as AnyListener);
+ this.openListeners.delete(listener as AnyListener);
this.recon.delete(listener as AnyListener);
+ this.closeListeners.delete(listener as AnyListener);
+ this.retryListeners.delete(listener as AnyListener);
+ this.heartbeatMissedListeners.delete(listener as AnyListener);
}
// test drivers
@@ -64,6 +80,28 @@ class FakeSocket implements SyncSocket {
const event = {} as Parameters<AnyListener>[1];
for (const listener of [...this.recon]) listener(this as never, event);
}
+ fireOpen() {
+ this.connectionState = WebsocketConnectionState.Open;
+ const event = {} as Parameters<AnyListener>[1];
+ for (const listener of [...this.openListeners])
+ listener(this as never, event);
+ }
+ fireClose(code: number, reason: string) {
+ const event = {
+ code,
+ reason,
+ wasClean: false,
+ } as Parameters<AnyListener>[1];
+ for (const listener of [...this.closeListeners])
+ listener(this as never, event);
+ }
+ fireRetry(retries: number, backoff: number) {
+ const event = {
+ detail: { retries, backoff },
+ } as Parameters<AnyListener>[1];
+ for (const listener of [...this.retryListeners])
+ listener(this as never, event);
+ }
}
const GUARDS = {
diff --git a/packages/collaboration/src/sync-service/source.ts b/packages/collaboration/src/sync-service/source.ts
index 209e21f8a5..502ebe80d1 100644
--- a/packages/collaboration/src/sync-service/source.ts
+++ b/packages/collaboration/src/sync-service/source.ts
@@ -133,6 +133,11 @@ export class SyncServiceSource implements LiveSyncSource {
});
this.ws.addEventListener(WebsocketEvent.Message, this.onMessage);
+ this.ws.addEventListener(WebsocketEvent.Close, this.onClose);
+ this.ws.addEventListener(
+ WebsocketEvent.HeartbeatMissed,
+ this.onHeartbeatMissed
+ );
this.ws.addEventListener(WebsocketEvent.Reconnect, this.onReconnect);
this.registerEnvironmentListeners();
@@ -243,6 +248,11 @@ export class SyncServiceSource implements LiveSyncSource {
document.removeEventListener('visibilitychange', this.onVisibilityChange);
}
this.ws.removeEventListener(WebsocketEvent.Message, this.onMessage);
+ this.ws.removeEventListener(WebsocketEvent.Close, this.onClose);
+ this.ws.removeEventListener(
+ WebsocketEvent.HeartbeatMissed,
+ this.onHeartbeatMissed
+ );
this.ws.removeEventListener(WebsocketEvent.Reconnect, this.onReconnect);
this.ws.close();
};
@@ -312,6 +322,45 @@ export class SyncServiceSource implements LiveSyncSource {
}
};
+ private onClose: WebsocketEventListener<
+ WebsocketEvent.Close,
+ IFromPeer,
+ FromRemote
+ > = (_ws, event) => {
+ logSyncService({
+ documentId: this.documentId,
+ level: 'warn',
+ context: {
+ misc: {
+ 'ws.close.code': event.code,
+ 'ws.close.clean': event.wasClean,
+ ...(event.reason && {
+ 'ws.close.reason': event.reason.slice(0, 256),
+ }),
+ },
+ },
+ message: 'sync source: WebSocket disconnected',
+ });
+ };
+
+ private onHeartbeatMissed: WebsocketEventListener<
+ WebsocketEvent.HeartbeatMissed,
+ IFromPeer,
+ FromRemote
+ > = (_ws, event) => {
+ if (!event.detail.willReconnect) return;
+ logSyncService({
+ documentId: this.documentId,
+ level: 'warn',
+ context: {
+ misc: {
+ 'ws.heartbeat.missed': event.detail.missedHeartbeats,
+ },
+ },
+ message: 'sync source: WebSocket heartbeat missed; reconnecting',
+ });
+ };
+
private onReconnect: WebsocketEventListener<
WebsocketEvent.Reconnect,
IFromPeer,
@@ -347,7 +396,7 @@ export class SyncServiceSource implements LiveSyncSource {
logSyncService({
documentId: this.documentId,
level: 'warn',
- context: { misc: { error } },
+ context: { misc: { error: String(error) } },
message: 'sync source: reconnect initial sync timed out',
});
// Heartbeat is already running, so the connection remains monitored
diff --git a/packages/observability/package.json b/packages/observability/package.json
new file mode 100644
index 0000000000..5f8e20465d
--- /dev/null
+++ b/packages/observability/package.json
@@ -0,0 +1,37 @@
+{
+ "name": "@macro-inc/observability",
+ "version": "0.0.1",
+ "private": true,
+ "type": "module",
+ "exports": {
+ ".": "./src/index.ts",
+ "./core": "./src/core.ts",
+ "./web": "./src/web.ts",
+ "./zone": "./src/zone.ts"
+ },
+ "scripts": {
+ "lint": "biome check .",
+ "test": "vitest run --config vitest.config.ts",
+ "type-check": "tsc --noEmit --project tsconfig.json"
+ },
+ "dependencies": {
+ "@opentelemetry/api": "^1.9.1",
+ "@opentelemetry/api-logs": "^0.220.0",
+ "@opentelemetry/context-zone": "^2.9.0",
+ "@opentelemetry/core": "^2.9.0",
+ "@opentelemetry/exporter-logs-otlp-http": "^0.220.0",
+ "@opentelemetry/exporter-trace-otlp-http": "^0.220.0",
+ "@opentelemetry/resources": "^2.9.0",
+ "@opentelemetry/sdk-logs": "^0.220.0",
+ "@opentelemetry/sdk-trace-base": "^2.9.0",
+ "@opentelemetry/sdk-trace-web": "^2.9.0",
+ "@opentelemetry/semantic-conventions": "^1.43.0"
+ },
+ "devDependencies": {
+ "@types/bun": "^1.3.14",
+ "@types/node": "^24.8.0",
+ "jsdom": "^26.1.0",
+ "typescript": "^5.9.3",
+ "vitest": "^3.2.4"
+ }
+}
diff --git a/packages/observability/src/config.ts b/packages/observability/src/config.ts
new file mode 100644
index 0000000000..d74e418644
--- /dev/null
+++ b/packages/observability/src/config.ts
@@ -0,0 +1,32 @@
+import type {
+ ContextManager,
+ AttributeValue as OtelAttributeValue,
+} from "@opentelemetry/api";
+import type { TelemetryTracingProviderFactory } from "./provider";
+
+export type Attribute = OtelAttributeValue;
+export type Attributes = Record<string, Attribute | undefined>;
+export type LogAttributes = Attributes;
+
+export interface TelemetryInitConfig {
+ serviceName: string;
+ environment: string;
+ tracesUrl?: string;
+ logsUrl?: string;
+ contextManager?: ContextManager;
+ tracingProvider?: TelemetryTracingProviderFactory;
+ enabled: () => Promise<boolean>;
+}
+
+/** Mutable attributes applied to telemetry created after they change. */
+export class TelemetryRuntimeConfig {
+ #userId: string | undefined;
+
+ get userId(): string | undefined {
+ return this.#userId;
+ }
+
+ setUser(userId: string | undefined): void {
+ this.#userId = userId;
+ }
+}
diff --git a/packages/observability/src/constants.ts b/packages/observability/src/constants.ts
new file mode 100644
index 0000000000..85010dd888
--- /dev/null
+++ b/packages/observability/src/constants.ts
@@ -0,0 +1,4 @@
+export const INSTRUMENTATION_SCOPE_NAME = "@macro-inc/observability";
+export const ATTR_USER_ID = "usr.id";
+export const ATTR_DEPLOYMENT_ENVIRONMENT = "deployment.environment";
+export const CONSOLE_PREFIX = "[observability]";
diff --git a/packages/observability/src/core.ts b/packages/observability/src/core.ts
new file mode 100644
index 0000000000..8a43a80583
--- /dev/null
+++ b/packages/observability/src/core.ts
@@ -0,0 +1,13 @@
+export type {
+ Attribute,
+ Attributes,
+ LogAttributes,
+ TelemetryInitConfig,
+} from "./config";
+export { TelemetryRuntimeConfig } from "./config";
+export type {
+ TelemetryTracingProvider,
+ TelemetryTracingProviderFactory,
+} from "./provider";
+export type { Span } from "./span";
+export { Telemetry } from "./telemetry";
diff --git a/packages/observability/src/index.ts b/packages/observability/src/index.ts
new file mode 100644
index 0000000000..cf304f99e2
--- /dev/null
+++ b/packages/observability/src/index.ts
@@ -0,0 +1,9 @@
+export type {
+ Attribute,
+ Attributes,
+ LogAttributes,
+ TelemetryInitConfig,
+} from "./config";
+export { TelemetryRuntimeConfig } from "./config";
+export type { Span } from "./span";
+export { Telemetry } from "./telemetry";
diff --git a/packages/observability/src/logging.ts b/packages/observability/src/logging.ts
new file mode 100644
index 0000000000..969b581133
--- /dev/null
+++ b/packages/observability/src/logging.ts
@@ -0,0 +1,115 @@
+import type { Attributes as OtelAttributes } from "@opentelemetry/api";
+import { type Logger, logs, SeverityNumber } from "@opentelemetry/api-logs";
+import { OTLPLogExporter } from "@opentelemetry/exporter-logs-otlp-http";
+import type { Resource } from "@opentelemetry/resources";
+import {
+ BatchLogRecordProcessor,
+ LoggerProvider,
+} from "@opentelemetry/sdk-logs";
+import {
+ ATTR_EXCEPTION_MESSAGE,
+ ATTR_EXCEPTION_STACKTRACE,
+ ATTR_EXCEPTION_TYPE,
+} from "@opentelemetry/semantic-conventions";
+import type { LogAttributes, TelemetryInitConfig } from "./config";
+import {
+ ATTR_USER_ID,
+ CONSOLE_PREFIX,
+ INSTRUMENTATION_SCOPE_NAME,
+} from "./constants";
+
+type LogLevel = "debug" | "info" | "warn" | "error";
+
+export class Logging {
+ #provider: LoggerProvider | undefined;
+ readonly #logger: Logger = logs.getLogger(INSTRUMENTATION_SCOPE_NAME);
+
+ init(
+ config: TelemetryInitConfig,
+ resource: Resource,
+ getUserId: () => string | undefined,
+ ): void {
+ if (!config.logsUrl) return;
+ this.#provider = new LoggerProvider({
+ resource,
+ processors: [
+ {
+ onEmit: (record) => {
+ const userId = getUserId();
+ if (userId !== undefined) record.setAttribute(ATTR_USER_ID, userId);
+ },
+ forceFlush: () => Promise.resolve(),
+ shutdown: () => Promise.resolve(),
+ },
+ new BatchLogRecordProcessor({
+ exporter: new OTLPLogExporter({ url: config.logsUrl }),
+ }),
+ ],
+ });
+ logs.setGlobalLoggerProvider(this.#provider);
+ }
+
+ debug(message: string, attributes?: LogAttributes): void {
+ this.#emit("debug", message, attributes);
+ }
+
+ info(message: string, attributes?: LogAttributes): void {
+ this.#emit("info", message, attributes);
+ }
+
+ warn(message: string, attributes?: LogAttributes): void {
+ this.#emit("warn", message, attributes);
+ }
+
+ error(error: unknown, attributes?: LogAttributes): void {
+ const normalized = Logging.#normalizeError(error);
+ this.#emit(
+ "error",
+ typeof normalized === "string"
+ ? normalized
+ : normalized.message || normalized.name,
+ {
+ ...attributes,
+ ...(normalized instanceof Error && {
+ [ATTR_EXCEPTION_TYPE]: normalized.name,
+ [ATTR_EXCEPTION_MESSAGE]: normalized.message,
+ [ATTR_EXCEPTION_STACKTRACE]: normalized.stack,
+ }),
+ },
+ );
+ }
+
+ async flush(): Promise<void> {
+ await this.#provider?.forceFlush();
+ }
+
+ async shutdown(): Promise<void> {
+ await this.#provider?.shutdown();
+ this.#provider = undefined;
+ }
+
+ #emit(level: LogLevel, message: string, attributes?: LogAttributes): void {
+ const severity = {
+ debug: SeverityNumber.DEBUG,
+ info: SeverityNumber.INFO,
+ warn: SeverityNumber.WARN,
+ error: SeverityNumber.ERROR,
+ }[level];
+ this.#logger.emit({
+ severityNumber: severity,
+ severityText: level,
+ body: message,
+ attributes: attributes as OtelAttributes,
+ });
+ const consoleMethod = level === "info" ? "log" : level;
+ console[consoleMethod](CONSOLE_PREFIX, message, attributes ?? {});
+ }
+
+ static #normalizeError(error: unknown): Error | string {
+ return error instanceof Error ||
+ typeof error === "string" ||
+ (typeof error === "object" && error !== null && "message" in error)
+ ? (error as Error | string)
+ : String(error);
+ }
+}
diff --git a/packages/observability/src/provider.ts b/packages/observability/src/provider.ts
new file mode 100644
index 0000000000..2893cfa0f3
--- /dev/null
+++ b/packages/observability/src/provider.ts
@@ -0,0 +1,14 @@
+import type { TracerProvider } from "@opentelemetry/api";
+import type { Resource } from "@opentelemetry/resources";
+
+/** A runtime-specific OpenTelemetry tracer provider. */
+export type TelemetryTracingProvider = TracerProvider & {
+ forceFlush(): Promise<void>;
+ shutdown(): Promise<void>;
+};
+
+/** Creates a tracer provider after telemetry has been enabled. */
+export type TelemetryTracingProviderFactory = (
+ resource: Resource,
+ getUserId: () => string | undefined,
+) => TelemetryTracingProvider;
diff --git a/packages/observability/src/span.ts b/packages/observability/src/span.ts
new file mode 100644
index 0000000000..178e9be3e5
--- /dev/null
+++ b/packages/observability/src/span.ts
@@ -0,0 +1,108 @@
+import {
+ type Context,
+ context,
+ isSpanContextValid,
+ type Attributes as OtelAttributes,
+ propagation,
+ type Span as OtelSpan,
+ SpanStatusCode,
+} from "@opentelemetry/api";
+import type { Attribute, Attributes } from "./config";
+
+/** A manually managed span. Calling {@link Span.end} ends it. */
+export interface Span {
+ /** Start a child span, or run and end one around an async operation. */
+ span<T>(name: string, operation: (span: Span) => Promise<T>): Promise<T>;
+ span(name: string): Span;
+ /** Run with this span as the active OpenTelemetry context. */
+ run<T>(operation: () => T): T;
+ /** Set one span attribute. */
+ setAttr(name: string, value: Attribute): void;
+ /** Add a point-in-time event. */
+ event(name: string, attributes?: Attributes): void;
+ /** Record an exception and mark the span as failed. */
+ error(error: unknown): void;
+ /** Return this span's W3C trace context for transports without headers. */
+ traceparent(): string | undefined;
+ /** Inject this span's W3C trace context into an HTTP header carrier. */
+ injectTraceHeaders(headers: Record<string, string>): void;
+ /** End the span. Safe to call more than once. */
+ end(): void;
+}
+
+export class SpanImpl implements Span {
+ #ended = false;
+
+ constructor(
+ private readonly otelSpan: OtelSpan,
+ private readonly ctx: Context,
+ private readonly startChild: (name: string, parent: Context) => Span,
+ ) {}
+
+ span<T>(name: string, operation: (span: Span) => Promise<T>): Promise<T>;
+ span(name: string): Span;
+ span<T>(
+ name: string,
+ operation?: (span: Span) => Promise<T>,
+ ): Span | Promise<T> {
+ const span = this.startChild(name, this.ctx);
+ if (!operation) return span;
+
+ return span.run(async () => {
+ try {
+ return await operation(span);
+ } finally {
+ span.end();
+ }
+ });
+ }
+
+ run<T>(operation: () => T): T {
+ return context.with(this.ctx, operation);
+ }
+
+ setAttr(name: string, value: Attribute): void {
+ if (!this.#ended) this.otelSpan.setAttribute(name, value);
+ }
+
+ event(name: string, attributes?: Attributes): void {
+ if (!this.#ended)
+ this.otelSpan.addEvent(name, attributes as OtelAttributes);
+ }
+
+ error(error: unknown): void {
+ if (this.#ended) return;
+ const exception = SpanImpl.normalizeError(error);
+ this.otelSpan.recordException(exception);
+ this.otelSpan.setStatus({
+ code: SpanStatusCode.ERROR,
+ message:
+ typeof exception === "string" ? exception : (exception.message ?? ""),
+ });
+ }
+
+ traceparent(): string | undefined {
+ const spanContext = this.otelSpan.spanContext();
+ if (!isSpanContextValid(spanContext)) return undefined;
+ const { traceId, spanId, traceFlags } = spanContext;
+ return `00-${traceId}-${spanId}-${traceFlags.toString(16).padStart(2, "0")}`;
+ }
+
+ injectTraceHeaders(headers: Record<string, string>): void {
+ if (!this.#ended) propagation.inject(this.ctx, headers);
+ }
+
+ end(): void {
+ if (this.#ended) return;
+ this.#ended = true;
+ this.otelSpan.end();
+ }
+
+ private static normalizeError(error: unknown): Error | string {
+ return error instanceof Error ||
+ typeof error === "string" ||
+ (typeof error === "object" && error !== null && "message" in error)
+ ? (error as Error | string)
+ : String(error);
+ }
+}
diff --git a/packages/observability/src/telemetry.test.ts b/packages/observability/src/telemetry.test.ts
new file mode 100644
index 0000000000..65820b54f3
--- /dev/null
+++ b/packages/observability/src/telemetry.test.ts
@@ -0,0 +1,189 @@
+// @vitest-environment jsdom
+import { context, trace } from "@opentelemetry/api";
+import { logs } from "@opentelemetry/api-logs";
+import {
+ InMemoryLogRecordExporter,
+ LoggerProvider,
+ SimpleLogRecordProcessor,
+} from "@opentelemetry/sdk-logs";
+import {
+ BasicTracerProvider,
+ InMemorySpanExporter,
+ SimpleSpanProcessor,
+} from "@opentelemetry/sdk-trace-base";
+import { beforeEach, describe, expect, test } from "vitest";
+import { Telemetry } from "./index";
+import { ZoneContextManager } from "./zone";
+
+context.setGlobalContextManager(new ZoneContextManager().enable());
+
+const spanExporter = new InMemorySpanExporter();
+trace.setGlobalTracerProvider(
+ new BasicTracerProvider({
+ spanProcessors: [new SimpleSpanProcessor(spanExporter)],
+ }),
+);
+
+const logExporter = new InMemoryLogRecordExporter();
+logs.setGlobalLoggerProvider(
+ new LoggerProvider({
+ processors: [new SimpleLogRecordProcessor({ exporter: logExporter })],
+ }),
+);
+
+describe("Telemetry", () => {
+ beforeEach(() => {
+ spanExporter.reset();
+ logExporter.reset();
+ });
+
+ test("awaits the async enablement decision", async () => {
+ let decide!: (enabled: boolean) => void;
+ let markDecisionRequested!: () => void;
+ const decisionRequested = new Promise<void>((resolve) => {
+ markDecisionRequested = resolve;
+ });
+ let initialized = false;
+ const initialization = Telemetry.init({
+ serviceName: "web-app",
+ environment: "test",
+ enabled: () =>
+ new Promise<boolean>((resolve) => {
+ decide = resolve;
+ markDecisionRequested();
+ }),
+ }).then(() => {
+ initialized = true;
+ });
+
+ await decisionRequested;
+ expect(initialized).toBe(false);
+
+ decide(false);
+ await initialization;
+ expect(initialized).toBe(true);
+ });
+
+ test("a span exports only when ended", () => {
+ const span = Telemetry.span("doc.open");
+ span.setAttr("document.id", "abc");
+ expect(spanExporter.getFinishedSpans()).toHaveLength(0);
+
+ span.end();
+ span.end();
+
+ expect(spanExporter.getFinishedSpans()).toHaveLength(1);
+ expect(spanExporter.getFinishedSpans()[0]?.attributes).toMatchObject({
+ "document.id": "abc",
+ });
+ });
+
+ test("ends a callback span after the async operation", async () => {
+ let complete: () => void = () => {};
+ const operation = Telemetry.span("doc.save", async (span) => {
+ span.setAttr("document.id", "abc");
+ await new Promise<void>((resolve) => {
+ complete = resolve;
+ });
+ return 42;
+ });
+
+ await Promise.resolve();
+ expect(spanExporter.getFinishedSpans()).toHaveLength(0);
+ complete();
+ expect(await operation).toBe(42);
+ expect(spanExporter.getFinishedSpans()).toHaveLength(1);
+ });
+
+ test("ends a callback span when the async operation throws", async () => {
+ const failure = new Error("failed");
+
+ await expect(
+ Telemetry.span("doc.save", async () => {
+ throw failure;
+ }),
+ ).rejects.toBe(failure);
+ expect(spanExporter.getFinishedSpans()).toHaveLength(1);
+ });
+
+ test("starts children from the parent span context", () => {
+ const root = Telemetry.span("doc.open");
+ const child = root.span("doc.load");
+ child.end();
+ root.end();
+
+ const spans = new Map(
+ spanExporter.getFinishedSpans().map((span) => [span.name, span]),
+ );
+ expect(spans.get("doc.load")?.parentSpanContext?.spanId).toBe(
+ spans.get("doc.open")?.spanContext().spanId,
+ );
+ });
+
+ test("ends a callback child span after the async operation", async () => {
+ const root = Telemetry.span("doc.open");
+ let complete: () => void = () => {};
+ const operation = root.span("doc.snapshot.dss-cache", async (span) => {
+ span.setAttr("snapshot.source", "dss");
+ await new Promise<void>((resolve) => {
+ complete = resolve;
+ });
+ return "unavailable";
+ });
+
+ await Promise.resolve();
+ expect(spanExporter.getFinishedSpans()).toHaveLength(0);
+ complete();
+ expect(await operation).toBe("unavailable");
+ root.end();
+
+ const spans = new Map(
+ spanExporter.getFinishedSpans().map((span) => [span.name, span]),
+ );
+ expect(spans.get("doc.snapshot.dss-cache")?.attributes).toMatchObject({
+ "snapshot.source": "dss",
+ });
+ expect(spans.get("doc.snapshot.dss-cache")?.parentSpanContext?.spanId).toBe(
+ spans.get("doc.open")?.spanContext().spanId,
+ );
+ });
+
+ test("run makes the span active", () => {
+ const root = Telemetry.span("doc.open");
+ const child = root.run(() => Telemetry.span("http request"));
+ child.end();
+ root.end();
+
+ const spans = new Map(
+ spanExporter.getFinishedSpans().map((span) => [span.name, span]),
+ );
+ expect(spans.get("http request")?.parentSpanContext?.spanId).toBe(
+ spans.get("doc.open")?.spanContext().spanId,
+ );
+ });
+
+ test("records span errors", () => {
+ const span = Telemetry.span("doc.open");
+ span.error(new Error("boom"));
+ span.end();
+
+ const [exported] = spanExporter.getFinishedSpans();
+ expect(exported?.status.code).not.toBe(0);
+ expect(exported?.events[0]?.name).toBe("exception");
+ });
+
+ test("emits structured logs", () => {
+ Telemetry.warn("WAL flush not acked", {
+ "document.id": "doc-1",
+ dirty: 3,
+ });
+
+ const [record] = logExporter.getFinishedLogRecords();
+ expect(record?.severityText).toBe("warn");
+ expect(record?.body).toBe("WAL flush not acked");
+ expect(record?.attributes).toMatchObject({
+ "document.id": "doc-1",
+ dirty: 3,
+ });
+ });
+});
diff --git a/packages/observability/src/telemetry.ts b/packages/observability/src/telemetry.ts
new file mode 100644
index 0000000000..c872a5472d
--- /dev/null
+++ b/packages/observability/src/telemetry.ts
@@ -0,0 +1,96 @@
+import { resourceFromAttributes } from "@opentelemetry/resources";
+import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";
+import {
+ type LogAttributes,
+ type TelemetryInitConfig,
+ TelemetryRuntimeConfig,
+} from "./config";
+import { ATTR_DEPLOYMENT_ENVIRONMENT, CONSOLE_PREFIX } from "./constants";
+import { Logging } from "./logging";
+import type { Span } from "./span";
+import { Tracing } from "./tracing";
+
+type SpanOperation<T> = (span: Span) => Promise<T>;
+
+export class Telemetry {
+ static readonly #tracing = new Tracing();
+ static readonly #logging = new Logging();
+ static #initialization: Promise<void> | undefined;
+
+ private constructor() {}
+
+ static readonly config = new TelemetryRuntimeConfig();
+
+ /** Initialize providers once, after the optional async enablement decision. */
+ static init(config: TelemetryInitConfig): Promise<void> {
+ Telemetry.#initialization ??= Telemetry.#initialize(config);
+ return Telemetry.#initialization;
+ }
+
+ static span<T>(name: string, operation: SpanOperation<T>): Promise<T>;
+ static span(name: string): Span;
+ static span<T>(
+ name: string,
+ operation?: SpanOperation<T>,
+ ): Span | Promise<T> {
+ const span = Telemetry.#tracing.span(name);
+ if (!operation) return span;
+
+ return span.run(async () => {
+ try {
+ return await operation(span);
+ } finally {
+ span.end();
+ }
+ });
+ }
+
+ /** Start an HTTP client span. Callers must end the returned span. */
+ static clientSpan(name: string): Span {
+ return Telemetry.#tracing.clientSpan(name);
+ }
+
+ static debug(message: string, attributes?: LogAttributes): void {
+ Telemetry.#logging.debug(message, attributes);
+ }
+
+ static info(message: string, attributes?: LogAttributes): void {
+ Telemetry.#logging.info(message, attributes);
+ }
+
+ static warn(message: string, attributes?: LogAttributes): void {
+ Telemetry.#logging.warn(message, attributes);
+ }
+
+ static error(error: unknown, attributes?: LogAttributes): void {
+ Telemetry.#logging.error(error, attributes);
+ }
+
+ static async flush(): Promise<void> {
+ await Promise.all([Telemetry.#tracing.flush(), Telemetry.#logging.flush()]);
+ }
+
+ static async shutdown(): Promise<void> {
+ await Promise.all([
+ Telemetry.#tracing.shutdown(),
+ Telemetry.#logging.shutdown(),
+ ]);
+ }
+
+ static async #initialize(config: TelemetryInitConfig): Promise<void> {
+ try {
+ if (!(await config.enabled())) return;
+ } catch (error) {
+ console.warn(`${CONSOLE_PREFIX} Failed to resolve enablement`, error);
+ return;
+ }
+
+ const resource = resourceFromAttributes({
+ [ATTR_SERVICE_NAME]: config.serviceName,
+ [ATTR_DEPLOYMENT_ENVIRONMENT]: config.environment,
+ });
+ const getUserId = () => Telemetry.config.userId;
+ Telemetry.#tracing.init(config, resource, getUserId);
+ Telemetry.#logging.init(config, resource, getUserId);
+ }
+}
diff --git a/packages/observability/src/tracing.ts b/packages/observability/src/tracing.ts
new file mode 100644
index 0000000000..37ad20ffb4
--- /dev/null
+++ b/packages/observability/src/tracing.ts
@@ -0,0 +1,46 @@
+import { type Context, context, SpanKind, trace } from "@opentelemetry/api";
+import type { Resource } from "@opentelemetry/resources";
+import type { TelemetryInitConfig } from "./config";
+import { INSTRUMENTATION_SCOPE_NAME } from "./constants";
+import type { TelemetryTracingProvider } from "./provider";
+import { type Span, SpanImpl } from "./span";
+
+export class Tracing {
+ #provider: TelemetryTracingProvider | undefined;
+
+ init(
+ config: TelemetryInitConfig,
+ resource: Resource,
+ getUserId: () => string | undefined,
+ ): void {
+ this.#provider = config.tracingProvider?.(resource, getUserId);
+ }
+
+ span(name: string): Span {
+ return this.#startSpan(name, context.active());
+ }
+
+ clientSpan(name: string): Span {
+ return this.#startSpan(name, context.active(), SpanKind.CLIENT);
+ }
+
+ async flush(): Promise<void> {
+ await this.#provider?.forceFlush();
+ }
+
+ async shutdown(): Promise<void> {
+ await this.#provider?.shutdown();
+ this.#provider = undefined;
+ }
+
+ #startSpan(name: string, parent: Context, kind?: SpanKind): Span {
+ const otelSpan = trace
+ .getTracer(INSTRUMENTATION_SCOPE_NAME)
+ .startSpan(name, kind === undefined ? undefined : { kind }, parent);
+ return new SpanImpl(
+ otelSpan,
+ trace.setSpan(parent, otelSpan),
+ (childName, childParent) => this.#startSpan(childName, childParent),
+ );
+ }
+}
diff --git a/packages/observability/src/web.ts b/packages/observability/src/web.ts
new file mode 100644
index 0000000000..3bff964281
--- /dev/null
+++ b/packages/observability/src/web.ts
@@ -0,0 +1,42 @@
+import { W3CTraceContextPropagator } from "@opentelemetry/core";
+import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
+import type { Resource } from "@opentelemetry/resources";
+import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-base";
+import { WebTracerProvider } from "@opentelemetry/sdk-trace-web";
+import type { TelemetryInitConfig } from "./config";
+import { ATTR_USER_ID } from "./constants";
+import type { TelemetryTracingProvider } from "./provider";
+
+/** Creates the browser OpenTelemetry provider used by the web application. */
+export function createWebTracingProvider(
+ config: TelemetryInitConfig,
+ resource: Resource,
+ getUserId: () => string | undefined,
+): TelemetryTracingProvider {
+ const provider = new WebTracerProvider({
+ resource,
+ spanProcessors: [
+ {
+ onStart: (span) => {
+ const userId = getUserId();
+ if (userId !== undefined) span.setAttribute(ATTR_USER_ID, userId);
+ },
+ onEnd: () => {},
+ forceFlush: () => Promise.resolve(),
+ shutdown: () => Promise.resolve(),
+ },
+ ...(config.tracesUrl
+ ? [
+ new BatchSpanProcessor(
+ new OTLPTraceExporter({ url: config.tracesUrl }),
+ ),
+ ]
+ : []),
+ ],
+ });
+ provider.register({
+ ...(config.contextManager && { contextManager: config.contextManager }),
+ propagator: new W3CTraceContextPropagator(),
+ });
+ return provider;
+}
diff --git a/packages/observability/src/zone.ts b/packages/observability/src/zone.ts
new file mode 100644
index 0000000000..2340ab0544
--- /dev/null
+++ b/packages/observability/src/zone.ts
@@ -0,0 +1,6 @@
+// Importing context-zone loads zone.js, which patches Promise & friends as a
+// module side effect, quarantined in its own entry so it never rides into
+// non-browser consumers. Hosts must import this statically at startup: patching
+// lazily is unreliable, since context would not flow through Promises created
+// before the patch.
+export { ZoneContextManager } from "@opentelemetry/context-zone";
diff --git a/packages/observability/tsconfig.json b/packages/observability/tsconfig.json
new file mode 100644
index 0000000000..b231a634a4
--- /dev/null
+++ b/packages/observability/tsconfig.json
@@ -0,0 +1,18 @@
+{
+ "compilerOptions": {
+ "allowImportingTsExtensions": true,
+ "esModuleInterop": true,
+ "forceConsistentCasingInFileNames": true,
+ "isolatedModules": true,
+ "lib": ["DOM", "DOM.Iterable", "ESNext"],
+ "module": "ESNext",
+ "moduleResolution": "Bundler",
+ "noEmit": true,
+ "skipLibCheck": true,
+ "strict": true,
+ "target": "ESNext",
+ "types": ["@types/bun", "@types/node"]
+ },
+ "include": ["src/**/*"],
+ "exclude": ["node_modules"]
+}
diff --git a/packages/observability/vitest.config.ts b/packages/observability/vitest.config.ts
new file mode 100644
index 0000000000..36acedb5d1
--- /dev/null
+++ b/packages/observability/vitest.config.ts
@@ -0,0 +1,9 @@
+import { defineConfig } from "vitest/config";
+
+export default defineConfig({
+ test: {
+ name: "observability",
+ environment: "jsdom",
+ include: ["src/**/*.test.ts"],
+ },
+});
diff --git a/services/analytics-proxy/bun.lock b/services/analytics-proxy/bun.lock
index ef140d2865..4585fc38b8 100644
--- a/services/analytics-proxy/bun.lock
+++ b/services/analytics-proxy/bun.lock
@@ -4,6 +4,9 @@
"workspaces": {
"": {
"name": "analytics-proxy",
+ "dependencies": {
+ "hono": "^4.6.20",
+ },
"devDependencies": {
"typescript": "^5.7.3",
"wrangler": "^4.25.1",
@@ -159,6 +162,8 @@
"fsevents": ["[email protected]", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="],
+ "hono": ["[email protected]", "", {}, "sha512-XcuyW9qE2kJn07PkecMOBd5Vq/hMy7mmGw+idz1yblbg9N17ijJODrvPkn7/dwL3Kulj8LcRJ69DLOWf91dRUg=="],
+
"kleur": ["[email protected]", "", {}, "sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ=="],
"miniflare": ["[email protected]", "", { "dependencies": { "@cspotcode/source-map-support": "0.8.1", "sharp": "^0.34.5", "undici": "7.18.2", "workerd": "1.20260312.1", "ws": "8.18.0", "youch": "4.1.0-beta.10" }, "bin": { "miniflare": "bootstrap.js" } }, "sha512-YSWxec9ssisqkQgaCgcIQxZlB41E9hMiq1nxUgxXHRrE9NsfyC6ptSt8yfgBobsKIseAVKLTB/iEDpMumBv8oA=="],
diff --git a/services/analytics-proxy/package.json b/services/analytics-proxy/package.json
index c1f4d7a347..98dc10cbd4 100644
--- a/services/analytics-proxy/package.json
+++ b/services/analytics-proxy/package.json
@@ -14,5 +14,8 @@
"devDependencies": {
"typescript": "^5.7.3",
"wrangler": "^4.25.1"
+ },
+ "dependencies": {
+ "hono": "^4.6.20"
}
}
diff --git a/services/analytics-proxy/src/index.ts b/services/analytics-proxy/src/index.ts
index 94756f6230..1ea15a3408 100644
--- a/services/analytics-proxy/src/index.ts
+++ b/services/analytics-proxy/src/index.ts
@@ -1,59 +1,132 @@
-const POSTHOG_PROVIDER_PREFIX = '/i/ph';
-const POSTHOG_HOST = 'us.i.posthog.com';
-// Privacy filter lists block the upstream filename; keep the browser-facing alias opaque.
+/// <reference path="../worker-configuration.d.ts" />
+
+import { Hono } from 'hono';
+import { cors } from 'hono/cors';
+
+// The OTLP traces intake needs a server-side Datadog API key the browser must
+// never hold, so it lives as a Worker secret (set via
+// `wrangler secret put DD_API_KEY`) and is injected here. The generated Env
+// only knows vars/bindings declared in wrangler.jsonc, so augment it with the
+// secret. It is absent locally (the dev OTLP collector trusts the container
+// network), so the injection is conditional. Other providers rely on the
+// client-side token already present on the request, so they need no
+// augmentation.
+declare global {
+ interface Env {
+ /** Datadog API key for the OTLP traces intake. Unset in local dev. */
+ DD_API_KEY?: string;
+ }
+}
+
+const POSTHOG_PREFIX = '/i/ph';
+const POSTHOG_ORIGIN = 'https://us.i.posthog.com';
+// Privacy filter lists block the upstream filename; keep the browser-facing
+// alias opaque and restore the real filename only for the upstream request.
const POSTHOG_RECORDER_SCRIPT_NAME = 'posthog-recorder.js';
const POSTHOG_RECORDER_PROXY_SCRIPT_NAME = 'runtime.js';
-const PROVIDERS: Record<string, string> = {
- [POSTHOG_PROVIDER_PREFIX]: POSTHOG_HOST,
- // Datadog browser logs intake for the us5 site (site: us5.datadoghq.com).
- // The observability SDK's `proxy` option targets the `/i/dd` prefix.
- '/i/dd': 'browser-intake-us5-datadoghq.com',
-};
-
-function getProvider(pathname: string): { apiHost: string; path: string } | null {
- if (
- pathname ===
- `${POSTHOG_PROVIDER_PREFIX}/static/${POSTHOG_RECORDER_PROXY_SCRIPT_NAME}`
- ) {
- return {
- apiHost: POSTHOG_HOST,
- path: `/static/${POSTHOG_RECORDER_SCRIPT_NAME}`,
- };
- }
+// OTLP telemetry is proxied to a per-signal intake origin (full origin,
+// scheme included, so it can be Datadog's https intakes in deployed envs or
+// the plaintext otel-collector in the local pool — where one collector
+// serves both signals). Split per signal because Datadog has no single OTLP
+// origin: traces and logs use different intake hosts. Unlike the static
+// providers the browser can't authenticate, so the Worker injects dd-api-key
+// when present.
+const OTLP_PREFIX = '/i/otlp';
- for (const [prefix, apiHost] of Object.entries(PROVIDERS)) {
- if (pathname.startsWith(prefix)) {
- return { apiHost, path: pathname.slice(prefix.length) || '/' };
- }
- }
+/** Intake origin for an OTLP subpath, or null for an unknown signal. */
+function otlpIntakeUrl(env: Env, path: string): string | null {
+ if (path.startsWith('/v1/traces')) return env.OTLP_TRACES_INTAKE_URL;
+ if (path.startsWith('/v1/logs')) return env.OTLP_LOGS_INTAKE_URL;
return null;
}
-async function handleProxy(request: Request, apiHost: string, pathWithSearch: string): Promise<Response> {
+async function handleProxy(
+ request: Request,
+ origin: string,
+ pathWithSearch: string,
+ extraHeaders?: Record<string, string>
+): Promise<Response> {
const originHeaders = new Headers(request.headers);
originHeaders.delete('cookie');
- originHeaders.set('X-Forwarded-For', request.headers.get('CF-Connecting-IP') || '');
+ originHeaders.set(
+ 'X-Forwarded-For',
+ request.headers.get('CF-Connecting-IP') || ''
+ );
+ for (const [key, value] of Object.entries(extraHeaders ?? {})) {
+ originHeaders.set(key, value);
+ }
- const originRequest = new Request(`https://${apiHost}${pathWithSearch}`, {
+ const originRequest = new Request(`${origin}${pathWithSearch}`, {
method: request.method,
headers: originHeaders,
- body: request.method !== 'GET' && request.method !== 'HEAD' ? await request.arrayBuffer() : null,
+ body:
+ request.method !== 'GET' && request.method !== 'HEAD'
+ ? await request.arrayBuffer()
+ : null,
redirect: request.redirect,
});
return await fetch(originRequest);
}
-export default {
- async fetch(request: Request): Promise<Response> {
- const url = new URL(request.url);
- const provider = getProvider(url.pathname);
+const app = new Hono<{ Bindings: Env }>();
+
+// OTLP uses protobuf cross-origin. Cookies are stripped by handleProxy, so
+// wildcard origins and headers are safe and preflights need no credentials.
+app.use(
+ `${OTLP_PREFIX}/*`,
+ cors({
+ origin: '*',
+ allowMethods: ['POST', 'OPTIONS'],
+ maxAge: 86400,
+ })
+);
+
+// DD is expensive and I don't want someone hammering it. not a perfect solution
+// but better than nothing.
+app.use('*', async (c, next) => {
+ const url = new URL(c.req.url);
+
+ // Cloudflare's edge sets CF-Connecting-IP on every real request; the local
+ // pool's Caddy route forwards it too.
+ const isLocalhost =
+ url.hostname === 'localhost' || url.hostname === '127.0.0.1';
+ const clientIp =
+ c.req.header('CF-Connecting-IP') ||
+ (isLocalhost ? 'local-development' : null);
+ if (!clientIp) return c.text('Missing CF-Connecting-IP', 400);
+
+ const { success } = await c.env.RATE_LIMITER.limit({ key: clientIp });
+ if (!success) return c.text('Too Many Requests', 429);
+
+ await next();
+});
+
+app.all(`${OTLP_PREFIX}/*`, async (c) => {
+ const url = new URL(c.req.url);
+ const path = url.pathname.slice(OTLP_PREFIX.length) || '/';
+ const intake = otlpIntakeUrl(c.env, path);
+ if (!intake) return c.text('Not found', 404);
+
+ return handleProxy(
+ c.req.raw,
+ intake,
+ path + url.search,
+ c.env.DD_API_KEY ? { 'dd-api-key': c.env.DD_API_KEY } : undefined
+ );
+});
+
+app.all(`${POSTHOG_PREFIX}/*`, (c) => {
+ const url = new URL(c.req.url);
+ const proxyPath = url.pathname.slice(POSTHOG_PREFIX.length) || '/';
+ const path =
+ proxyPath === `/static/${POSTHOG_RECORDER_PROXY_SCRIPT_NAME}`
+ ? `/static/${POSTHOG_RECORDER_SCRIPT_NAME}`
+ : proxyPath;
+ return handleProxy(c.req.raw, POSTHOG_ORIGIN, path + url.search);
+});
- if (!provider) {
- return new Response('Not found', { status: 404 });
- }
+app.notFound((c) => c.text('Not found', 404));
- return handleProxy(request, provider.apiHost, provider.path + url.search);
- },
-};
+export default app;
diff --git a/services/analytics-proxy/wrangler.jsonc b/services/analytics-proxy/wrangler.jsonc
index 89e288f74b..b279468698 100644
--- a/services/analytics-proxy/wrangler.jsonc
+++ b/services/analytics-proxy/wrangler.jsonc
@@ -6,23 +6,70 @@
"observability": {
"enabled": true
},
+ // Cap each client to 30 req/s. The binding only allows a 10s or 60s window,
+ // so 300 per 10s expresses 30/s. Bindings are not inherited by named
+ // environments, so this is repeated under each env below.
+ "ratelimits": [
+ {
+ "name": "RATE_LIMITER",
+ "namespace_id": "1001",
+ "simple": { "limit": 300, "period": 10 }
+ }
+ ],
"vars": {
- "POSTHOG_HOST": "https://us.i.posthog.com"
+ "POSTHOG_HOST": "https://us.i.posthog.com",
+ // Full origins (scheme included) the `/i/otlp` route forwards to, split
+ // per signal because Datadog has no single OTLP origin: traces and logs
+ // use different intake hosts. In deployed envs the server-side dd-api-key
+ // is injected (a secret set via `wrangler secret put DD_API_KEY`, never
+ // committed here). In the local pool the docker command overrides both to
+ // the otel-collector alias via --var (one collector serves both paths).
+ "OTLP_TRACES_INTAKE_URL": "https://trace.agent.us5.datadoghq.com",
+ "OTLP_LOGS_INTAKE_URL": "https://otlp-http-intake.logs.us5.datadoghq.com"
},
"env": {
"local": {
+ "ratelimits": [
+ {
+ "name": "RATE_LIMITER",
+ "namespace_id": "1001",
+ "simple": { "limit": 300, "period": 10 }
+ }
+ ],
"vars": {
- "POSTHOG_HOST": "https://us.i.posthog.com"
+ "POSTHOG_HOST": "https://us.i.posthog.com",
+ // Host defaults for a bare `bun run dev`; the docker pool overrides
+ // both to http://otel-collector:4318 via --var (see docker-compose.yml).
+ "OTLP_TRACES_INTAKE_URL": "http://localhost:4318",
+ "OTLP_LOGS_INTAKE_URL": "http://localhost:4318"
}
},
"dev": {
+ "ratelimits": [
+ {
+ "name": "RATE_LIMITER",
+ "namespace_id": "1001",
+ "simple": { "limit": 300, "period": 10 }
+ }
+ ],
"vars": {
- "POSTHOG_HOST": "https://us.i.posthog.com"
+ "POSTHOG_HOST": "https://us.i.posthog.com",
+ "OTLP_TRACES_INTAKE_URL": "https://trace.agent.us5.datadoghq.com",
+ "OTLP_LOGS_INTAKE_URL": "https://otlp-http-intake.logs.us5.datadoghq.com"
}
},
"prod": {
+ "ratelimits": [
+ {
+ "name": "RATE_LIMITER",
+ "namespace_id": "1001",
+ "simple": { "limit": 300, "period": 10 }
+ }
+ ],
"vars": {
- "POSTHOG_HOST": "https://us.i.posthog.com"
+ "POSTHOG_HOST": "https://us.i.posthog.com",
+ "OTLP_TRACES_INTAKE_URL": "https://trace.agent.us5.datadoghq.com",
+ "OTLP_LOGS_INTAKE_URL": "https://otlp-http-intake.logs.us5.datadoghq.com"
}
}
},
diff --git a/services/sync-service/Cargo.toml b/services/sync-service/Cargo.toml
index 262ff6767b..46687da287 100644
--- a/services/sync-service/Cargo.toml
+++ b/services/sync-service/Cargo.toml
@@ -28,20 +28,22 @@ release = false
wasm-opt = true
[dependencies]
-worker = { version = "0.8.1", features = ['d1'] }
+worker = { workspace = true, features = ['d1'] }
tracing.workspace = true
tracing-subscriber = { workspace = true, features = [
'time',
] }
tracing-web = "0.1"
-web-time = "1.1.0"
+web-time.workspace = true
time = { workspace = true, features = ['wasm-bindgen'] }
bebop.workspace = true
sync_service_bebop_schema = { path = "bebop-schema" }
serde_json.workspace = true
serde.workspace = true
+getrandom = { workspace = true, features = ["wasm_js"] }
wasm-bindgen.workspace = true
uuid = { workspace = true, features = ["js"] }
loro = "1.13"
macro_sync_service_jwt = { path = "../../crates/macro_sync_service_jwt" }
matchit = "0.8.4"
+worker-rs-otel = { path = "../../crates/worker-rs-otel" }
diff --git a/services/sync-service/src/cf_worker.rs b/services/sync-service/src/cf_worker.rs
index e2137c89fa..f4f9778ad8 100644
--- a/services/sync-service/src/cf_worker.rs
+++ b/services/sync-service/src/cf_worker.rs
@@ -1,5 +1,5 @@
use bebop::{Record, SubRecord};
-use tracing::error;
+use tracing::{Instrument, error};
use wasm_bindgen::JsValue;
use worker::{Env, Error, Headers, Method, Request, RequestInit, Response, Result, Stub};
@@ -76,6 +76,7 @@ pub async fn copy_handler(env: Env, mut req: Request, document_id: &str) -> Resu
&headers,
MACRO_INTERNAL_AUTH_KEY_HEADER_KEY,
)?;
+ mv_header(og_req.headers(), &headers, worker_rs_otel::TRACEPARENT)?;
ss_req.with_headers(headers);
let mut url = og_req.url()?;
@@ -140,8 +141,13 @@ pub async fn pass_to_durable_object(
document_id: &str,
) -> Result<Response> {
let stub = get_durable_object(env, document_id)?;
+ let span = tracing::info_span!("do.fetch", document.id = %document_id);
+ if let Some(traceparent) = worker_rs_otel::traceparent_for_span(&span) {
+ req.headers()
+ .set(worker_rs_otel::TRACEPARENT, &traceparent)?;
+ }
- let fut = timeout(stub.fetch_with_request(req), DEFAULT_TIMEOUT_MS);
+ let fut = timeout(stub.fetch_with_request(req), DEFAULT_TIMEOUT_MS).instrument(span);
let res = timeit_log!("worker -> do_fetch", fut.await);
Ok(match res {
crate::timeout::TimeoutResult::Ok(x) => x?,
diff --git a/services/sync-service/src/durable_object.rs b/services/sync-service/src/durable_object.rs
index fd206e0f0b..1b04830f7a 100644
--- a/services/sync-service/src/durable_object.rs
+++ b/services/sync-service/src/durable_object.rs
@@ -9,7 +9,7 @@ use bebop::Record;
use loro::{ExportMode, awareness::EphemeralStore};
use matchit::Router;
use serde::{Deserialize, Serialize};
-use tracing::{debug, error, info, instrument, trace, warn};
+use tracing::{Instrument, debug, error, info, instrument, trace, warn};
use worker::{
Cors, Date, DurableObject, Env, Error, Method, Request, Response, ResponseBody,
ResponseBuilder, Result, ScheduledTime, State, WebSocket, WebSocketIncomingMessage,
@@ -354,6 +354,7 @@ impl DocumentSyncSession {
match (
*matched.value,
matched.params.get("document_id").inspect(|document_id| {
+ tracing::Span::current().record("document.id", *document_id);
trace!(route =? matched.value, document_id = document_id, "matched route");
}),
) {
@@ -682,6 +683,9 @@ impl DocumentSyncSession {
.and_then(|state| state.export_shallow_snapshot());
if let Ok(snapshot) = snapshot {
+ // Size of the initial sync this connect sent — the server end
+ // of the client's `doc.sync.initial-sync` span.
+ tracing::Span::current().record("snapshot.bytes", snapshot.len());
websocket::send_initial_sync(
&pair.server,
snapshot.as_slice(),
@@ -936,11 +940,26 @@ impl DurableObject for DocumentSyncSession {
.with_cors(&cors(set_allow_origin.as_deref()))?
.empty());
}
- let res = self
- .inner_fetch(req)
- .await
- .context("DurableObject::fetch error")?;
- res.with_cors(&cors(set_allow_origin.as_deref()))
+ let traceparent = worker_rs_otel::traceparent_from_request(&req);
+ let (remote_id, remote_parent) = worker_rs_otel::remote_fields(traceparent.as_ref());
+ let path = req.path();
+ let span = tracing::info_span!(
+ "do.request",
+ http.path = %path,
+ document.id = tracing::field::Empty,
+ snapshot.bytes = tracing::field::Empty,
+ trace.remote_id = %remote_id,
+ trace.remote_parent = %remote_parent,
+ );
+
+ let res = worker_rs_otel::scope(
+ &self.env,
+ &self.state,
+ self.inner_fetch(req).instrument(span),
+ )
+ .await;
+ res.context("DurableObject::fetch error")?
+ .with_cors(&cors(set_allow_origin.as_deref()))
}
async fn websocket_message(&self, ws: WebSocket, msg: WebSocketIncomingMessage) -> Result<()> {
@@ -948,41 +967,82 @@ impl DurableObject for DocumentSyncSession {
const PING: &str = "ping";
let binary_message = match msg {
WebSocketIncomingMessage::String(message) => {
- // TODO do keepalive?
+ // Heartbeat: deliberately unspanned to keep it out of traces.
if message == PING {
ws.send_with_str(PONG).ok();
} else {
- warn!("Received unknown 'String' message: {message:?}");
+ return worker_rs_otel::scope(&self.env, &self.state, async {
+ warn!("Received unknown 'String' message: {message:?}");
+ Ok(())
+ })
+ .await;
}
return Ok(());
}
WebSocketIncomingMessage::Binary(bm) => bm,
};
+ worker_rs_otel::scope(&self.env, &self.state, async {
+ let mut telemetry = websocket::InboundMessageTelemetry::new(binary_message.len());
+
+ let res: Result<()> = async {
+ let document_id = self.document_id().await.inspect_err(|_| {
+ telemetry.record_error_stage("document_context");
+ })?;
+ let ws_id = get_ws_id_from_tags(&self.state.get_tags(&ws)).ok();
+ telemetry.record_context(&document_id, ws_id);
+
+ let message =
+ websocket::deserialize_message(&binary_message).inspect_err(|_| {
+ telemetry.record_message_type("invalid");
+ telemetry.record_error_stage("deserialize");
+ })?;
+ telemetry.record_message_type(websocket::message_type(&message));
+
+ websocket::process_message(
+ &ws,
+ &document_id,
+ &*self.document_state().await?,
+ &*self.session_storage().await?,
+ &self.awareness,
+ message,
+ self.msg_buffer.clone(),
+ self,
+ &mut telemetry,
+ )
+ .await
+ .inspect_err(|_| {
+ telemetry.record_error_stage("process");
+ })
+ .context("failed to process websocket message")?;
+
+ bump_alarm(&self.state)
+ .await
+ .inspect_err(|_| {
+ telemetry.record_error_stage("alarm");
+ })
+ .context("failed to keep document alive")?;
+
+ Ok(())
+ }
+ .await;
- websocket::process_message(
- &ws,
- &self.document_id().await?,
- &*self.document_state().await?,
- &*self.session_storage().await?,
- &self.awareness,
- binary_message,
- self.msg_buffer.clone(),
- self,
- )
+ if let Err(error) = &res {
+ let span = telemetry.error_span();
+ {
+ let _entered = span.enter();
+ tracing::error!(error = ?error, "failed to handle websocket message");
+ }
+ }
+ res
+ })
.await
- .context("failed to process websocket message")?;
-
- bump_alarm(&self.state)
- .await
- .context("failed to keep document alive")?;
-
- Ok(())
}
/// Save document if needed
- #[instrument(skip_all, err)]
async fn alarm(&self) -> Result<Response> {
- let state = match self
+ let span = tracing::info_span!("do.alarm");
+ worker_rs_otel::scope(&self.env, &self.state, async {
+ let state = match self
.document_state()
.await
.context("failed to get document_state")
@@ -1046,7 +1106,10 @@ impl DurableObject for DocumentSyncSession {
info!("durable object has reached 0 connections")
}
- Response::ok("ok")
+ Response::ok("ok")
+ }
+ .instrument(span))
+ .await
}
async fn websocket_close(
@@ -1056,40 +1119,46 @@ impl DurableObject for DocumentSyncSession {
_reason: String,
_was_clean: bool,
) -> Result<()> {
- let peer_ids = Wsm::new(self, &ws).get_peer_ids().await?;
- for peer_id in peer_ids {
- self.awareness.delete(&peer_id.to_string());
- let update = self.awareness.encode(&peer_id.to_string());
-
- // Don't silently discard the error
- websocket::broadcast_awareness(
- &ws,
- self.state.get_websockets().as_slice(),
- update.as_slice(),
- self.msg_buffer.clone(),
- )
- .context("failed to broadcast awareness")?;
- }
+ worker_rs_otel::scope(&self.env, &self.state, async {
+ let peer_ids = Wsm::new(self, &ws).get_peer_ids().await?;
+ for peer_id in peer_ids {
+ self.awareness.delete(&peer_id.to_string());
+ let update = self.awareness.encode(&peer_id.to_string());
+
+ // Don't silently discard the error
+ websocket::broadcast_awareness(
+ &ws,
+ self.state.get_websockets().as_slice(),
+ update.as_slice(),
+ self.msg_buffer.clone(),
+ )
+ .context("failed to broadcast awareness")?;
+ }
- if self.state.get_websockets().len() == 1
- && let Ok(document_id) = self.document_id().await
- && let Ok(state) = self.document_state().await
- && let Ok(snapshot) = state.export_shallow_snapshot()
- {
- let env = self.env.clone();
- self.state.wait_until(async move {
- report_new_doc_state(&document_id, &snapshot, &env).await;
- report_interaction(&document_id, &env, InteractionReason::LastLeave).await;
- });
- }
- Ok(())
+ if self.state.get_websockets().len() == 1
+ && let Ok(document_id) = self.document_id().await
+ && let Ok(state) = self.document_state().await
+ && let Ok(snapshot) = state.export_shallow_snapshot()
+ {
+ let env = self.env.clone();
+ self.state.wait_until(async move {
+ report_new_doc_state(&document_id, &snapshot, &env).await;
+ report_interaction(&document_id, &env, InteractionReason::LastLeave).await;
+ });
+ }
+ Ok(())
+ })
+ .await
}
async fn websocket_error(&self, ws: WebSocket, error: Error) -> Result<()> {
- let ws_id = get_ws_id(&self.state, &ws)?;
- error!(ws_id = ws_id, error = ?error, "websocket error");
- // TODO update awareness stuff
- Ok(())
+ worker_rs_otel::scope(&self.env, &self.state, async {
+ let ws_id = get_ws_id(&self.state, &ws)?;
+ error!(ws_id = ws_id, error = ?error, "websocket error");
+ // TODO update awareness stuff
+ Ok(())
+ })
+ .await
}
}
diff --git a/services/sync-service/src/lib.rs b/services/sync-service/src/lib.rs
index 6285506a80..c473781bf0 100644
--- a/services/sync-service/src/lib.rs
+++ b/services/sync-service/src/lib.rs
@@ -49,6 +49,7 @@ fn inner_start() {
performance_layer()
.with_details_from_fields(tracing_subscriber::fmt::format::Pretty::default()),
)
+ .with(worker_rs_otel::OtelLayer::new("sync-service"))
.init();
}
#[cfg(not(target_arch = "wasm32"))]
@@ -64,7 +65,21 @@ fn start() {
}
#[event(fetch)]
-async fn fetch(req: worker::Request, env: Env, _ctx: Context) -> Result<worker::Response> {
+async fn fetch(req: worker::Request, env: Env, ctx: Context) -> Result<worker::Response> {
use crate::cf_worker::router;
- router(env, req).await
+ use tracing::Instrument;
+
+ let traceparent = worker_rs_otel::traceparent_from_request(&req);
+ let (remote_id, remote_parent) = worker_rs_otel::remote_fields(traceparent.as_ref());
+ let method = req.method().to_string();
+ let path = req.path();
+ let span = tracing::info_span!(
+ "request",
+ http.method = %method,
+ http.path = %path,
+ trace.remote_id = %remote_id,
+ trace.remote_parent = %remote_parent,
+ );
+
+ worker_rs_otel::scope(&env, &ctx, router(env.clone(), req).instrument(span)).await
}
diff --git a/services/sync-service/src/websocket.rs b/services/sync-service/src/websocket.rs
index 45ddf378cb..b73c5ad7e6 100644
--- a/services/sync-service/src/websocket.rs
+++ b/services/sync-service/src/websocket.rs
@@ -23,6 +23,118 @@ fn serialize<'a, T: bebop::Record<'a>>(
Ok(msg_buf)
}
+/// Decodes an inbound WebSocket payload once, before its trace span is created.
+pub fn deserialize_message(message: &[u8]) -> Result<FromPeer<'_>> {
+ if message.len() > MAX_MESSAGE_SIZE {
+ tracing::warn!("received message might be too large {}", message.len());
+ }
+ FromPeer::deserialize(message).context(format!(
+ "failed to deserialize message, message length {}",
+ message.len()
+ ))
+}
+
+/// Returns the protocol-level type of an inbound WebSocket message.
+pub fn message_type(message: &FromPeer<'_>) -> &'static str {
+ match message {
+ FromPeer::PeerRegisterId { .. } => "register_id",
+ FromPeer::PeerUpdate { .. } => "update",
+ FromPeer::PeerAwareness { .. } => "awareness",
+ FromPeer::PeerRequestSince { .. } => "request_since",
+ FromPeer::PeerRequestSnapshot { .. } => "request_snapshot",
+ FromPeer::Unknown => "unknown",
+ }
+}
+
+/// Diagnostic fields collected while handling an inbound WebSocket message.
+pub struct InboundMessageTelemetry {
+ document_id: Option<String>,
+ ws_id: Option<String>,
+ message_type: &'static str,
+ message_bytes: usize,
+ error_stage: &'static str,
+ op_id: Option<String>,
+ peer_id: Option<u64>,
+ update_bytes: Option<usize>,
+ response_bytes: Option<usize>,
+ broadcast_targets: Option<usize>,
+}
+
+impl InboundMessageTelemetry {
+ /// Start collecting diagnostics without creating a span.
+ pub fn new(message_bytes: usize) -> Self {
+ Self {
+ document_id: None,
+ ws_id: None,
+ message_type: "unknown",
+ message_bytes,
+ error_stage: "unknown",
+ op_id: None,
+ peer_id: None,
+ update_bytes: None,
+ response_bytes: None,
+ broadcast_targets: None,
+ }
+ }
+
+ /// Record identifiers available from the Durable Object callback.
+ pub fn record_context(&mut self, document_id: &str, ws_id: Option<String>) {
+ self.document_id = Some(document_id.to_string());
+ self.ws_id = ws_id;
+ }
+
+ /// Record the decoded protocol message type.
+ pub fn record_message_type(&mut self, message_type: &'static str) {
+ self.message_type = message_type;
+ }
+
+ /// Record which stage of message handling failed.
+ pub fn record_error_stage(&mut self, error_stage: &'static str) {
+ self.error_stage = error_stage;
+ }
+
+ /// Create the only span emitted for this message, after handling fails.
+ pub fn error_span(&self) -> tracing::Span {
+ let span = tracing::error_span!(
+ "ws.message.error",
+ document.id = tracing::field::Empty,
+ ws.id = tracing::field::Empty,
+ message.type = self.message_type,
+ message.bytes = self.message_bytes,
+ error.stage = self.error_stage,
+ op.id = tracing::field::Empty,
+ peer.id = tracing::field::Empty,
+ update.bytes = tracing::field::Empty,
+ response.bytes = tracing::field::Empty,
+ broadcast.targets = tracing::field::Empty,
+ );
+
+ if let Some(document_id) = &self.document_id {
+ span.record("document.id", document_id.as_str());
+ }
+ if let Some(ws_id) = &self.ws_id {
+ span.record("ws.id", ws_id.as_str());
+ }
+ if let Some(op_id) = &self.op_id {
+ span.record("op.id", op_id.as_str());
+ }
+ if let Some(peer_id) = self.peer_id {
+ span.record("peer.id", peer_id);
+ }
+ if let Some(update_bytes) = self.update_bytes {
+ span.record("update.bytes", update_bytes);
+ }
+ if let Some(response_bytes) = self.response_bytes {
+ span.record("response.bytes", response_bytes);
+ }
+ if let Some(broadcast_targets) = self.broadcast_targets {
+ span.record("broadcast.targets", broadcast_targets);
+ }
+
+ span
+ }
+}
+
/// Sends the initial sync message to the client over the websocket
/// The initial sync message contains the snapshot of the current state of the document
pub fn send_initial_sync(
@@ -79,19 +191,11 @@ pub async fn process_message(
document_state: &DocumentState,
session_storage: &SessionStorage,
awareness: &EphemeralStore,
- message: Vec<u8>,
+ message: FromPeer<'_>,
buf: Arc<Mutex<Vec<u8>>>,
dss: &DocumentSyncSession,
+ telemetry: &mut InboundMessageTelemetry,
) -> Result<()> {
- if message.len() > MAX_MESSAGE_SIZE {
- tracing::warn!("received message might be too large {}", message.len());
- }
-
- let message: FromPeer = FromPeer::deserialize(message.as_slice()).context(format!(
- "failed to deserialize message, message length {}",
- message.len()
- ))?;
-
trace!(
message = tracing::field::display(&message),
"process websocket message"
@@ -100,6 +204,7 @@ pub async fn process_message(
// Handle peer id registration
// This registers a peer id to the owner of the current websocket
FromPeer::PeerRegisterId { peerid } => {
+ telemetry.peer_id = Some(peerid);
Wsm::new(dss, ws)
.add_new_peerid(peerid, document_id)
.await?;
@@ -108,6 +213,8 @@ pub async fn process_message(
// Should extract binary update and broadcast it to all other connected peers
// Should also store the update in the operation log to be applied to the remote doc
FromPeer::PeerUpdate { updates, id } => {
+ telemetry.op_id = Some(id.to_string());
+ telemetry.update_bytes = Some(updates.iter().map(|u| u.len()).sum());
if !Wsm::new(dss, ws).can_edit().await? {
tracing::warn!("received update from peer without edit permission");
return Ok(());
@@ -115,6 +222,7 @@ pub async fn process_message(
let peer_ids = Wsm::new(dss, ws).get_peer_ids().await.unwrap_or_default();
let peer_id = peer_ids.first().copied();
+ telemetry.peer_id = peer_id;
let now_ms = web_time::SystemTime::now()
.duration_since(web_time::UNIX_EPOCH)
.map(|d| d.as_millis() as i64)
@@ -148,9 +256,19 @@ pub async fn process_message(
let mut buf = buf.lock("serialize RemoteUpdateAck in PeerUpdate handler");
let serialized =
serialize(message, &mut buf).context("Failed serializing update")?;
- ws.send_with_bytes(serialized).context("failed to send")?;
+ ws.send_with_bytes(serialized)
+ .inspect_err(|error| {
+ tracing::error!(error = ?error, op.id = %id, "failed to send update ack");
+ })
+ .context("failed to send update ack")?;
}
+ // Peers receiving the rebroadcast (everyone but the sender). The
+ // count is the key Teo/Hutch diagnostic: if it excludes a peer the
+ // DO thinks isn't connected, that's the dropped-update bug.
+ let sockets = dss.get_websockets();
+ let targets: Vec<&WebSocket> = sockets.iter().filter(|w| w != &ws).collect();
+ telemetry.broadcast_targets = Some(targets.len());
for update in &updates {
// broadcast each update to other peers
let message = FromRemote::RemoteUpdate {
@@ -159,7 +277,7 @@ pub async fn process_message(
let mut buf = buf.lock("serialize RemoteUpdate in PeerUpdate handler");
let serialized =
serialize(message, &mut buf).context("Failed serializing update")?;
- for w in dss.get_websockets().iter().filter(|w| w != &ws) {
+ for w in &targets {
// A dead peer socket must not abort delivery to the
// remaining peers.
if let Err(e) = w.send_with_bytes(serialized) {
@@ -178,7 +296,10 @@ pub async fn process_message(
return Ok(());
}
let encodede = awareness.encode_all();
- broadcast_awareness(ws, &dss.get_websockets(), &encodede, buf)
+ let sockets = dss.get_websockets();
+ telemetry.broadcast_targets =
+ Some(sockets.iter().filter(|socket| socket != &ws).count());
+ broadcast_awareness(ws, &sockets, &encodede, buf)
.context("failed to broadcast awareness")?;
}
// Handle a peer requesting a specific set of updates from the document.
@@ -191,6 +312,8 @@ pub async fn process_message(
let update = document_state
.export_updates_since(&decoded)
.context("failed to export updates")?;
+ // Server end of the client's `doc.sync.catchup` span.
+ telemetry.response_bytes = Some(update.len());
// Echo the client's *original* vv bytes back, not a re-encoded copy.
// The client correlates the response by byte-exact match on the vv it
@@ -211,6 +334,7 @@ pub async fn process_message(
// Peer is requesting a snapshot from the remote
FromPeer::PeerRequestSnapshot {} => {
let snapshot = document_state.export_shallow_snapshot()?;
+ telemetry.response_bytes = Some(snapshot.len());
let message = FromRemote::RemoteSnapshot {
snapshot: SliceWrapper::Raw(&snapshot),
diff --git a/services/sync-service/wrangler.docker.toml b/services/sync-service/wrangler.docker.toml
index 2a653af9f9..68c1342960 100644
--- a/services/sync-service/wrangler.docker.toml
+++ b/services/sync-service/wrangler.docker.toml
@@ -33,6 +33,12 @@ INTERNAL_API_SECRET_KEY = "LOCAL_API_KEY"
SPS_URL = "http://search-processing-service:8080"
DSS_URL = "http://document_storage_service:8080"
DSS_INTERNAL_AUTH_KEY = "local"
+# Tags exported spans with deployment.environment=local, matching the
+# Datadog agent's DD_ENV so the env:local trace/log views include them.
+ENVIRONMENT = "local"
+# jaeger/datadog-agent alias on the compose `services` network.
+# Unset in deployed envs = trace and log export disabled.
+OTEL_EXPORTER_OTLP_ENDPOINT = "http://otel-collector:4318"
[[kv_namespaces]]
binding = "DOCUMENT_VERSIONING_KV"
diff --git a/services/sync-service/wrangler.toml b/services/sync-service/wrangler.toml
index 8b49e8f9ef..9252dfcd7d 100644
--- a/services/sync-service/wrangler.toml
+++ b/services/sync-service/wrangler.toml
@@ -34,6 +34,12 @@ INTERNAL_API_SECRET_KEY = "LOCAL_API_KEY"
SPS_URL = "https://search-processing-dev.macro.com"
DSS_URL = "http://localhost:8086"
DSS_INTERNAL_AUTH_KEY = "local"
+# Tags exported spans with deployment.environment=local, matching the
+# Datadog agent's DD_ENV so the env:local trace/log views include them.
+ENVIRONMENT = "local"
+# Local jaeger/datadog-agent OTLP receiver (docker port-mapped to the host).
+# Unset in deployed envs = trace and log export disabled.
+OTEL_EXPORTER_OTLP_ENDPOINT = "http://localhost:4318"
[[kv_namespaces]]
binding = "DOCUMENT_VERSIONING_KV"
diff --git a/tooling/xtask/crates/xtask_local/src/local/cli.rs b/tooling/xtask/crates/xtask_local/src/local/cli.rs
index 55d14e8c9d..4d207609e8 100644
--- a/tooling/xtask/crates/xtask_local/src/local/cli.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/cli.rs
@@ -118,6 +118,48 @@ pub struct RunArgs {
/// Stream subprocess output and show per-step timings.
#[arg(long, short)]
pub verbose: bool,
+ /// Start a local OTLP trace collector and wire services to export to it.
+ /// Omit to leave tracing off (the default) — see `docker/docker-compose.yml`
+ /// for what each backend does.
+ #[arg(long)]
+ pub traces: Option<TracesBackend>,
+}
+
+/// Which OTLP trace collector `--traces` should bring up.
+#[derive(clap::ValueEnum, Clone, Copy, Debug, PartialEq, Eq)]
+pub enum TracesBackend {
+ /// Fully local trace viewer at http://localhost:16686, no account needed.
+ Jaeger,
+ /// Forwards to Datadog APM (us5); requires `DD_API_KEY`.
+ Datadog,
+}
+
+impl TracesBackend {
+ /// The compose service name gated by this backend's profile.
+ pub fn compose_service(self) -> &'static str {
+ match self {
+ TracesBackend::Jaeger => "jaeger",
+ TracesBackend::Datadog => "datadog-agent",
+ }
+ }
+
+ /// The compose profile gating this backend's service.
+ pub fn compose_profile(self) -> &'static str {
+ match self {
+ TracesBackend::Jaeger => "jaeger",
+ TracesBackend::Datadog => "datadog",
+ }
+ }
+
+ /// Env var the backend needs to forward telemetry, if any. Checked
+ /// before starting the collector so a missing key fails loud instead of
+ /// silently dropping everything at the intake.
+ pub fn required_env(self) -> Option<&'static str> {
+ match self {
+ TracesBackend::Jaeger => None,
+ TracesBackend::Datadog => Some("DD_API_KEY"),
+ }
+ }
}
#[derive(Args, Clone, Default)]
diff --git a/tooling/xtask/crates/xtask_local/src/local/e2e.rs b/tooling/xtask/crates/xtask_local/src/local/e2e.rs
index 0f9118a0a4..2a67f8d86a 100644
--- a/tooling/xtask/crates/xtask_local/src/local/e2e.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/e2e.rs
@@ -122,6 +122,7 @@ pub fn run(args: &LocalE2eArgs) -> Result<()> {
env: EnvArgs::default(),
build: BuildArgs::default(),
no_frontend: true,
+ traces: None,
verbose: false,
},
..stack::UpArgs::default()
diff --git a/tooling/xtask/crates/xtask_local/src/local/env_layer.rs b/tooling/xtask/crates/xtask_local/src/local/env_layer.rs
index 0d35e668da..20227761a8 100644
--- a/tooling/xtask/crates/xtask_local/src/local/env_layer.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/env_layer.rs
@@ -60,6 +60,18 @@ pub fn resolve(
for (k, v) in local_env::LocalEnv::for_instance(mode, instance).to_env() {
env.insert(k, v);
}
+ // Opt-in local trace export: point services at the local OTLP collector
+ // (docker-network alias `otel-collector`) only when one answers on the
+ // OTLP HTTP port, so services don't spam export errors when none is
+ // running. Both viewers bind 4318 — Jaeger (compose profile `jaeger`)
+ // and the Datadog agent (profile `datadog`) — so (re)start the stack
+ // after starting one to pick up tracing.
+ if super::summary::port_open(4318) {
+ env.insert(
+ "OTEL_EXPORTER_OTLP_ENDPOINT".into(),
+ "http://otel-collector:4317".into(),
+ );
+ }
}
// `--env-file` overlay (the only dotenv we still parse).
diff --git a/tooling/xtask/crates/xtask_local/src/local/frontend.rs b/tooling/xtask/crates/xtask_local/src/local/frontend.rs
index 804e19dbbd..d8197eb8bf 100644
--- a/tooling/xtask/crates/xtask_local/src/local/frontend.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/frontend.rs
@@ -90,8 +90,15 @@ pub fn build_static(stage: &Stage, instance: &Instance, prebuilt: Option<&Path>)
/// The env the dev server runs with. Both local and dev point the whole app at
/// the local proxy origin (single backend origin); the modes differ only in
/// what the *services* behind the proxy talk to (local infra vs dev resources).
-fn dev_env(instance: &Instance) -> Vec<(String, String)> {
- vec![
+///
+/// When a trace collector is up (`--traces`), point the OTel exporter at the
+/// analytics-proxy through the same proxy origin (`/i/otlp`), so tracing works
+/// for any instance (named instances have no host-port binding for the proxy)
+/// and exercises the real browser -> proxy -> collector path. Left unset
+/// otherwise. This overrides the bare-dev defaults in apps/web/.env.local
+/// (Vite lets process env win).
+fn dev_env(instance: &Instance, traces_enabled: bool) -> Vec<(String, String)> {
+ let mut env = vec![
(
"PORT".to_string(),
instance.port(Port::Frontend).to_string(),
@@ -101,7 +108,25 @@ fn dev_env(instance: &Instance) -> Vec<(String, String)> {
"VITE_LOCAL_BACKEND_ORIGIN".to_string(),
proxy::url(instance),
),
- ]
+ ];
+ if traces_enabled {
+ env.push((
+ "VITE_OTEL_EXPORTER_URL".to_string(),
+ format!("{}/i/otlp/v1/traces", proxy::url(instance)),
+ ));
+ // Tag frontend telemetry with the same env the Datadog agent uses
+ // (DD_ENV, default `local`), so the summary's traces/logs links —
+ // filtered by that env — actually match the emitted spans/records.
+ env.push((
+ "VITE_OTEL_ENV".to_string(),
+ macro_env_var::maybe_read_env("DD_ENV").unwrap_or_else(|| "local".into()),
+ ));
+ }
+ env.push((
+ "VITE_ENABLE_BROWSER_OTEL".to_string(),
+ traces_enabled.to_string(),
+ ));
+ env
}
/// Poll the backend (auth health, through the proxy) until ready.
@@ -163,7 +188,12 @@ fn tail_str(s: &str, n: usize) -> String {
/// is captured (suppressed — the vite banner would duplicate the summary's URL)
/// but retained so an unexpected exit can be diagnosed. Returns `None` in
/// dry-run mode.
-pub fn start(stage: &Stage, instance: &Instance, mode: Mode) -> Result<Option<Frontend>> {
+pub fn start(
+ stage: &Stage,
+ instance: &Instance,
+ mode: Mode,
+ traces_enabled: bool,
+) -> Result<Option<Frontend>> {
if mode.spec().wait_backend_before_frontend {
wait_backend_ready(stage, instance)?;
}
@@ -205,7 +235,7 @@ pub fn start(stage: &Stage, instance: &Instance, mode: Mode) -> Result<Option<Fr
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped());
- for (k, v) in dev_env(instance) {
+ for (k, v) in dev_env(instance, traces_enabled) {
cmd.env(k, v);
}
let mut child = cmd.spawn().context("launching `bun run dev`")?;
diff --git a/tooling/xtask/crates/xtask_local/src/local/gen_compose.rs b/tooling/xtask/crates/xtask_local/src/local/gen_compose.rs
index b6c0e847cd..09d8f9d844 100644
--- a/tooling/xtask/crates/xtask_local/src/local/gen_compose.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/gen_compose.rs
@@ -33,12 +33,14 @@ pub const CADDY_IMAGE: &str = "caddy:2-alpine";
/// Base-compose services that bind fixed host ports but aren't in our inventory
/// (reached via the proxy / container network, not the host). For named
/// instances we drop their host-port bindings so concurrent stacks don't collide
-/// on 8787/6969/8096/8100.
+/// on 8787/6969/8096/8100/8933.
const AUX_HOST_PORT_SERVICES: &[&str] = &[
"sync_service",
"websocket_service",
"lexical_service",
"static_file_cdn",
+ "ai_editing_worker",
+ "analytics_proxy",
];
/// The host directory bind-mounted into FusionAuth at
diff --git a/tooling/xtask/crates/xtask_local/src/local/mod.rs b/tooling/xtask/crates/xtask_local/src/local/mod.rs
index b56e569520..d374b5ce62 100644
--- a/tooling/xtask/crates/xtask_local/src/local/mod.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/mod.rs
@@ -185,6 +185,13 @@ pub fn run_stack(mode: Mode, args: &cli::RunArgs) -> Result<()> {
stack::clear_state(&instance)?;
}
+ // Before `prepare` (which resolves env and reads the OTLP port to decide
+ // whether to wire `OTEL_EXPORTER_OTLP_ENDPOINT`), so a `--traces` run gets
+ // the same auto-wiring as a collector started manually beforehand.
+ if let Some(backend) = args.traces {
+ ensure_tracing_backend(&stage, backend)?;
+ }
+
// `run_local`/`run_dev` are full delete + full create: tear the previous
// stack and ALL its stateful volumes down so the bring-up is always from a
// clean slate. That makes the command unconditionally idempotent — no
@@ -240,7 +247,7 @@ pub fn run_stack(mode: Mode, args: &cli::RunArgs) -> Result<()> {
stage.note("frontend disabled (--no-frontend)");
None
} else {
- frontend::start(&stage, &instance, mode)?
+ frontend::start(&stage, &instance, mode, args.traces.is_some())?
};
summary::print(
@@ -650,7 +657,93 @@ fn bring_up_app(
}
up.arg("proxy");
}
- stage.run("Starting services (docker compose up -d)", &mut up)
+ stage.run("Starting services (docker compose up -d)", &mut up)?;
+ connect_tracing_network(instance);
+ Ok(())
+}
+
+/// Start the requested trace collector (`--traces`) under its own compose
+/// project, the same one a developer would use manually — see
+/// `docker/docker-compose.yml`'s `jaeger`/`datadog-agent` services. Global and
+/// idempotent (like `start_localstack`): one collector per machine, shared
+/// across instances, left running across `run_local` invocations.
+fn ensure_tracing_backend(stage: &Stage, backend: cli::TracesBackend) -> Result<()> {
+ // A keyed backend with a missing key accepts telemetry locally and drops
+ // every payload at the vendor intake (403), which looks like "traces are
+ // broken" rather than "key is missing" — so fail loud up front.
+ if let Some(var) = backend.required_env()
+ && macro_env_var::maybe_read_env(var).is_none_or(|v| v.is_empty())
+ {
+ anyhow::bail!(
+ "--traces {} requires the {var} env var to be set (export it in \
+ the shell you run this from)",
+ backend.compose_profile()
+ );
+ }
+ let compose = repo_root().join("docker/docker-compose.yml");
+ let mut up = Command::new("docker");
+ up.arg("compose")
+ .arg("--project-directory")
+ .arg(repo_root())
+ .arg("-f")
+ .arg(&compose)
+ .arg("--profile")
+ .arg(backend.compose_profile())
+ .arg("up")
+ .arg("-d")
+ .arg("--remove-orphans")
+ .arg(backend.compose_service());
+ stage.run(
+ &format!("Starting {} trace collector", backend.compose_profile()),
+ &mut up,
+ )?;
+
+ // `up -d` returns once the container starts, not once it's accepting
+ // connections; `env_layer::resolve` (which runs right after this, in
+ // `prepare`) needs the OTLP port live NOW to decide whether to wire
+ // `OTEL_EXPORTER_OTLP_ENDPOINT`, so wait for it here instead of racing.
+ for _ in 0..50 {
+ if summary::port_open(4318) {
+ return Ok(());
+ }
+ std::thread::sleep(std::time::Duration::from_millis(100));
+ }
+ anyhow::bail!(
+ "{} trace collector did not come up on port 4318 in time",
+ backend.compose_profile()
+ )
+}
+
+/// If the global trace collector (Jaeger or the Datadog agent) is running,
+/// attach it to this instance's `services` network under the `otel-collector`
+/// alias that the injected `OTEL_EXPORTER_OTLP_ENDPOINT` points at (see
+/// `env_layer::resolve`). The collectors are started under their own compose
+/// project (`docker/docker-compose.yml`, profiles `jaeger`/`datadog`), and
+/// Compose prefixes networks per project — so without this, instance
+/// containers can't resolve `otel-collector` and span exports fail with DNS
+/// errors. Best-effort: "already connected" (rerun) and other failures are
+/// ignored, they only mean traces don't flow.
+fn connect_tracing_network(instance: &Instance) {
+ if !summary::port_open(4318) {
+ return;
+ }
+ // Find the collector container (Jaeger or Datadog agent — both publish the
+ // OTLP HTTP port) by that port rather than assuming a container name, in
+ // case it was started under a different project.
+ let Some(collector) = Command::new("docker")
+ .args(["ps", "--filter", "publish=4318", "--format", "{{.Names}}"])
+ .output()
+ .ok()
+ .map(|out| String::from_utf8_lossy(&out.stdout).trim().to_string())
+ .filter(|name| !name.is_empty())
+ else {
+ return;
+ };
+ let _ = Command::new("docker")
+ .args(["network", "connect", "--alias", "otel-collector"])
+ .arg(format!("{}_services", instance.project_name()))
+ .arg(collector)
+ .output();
}
/// Restart the given services' containers so they re-exec their freshly built
diff --git a/tooling/xtask/crates/xtask_local/src/local/proxy.rs b/tooling/xtask/crates/xtask_local/src/local/proxy.rs
index b809517aa6..062689a36a 100644
--- a/tooling/xtask/crates/xtask_local/src/local/proxy.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/proxy.rs
@@ -121,6 +121,16 @@ const SPECIAL_ROUTES: &str = r#" @websocket path /websocket /websocket/*
uri strip_prefix /sync
reverse_proxy sync-service:8787
}
+ # Analytics/telemetry proxy worker (PostHog and OTLP traces/logs).
+ # No prefix strip: the worker itself routes on the /i/{ph,dd,otlp} prefix,
+ # and it listens on 8098, not the :8080 the generated routes assume.
+ # Set CF-Connecting-IP (absent without Cloudflare's edge in front) so the
+ # worker's rate-limit keying has a client IP instead of erroring.
+ handle /i/* {
+ reverse_proxy analytics-proxy:8098 {
+ header_up CF-Connecting-IP {http.request.remote.host}
+ }
+ }
handle_path /lexical/* {
reverse_proxy lexical-service:8096
}
diff --git a/tooling/xtask/crates/xtask_local/src/local/proxy/test.rs b/tooling/xtask/crates/xtask_local/src/local/proxy/test.rs
index d7917ca37d..070ffce3e1 100644
--- a/tooling/xtask/crates/xtask_local/src/local/proxy/test.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/proxy/test.rs
@@ -35,6 +35,15 @@ fn websocket_services_use_a_matcher() {
assert!(caddy.contains("handle_path /auth/* {"));
}
+/// The analytics-proxy worker is reached through the single origin at `/i/*`
+/// (PostHog and OTLP traces/logs), un-stripped, on its own :8098 port.
+#[test]
+fn analytics_proxy_route_is_present() {
+ let caddy = caddyfile(Mode::Local, false);
+ assert!(caddy.contains("handle /i/* {"));
+ assert!(caddy.contains("reverse_proxy analytics-proxy:8098"));
+}
+
#[test]
fn document_content_services_are_available_through_the_proxy() {
let caddy = caddyfile(Mode::Local, false);
diff --git a/tooling/xtask/crates/xtask_local/src/local/summary.rs b/tooling/xtask/crates/xtask_local/src/local/summary.rs
index bd35529afe..43e638ae8b 100644
--- a/tooling/xtask/crates/xtask_local/src/local/summary.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/summary.rs
@@ -146,7 +146,60 @@ pub fn print(
};
row(label, value);
}
+ if let Some(url) = traces_url() {
+ row("traces", url);
+ }
+ if let Some(url) = dd_logs_url() {
+ row("dd logs", url);
+ }
+
row("logs", logs_command(instance, &env.generated_path));
row("stop", stop_command(instance));
println!();
}
+
+/// The trace viewer for the OTel spans the web app emits, if one is running.
+///
+/// The viewers are global (fixed ports, one per machine, started manually via
+/// compose profiles — see `docker/docker-compose.yml`), so this probes rather
+/// than consulting the instance: the Jaeger UI on 16686, else a Datadog agent
+/// on the OTLP port 4318, whose traces land in the Datadog APM UI under the
+/// `env:` its compose profile sets (`DD_ENV`, default `local`).
+fn traces_url() -> Option<String> {
+ if port_open(16686) {
+ return Some("http://localhost:16686".into());
+ }
+ if port_open(4318) {
+ return Some(format!(
+ "https://us5.datadoghq.com/apm/traces?query=env%3A{}",
+ dd_env()
+ ));
+ }
+ None
+}
+
+/// The Datadog Logs Explorer for the OTel log records the web app emits —
+/// only when the Datadog agent is the running collector (Jaeger has no log
+/// UI; its OTLP logs are dropped).
+fn dd_logs_url() -> Option<String> {
+ if port_open(16686) || !port_open(4318) {
+ return None;
+ }
+ Some(format!(
+ "https://us5.datadoghq.com/logs?query=env%3A{}",
+ dd_env()
+ ))
+}
+
+fn dd_env() -> String {
+ macro_env_var::maybe_read_env("DD_ENV").unwrap_or_else(|| "local".into())
+}
+
+/// Whether something is listening on `port` on localhost.
+pub(super) fn port_open(port: u16) -> bool {
+ std::net::TcpStream::connect_timeout(
+ &([127, 0, 0, 1], port).into(),
+ std::time::Duration::from_millis(150),
+ )
+ .is_ok()
+}
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/deploy_preview.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/deploy_preview.rs
index eb8e7cfc1c..89fb7a2a5c 100644
--- a/tooling/xtask/crates/xtask_workflows/src/workflows/deploy_preview.rs
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/deploy_preview.rs
@@ -74,8 +74,6 @@ fn build() -> Step<Run> {
Step::new("Build")
.run("just build-dev")
.working_directory(xtask_paths::repo_dir!("apps/web"))
- .add_env(Env::new("VITE_DD_WEB_APP_TOKEN", vars::DD_WEB_APP_TOKEN))
- .add_env(Env::new("VITE_DD_HASH", "${{ github.sha }}"))
.add_env(Env::new(
"VITE_SEGMENT_WRITE_KEY",
vars::SEGMENT_WRITE_KEY_PRODUCTION,
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/deploy_web_app.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/deploy_web_app.rs
index 9a720382c3..07635b6ca3 100644
--- a/tooling/xtask/crates/xtask_workflows/src/workflows/deploy_web_app.rs
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/deploy_web_app.rs
@@ -81,8 +81,6 @@ pub fn patch(root: &mut serde_yaml::Value) -> Result<()> {
required: true
DD_API_KEY:
required: true
- DD_WEB_APP_TOKEN:
- required: true
SEGMENT_WRITE_KEY:
required: true
POSTHOG_API_KEY:
@@ -123,8 +121,6 @@ fn build() -> Step<Run> {
Step::new("Build")
.run("just build-${{ inputs.environment }}")
.working_directory(xtask_paths::repo_dir!("apps/web"))
- .add_env(Env::new("VITE_DD_WEB_APP_TOKEN", vars::DD_WEB_APP_TOKEN))
- .add_env(Env::new("VITE_DD_HASH", "${{ github.sha }}"))
.add_env(Env::new("VITE_SEGMENT_WRITE_KEY", vars::SEGMENT_WRITE_KEY))
.add_env(Env::new("VITE_POSTHOG_API_KEY", vars::POSTHOG_API_KEY))
}
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/deploy_web_app_dev_push.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/deploy_web_app_dev_push.rs
index 9b136d81b2..1f34f88e63 100644
--- a/tooling/xtask/crates/xtask_workflows/src/workflows/deploy_web_app_dev_push.rs
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/deploy_web_app_dev_push.rs
@@ -42,7 +42,6 @@ pub fn patch(root: &mut serde_yaml::Value) -> Result<()> {
PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }}
DD_APP_KEY: ${{ secrets.DD_APP_KEY }}
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- DD_WEB_APP_TOKEN: ${{ secrets.DD_WEB_APP_TOKEN }}
SEGMENT_WRITE_KEY: ${{ secrets.SEGMENT_WRITE_KEY_PRODUCTION }}
POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}
"#})?,
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/vars.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/vars.rs
index cc59718369..db6e6759d3 100644
--- a/tooling/xtask/crates/xtask_workflows/src/workflows/vars.rs
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/vars.rs
@@ -17,7 +17,6 @@ secret!(CACHIX_AUTH_TOKEN);
secret!(CLOUDFLARE_API_TOKEN);
secret!(DD_API_KEY);
secret!(DD_APP_KEY);
-secret!(DD_WEB_APP_TOKEN);
secret!(DOPPLER_PREVIEW_TOKEN);
secret!(DOPPLER_TOKEN);
secret!(FLY_API_TOKEN);