Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat: otel for ai workers (#5358)
Wolf Mermelsteindd314b29a666
diff --git a/.github/workspace-dep-closures.json b/.github/workspace-dep-closures.jsonindex e3f6adedb9..8150cc8bea 100644--- a/.github/workspace-dep-closures.json+++ b/.github/workspace-dep-closures.json@@ -80,6 +80,7 @@"crates/macro_queues","crates/macro_service_urls","crates/macro_sync_service_jwt",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/mcp_client",@@ -247,6 +248,7 @@"crates/macro_queues","crates/macro_service_urls","crates/macro_sync_service_jwt",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/mcp_client",@@ -686,6 +688,7 @@"crates/macro_queues","crates/macro_service_urls","crates/macro_sync_service_jwt",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/mcp_client",@@ -941,6 +944,7 @@"crates/macro_service_urls","crates/macro_sha_count_client","crates/macro_sync_service_jwt",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/mention_utils",@@ -1338,6 +1342,7 @@"crates/macro_service_urls","crates/macro_sha_count_client","crates/macro_sync_service_jwt",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/mention_utils",@@ -1445,6 +1450,7 @@"crates/macro_queues","crates/macro_service_urls","crates/macro_sync_service_jwt",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/mcp_client",@@ -1747,6 +1753,7 @@"crates/macro_event_topics","crates/macro_service_urls","crates/macro_sync_service_jwt",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/mention_utils",@@ -1828,6 +1835,7 @@"crates/macro_event_topics","crates/macro_service_urls","crates/macro_sync_service_jwt",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/mention_utils",@@ -2622,6 +2630,7 @@"crates/macro_event_topics","crates/macro_service_urls","crates/macro_sync_service_jwt",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/mention_utils",@@ -2851,6 +2860,7 @@"crates/macro_service_urls","crates/macro_sha_count_client","crates/macro_sync_service_jwt",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/mention_utils",@@ -3084,6 +3094,7 @@"crates/macro_service_urls","crates/macro_sha_count_client","crates/macro_sync_service_jwt",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/mention_utils",@@ -3759,6 +3770,7 @@"crates/macro_queues","crates/macro_service_urls","crates/macro_sync_service_jwt",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/mcp_client",@@ -3864,6 +3876,7 @@"crates/macro_queues","crates/macro_service_urls","crates/macro_sync_service_jwt",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/mcp_client",@@ -4624,6 +4637,7 @@"crates/macro_service_urls","crates/macro_sha_count_client","crates/macro_sync_service_jwt",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/mention_utils",@@ -4873,6 +4887,7 @@"crates/macro_queues","crates/macro_service_urls","crates/macro_sync_service_jwt",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/mcp_client",@@ -4981,6 +4996,7 @@"crates/macro_service_urls","crates/macro_sha_count_client","crates/macro_sync_service_jwt",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/mention_utils",@@ -5449,6 +5465,7 @@"crates/macro_service_urls","crates/macro_sha_count_client","crates/macro_sync_service_jwt",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/mention_utils",@@ -6045,6 +6062,7 @@"crates/macro_event_topics","crates/macro_service_urls","crates/macro_sync_service_jwt",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/mention_utils",diff --git a/Cargo.lock b/Cargo.lockindex 263a0e5416..44a26b4e57 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -4634,6 +4634,7 @@ dependencies = ["macro_event_topics","macro_service_urls","macro_sync_service_jwt",+ "macro_tower_layers","macro_user_id","macro_uuid","mockall",diff --git a/apps/web/src/lib/service-clients/ai-editing-worker/client.ts b/apps/web/src/lib/service-clients/ai-editing-worker/client.tsindex 98429ae98d..d5e7fe76cc 100644--- a/apps/web/src/lib/service-clients/ai-editing-worker/client.ts+++ b/apps/web/src/lib/service-clients/ai-editing-worker/client.ts@@ -1,5 +1,7 @@import type { DocumentOp } from '@ai-ops/editor';+import { resumeDocumentSpan } from '@block-md/observability';import { toast } from '@core/component/Toast/Toast';+import { Telemetry } from '@macro-inc/observability';import { getDocumentPermissionToken } from '@service-storage/client';import { createSignal } from 'solid-js';@@ -78,11 +80,25 @@ export async function requestAiEdit(args: {const controller = new AbortController();editControllers.set(args.documentId, controller);syncActiveEditDocs();+ // Parent under the document's long-lived span when one exists, so the+ // worker's whole edit trace joins the user's editing session; the injected+ // traceparent makes the worker's request span a child of this one.+ const documentSpan = resumeDocumentSpan(args.documentId);+ const span = documentSpan+ ? documentSpan.run(() => Telemetry.clientSpan('http POST /edit'))+ : Telemetry.clientSpan('http POST /edit');+ span.setAttr('http.method', 'POST');+ span.setAttr('http.url', `${AI_EDITING_WORKER_HOST}/edit`);+ span.setAttr('document.id', args.documentId);try {const token = await getDocumentPermissionToken(args.documentId);+ const headers: Record<string, string> = {+ 'Content-Type': 'application/json',+ };+ span.injectTraceHeaders(headers);const res = await fetch(`${AI_EDITING_WORKER_HOST}/edit`, {method: 'POST',- headers: { 'Content-Type': 'application/json' },+ headers,body: JSON.stringify({documentToken: token,documentId: args.documentId,@@ -93,9 +109,16 @@ export async function requestAiEdit(args: {}),signal: controller.signal,});+ span.setAttr('http.status_code', res.status);if (!res.ok) {const body = await res.text().catch(() => '');console.error('ai edit request failed', res.status, body);+ const message = `HTTP ${res.status} for POST /edit`;+ span.error({+ name: 'HttpError',+ message,+ stack: new Error(message).stack,+ });return 'failed';}if (args.onOps) {@@ -104,10 +127,15 @@ export async function requestAiEdit(args: {}return 'ok';} catch (e) {- if (controller.signal.aborted) return 'cancelled';+ if (controller.signal.aborted) {+ span.setAttr('http.aborted', true);+ return 'cancelled';+ }console.error('ai edit request failed', e);+ span.error(e);return 'failed';} finally {+ span.end();if (editControllers.get(args.documentId) === controller) {editControllers.delete(args.documentId);syncActiveEditDocs();diff --git a/bun.lock b/bun.lockindex 743ef93388..6ec343a540 100644--- a/bun.lock+++ b/bun.lock@@ -254,6 +254,7 @@"name": "@macro-inc/observability","version": "0.0.1","dependencies": {+ "@microlabs/otel-cf-workers": "^1.0.0-rc.52","@opentelemetry/api": "^1.9.1","@opentelemetry/api-logs": "^0.220.0","@opentelemetry/context-zone": "^2.9.0",@@ -285,6 +286,7 @@"@loro-mirror/core": "workspace:*","@macro-inc/collaboration": "workspace:*","@macro-inc/lexical-core": "workspace:*",+ "@macro-inc/observability": "workspace:*","ai": "^6.0.207","ai-fallback": "2","bitf": "^1.0.2",@@ -1069,6 +1071,8 @@"@mediapipe/tasks-vision": ["@mediapipe/[email protected]", "", {}, "sha512-vOifgZhkndgybdvoRITzRkIueWWSiCKuEUXXK6Q4FaJsFvRJuwgg++vqFUMlL0Uox62U5aEXFhHxlhV7Ja5e3Q=="],+ "@microlabs/otel-cf-workers": ["@microlabs/[email protected]", "", { "dependencies": { "@opentelemetry/core": "^2.0.0", "@opentelemetry/exporter-trace-otlp-http": "^0.200.0", "@opentelemetry/otlp-exporter-base": "^0.200.0", "@opentelemetry/otlp-transformer": "^0.200.0", "@opentelemetry/resources": "^2.0.0", "@opentelemetry/sdk-trace-base": "^2.0.0", "@opentelemetry/semantic-conventions": "^1.33.0" }, "peerDependencies": { "@opentelemetry/api": "~1.9.0" } }, "sha512-7jf4sBNoaZCz2RSayefcvc/CZX0PDVA9aVk44bHpUebLrap+xjzDDw4T03klcYrBl/FdC7NamwvqjUMCtYdBvQ=="],+"@napi-rs/wasm-runtime": ["@napi-rs/[email protected]", "", { "dependencies": { "@tybys/wasm-util": "^0.10.3" }, "peerDependencies": { "@emnapi/core": "^1.7.1", "@emnapi/runtime": "^1.7.1" } }, "sha512-ZLv/JdUfkvOy9eCnnBaGfiO+XimbjebAeO+MRQqD/B+FR1tnRN0tpKSJHRbE8sFfS6aqsXZ67TQjfwfsxULVbg=="],"@noble/hashes": ["@noble/[email protected]", "", {}, "sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A=="],@@ -1097,15 +1101,15 @@"@opentelemetry/exporter-trace-otlp-http": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/core": "2.9.0", "@opentelemetry/otlp-exporter-base": "0.220.0", "@opentelemetry/otlp-transformer": "0.220.0", "@opentelemetry/resources": "2.9.0", "@opentelemetry/sdk-trace": "2.9.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-/+ExB3lRkf+erv4PnoywyL7RHKITidxtUpUTS55k7OQ0dB42S7gEF1gry7swb9MSm1hYLUhJg4QQh9W8SpwwqA=="],- "@opentelemetry/otlp-exporter-base": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/core": "2.9.0", "@opentelemetry/otlp-transformer": "0.220.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-CXYo8UD5Mn9YbgebO2EL4wejtA+gxLmLiu6HCk2KH2BR7XhFN6/6p1UlCb23DYCjeYkndevLHuejCCN1yx4+OQ=="],+ "@opentelemetry/otlp-exporter-base": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/core": "2.0.0", "@opentelemetry/otlp-transformer": "0.200.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-IxJgA3FD7q4V6gGq4bnmQM5nTIyMDkoGFGrBrrDjB6onEiq1pafma55V+bHvGYLWvcqbBbRfezr1GED88lacEQ=="],- "@opentelemetry/otlp-transformer": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/api-logs": "0.220.0", "@opentelemetry/core": "2.9.0", "@opentelemetry/resources": "2.9.0", "@opentelemetry/sdk-logs": "0.220.0", "@opentelemetry/sdk-metrics": "2.9.0", "@opentelemetry/sdk-trace": "2.9.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-lXGrv7KXZ0gNH9SVNUaa6vv6phVYGvJxfXAlMbzbakiXru75f5MZl8Z7oqiMMQD77riVHJCFlQvbZs/VVN2/4A=="],+ "@opentelemetry/otlp-transformer": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/api-logs": "0.200.0", "@opentelemetry/core": "2.0.0", "@opentelemetry/resources": "2.0.0", "@opentelemetry/sdk-logs": "0.200.0", "@opentelemetry/sdk-metrics": "2.0.0", "@opentelemetry/sdk-trace-base": "2.0.0", "protobufjs": "^7.3.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-+9YDZbYybOnv7sWzebWOeK6gKyt2XE7iarSyBFkwwnP559pEevKOUD8NyDHhRjCSp13ybh9iVXlMfcj/DwF/yw=="],"@opentelemetry/resources": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/core": "2.9.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-jyA5MBLQ+Dkl3+JsZkUoUvL7yHvU64kLsvpXKarWm6347Sl1t1bXFTFykUePNpT5WH5pm9a2Qtt03iIYQhZ1Fg=="],"@opentelemetry/sdk-logs": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/api-logs": "0.220.0", "@opentelemetry/core": "2.9.0", "@opentelemetry/resources": "2.9.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.4.0 <1.10.0" } }, "sha512-WywcTkQtv2iNmt+6y5Kcd4rzvx9bLVsBa2Nwcmg01IUaBTkTow3W4d9KE5vNBpEDtb9tp21WcRBY/lANRrApYA=="],- "@opentelemetry/sdk-metrics": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/core": "2.9.0", "@opentelemetry/resources": "2.9.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.9.0 <1.10.0" } }, "sha512-Xx8RGS4H5XEBl01WuCreMIpiah9cCXMbSkeuIePPdD2cUpq/vUzYmj8E/MK1OsbOc93FuAD4jfn2WOacKwLn7Q=="],+ "@opentelemetry/sdk-metrics": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/core": "2.0.0", "@opentelemetry/resources": "2.0.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.9.0 <1.10.0" } }, "sha512-Bvy8QDjO05umd0+j+gDeWcTaVa1/R2lDj/eOvjzpm8VQj1K1vVZJuyjThpV5/lSHyYW2JaHF2IQ7Z8twJFAhjA=="],"@opentelemetry/sdk-trace": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/core": "2.9.0", "@opentelemetry/resources": "2.9.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-sGA19HvtrrSKYsseHphluH6j3p6Xa3fqc7c7y8f/7mYWejc1lyDFcpSdD1kYa50HCLUeEo4zA5bW0pniaPszuw=="],@@ -3305,6 +3309,32 @@"@macro-inc/lexical-core/nanoid": ["[email protected]", "", { "bin": { "nanoid": "bin/nanoid.js" } }, "sha512-kVrnsrJqMR8+oLJnGEmSWw9BivK5mt7H3FZatVRjrc5wGqFYuBxX1yG7+A7Gi5AefkX6t/oCkizcQgpu0cY1dQ=="],+ "@microlabs/otel-cf-workers/@opentelemetry/exporter-trace-otlp-http": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/core": "2.0.0", "@opentelemetry/otlp-exporter-base": "0.200.0", "@opentelemetry/otlp-transformer": "0.200.0", "@opentelemetry/resources": "2.0.0", "@opentelemetry/sdk-trace-base": "2.0.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-Goi//m/7ZHeUedxTGVmEzH19NgqJY+Bzr6zXo1Rni1+hwqaksEyJ44gdlEMREu6dzX1DlAaH/qSykSVzdrdafA=="],++ "@opentelemetry/exporter-logs-otlp-http/@opentelemetry/otlp-exporter-base": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/core": "2.9.0", "@opentelemetry/otlp-transformer": "0.220.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-CXYo8UD5Mn9YbgebO2EL4wejtA+gxLmLiu6HCk2KH2BR7XhFN6/6p1UlCb23DYCjeYkndevLHuejCCN1yx4+OQ=="],++ "@opentelemetry/exporter-logs-otlp-http/@opentelemetry/otlp-transformer": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/api-logs": "0.220.0", "@opentelemetry/core": "2.9.0", "@opentelemetry/resources": "2.9.0", "@opentelemetry/sdk-logs": "0.220.0", "@opentelemetry/sdk-metrics": "2.9.0", "@opentelemetry/sdk-trace": "2.9.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-lXGrv7KXZ0gNH9SVNUaa6vv6phVYGvJxfXAlMbzbakiXru75f5MZl8Z7oqiMMQD77riVHJCFlQvbZs/VVN2/4A=="],++ "@opentelemetry/exporter-trace-otlp-http/@opentelemetry/otlp-exporter-base": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/core": "2.9.0", "@opentelemetry/otlp-transformer": "0.220.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-CXYo8UD5Mn9YbgebO2EL4wejtA+gxLmLiu6HCk2KH2BR7XhFN6/6p1UlCb23DYCjeYkndevLHuejCCN1yx4+OQ=="],++ "@opentelemetry/exporter-trace-otlp-http/@opentelemetry/otlp-transformer": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/api-logs": "0.220.0", "@opentelemetry/core": "2.9.0", "@opentelemetry/resources": "2.9.0", "@opentelemetry/sdk-logs": "0.220.0", "@opentelemetry/sdk-metrics": "2.9.0", "@opentelemetry/sdk-trace": "2.9.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-lXGrv7KXZ0gNH9SVNUaa6vv6phVYGvJxfXAlMbzbakiXru75f5MZl8Z7oqiMMQD77riVHJCFlQvbZs/VVN2/4A=="],++ "@opentelemetry/otlp-exporter-base/@opentelemetry/core": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-SLX36allrcnVaPYG3R78F/UZZsBsvbc7lMCLx37LyH5MJ1KAAZ2E3mW9OAD3zGz0G8q/BtoS5VUrjzDydhD6LQ=="],++ "@opentelemetry/otlp-transformer/@opentelemetry/api-logs": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-IKJBQxh91qJ+3ssRly5hYEJ8NDHu9oY/B1PXVSCWf7zytmYO9RNLB0Ox9XQ/fJ8m6gY6Q6NtBWlmXfaXt5Uc4Q=="],++ "@opentelemetry/otlp-transformer/@opentelemetry/core": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-SLX36allrcnVaPYG3R78F/UZZsBsvbc7lMCLx37LyH5MJ1KAAZ2E3mW9OAD3zGz0G8q/BtoS5VUrjzDydhD6LQ=="],++ "@opentelemetry/otlp-transformer/@opentelemetry/resources": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/core": "2.0.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-rnZr6dML2z4IARI4zPGQV4arDikF/9OXZQzrC01dLmn0CZxU5U5OLd/m1T7YkGRj5UitjeoCtg/zorlgMQcdTg=="],++ "@opentelemetry/otlp-transformer/@opentelemetry/sdk-logs": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/api-logs": "0.200.0", "@opentelemetry/core": "2.0.0", "@opentelemetry/resources": "2.0.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.4.0 <1.10.0" } }, "sha512-VZG870063NLfObmQQNtCVcdXXLzI3vOjjrRENmU37HYiPFa0ZXpXVDsTD02Nh3AT3xYJzQaWKl2X2lQ2l7TWJA=="],++ "@opentelemetry/otlp-transformer/@opentelemetry/sdk-trace-base": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/core": "2.0.0", "@opentelemetry/resources": "2.0.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-qQnYdX+ZCkonM7tA5iU4fSRsVxbFGml8jbxOgipRGMFHKaXKHQ30js03rTobYjKjIfnOsZSbHKWF0/0v0OQGfw=="],++ "@opentelemetry/sdk-metrics/@opentelemetry/core": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-SLX36allrcnVaPYG3R78F/UZZsBsvbc7lMCLx37LyH5MJ1KAAZ2E3mW9OAD3zGz0G8q/BtoS5VUrjzDydhD6LQ=="],++ "@opentelemetry/sdk-metrics/@opentelemetry/resources": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/core": "2.0.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-rnZr6dML2z4IARI4zPGQV4arDikF/9OXZQzrC01dLmn0CZxU5U5OLd/m1T7YkGRj5UitjeoCtg/zorlgMQcdTg=="],+"@orval/core/chalk": ["[email protected]", "", { "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" } }, "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA=="],"@orval/core/typedoc": ["[email protected]", "", { "dependencies": { "@gerrit0/mini-shiki": "^3.23.0", "lunr": "^2.3.9", "markdown-it": "^14.3.0", "minimatch": "^10.2.5", "yaml": "^2.9.0" }, "peerDependencies": { "typescript": "5.0.x || 5.1.x || 5.2.x || 5.3.x || 5.4.x || 5.5.x || 5.6.x || 5.7.x || 5.8.x || 5.9.x || 6.0.x" }, "bin": { "typedoc": "bin/typedoc" } }, "sha512-uSKqkh8Cr48vllnEy+jdaAgOeR6Y+QCBW7usgUsKj7gJEfR7stw9U/fE49LBnj2tPRKPY0c0EBJSWe9Appmplg=="],@@ -3645,6 +3675,16 @@"@macro-inc/lexical-core/@types/node/undici-types": ["[email protected]", "", {}, "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg=="],+ "@microlabs/otel-cf-workers/@opentelemetry/exporter-trace-otlp-http/@opentelemetry/core": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-SLX36allrcnVaPYG3R78F/UZZsBsvbc7lMCLx37LyH5MJ1KAAZ2E3mW9OAD3zGz0G8q/BtoS5VUrjzDydhD6LQ=="],++ "@microlabs/otel-cf-workers/@opentelemetry/exporter-trace-otlp-http/@opentelemetry/resources": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/core": "2.0.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-rnZr6dML2z4IARI4zPGQV4arDikF/9OXZQzrC01dLmn0CZxU5U5OLd/m1T7YkGRj5UitjeoCtg/zorlgMQcdTg=="],++ "@microlabs/otel-cf-workers/@opentelemetry/exporter-trace-otlp-http/@opentelemetry/sdk-trace-base": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/core": "2.0.0", "@opentelemetry/resources": "2.0.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-qQnYdX+ZCkonM7tA5iU4fSRsVxbFGml8jbxOgipRGMFHKaXKHQ30js03rTobYjKjIfnOsZSbHKWF0/0v0OQGfw=="],++ "@opentelemetry/exporter-logs-otlp-http/@opentelemetry/otlp-transformer/@opentelemetry/sdk-metrics": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/core": "2.9.0", "@opentelemetry/resources": "2.9.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.9.0 <1.10.0" } }, "sha512-Xx8RGS4H5XEBl01WuCreMIpiah9cCXMbSkeuIePPdD2cUpq/vUzYmj8E/MK1OsbOc93FuAD4jfn2WOacKwLn7Q=="],++ "@opentelemetry/exporter-trace-otlp-http/@opentelemetry/otlp-transformer/@opentelemetry/sdk-metrics": ["@opentelemetry/[email protected]", "", { "dependencies": { "@opentelemetry/core": "2.9.0", "@opentelemetry/resources": "2.9.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.9.0 <1.10.0" } }, "sha512-Xx8RGS4H5XEBl01WuCreMIpiah9cCXMbSkeuIePPdD2cUpq/vUzYmj8E/MK1OsbOc93FuAD4jfn2WOacKwLn7Q=="],+"@orval/core/chalk/supports-color": ["[email protected]", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="],"@orval/query/chalk/supports-color": ["[email protected]", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="],diff --git a/crates/documents/Cargo.toml b/crates/documents/Cargo.tomlindex b513483172..16d618320e 100644--- a/crates/documents/Cargo.toml+++ b/crates/documents/Cargo.toml@@ -14,6 +14,7 @@ ai_tools = ["dep:ai_toolset","dep:ai_usage","dep:async-trait",+ "dep:macro_tower_layers","dep:model_file_type","dep:schemars","dep:sha2",@@ -141,6 +142,7 @@ ai_toolset = { path = "../ai_toolset", optional = true }ai_usage = { path = "../ai_usage", optional = true }async-trait = { workspace = true, optional = true }lexical_client = { path = "../lexical_client", optional = true }+macro_tower_layers = { path = "../macro_tower_layers", optional = true }model_file_type = { path = "../model_file_type", optional = true }reqwest = { workspace = true, optional = true }schemars = { workspace = true, optional = true }diff --git a/crates/documents/src/outbound/editing_worker_client.rs b/crates/documents/src/outbound/editing_worker_client.rsindex 36f3490e73..c9922702df 100644--- a/crates/documents/src/outbound/editing_worker_client.rs+++ b/crates/documents/src/outbound/editing_worker_client.rs@@ -65,9 +65,15 @@ impl EditingWorkerService for ReqwestEditingWorkerClient {"interpret": false,});+ // Propagate the current trace so the worker's spans join this+ // service's trace instead of rooting their own.+ let mut headers = reqwest::header::HeaderMap::new();+ macro_tower_layers::inject_trace_headers(&mut headers);+let edit_resp = self.client.post(format!("{}/edit", self.worker_url))+ .headers(headers).json(&request_body).send().await?;@@ -112,9 +118,13 @@ impl EditingWorkerService for ReqwestEditingWorkerClient {anyhow::bail!("editing worker client has no internal auth key configured");};+ let mut headers = reqwest::header::HeaderMap::new();+ macro_tower_layers::inject_trace_headers(&mut headers);+let resp = self.client.delete(format!("{}/traces/{}", self.worker_url, document_id))+ .headers(headers).header("x-internal-auth-key", internal_auth_key).send().await?;diff --git a/crates/macro_tower_layers/src/lib.rs b/crates/macro_tower_layers/src/lib.rsindex cd228577cf..009577f384 100644--- a/crates/macro_tower_layers/src/lib.rs+++ b/crates/macro_tower_layers/src/lib.rs@@ -47,6 +47,19 @@ impl<B> MakeSpan<B> for MakeSpanWithRemoteParent {}}+/// Injects the current span's W3C trace context into outbound request headers.+///+/// Outbound counterpart of [`MakeSpanWithRemoteParent`]: a downstream service+/// that adopts `traceparent` joins this service's trace. The global text-map+/// propagator is registered by `macro_entrypoint`; without one, or outside any+/// span, this is a no-op.+pub fn inject_trace_headers(headers: &mut http::HeaderMap) {+ let context = Span::current().context();+ opentelemetry::global::get_text_map_propagator(|propagator| {+ propagator.inject_context(&context, &mut opentelemetry_http::HeaderInjector(headers));+ });+}+/// A very simple builder for x-request-ids#[derive(Default, Clone)]pub struct RequestIdBuilder(Arc<AtomicU64>);diff --git a/crates/macro_tower_layers/src/test.rs b/crates/macro_tower_layers/src/test.rsindex b1cfb78cea..93bd9483c7 100644--- a/crates/macro_tower_layers/src/test.rs+++ b/crates/macro_tower_layers/src/test.rs@@ -300,6 +300,45 @@ fn request_span_without_traceparent_stays_root() {});}+#[test]+fn inject_trace_headers_emits_current_span_traceparent() {+ use opentelemetry::trace::TracerProvider as _;+ use tracing_subscriber::layer::SubscriberExt;++ opentelemetry::global::set_text_map_propagator(+ opentelemetry_sdk::propagation::TraceContextPropagator::new(),+ );+ let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder().build();+ let subscriber = tracing_subscriber::registry()+ .with(tracing_opentelemetry::layer().with_tracer(provider.tracer("test")));++ with_default(subscriber, || {+ let span = tracing::info_span!("outbound_call");+ let _guard = span.enter();++ let mut headers = http::HeaderMap::new();+ inject_trace_headers(&mut headers);++ let traceparent = headers+ .get("traceparent")+ .expect("traceparent header should be injected inside a span")+ .to_str()+ .unwrap();+ let trace_id = span.context().span().span_context().trace_id().to_string();+ assert!(+ traceparent.contains(&trace_id),+ "traceparent {traceparent} should carry the current trace id {trace_id}"+ );+ });+}++#[test]+fn inject_trace_headers_is_a_noop_outside_a_span() {+ let mut headers = http::HeaderMap::new();+ inject_trace_headers(&mut headers);+ assert!(headers.get("traceparent").is_none());+}+#[tokio::test]async fn starvation_detector_does_not_warn_within_grace_period() {tokio::time::pause();diff --git a/packages/observability/package.json b/packages/observability/package.jsonindex 5f8e20465d..836656b00b 100644--- a/packages/observability/package.json+++ b/packages/observability/package.json@@ -7,6 +7,7 @@".": "./src/index.ts","./core": "./src/core.ts","./web": "./src/web.ts",+ "./worker": "./src/worker.ts","./zone": "./src/zone.ts"},"scripts": {@@ -15,6 +16,7 @@"type-check": "tsc --noEmit --project tsconfig.json"},"dependencies": {+ "@microlabs/otel-cf-workers": "^1.0.0-rc.52","@opentelemetry/api": "^1.9.1","@opentelemetry/api-logs": "^0.220.0","@opentelemetry/context-zone": "^2.9.0",diff --git a/packages/observability/src/span.ts b/packages/observability/src/span.tsindex 178e9be3e5..366e37ab7a 100644--- a/packages/observability/src/span.ts+++ b/packages/observability/src/span.ts@@ -3,8 +3,8 @@ import {context,isSpanContextValid,type Attributes as OtelAttributes,- propagation,type Span as OtelSpan,+ propagation,SpanStatusCode,} from "@opentelemetry/api";import type { Attribute, Attributes } from "./config";diff --git a/packages/observability/src/worker.ts b/packages/observability/src/worker.tsnew file mode 100644index 0000000000..8398603e5e--- /dev/null+++ b/packages/observability/src/worker.ts@@ -0,0 +1,70 @@+// Cloudflare Workers entry. Importing @microlabs/otel-cf-workers loads the+// `cloudflare:workers` module at module scope, so this stays in its own entry+// (like zone.ts) and must only be imported from code that runs in workerd.+import {+ BatchTraceSpanProcessor,+ instrument,+ instrumentDO,+ OTLPExporter,+ type TraceConfig,+} from "@microlabs/otel-cf-workers";+import type { Span } from "@opentelemetry/api";+import { ATTR_DEPLOYMENT_ENVIRONMENT, ATTR_USER_ID } from "./constants";++export type { TraceConfig };+export { instrument, instrumentDO };++export interface WorkerTraceConfigOptions {+ serviceName: string;+ environment: string;+ /** OTLP/HTTP traces endpoint. Spans are not exported when unset. */+ tracesUrl?: string;+ /** Extra headers for the export request (e.g. an intake API key). */+ tracesHeaders?: Record<string, string>;+ /** Read the current user id; called once per span when it starts. */+ getUserId?: () => string | undefined;+}++/**+ * Trace config for otel-cf-workers' `instrument()`/`instrumentDO()`.+ *+ * `instrument()` registers the global tracer provider, an AsyncLocalStorage+ * context manager, and the W3C propagator. `Telemetry.span()` resolves its+ * tracer through the global OpenTelemetry API, so spans parent and export+ * correctly with no `tracingProvider` passed to `Telemetry.init`. Export and+ * flushing are owned by otel-cf-workers (via `ctx.waitUntil`), not+ * `Telemetry.flush`.+ *+ * otel-cf-workers' service config has no environment field, so unlike the+ * web provider — which sets `deployment.environment` on the resource —+ * environment (and user id) are stamped onto every span here.+ */+export function createWorkerTraceConfig(+ options: WorkerTraceConfigOptions,+): TraceConfig {+ return {+ service: { name: options.serviceName },+ spanProcessors: [+ {+ onStart: (span: Span) => {+ span.setAttribute(ATTR_DEPLOYMENT_ENVIRONMENT, options.environment);+ const userId = options.getUserId?.();+ if (userId !== undefined) span.setAttribute(ATTR_USER_ID, userId);+ },+ onEnd: () => {},+ forceFlush: () => Promise.resolve(),+ shutdown: () => Promise.resolve(),+ },+ ...(options.tracesUrl+ ? [+ new BatchTraceSpanProcessor(+ new OTLPExporter({+ url: options.tracesUrl,+ headers: options.tracesHeaders,+ }),+ ),+ ]+ : []),+ ],+ };+}diff --git a/services/ai-editing-worker/package.json b/services/ai-editing-worker/package.jsonindex 202a4fcb30..6ef0add7a1 100644--- a/services/ai-editing-worker/package.json+++ b/services/ai-editing-worker/package.json@@ -16,6 +16,7 @@"@ai-sdk/openai": "^3.0.73","@hono/zod-validator": "^0.8.0","@loro-mirror/core": "workspace:*",+ "@macro-inc/observability": "workspace:*","@macro-inc/collaboration": "workspace:*","@macro-inc/lexical-core": "workspace:*","ai": "^6.0.207",diff --git a/services/ai-editing-worker/src/ai-editing/agents/coder.ts b/services/ai-editing-worker/src/ai-editing/agents/coder.tsindex 40281e793a..a558dbc2b8 100644--- a/services/ai-editing-worker/src/ai-editing/agents/coder.ts+++ b/services/ai-editing-worker/src/ai-editing/agents/coder.ts@@ -41,6 +41,7 @@ export async function coder(runner: deps.runner,onOps: deps.onOps,onRunCode: deps.onRunCode,+ span: deps.span,}),readDocument: createReadDocumentTool({ session }),reportBlocked: createImBlockedTool(diff --git a/services/ai-editing-worker/src/ai-editing/agents/supervisor.ts b/services/ai-editing-worker/src/ai-editing/agents/supervisor.tsindex b90950bfe7..9ef56e9a73 100644--- a/services/ai-editing-worker/src/ai-editing/agents/supervisor.ts+++ b/services/ai-editing-worker/src/ai-editing/agents/supervisor.ts@@ -1,3 +1,4 @@+import { type Span, Telemetry } from '@macro-inc/observability';import { hasToolCall, stepCountIs, streamText } from 'ai';import type { ResolvedModels } from '../../run-edit';import type { LexicalSession } from '../ai-toolkit';@@ -38,11 +39,22 @@ export async function supervisor(let interpretDurationMs: number | undefined;if (opts.interpret) {const interpretStartedAt = Date.now();- const interpretation = await interpreter(- docContext,- request,- models.interpret,- INTERPRET_SYSTEM+ const interpretation = await Telemetry.span(+ 'edit.interpret',+ async (span) => {+ const result = await interpreter(+ docContext,+ request,+ models.interpret,+ INTERPRET_SYSTEM+ );+ span.setAttr(+ 'gen_ai.request.model',+ (models.interpret as { modelId: string }).modelId+ );+ span.setAttr('intent.chars', result.text.length);+ return result;+ });interpretDurationMs = Date.now() - interpretStartedAt;tracker.add(@@ -52,84 +64,127 @@ export async function supervisor(intent = interpretation.text;}- const dispatch = createDispatchTool({- session,- makeChildModel: models.coding,- tracker,- request: intent ? `${request}\n\n<intent>\n${intent}\n</intent>` : request,- params: opts.params,- typingAnimations: opts.typingAnimations,- sleep: opts.sleep,- signal: opts.signal,- makeWriter: opts.borrowWriter,- runTask: coder,- serialize,- runner: opts.runner,- onOps: opts.onOps,- onCoderResult: opts.onCoderResult,- onEditTrace: opts.onEditTrace,- });-- const tools = {- reportBlocked: createImBlockedTool(- 'Call this when you cannot proceed without more information or when the task is impossible.',- true- ),- dispatch: dispatch.tool,+ // Manual span: dispatched coders launch from AI SDK stream callbacks where+ // the ambient context is unreliable, so they parent off this span explicitly.+ const superviseSpan = Telemetry.span('edit.supervise');+ // Open in `prepareStep`, closed in `onStepFinish`. Coders dispatched during a+ // turn parent off it, so the trace shows which turn spawned which edits.+ let turnSpan: Span | undefined;+ const endTurn = () => {+ turnSpan?.end();+ turnSpan = undefined;};+ try {+ const dispatch = createDispatchTool({+ session,+ makeChildModel: models.coding,+ tracker,+ request: intent+ ? `${request}\n\n<intent>\n${intent}\n</intent>`+ : request,+ params: opts.params,+ typingAnimations: opts.typingAnimations,+ sleep: opts.sleep,+ signal: opts.signal,+ makeWriter: opts.borrowWriter,+ runTask: coder,+ serialize,+ runner: opts.runner,+ onOps: opts.onOps,+ onCoderResult: opts.onCoderResult,+ onEditTrace: opts.onEditTrace,+ parentSpan: () => turnSpan ?? superviseSpan,+ });- const intentBlock = intent ? `<intent>\n${intent}\n</intent>\n\n` : '';- const prompt = `Request: ${request}\n\n${intentBlock}${docContext}`;+ const tools = {+ reportBlocked: createImBlockedTool(+ 'Call this when you cannot proceed without more information or when the task is impossible.',+ true+ ),+ dispatch: dispatch.tool,+ };- // Wall-clock duration of each supervisor step, measured between step- // boundaries. Best-effort, but since between model calls it's probably good- // enough.- const stepDurationsMs: number[] = [];- let lastStepAt = Date.now();- const result = streamText({- model: models.supervisor,- stopWhen: [stepCountIs(7), hasToolCall('reportBlocked')],- system: MASTER_SYSTEM,- prompt,- tools,- providerOptions: EDIT_PROVIDER_OPTIONS,- abortSignal: opts.signal,- prepareStep: ({ stepNumber }) =>- // require that the very first thing it does is a tool call- stepNumber === 0 ? { toolChoice: 'required' } : undefined,- onStepFinish: () => {- const now = Date.now();- stepDurationsMs.push(now - lastStepAt);- lastStepAt = now;- },- });+ const intentBlock = intent ? `<intent>\n${intent}\n</intent>\n\n` : '';+ const prompt = `Request: ${request}\n\n${intentBlock}${docContext}`;- for await (const part of result.fullStream) {- if (part.type === 'error') throw part.error;- if (part.type === 'abort')- throw opts.signal?.reason ?? new Error('edit session aborted');- }+ // Wall-clock duration of each supervisor step, measured between step+ // boundaries. Best-effort, but since between model calls it's probably good+ // enough.+ const stepDurationsMs: number[] = [];+ let lastStepAt = Date.now();+ const result = streamText({+ model: models.supervisor,+ stopWhen: [stepCountIs(7), hasToolCall('reportBlocked')],+ system: MASTER_SYSTEM,+ prompt,+ tools,+ providerOptions: EDIT_PROVIDER_OPTIONS,+ abortSignal: opts.signal,+ prepareStep: ({ stepNumber }) => {+ // A step that throws mid-flight never reaches onStepFinish; close any+ // straggler so turns stay one-to-one with spans.+ endTurn();+ turnSpan = superviseSpan.span('edit.supervise.turn');+ turnSpan.setAttr('turn.index', stepNumber);+ turnSpan.setAttr('gen_ai.operation.name', 'chat');+ // require that the very first thing it does is a tool call+ return stepNumber === 0 ? { toolChoice: 'required' } : undefined;+ },+ onStepFinish: (step) => {+ const now = Date.now();+ stepDurationsMs.push(now - lastStepAt);+ turnSpan?.setAttr('turn.tool_calls', step.toolCalls.length);+ turnSpan?.setAttr('turn.finish_reason', step.finishReason);+ endTurn();+ lastStepAt = now;+ },+ });- const [steps, totalUsage, text] = await Promise.all([- result.steps,- result.totalUsage,- result.text,- ]);- tracker.add(models.supervisor as { modelId: string }, totalUsage);+ for await (const part of result.fullStream) {+ if (part.type === 'error') throw part.error;+ if (part.type === 'abort')+ throw opts.signal?.reason ?? new Error('edit session aborted');+ }- const blocked = steps- .flatMap((s) => s.toolCalls)- .find((c) => c.toolName === 'reportBlocked');- const clarification = (blocked?.input as { message: string } | undefined)- ?.message;+ const [steps, totalUsage, text] = await Promise.all([+ result.steps,+ result.totalUsage,+ result.text,+ ]);+ tracker.add(models.supervisor as { modelId: string }, totalUsage);- return {- text: text || 'Applied edits.',- totalUsage: tracker,- steps,- stepDurationsMs,- intent,- interpretDurationMs,- clarification,- };+ const blocked = steps+ .flatMap((s) => s.toolCalls)+ .find((c) => c.toolName === 'reportBlocked');+ const clarification = (blocked?.input as { message: string } | undefined)+ ?.message;++ superviseSpan.setAttr(+ 'gen_ai.request.model',+ (models.supervisor as { modelId: string }).modelId+ );+ superviseSpan.setAttr('steps.count', steps.length);+ superviseSpan.setAttr('edit.blocked', blocked !== undefined);+ if (clarification !== undefined)+ superviseSpan.setAttr('clarification.chars', clarification.length);++ return {+ text: text || 'Applied edits.',+ totalUsage: tracker,+ steps,+ stepDurationsMs,+ intent,+ interpretDurationMs,+ clarification,+ };+ } catch (e) {+ // A turn is still open when the failure happened mid-step; blame it there+ // too so the trace points at the turn, not just the whole supervisor run.+ turnSpan?.error(e);+ superviseSpan.error(e);+ throw e;+ } finally {+ endTurn();+ superviseSpan.end();+ }}diff --git a/services/ai-editing-worker/src/ai-editing/agents/types.ts b/services/ai-editing-worker/src/ai-editing/agents/types.tsindex 6eede290a7..f22219f1cd 100644--- a/services/ai-editing-worker/src/ai-editing/agents/types.ts+++ b/services/ai-editing-worker/src/ai-editing/agents/types.ts@@ -1,3 +1,4 @@+import type { Span } from '@macro-inc/observability';import type { AwarenessSource } from '../awareness';import type { Doc } from '../doc';import type { DocumentOp } from '../editor';@@ -23,6 +24,8 @@ export type RunTaskDeps = {signal?: AbortSignal;onOps?: RunCodeToolOptions['onOps'];onRunCode?: RunCodeToolOptions['onRunCode'];+ /** This coder's `edit.dispatch` span; parents its `edit.run_code` children. */+ span?: Span;};export type RunAgentOptions = {diff --git a/services/ai-editing-worker/src/ai-editing/runtime.ts b/services/ai-editing-worker/src/ai-editing/runtime.tsindex 9c5126ea12..56ba648437 100644--- a/services/ai-editing-worker/src/ai-editing/runtime.ts+++ b/services/ai-editing-worker/src/ai-editing/runtime.ts@@ -1,4 +1,5 @@import { $getId } from '@macro-inc/lexical-core/plugins/nodeIdPlugin';+import type { Span } from '@macro-inc/observability';import { $getRoot, $isElementNode, type LexicalNode } from 'lexical';import type { LexicalSession } from './ai-toolkit/session';import type { AwarenessSource } from './awareness';@@ -59,6 +60,9 @@ export type RunEditorCodeArgs = {typingAnimations?: boolean;/** Called with the raw ops before they are queued/applied -- use to collect them. */onOps?: (ops: DocumentOp[]) => void;+ /** Annotated (never created/ended here) with sandbox timing, op counts, and+ * a failure classification (`error.kind`). */+ span?: Span;};/**@@ -67,6 +71,7 @@ export type RunEditorCodeArgs = {* and nothing is applied.*/export async function runEditorCode(args: RunEditorCodeArgs): Promise<string> {+ const span = args.span;let ops: DocumentOp[];try {// Every key the code references must come with the call — catch a missing@@ -76,40 +81,52 @@ export async function runEditorCode(args: RunEditorCodeArgs): Promise<string> {);if (missing.length > 0) {const list = missing.map((k) => `"${k}"`).join(', ');+ span?.setAttr('error.kind', 'snippet_missing');throw new Error(`snippet${missing.length > 1 ? 's' : ''} ${list} ${missing.length > 1 ? 'are' : 'is'} not defined -- pass their text in the \`snippets\` argument of this runCode call`);}- ops = await args.runner(docIds(args.session), args.code, args.snippets);+ const sandboxStartedAt = Date.now();+ try {+ ops = await args.runner(docIds(args.session), args.code, args.snippets);+ } catch (e) {+ span?.setAttr('error.kind', 'sandbox');+ throw e;+ } finally {+ span?.setAttr('sandbox.ms', Date.now() - sandboxStartedAt);+ }} catch (e) {if (!(e instanceof Error)) throw new Error(String(e));return `error: ${e.message}`;}args.onOps?.(ops);+ span?.setAttr('ops.count', ops.length);+ span?.setAttr('animated', args.typingAnimations !== false);if (ops.length === 0) return 'ok';- if (args.typingAnimations === false) {- const results: OpResult[] = ops.map((op) => {- try {- args.doc.apply(op);- return { ok: true, op };- } catch (e) {- if (!(e instanceof Error)) throw new Error(String(e));- return { ok: false, op, error: e.message };- }- });- return summarize(results);- }-- const results = await runQueue({- ops,- params: args.params,- randomSource: realRandomSource(),- docReader: args.doc,- docWriter: args.doc,- awarenessSource: args.awarenessSource,- resolveNode: (n) => args.doc.resolveRef(n), // point cursors at inserted nodes- sleep: args.sleep,- });+ const results: OpResult[] =+ args.typingAnimations === false+ ? ops.map((op) => {+ try {+ args.doc.apply(op);+ return { ok: true, op };+ } catch (e) {+ if (!(e instanceof Error)) throw new Error(String(e));+ return { ok: false, op, error: e.message };+ }+ })+ : await runQueue({+ ops,+ params: args.params,+ randomSource: realRandomSource(),+ docReader: args.doc,+ docWriter: args.doc,+ awarenessSource: args.awarenessSource,+ resolveNode: (n) => args.doc.resolveRef(n), // point cursors at inserted nodes+ sleep: args.sleep,+ });+ const failed = results.filter((r) => !r.ok).length;+ span?.setAttr('ops.failed', failed);+ if (failed > 0) span?.setAttr('error.kind', 'apply');return summarize(results);}diff --git a/services/ai-editing-worker/src/ai-editing/tools/dispatch.ts b/services/ai-editing-worker/src/ai-editing/tools/dispatch.tsindex 4d0dd475b5..226a8d9c3e 100644--- a/services/ai-editing-worker/src/ai-editing/tools/dispatch.ts+++ b/services/ai-editing-worker/src/ai-editing/tools/dispatch.ts@@ -1,3 +1,4 @@+import { type Span, Telemetry } from '@macro-inc/observability';import { type LanguageModel, tool } from 'ai';import { z } from 'zod';import type { coder } from '../agents';@@ -227,6 +228,12 @@ export type DispatchToolOptions = {onCoderResult?: (codes: CoderRunCode[]) => void;/** Called after each dispatch call with the edit's timing trace. */onEditTrace?: (edit: DispatchEditTrace) => void;+ /** Resolves the parent for each `edit.dispatch` span at launch time — the+ * supervisor turn that issued the call. Passed explicitly, and as a+ * function, because coders launch from AI SDK stream callbacks: the+ * ambient context is not trustworthy there, and the parent changes+ * from one turn to the next. */+ parentSpan?: () => Span | undefined;};export const DispatchEditSchema = z.object({@@ -271,34 +278,57 @@ export function createDispatchTool(opts: DispatchToolOptions) {async function runEdit(edit: DispatchEdit,trace: DispatchEditTrace,- childModel: LanguageModel+ childModel: LanguageModel,+ span: Span): Promise<CoderRun> {- // Serialized at launch time so an edit launched early sees whatever earlier- // coders have already applied.- const xml = serializeWithXml(session);- const context = xmlWindow(- xml,- computeContextRange(xml, edit.editing_instruction)- );- const writer = await makeWriter();- const { doc, awarenessSource } = writer;try {- return await runTask(session, edit.editing_instruction, childModel, {- doc,- awarenessSource,- context,- request,- params,- typingAnimations,- sleep,- signal,- runner,- onOps,- onRunCode: () => trace.runCodeAt.push(Date.now()),- });+ // Serialized at launch time so an edit launched early sees whatever earlier+ // coders have already applied.+ const xml = serializeWithXml(session);+ const range = computeContextRange(xml, edit.editing_instruction);+ const context = xmlWindow(xml, range);+ span.setAttr('context.source', range.source);+ span.setAttr('context.lines', range.endLine - range.startLine + 1);+ const borrowStartedAt = Date.now();+ const writer = await makeWriter();+ span.setAttr('writer.wait_ms', Date.now() - borrowStartedAt);+ const { doc, awarenessSource } = writer;+ try {+ const result = await runTask(+ session,+ edit.editing_instruction,+ childModel,+ {+ doc,+ awarenessSource,+ context,+ request,+ params,+ typingAnimations,+ sleep,+ signal,+ runner,+ onOps,+ onRunCode: () => trace.runCodeAt.push(Date.now()),+ span,+ }+ );+ span.setAttr(+ 'gen_ai.request.model',+ (childModel as { modelId: string }).modelId+ );+ span.setAttr('run_code.count', trace.runCodeAt.length);+ span.setAttr('coder.blocked', findBlocked(result) !== null);+ return result;+ } finally {+ trace.coderFinishedAt = Date.now();+ writer.release();+ }+ } catch (e) {+ span.error(e);+ throw e;} finally {- trace.coderFinishedAt = Date.now();- writer.release();+ span.end();}}@@ -316,8 +346,21 @@ export function createDispatchTool(opts: DispatchToolOptions) {coderFinishedAt: 0,runCodeAt: [],};+ // runEdit ends the span in its finally, so streamed runs that reject+ // before `execute` joins them still close it.+ const parent = opts.parentSpan?.();+ const span = parent+ ? parent.span('edit.dispatch')+ : Telemetry.span('edit.dispatch');+ span.setAttr('dispatch.tool_call_id', toolCallId);+ span.setAttr('dispatch.streamed', streamed);+ span.setAttr('instruction.chars', edit.editing_instruction.length);const model = makeChildModel();- const entry: EditRun = { run: runEdit(edit, trace, model), trace, model };+ const entry: EditRun = {+ run: runEdit(edit, trace, model, span),+ trace,+ model,+ };// A streamed run can reject before `execute` awaits it; keep that rejection// for the join instead of letting it escape as an unhandled rejection.entry.run.catch(() => {});diff --git a/services/ai-editing-worker/src/ai-editing/tools/run-code.ts b/services/ai-editing-worker/src/ai-editing/tools/run-code.tsindex b8578f1526..ea86b937c7 100644--- a/services/ai-editing-worker/src/ai-editing/tools/run-code.ts+++ b/services/ai-editing-worker/src/ai-editing/tools/run-code.ts@@ -1,3 +1,4 @@+import { type Span, Telemetry } from '@macro-inc/observability';import { tool } from 'ai';import { z } from 'zod';import type { LexicalSession } from '../ai-toolkit';@@ -18,6 +19,8 @@ export type RunCodeToolOptions = {onOps?: (ops: DocumentOp[]) => void;/** Called when a runCode call begins executing. */onRunCode?: (snippets?: Record<string, string>) => void;+ /** Parent for the per-call `edit.run_code` span (the dispatch span). */+ span?: Span;};/** The writer's one tool: run a JS snippet against `editor`, returning compact@@ -38,19 +41,33 @@ export function createRunCodeTool(opts: RunCodeToolOptions) {}),execute: async ({ code, snippets }) => {opts.onRunCode?.(snippets);- const result = await runEditorCode({- session: opts.session,- doc: opts.doc,- code,- awarenessSource: opts.awarenessSource,- snippets,- params: opts.params,- typingAnimations: opts.typingAnimations,- sleep: opts.sleep,- runner: opts.runner,- onOps: opts.onOps,- });- return result;+ const span = opts.span+ ? opts.span.span('edit.run_code')+ : Telemetry.span('edit.run_code');+ span.setAttr('code.bytes', code.length);+ span.setAttr('snippets.count', Object.keys(snippets ?? {}).length);+ try {+ const result = await runEditorCode({+ session: opts.session,+ doc: opts.doc,+ code,+ awarenessSource: opts.awarenessSource,+ snippets,+ params: opts.params,+ typingAnimations: opts.typingAnimations,+ sleep: opts.sleep,+ runner: opts.runner,+ onOps: opts.onOps,+ span,+ });+ span.setAttr('result', result.startsWith('error:') ? 'error' : 'ok');+ return result;+ } catch (e) {+ span.error(e);+ throw e;+ } finally {+ span.end();+ }},});}diff --git a/services/ai-editing-worker/src/endpoints/edit.ts b/services/ai-editing-worker/src/endpoints/edit.tsindex 4fe87c747c..954dc98e54 100644--- a/services/ai-editing-worker/src/endpoints/edit.ts+++ b/services/ai-editing-worker/src/endpoints/edit.ts@@ -2,6 +2,7 @@ import { createAnthropic } from '@ai-sdk/anthropic';import { createCerebras } from '@ai-sdk/cerebras';import { createOpenAI } from '@ai-sdk/openai';import { zValidator } from '@hono/zod-validator';+import { Telemetry } from '@macro-inc/observability';import type { LanguageModel } from 'ai';import { createFallback } from 'ai-fallback';import { Hono } from 'hono';@@ -64,7 +65,8 @@ const EditBody = z.object({* chain (multiple, advancing on provider errors/rate limits). */function buildModels(env: ReturnType<typeof getEnv>,- models: EditModels+ models: EditModels,+ onFallback?: () => void): ResolvedModels {const resolveOne = ({ provider, model }: Model) => {const apiKey = env[PROVIDERS[provider].key];@@ -75,8 +77,10 @@ function buildModels(if (resolved.length === 1) return resolved[0];return createFallback({models: resolved,- onError: (error, modelId) =>- console.error(`edit model ${modelId} failed, falling back:`, error),+ onError: (error, modelId) => {+ onFallback?.();+ console.error(`edit model ${modelId} failed, falling back:`, error);+ },});};return {@@ -131,19 +135,43 @@ edit.post('/', zValidator('json', EditBody), async (c) => {setTimeout(resolve, ms / presence.multiplier()));- const { usage, ops, session, clarification } = await runEditSession({- source,- documentId,- prompt,- models: buildModels(env, models),- typingAnimations,- sleep,- interpret,- debug,- propagate,- runner: runInSandbox,- signal,- }).finally(presence.stop);+ const { usage, ops, session, clarification } = await Telemetry.span(+ 'edit.session',+ async (span) => {+ span.setAttr('document.id', documentId);+ span.setAttr('edit.interpret', interpret);+ span.setAttr('edit.propagate', propagate);+ let modelFallbacks = 0;+ try {+ const result = await runEditSession({+ source,+ documentId,+ prompt,+ models: buildModels(env, models, () => modelFallbacks++),+ typingAnimations,+ sleep,+ interpret,+ debug,+ propagate,+ runner: runInSandbox,+ signal,+ }).finally(presence.stop);+ span.setAttr(+ 'edit.dispatch_count',+ result.session.dispatchEditTraces?.length ?? 0+ );+ span.setAttr('edit.ops_total', result.ops.length);+ span.setAttr('edit.blocked', result.clarification !== undefined);+ return result;+ } catch (err) {+ span.setAttr('edit.aborted', signal.aborted);+ span.error(err);+ throw err;+ } finally {+ span.setAttr('model.fallbacks', modelFallbacks);+ }+ }+ );const db = c.env.TRACES_DB;if (db) {diff --git a/services/ai-editing-worker/src/env.ts b/services/ai-editing-worker/src/env.tsindex 546f785ffd..ae5aa3404f 100644--- a/services/ai-editing-worker/src/env.ts+++ b/services/ai-editing-worker/src/env.ts@@ -2,6 +2,13 @@ import { envsafe, str } from 'envsafe';import type { D1Database } from './traces-db';export type Bindings = {+ /** Tags exported spans with deployment.environment (matches DD_ENV). */+ ENVIRONMENT: string | undefined;+ /** OTLP/HTTP traces endpoint; unset disables trace export. */+ OTEL_EXPORTER_OTLP_ENDPOINT: string | undefined;+ /** Datadog API key attached to trace exports when the endpoint is a+ * Datadog intake; secret, set via `wrangler secret put DD_API_KEY`. */+ DD_API_KEY: string | undefined;ANTHROPIC_API_KEY: string | undefined;CEREBRAS_API_KEY: string | undefined;OPENAI_API_KEY: string | undefined;diff --git a/services/ai-editing-worker/src/index.ts b/services/ai-editing-worker/src/index.tsindex 5d143d4eac..5628157e8f 100644--- a/services/ai-editing-worker/src/index.ts+++ b/services/ai-editing-worker/src/index.ts@@ -1,4 +1,8 @@import './globals';+import {+ createWorkerTraceConfig,+ instrument,+} from '@macro-inc/observability/worker';import { Hono } from 'hono';import { cors } from 'hono/cors';import endpoints from './endpoints';@@ -53,4 +57,16 @@ app.use();app.route('/', endpoints);-export default app;+// instrument() wraps fetch with a request root span, registers the global+// tracer provider/context manager (which Telemetry.span resolves through),+// propagates traceparent, and flushes spans via waitUntil after the response.+export default instrument({ fetch: app.fetch }, (env: Bindings) =>+ createWorkerTraceConfig({+ serviceName: 'ai-editing-worker',+ environment: env.ENVIRONMENT ?? 'local',+ tracesUrl: env.OTEL_EXPORTER_OTLP_ENDPOINT,+ tracesHeaders: env.DD_API_KEY+ ? { 'dd-api-key': env.DD_API_KEY }+ : undefined,+ })+);diff --git a/services/ai-editing-worker/src/run-edit.ts b/services/ai-editing-worker/src/run-edit.tsindex 1a29efc736..17965d32e8 100644--- a/services/ai-editing-worker/src/run-edit.ts+++ b/services/ai-editing-worker/src/run-edit.ts@@ -10,6 +10,7 @@ import {} from '@macro-inc/collaboration/collab/wal';import type { SyncServiceSource } from '@macro-inc/collaboration/sync-service/source';import { MARKDOWN_LORO_SCHEMA } from '@macro-inc/lexical-core/markdown-loro-schema';+import { Telemetry } from '@macro-inc/observability';import type { LanguageModel } from 'ai';import { supervisor } from './ai-editing/agents';import type { DocumentOp } from './ai-editing/editor';@@ -74,26 +75,34 @@ export async function runEditSession(args: RunEditArgs): Promise<RunEditResult> {const source = args.source;- const initialResult = await source.doInitialSync();- if (initialResult.isErr()) {- source.cleanup();- const e = initialResult.error;- throw new Error(`initial sync failed: ${e.type} (${e.duration}ms)`);- }- const initial = initialResult.value;// The manager owns the one true (merged) doc + mirror.const manager = new LoroManager(MARKDOWN_LORO_SCHEMA, {documentId: args.documentId,});- const initResult = await manager.initializeFromSnapshot(initial.snapshot);- if (initResult.isErr()) {- source.cleanup();- throw new Error(- `failed to initialize from snapshot: ${initResult.error[0]?.message}`- );- }+ await Telemetry.span('edit.sync_init', async (span) => {+ const initialResult = await source.doInitialSync();+ if (initialResult.isErr()) {+ source.cleanup();+ const e = initialResult.error;+ const err = new Error(`initial sync failed: ${e.type} (${e.duration}ms)`);+ span.error(err);+ throw err;+ }+ const initial = initialResult.value;+ span.setAttr('snapshot.bytes', initial.snapshot.byteLength);++ const initResult = await manager.initializeFromSnapshot(initial.snapshot);+ if (initResult.isErr()) {+ source.cleanup();+ const err = new Error(+ `failed to initialize from snapshot: ${initResult.error[0]?.message}`+ );+ span.error(err);+ throw err;+ }+ });// When the caller applies the returned ops locally instead, propagating// here too would double the content — so route live sync to a no-op sink@@ -110,14 +119,21 @@ export async function runEditSession(// The workspace owns the editing surface + its two-way sync with Loro, and// hands out per-coder writers. Under debug it also records a replay trace.- const workspace = new EditingWorkspace(manager, liveSource, wal);+ const { workspace, initialDocument } = await Telemetry.span(+ 'edit.hydrate',+ async (span) => {+ const workspace = new EditingWorkspace(manager, liveSource, wal);+ const initialDocument = serializeWithXml(workspace.session);+ span.setAttr('document.chars', initialDocument.length);+ return { workspace, initialDocument };+ }+ );const allOps: DocumentOp[] = [];const coderCodeBlocks: CoderRunCode[][][] = [];const dispatchEditTraces: DispatchEditTrace[][] = [];const sessionId = crypto.randomUUID();const startedAt = new Date();- const initialDocument = serializeWithXml(workspace.session);try {const {totalUsage,@@ -141,8 +157,10 @@ export async function runEditSession(// Drain the queued propagates (plus a final catch-all sync) and ensure every// commit reached the server before we disconnect. No-op sink when not// propagating, so this is harmless either way.- await workspace.flush();- await wal.flush();+ await Telemetry.span('edit.flush', async () => {+ await workspace.flush();+ await wal.flush();+ });const usage = totalUsage.toEntries();diff --git a/services/ai-editing-worker/wrangler.toml b/services/ai-editing-worker/wrangler.tomlindex de4ee70c57..d0a5f40435 100644--- a/services/ai-editing-worker/wrangler.toml+++ b/services/ai-editing-worker/wrangler.toml@@ -27,6 +27,12 @@ database_name = "ai-editing-traces"database_id = "16d34e99-3bc6-44e4-a287-1ea65183bd28"[vars]+# Tags exported spans with deployment.environment, matching the Datadog+# agent's DD_ENV so the env-scoped trace views include them.+ENVIRONMENT = "local"+# Local jaeger/datadog-agent OTLP receiver (docker port-mapped to the host).+# Unset in deployed envs = trace export disabled.+OTEL_EXPORTER_OTLP_ENDPOINT = "http://localhost:4318/v1/traces"SYNC_WS_BASE = "wss://sync-service-prod2.macroverse.workers.dev"DSS_BASE = "https://cloud-storage.macro.com"SEARCH_SERVICE_BASE = "https://cloud-storage.macro.com"@@ -37,6 +43,8 @@ AUTH_SERVICE_BASE = "https://auth-service.macro.com"port = 8933[env.local.vars]+ENVIRONMENT = "local"+OTEL_EXPORTER_OTLP_ENDPOINT = "http://otel-collector:4318/v1/traces"SYNC_WS_BASE = "ws://sync-service:8787"DSS_BASE = "http://document-storage-service:8080"SEARCH_SERVICE_BASE = "http://search-processing-service:8080"@@ -50,6 +58,7 @@ database_name = "ai-editing-traces"database_id = "16d34e99-3bc6-44e4-a287-1ea65183bd28"[env.dev.vars]+ENVIRONMENT = "development"SYNC_WS_BASE = "wss://sync-service-dev3.macroverse.workers.dev"DSS_BASE = "https://cloud-storage-dev.macro.com"SEARCH_SERVICE_BASE = "https://cloud-storage-dev.macro.com"@@ -65,6 +74,7 @@ database_id = "16d34e99-3bc6-44e4-a287-1ea65183bd28"name = "ai-editing-worker-playground"[env.playground.vars]+ENVIRONMENT = "development"SYNC_WS_BASE = "wss://sync-service-dev3.macroverse.workers.dev"DSS_BASE = "https://cloud-storage-dev.macro.com"SEARCH_SERVICE_BASE = "https://cloud-storage-dev.macro.com"@@ -80,6 +90,7 @@ database_id = "28cdeacb-6468-449b-a1c7-a193b220501e"name = "ai-editing-worker"[env.prod.vars]+ENVIRONMENT = "production"SYNC_WS_BASE = "wss://sync-service-prod2.macroverse.workers.dev"DSS_BASE = "https://cloud-storage.macro.com"SEARCH_SERVICE_BASE = "https://cloud-storage.macro.com"diff --git a/tooling/xtask/crates/xtask_local/src/local/frontend.rs b/tooling/xtask/crates/xtask_local/src/local/frontend.rsindex d8197eb8bf..477f4ef9a1 100644--- a/tooling/xtask/crates/xtask_local/src/local/frontend.rs+++ b/tooling/xtask/crates/xtask_local/src/local/frontend.rs@@ -45,7 +45,12 @@ pub fn static_dir(instance: &Instance) -> std::path::PathBuf {/// (dev-mode bundle, production optimizations), except the backend origin is/// the `same-origin` sentinel — resolved from `location.origin` at runtime — so/// the one bundle works on localhost and through any tunnel/preview hostname.-pub fn build_static(stage: &Stage, instance: &Instance, prebuilt: Option<&Path>) -> Result<()> {+pub fn build_static(+ stage: &Stage,+ instance: &Instance,+ mode: Mode,+ prebuilt: Option<&Path>,+) -> Result<()> {let dist = match prebuilt {Some(dir) => dir.to_owned(),None => {@@ -56,6 +61,9 @@ pub fn build_static(stage: &Stage, instance: &Instance, prebuilt: Option<&Path>).env("NODE_ENV", "production").env("VITE_LOCAL_SERVERS", "ALL").env("VITE_LOCAL_BACKEND_ORIGIN", "same-origin");+ if mode.spec().runs_local_infra {+ cmd.env("VITE_AI_EDITING_WORKER_URL", "/ai-editing");+ }stage.run("Building frontend bundle", &mut cmd)?;app_dir().join("dist")}@@ -97,7 +105,7 @@ pub fn build_static(stage: &Stage, instance: &Instance, prebuilt: Option<&Path>)/// and exercises the real browser -> proxy -> collector path. Left unset/// otherwise. This overrides the bare-dev defaults in apps/web/.env.local/// (Vite lets process env win).-fn dev_env(instance: &Instance, traces_enabled: bool) -> Vec<(String, String)> {+fn dev_env(instance: &Instance, mode: Mode, traces_enabled: bool) -> Vec<(String, String)> {let mut env = vec![("PORT".to_string(),@@ -109,6 +117,12 @@ fn dev_env(instance: &Instance, traces_enabled: bool) -> Vec<(String, String)> {proxy::url(instance),),];+ if mode.spec().runs_local_infra {+ env.push((+ "VITE_AI_EDITING_WORKER_URL".to_string(),+ format!("{}/ai-editing", proxy::url(instance)),+ ));+ }if traces_enabled {env.push(("VITE_OTEL_EXPORTER_URL".to_string(),@@ -235,7 +249,7 @@ pub fn start(.stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::piped());- for (k, v) in dev_env(instance, traces_enabled) {+ for (k, v) in dev_env(instance, mode, traces_enabled) {cmd.env(k, v);}let mut child = cmd.spawn().context("launching `bun run dev`")?;diff --git a/tooling/xtask/crates/xtask_local/src/local/proxy.rs b/tooling/xtask/crates/xtask_local/src/local/proxy.rsindex 3ea67b2eb1..3522116416 100644--- a/tooling/xtask/crates/xtask_local/src/local/proxy.rs+++ b/tooling/xtask/crates/xtask_local/src/local/proxy.rs@@ -134,6 +134,9 @@ const SPECIAL_ROUTES: &str = r#" @websocket path /websocket /websocket/*handle_path /lexical/* {reverse_proxy lexical-service:8096}+ handle_path /ai-editing/* {+ reverse_proxy ai-editing-worker:8933+ }"#;const MAILPIT_ROUTE: &str = r#" # Mailpit serves itself under /mailpit (MP_WEBROOT), so no prefix strip —diff --git a/tooling/xtask/crates/xtask_local/src/local/proxy/test.rs b/tooling/xtask/crates/xtask_local/src/local/proxy/test.rsindex df6d20e0f5..466bf804e8 100644--- a/tooling/xtask/crates/xtask_local/src/local/proxy/test.rs+++ b/tooling/xtask/crates/xtask_local/src/local/proxy/test.rs@@ -52,6 +52,8 @@ fn document_content_services_are_available_through_the_proxy() {assert!(caddy.contains("reverse_proxy sync-service:8787"));assert!(caddy.contains("handle_path /lexical/*"));assert!(caddy.contains("reverse_proxy lexical-service:8096"));+ assert!(caddy.contains("handle_path /ai-editing/*"));+ assert!(caddy.contains("reverse_proxy ai-editing-worker:8933"));}/// The static-file block is the one route that differs by mode: LocalStack S3diff --git a/tooling/xtask/crates/xtask_local/src/local/stack.rs b/tooling/xtask/crates/xtask_local/src/local/stack.rsindex 5e445c3ee6..175e9e6a62 100644--- a/tooling/xtask/crates/xtask_local/src/local/stack.rs+++ b/tooling/xtask/crates/xtask_local/src/local/stack.rs@@ -197,13 +197,18 @@ pub fn up(mode: Mode, args: &UpArgs) -> Result<Instance> {// joined just before `bring_up_app` creates that container. (Dry run:// synchronously, so the command preview prints in order.)if static_frontend && stage.is_dry_run() {- frontend::build_static(&stage, &instance, args.frontend_dist.as_deref())?;+ frontend::build_static(&stage, &instance, mode, args.frontend_dist.as_deref())?;}let fe_build = (static_frontend && !stage.is_dry_run()).then(|| {let instance = instance.clone();let prebuilt = args.frontend_dist.clone();std::thread::spawn(move || {- frontend::build_static(&Stage::from_env().quiet(), &instance, prebuilt.as_deref())+ frontend::build_static(+ &Stage::from_env().quiet(),+ &instance,+ mode,+ prebuilt.as_deref(),+ )})});@@ -359,6 +364,7 @@ pub fn update(args: &UpdateArgs) -> Result<()> {reload_static_frontend(&stage,&instance,+ mode,&env,&state,args.frontend_dist.as_deref(),@@ -405,6 +411,7 @@ fn apply_prebuilt_binaries(fn reload_static_frontend(stage: &Stage,instance: &Instance,+ mode: Mode,env: &env_layer::ResolvedEnv,state: &StackState,frontend_dist: Option<&Path>,@@ -415,7 +422,7 @@ fn reload_static_frontend(re-run `just stack up` to serve one");}- frontend::build_static(stage, instance, frontend_dist)?;+ frontend::build_static(stage, instance, mode, frontend_dist)?;// build_static replaces the staged directory, so the container must be// recreated to establish a bind mount to the new inode.let mut up = super::compose_cmd(instance, env);