Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

feat(api): add workflow collaboration metrics (#45446)

非法操作cf38e6ad67b1
diff --git a/api/controllers/console/socketio/workflow.py b/api/controllers/console/socketio/workflow.py
index 13555b1a973..e062c0cd5a3 100644
--- a/api/controllers/console/socketio/workflow.py
+++ b/api/controllers/console/socketio/workflow.py
@@ -7,6 +7,7 @@ from flask import Request as FlaskRequest
from core.db.session_factory import session_factory
from extensions.ext_application_services import application_services
from extensions.ext_socketio import sio
+from extensions.otel.workflow_collaboration import observe_workflow_event
from libs.passport import PassportService
from libs.token import extract_access_token
from repositories.workflow_collaboration_repository import WorkflowCollaborationRepository
@@ -21,9 +22,10 @@ def _sio_on(event: str) -> Callable[[Callable[..., object]], Callable[..., objec
@_sio_on("connect")
-def socket_connect(sid, environ, auth):
+@observe_workflow_event("connect", sio)
+def socket_connect(sid, environ, auth=None):
"""
- WebSocket connect event, do authentication here.
+ Authenticate a connect with optional Socket.IO auth data; preserve token-based admission.
"""
try:
request_environ = FlaskRequest(environ)
@@ -61,6 +63,7 @@ def socket_connect(sid, environ, auth):
@_sio_on("user_connect")
+@observe_workflow_event("user_connect", sio)
def handle_user_connect(sid, data):
"""
Handle user connect event. Each session (tab) is treated as an independent collaborator.
@@ -78,15 +81,23 @@ def handle_user_connect(sid, data):
return {"msg": "connected", "user_id": user_id, "sid": sid, "isLeader": is_leader}
+@observe_workflow_event("disconnect", sio)
+def _handle_disconnect(sid):
+ """Remove the session from online users and observe the result."""
+ collaboration_service.disconnect_session(sid)
+
+
@_sio_on("disconnect")
def handle_disconnect(sid):
+ """Let Socket.IO discard its reason before starting observable cleanup.
+
+ A cleanup TypeError must propagate without Socket.IO retrying the cleanup.
"""
- Handle session disconnect event. Remove the specific session from online users.
- """
- collaboration_service.disconnect_session(sid)
+ _handle_disconnect(sid)
@_sio_on("collaboration_event")
+@observe_workflow_event("collaboration_event", sio)
def handle_collaboration_event(sid, data):
"""
Handle general collaboration events, include:
@@ -102,6 +113,7 @@ def handle_collaboration_event(sid, data):
@_sio_on("graph_event")
+@observe_workflow_event("graph_event", sio)
def handle_graph_event(sid, data):
"""
Handle graph events - simple broadcast relay.
diff --git a/api/extensions/otel/instrumentation.py b/api/extensions/otel/instrumentation.py
index 1180e658ca9..023fa2e8c03 100644
--- a/api/extensions/otel/instrumentation.py
+++ b/api/extensions/otel/instrumentation.py
@@ -19,6 +19,7 @@ from opentelemetry.trace.status import StatusCode
from configs import dify_config
from dify_app import DifyApp
from extensions.otel.runtime import is_celery_worker
+from extensions.otel.workflow_collaboration import init_workflow_collaboration_metrics
logger = logging.getLogger(__name__)
@@ -152,6 +153,7 @@ def init_httpx_instrumentor() -> None:
def init_instruments(app: DifyApp) -> None:
if not is_celery_worker():
init_flask_instrumentor(app)
+ init_workflow_collaboration_metrics(app)
_new_celery_instrumentor().instrument(
tracer_provider=get_tracer_provider(), meter_provider=get_meter_provider()
)
diff --git a/api/extensions/otel/workflow_collaboration.py b/api/extensions/otel/workflow_collaboration.py
new file mode 100644
index 00000000000..33df3cc8d00
--- /dev/null
+++ b/api/extensions/otel/workflow_collaboration.py
@@ -0,0 +1,271 @@
+"""Worker-local Socket.IO load metrics without persistence or per-message network I/O.
+
+Lifecycle updates run on the Socket.IO gevent hub and never yield. Observable callbacks
+only read immutable numeric snapshots, including when the SDK collects on a native thread.
+Room counts describe local membership, not globally distinct rooms or distinct people.
+No room, socket, tenant, user, payload, or exception values become metric attributes.
+"""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Callable, Iterable, Mapping, Sequence
+from functools import wraps
+from time import perf_counter
+from typing import Literal, NamedTuple, Protocol, runtime_checkable
+
+from flask import Flask
+from opentelemetry.metrics import CallbackOptions, Meter, Observation, get_meter
+
+from configs import dify_config
+
+logger = logging.getLogger(__name__)
+
+type SocketEvent = Literal["connect", "user_connect", "disconnect", "collaboration_event", "graph_event"]
+type Outcome = Literal["success", "rejected", "error"]
+
+_EVENT_TYPES = frozenset(
+ {
+ "mouse_move",
+ "vars_and_features_update",
+ "sync_request",
+ "workflow_update",
+ "comments_update",
+ "node_panel_presence",
+ "graph_view_state",
+ "graph_resync_request",
+ }
+)
+_SIZE_RANGES = ("1", "2-5", "6-10", "11-20", "21+")
+_WORKER_ATTRIBUTES = {"aggregation_scope": "worker"}
+_EXTENSION_KEY = "workflow_collaboration_metrics"
+
+
+class _SocketManager(Protocol):
+ def is_connected(self, sid: str, namespace: str) -> bool: ...
+
+
+@runtime_checkable
+class _SocketIO(Protocol):
+ @property
+ def app(self) -> Flask: ...
+
+ @property
+ def manager(self) -> _SocketManager: ...
+
+ def rooms(self, sid: str, namespace: str) -> list[str]: ...
+
+
+class _Snapshot(NamedTuple):
+ connections: int
+ rooms: int
+ size_ranges: tuple[int, ...]
+
+
+def _size_range(size: int) -> str:
+ """Given a positive local room size, return one bounded attribute value without I/O."""
+ if size == 1:
+ return "1"
+ if size <= 5:
+ return "2-5"
+ if size <= 10:
+ return "6-10"
+ if size <= 20:
+ return "11-20"
+ return "21+"
+
+
+class WorkflowCollaborationMetrics:
+ """Own one worker's instruments and an idempotent mirror of accepted socket membership.
+
+ Writers must run in Socket.IO handlers on the gevent hub; they perform no I/O or
+ yielding operations. SDK collection never traverses the mutable membership maps.
+ Unknown/late joins cannot recreate disconnected sessions, and duplicate lifecycle
+ notifications cannot inflate counts. Instrument failures are isolated by the decorator.
+ """
+
+ def __init__(self, meter: Meter | None = None) -> None:
+ """Register instruments with the supplied/default meter and start with zero load."""
+ self._connections: dict[str, frozenset[str]] = {}
+ self._room_sizes: dict[str, int] = {}
+ self._size_counts = dict.fromkeys(_SIZE_RANGES, 0)
+ self._snapshot = _Snapshot(0, 0, (0,) * len(_SIZE_RANGES))
+ meter = meter if meter is not None else get_meter("dify.collaboration")
+ meter.create_observable_gauge(
+ "dify.collaboration.connections",
+ callbacks=(self._observe_connections,),
+ description="Authenticated Socket.IO sessions connected to this worker; not distinct users",
+ unit="{connection}",
+ )
+ meter.create_observable_gauge(
+ "dify.collaboration.rooms",
+ callbacks=(self._observe_rooms,),
+ description="Workflow rooms with local members; summing workers does not give globally distinct rooms",
+ unit="{room}",
+ )
+ meter.create_observable_gauge(
+ "dify.collaboration.room_sessions",
+ callbacks=(self._observe_room_sessions,),
+ description="Current local room count by local session-count range; not a join-weighted histogram",
+ unit="{room}",
+ )
+ self._events = meter.create_counter(
+ "dify.collaboration.events", description="Completed Socket.IO handlers by event and outcome", unit="{event}"
+ )
+ self._duration = meter.create_histogram(
+ "dify.collaboration.event_duration",
+ description="Socket.IO handler wall time, including dependency waits; not peer delivery or client ACK time",
+ unit="s",
+ explicit_bucket_boundaries_advisory=(0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 15, 30),
+ )
+
+ def _observe_connections(self, _: CallbackOptions) -> Iterable[Observation]:
+ """Return a numeric snapshot for any collector thread without accessing live sockets."""
+ return (Observation(self._snapshot.connections, attributes=_WORKER_ATTRIBUTES),)
+
+ def _observe_rooms(self, _: CallbackOptions) -> Iterable[Observation]:
+ """Return the local room snapshot, including zero after the last member leaves."""
+ return (Observation(self._snapshot.rooms, attributes=_WORKER_ATTRIBUTES),)
+
+ def _observe_room_sessions(self, _: CallbackOptions) -> Iterable[Observation]:
+ """Return all fixed range counts from one snapshot without exposing room identifiers."""
+ snapshot = self._snapshot
+ return tuple(
+ Observation(count, attributes={**_WORKER_ATTRIBUTES, "size_range": size_range})
+ for size_range, count in zip(_SIZE_RANGES, snapshot.size_ranges, strict=True)
+ )
+
+ def _publish_snapshot(self) -> None:
+ """After a lifecycle update, atomically replace the collector's immutable numeric view."""
+ self._snapshot = _Snapshot(len(self._connections), len(self._room_sizes), tuple(self._size_counts.values()))
+
+ def _change_room_size(self, room: str, delta: Literal[-1, 1]) -> None:
+ """Apply one known member addition/removal and keep current room buckets consistent."""
+ previous = self._room_sizes.get(room, 0)
+ if previous:
+ self._size_counts[_size_range(previous)] -= 1
+ current = previous + delta
+ if current:
+ self._room_sizes[room] = current
+ self._size_counts[_size_range(current)] += 1
+ else:
+ del self._room_sizes[room]
+
+ def connected(self, sid: str) -> None:
+ """Register a live, authenticated SID once; a duplicate preserves existing memberships."""
+ if sid not in self._connections:
+ self._connections[sid] = frozenset()
+ self._publish_snapshot()
+
+ def update_rooms(self, sid: str, rooms: Sequence[str]) -> None:
+ """Mirror actual rooms for a registered SID; ignore late joins and exclude its private room."""
+ previous = self._connections.get(sid)
+ if previous is None:
+ return
+ current = frozenset(room for room in rooms if room != sid)
+ for room in previous - current:
+ self._change_room_size(room, -1)
+ for room in current - previous:
+ self._change_room_size(room, 1)
+ self._connections[sid] = current
+ self._publish_snapshot()
+
+ def disconnected(self, sid: str) -> None:
+ """Remove a SID and all local memberships, tolerating duplicate or rejected disconnects."""
+ previous = self._connections.pop(sid, None)
+ if previous is not None:
+ for room in previous:
+ self._change_room_size(room, -1)
+ self._publish_snapshot()
+
+ def record_event(self, event: SocketEvent, event_type: str, outcome: Outcome, duration_seconds: float) -> None:
+ """Record one completed handler with bounded labels; propagate SDK failures to the isolation boundary."""
+ bounded_type = event_type if event_type in _EVENT_TYPES else "other"
+ attributes = {
+ "event": event,
+ "event_type": bounded_type if event == "collaboration_event" else "none",
+ "outcome": outcome,
+ }
+ self._events.add(1, attributes=attributes)
+ self._duration.record(duration_seconds, attributes=attributes)
+
+
+def init_workflow_collaboration_metrics(app: Flask) -> None:
+ """Bind metrics during enabled OTel startup, after the provider and before serving events.
+
+ The application owns one instance; repeated startup calls preserve its state.
+ Registration failure leaves instrumentation absent and business handlers usable.
+ """
+ if _EXTENSION_KEY not in app.extensions:
+ try:
+ app.extensions[_EXTENSION_KEY] = WorkflowCollaborationMetrics()
+ except Exception:
+ logger.exception("Failed to initialize workflow collaboration metrics")
+
+
+def _outcome(result: object) -> Outcome:
+ """Classify existing handler results without modifying their body, status, or identity."""
+ if result is False:
+ return "rejected"
+ if isinstance(result, tuple) and len(result) == 2 and isinstance(result[1], int):
+ if result[1] >= 500:
+ return "error"
+ if result[1] >= 400:
+ return "rejected"
+ return "success"
+
+
+def observe_workflow_event[**P, R](
+ event: SocketEvent, socketio: _SocketIO
+) -> Callable[[Callable[P, R]], Callable[P, R]]:
+ """Observe an existing synchronous Socket.IO handler without changing returns or exceptions.
+
+ The caller supplies the event and Socket.IO server bound to its initialized app.
+ Disabled or unavailable instrumentation calls the handler directly. Otherwise capture
+ membership in the handler thread, even when a join partially succeeds or Redis cleanup
+ fails, without additional Redis/SQL queries. Telemetry failures never alter authentication,
+ room membership, ACK payloads, or the handler's exception.
+ """
+
+ def decorate(handler: Callable[P, R]) -> Callable[P, R]:
+ """Preserve the handler signature and metadata while observing its completion."""
+
+ @wraps(handler)
+ def observed(*args: P.args, **kwargs: P.kwargs) -> R:
+ """Run the original handler once and isolate all metric-side failures in finally."""
+ if not dify_config.ENABLE_OTEL:
+ return handler(*args, **kwargs)
+ metrics = socketio.app.extensions.get(_EXTENSION_KEY)
+ if not isinstance(metrics, WorkflowCollaborationMetrics):
+ return handler(*args, **kwargs)
+
+ started = perf_counter()
+ outcome: Outcome = "error"
+ try:
+ result = handler(*args, **kwargs)
+ outcome = _outcome(result)
+ return result
+ finally:
+ duration = perf_counter() - started
+ try:
+ sid = args[0] if args else kwargs.get("sid")
+ try:
+ if isinstance(sid, str):
+ if event == "connect" and outcome == "success" and socketio.manager.is_connected(sid, "/"):
+ metrics.connected(sid)
+ elif event == "user_connect":
+ metrics.update_rooms(sid, socketio.rooms(sid, "/"))
+ elif event == "disconnect":
+ metrics.disconnected(sid)
+ finally:
+ data = args[1] if len(args) > 1 else kwargs.get("data")
+ event_type = data.get("type") if isinstance(data, Mapping) else None
+ metrics.record_event(
+ event, event_type if isinstance(event_type, str) else "other", outcome, duration
+ )
+ except Exception:
+ logger.warning("Failed to record workflow collaboration metrics", exc_info=True)
+
+ return observed
+
+ return decorate
diff --git a/api/tests/unit_tests/controllers/console/socketio/test_workflow.py b/api/tests/unit_tests/controllers/console/socketio/test_workflow.py
new file mode 100644
index 00000000000..f4a4ecaa89c
--- /dev/null
+++ b/api/tests/unit_tests/controllers/console/socketio/test_workflow.py
@@ -0,0 +1,157 @@
+"""Exercise collaboration metrics through Socket.IO's real lifecycle dispatch."""
+
+from collections.abc import Callable, Iterator
+from unittest.mock import MagicMock
+
+import pytest
+import socketio
+from engineio.socket import Socket
+from flask import Flask
+from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.sdk.metrics.export import Gauge, Histogram, InMemoryMetricReader, Metric, Sum
+from socketio.packet import CONNECT_ERROR, Packet
+from werkzeug.test import EnvironBuilder
+
+from controllers.console.socketio import workflow
+from extensions.ext_redis import RedisClientWrapper
+from extensions.otel import workflow_collaboration
+from extensions.otel.workflow_collaboration import WorkflowCollaborationMetrics
+
+
+def collaboration_server(
+ monkeypatch: pytest.MonkeyPatch,
+ redis_transport: tuple[RedisClientWrapper, MagicMock],
+ config_overrides: Callable[..., None],
+) -> Iterator[tuple[socketio.Server, WorkflowCollaborationMetrics, InMemoryMetricReader]]:
+ """Initialize app-owned metrics for registered handlers while replacing only transports and metric export."""
+ config_overrides(ENABLE_OTEL=True)
+ app = Flask(__name__)
+ server = workflow.sio
+ manager = socketio.Manager()
+ manager.set_server(server)
+ monkeypatch.setattr(server, "manager", manager)
+ monkeypatch.setattr(server, "environ", {})
+ monkeypatch.setattr(server, "app", app, raising=False)
+ monkeypatch.setattr(server.eio, "sockets", {})
+ monkeypatch.setattr(workflow.repository, "_redis", redis_transport[0])
+
+ reader = InMemoryMetricReader()
+ provider = MeterProvider(metric_readers=[reader])
+ monkeypatch.setattr(workflow_collaboration, "get_meter", provider.get_meter)
+ workflow_collaboration.init_workflow_collaboration_metrics(app)
+ metrics = app.extensions["workflow_collaboration_metrics"]
+ assert isinstance(metrics, WorkflowCollaborationMetrics)
+ try:
+ yield server, metrics, reader
+ finally:
+ provider.shutdown()
+
+
+def _read_metrics(reader: InMemoryMetricReader) -> dict[str, Metric]:
+ """Collect the real SDK instruments and return their exported values by metric name."""
+ data = reader.get_metrics_data()
+ assert data is not None
+ return {
+ metric.name: metric
+ for resource in data.resource_metrics
+ for scope in resource.scope_metrics
+ for metric in scope.metrics
+ }
+
+
+def _assert_single_event(metrics: dict[str, Metric], event: str, outcome: str) -> None:
+ """Require exactly one completed handler observation and no compatibility-retry error series."""
+ attributes = {"event": event, "event_type": "none", "outcome": outcome}
+ events = metrics["dify.collaboration.events"].data
+ assert isinstance(events, Sum)
+ assert len(events.data_points) == 1
+ assert events.data_points[0].attributes == attributes
+ assert events.data_points[0].value == 1
+
+ duration = metrics["dify.collaboration.event_duration"].data
+ assert isinstance(duration, Histogram)
+ assert len(duration.data_points) == 1
+ assert duration.data_points[0].attributes == attributes
+ assert duration.data_points[0].count == 1
+
+ connections = metrics["dify.collaboration.connections"].data
+ assert isinstance(connections, Gauge)
+ assert connections.data_points[0].value == 0
+
+
+def test_connect_without_auth_payload_records_one_rejection(
+ collaboration_server: tuple[socketio.Server, WorkflowCollaborationMetrics, InMemoryMetricReader],
+) -> None:
+ """A tokenless client omitting auth is rejected once without an argument-compatibility error."""
+ server, _, reader = collaboration_server
+ eio_sid = "connect-without-auth"
+ transport = Socket(server.eio, eio_sid)
+ server.eio.sockets[eio_sid] = transport
+ builder = EnvironBuilder(path="/socket.io")
+ try:
+ server.environ[eio_sid] = builder.get_environ()
+ finally:
+ builder.close()
+
+ assert server.handlers["/"]["connect"] is workflow.socket_connect
+ server._handle_connect(eio_sid, "/", None)
+
+ response = Packet(encoded_packet=transport.queue.get_nowait().data)
+ assert response.packet_type == CONNECT_ERROR
+ assert server.manager.sid_from_eio_sid(eio_sid, "/") is None
+ _assert_single_event(_read_metrics(reader), "connect", "rejected")
+
+
+def test_disconnect_with_reason_records_one_success(
+ collaboration_server: tuple[socketio.Server, WorkflowCollaborationMetrics, InMemoryMetricReader],
+ redis_transport: tuple[RedisClientWrapper, MagicMock],
+) -> None:
+ """A reason-bearing disconnect executes real cleanup once and removes the worker's live connection."""
+ server, metrics, reader = collaboration_server
+ eio_sid = "disconnect-with-reason"
+ sid = server.manager.connect(eio_sid, "/")
+ assert isinstance(sid, str)
+ metrics.connected(sid)
+
+ assert server.handlers["/"]["disconnect"] is workflow.handle_disconnect
+ server._handle_disconnect(eio_sid, "/", server.reason.PING_TIMEOUT)
+
+ redis_transport[1].assert_called_once_with(
+ "GET", workflow.repository.sid_key(sid), keys=[workflow.repository.sid_key(sid)]
+ )
+ assert not server.manager.is_connected(sid, "/")
+ _assert_single_event(_read_metrics(reader), "disconnect", "success")
+
+
[email protected]("otel_enabled", [True, False])
+def test_disconnect_type_error_is_not_retried(
+ collaboration_server: tuple[socketio.Server, WorkflowCollaborationMetrics, InMemoryMetricReader],
+ monkeypatch: pytest.MonkeyPatch,
+ config_overrides: Callable[..., None],
+ otel_enabled: bool,
+) -> None:
+ """A cleanup TypeError propagates after one call, regardless of instrumentation."""
+ server, metrics, reader = collaboration_server
+ config_overrides(ENABLE_OTEL=otel_enabled)
+ eio_sid = "disconnect-cleanup-error"
+ sid = server.manager.connect(eio_sid, "/")
+ assert isinstance(sid, str)
+ metrics.connected(sid)
+ failure = TypeError("cleanup failed after partial work")
+ calls: list[str] = []
+
+ def cleanup(disconnected_sid: str) -> None:
+ calls.append(disconnected_sid)
+ if len(calls) == 1:
+ raise failure
+
+ monkeypatch.setattr(workflow.collaboration_service, "disconnect_session", cleanup)
+
+ with pytest.raises(TypeError) as caught:
+ server._handle_disconnect(eio_sid, "/", server.reason.CLIENT_DISCONNECT)
+
+ assert caught.value is failure
+ assert calls == [sid]
+ if otel_enabled:
+ _assert_single_event(_read_metrics(reader), "disconnect", "error")
diff --git a/api/tests/unit_tests/extensions/otel/test_instrumentation.py b/api/tests/unit_tests/extensions/otel/test_instrumentation.py
new file mode 100644
index 00000000000..350759830c7
--- /dev/null
+++ b/api/tests/unit_tests/extensions/otel/test_instrumentation.py
@@ -0,0 +1,86 @@
+"""Exercise collaboration metric startup through the existing OTel composition root."""
+
+import os
+import subprocess
+import sys
+from pathlib import Path
+from textwrap import dedent
+
+
+def test_init_instruments_registers_app_owned_collaboration_metrics_before_events() -> None:
+ """Real instrumentors bind one app-owned instance and collect zero gauges before traffic.
+
+ A child interpreter isolates gevent patching, SDK providers, and global instrumentors
+ from other tests. SQLite supplies a real SQLAlchemy engine without external services.
+ """
+ script = dedent(
+ """
+ from gevent import monkey
+ monkey.patch_all()
+
+ from flask_sqlalchemy import SQLAlchemy
+ from opentelemetry.metrics import set_meter_provider
+ from opentelemetry.sdk.metrics import MeterProvider
+ from opentelemetry.sdk.metrics.export import Gauge, InMemoryMetricReader
+
+ from dify_app import DifyApp
+ from extensions.otel.instrumentation import init_instruments
+ from extensions.otel.workflow_collaboration import WorkflowCollaborationMetrics
+
+ app = DifyApp(__name__)
+ app.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///:memory:"
+ database = SQLAlchemy(app)
+ reader = InMemoryMetricReader()
+ provider = MeterProvider(metric_readers=[reader])
+ set_meter_provider(provider)
+ try:
+ init_instruments(app)
+ metrics = app.extensions["workflow_collaboration_metrics"]
+ assert isinstance(metrics, WorkflowCollaborationMetrics)
+ assert [
+ extension for extension in app.extensions.values()
+ if isinstance(extension, WorkflowCollaborationMetrics)
+ ] == [metrics]
+
+ data = reader.get_metrics_data()
+ assert data is not None
+ exported = {
+ metric.name: metric
+ for resource in data.resource_metrics
+ for scope in resource.scope_metrics
+ for metric in scope.metrics
+ if metric.name.startswith("dify.collaboration.")
+ }
+ assert set(exported) == {
+ "dify.collaboration.connections",
+ "dify.collaboration.rooms",
+ "dify.collaboration.room_sessions",
+ }
+ for name in ("connections", "rooms"):
+ gauge = exported[f"dify.collaboration.{name}"].data
+ assert isinstance(gauge, Gauge)
+ assert len(gauge.data_points) == 1
+ assert gauge.data_points[0].value == 0
+ assert gauge.data_points[0].attributes == {"aggregation_scope": "worker"}
+ distribution = exported["dify.collaboration.room_sessions"].data
+ assert isinstance(distribution, Gauge)
+ assert len(distribution.data_points) == 5
+ assert {point.attributes["size_range"]: point.value for point in distribution.data_points} == {
+ "1": 0, "2-5": 0, "6-10": 0, "11-20": 0, "21+": 0
+ }
+ finally:
+ provider.shutdown()
+ with app.app_context():
+ database.engine.dispose()
+ """
+ )
+ completed = subprocess.run(
+ [sys.executable, "-B", "-c", script],
+ cwd=Path(__file__).resolve().parents[4],
+ env={**os.environ, "DEBUG": "false", "ENABLE_OTEL": "true"},
+ capture_output=True,
+ text=True,
+ timeout=30,
+ check=False,
+ )
+ assert completed.returncode == 0, completed.stdout + completed.stderr
diff --git a/api/tests/unit_tests/extensions/otel/test_workflow_collaboration.py b/api/tests/unit_tests/extensions/otel/test_workflow_collaboration.py
new file mode 100644
index 00000000000..95232d972b6
--- /dev/null
+++ b/api/tests/unit_tests/extensions/otel/test_workflow_collaboration.py
@@ -0,0 +1,605 @@
+"""Exercise worker-local collaboration metrics with real SDK collection and room membership."""
+
+from collections.abc import Callable, Iterator
+from concurrent.futures import ThreadPoolExecutor
+from threading import Barrier, get_ident
+from typing import Never
+
+import pytest
+import socketio
+from flask import Flask
+from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.sdk.metrics.export import Gauge, Histogram, InMemoryMetricReader, Metric, Sum
+
+from extensions.otel import workflow_collaboration as instrumentation
+
+SIZE_RANGES = ("1", "2-5", "6-10", "11-20", "21+")
+type ObservedMetrics = tuple[instrumentation.WorkflowCollaborationMetrics, InMemoryMetricReader]
+
+
+def metrics_app() -> Flask:
+ """Create a real app with isolated extension ownership and no external services."""
+ return Flask(__name__)
+
+
+def observed_metrics(
+ monkeypatch: pytest.MonkeyPatch,
+ config_overrides: Callable[..., None],
+ metrics_app: Flask,
+) -> Iterator[ObservedMetrics]:
+ """Initialize app-owned instruments with a real SDK meter and isolate global configuration."""
+ config_overrides(ENABLE_OTEL=True)
+ reader = InMemoryMetricReader()
+ provider = MeterProvider(metric_readers=[reader])
+ monkeypatch.setattr(instrumentation, "get_meter", provider.get_meter)
+ instrumentation.init_workflow_collaboration_metrics(metrics_app)
+ metrics = metrics_app.extensions["workflow_collaboration_metrics"]
+ assert isinstance(metrics, instrumentation.WorkflowCollaborationMetrics)
+ yield metrics, reader
+ provider.shutdown()
+
+
+def socket_server(metrics_app: Flask, monkeypatch: pytest.MonkeyPatch) -> socketio.Server:
+ """Attach the real app to a network-free server as the application factory does."""
+ server = socketio.Server(async_mode="threading")
+ monkeypatch.setattr(server, "app", metrics_app, raising=False)
+ return server
+
+
+def _metrics(reader: InMemoryMetricReader) -> dict[str, Metric]:
+ """Collect actual SDK data, requiring a successful observable callback collection."""
+ data = reader.get_metrics_data()
+ assert data is not None
+ return {
+ metric.name: metric
+ for resource in data.resource_metrics
+ for scope in resource.scope_metrics
+ for metric in scope.metrics
+ }
+
+
+def _assert_population(reader: InMemoryMetricReader, connections: int, rooms: int, sizes: dict[str, int]) -> None:
+ """Require exact worker-scope labels and all fixed buckets, including zeroes."""
+ collected = _metrics(reader)
+ for name, expected in (("connections", connections), ("rooms", rooms)):
+ data = collected[f"dify.collaboration.{name}"].data
+ assert isinstance(data, Gauge)
+ assert len(data.data_points) == 1
+ point = data.data_points[0]
+ assert point.attributes == {"aggregation_scope": "worker"}
+ assert point.value == expected
+
+ distribution = collected["dify.collaboration.room_sessions"].data
+ assert isinstance(distribution, Gauge)
+ assert len(distribution.data_points) == len(SIZE_RANGES)
+ actual: dict[str, int | float] = {}
+ for point in distribution.data_points:
+ assert point.attributes is not None
+ assert set(point.attributes) == {"aggregation_scope", "size_range"}
+ assert point.attributes["aggregation_scope"] == "worker"
+ size_range = point.attributes["size_range"]
+ assert isinstance(size_range, str)
+ actual[size_range] = point.value
+ assert actual == {size_range: sizes.get(size_range, 0) for size_range in SIZE_RANGES}
+
+
+def _assert_event(reader: InMemoryMetricReader, event: str, event_type: str, outcome: str) -> None:
+ """Require one matching counter and duration sample without payload-derived labels."""
+ collected = _metrics(reader)
+ expected = {"event": event, "event_type": event_type, "outcome": outcome}
+ events = collected["dify.collaboration.events"].data
+ durations = collected["dify.collaboration.event_duration"].data
+ assert isinstance(events, Sum)
+ assert isinstance(durations, Histogram)
+ assert len(events.data_points) == len(durations.data_points) == 1
+ assert events.data_points[0].attributes == expected
+ assert events.data_points[0].value == 1
+ assert durations.data_points[0].attributes == expected
+ assert durations.data_points[0].count == 1
+ assert durations.data_points[0].sum >= 0
+
+
+def test_empty_worker_emits_all_zero_gauges(observed_metrics: ObservedMetrics) -> None:
+ """An unused worker still exports zero-valued populations and every size bucket."""
+ _, reader = observed_metrics
+ _assert_population(reader, connections=0, rooms=0, sizes={})
+
+
+def test_duplicate_connections_and_joins_do_not_inflate_population(observed_metrics: ObservedMetrics) -> None:
+ """Repeated lifecycle notifications preserve unique local SID and room membership."""
+ metrics, reader = observed_metrics
+ metrics.connected("sid-a")
+ metrics.connected("sid-a")
+ metrics.update_rooms("sid-a", ["sid-a", "workflow-a", "workflow-a"])
+ metrics.update_rooms("sid-a", ["sid-a", "workflow-a"])
+ _assert_population(reader, connections=1, rooms=1, sizes={"1": 1})
+
+ metrics.connected("sid-b")
+ metrics.update_rooms("sid-b", ["sid-b", "workflow-a"])
+ _assert_population(reader, connections=2, rooms=1, sizes={"2-5": 1})
+
+
+def test_room_snapshot_replaces_membership_and_disconnect_clears_it(observed_metrics: ObservedMetrics) -> None:
+ """Multiple actual rooms are counted until removed; late joins cannot resurrect a SID."""
+ metrics, reader = observed_metrics
+ metrics.connected("sid-a")
+ metrics.connected("sid-b")
+ metrics.update_rooms("sid-a", ["sid-a", "workflow-a", "workflow-b"])
+ metrics.update_rooms("sid-b", ["sid-b", "workflow-a"])
+ _assert_population(reader, connections=2, rooms=2, sizes={"1": 1, "2-5": 1})
+
+ metrics.update_rooms("sid-a", ["sid-a", "workflow-b"])
+ _assert_population(reader, connections=2, rooms=2, sizes={"1": 2})
+ metrics.disconnected("sid-a")
+ metrics.disconnected("sid-a")
+ metrics.update_rooms("sid-a", ["workflow-b"])
+ _assert_population(reader, connections=1, rooms=1, sizes={"1": 1})
+
+ metrics.disconnected("sid-b")
+ metrics.update_rooms("unknown-sid", ["new-workflow"])
+ _assert_population(reader, connections=0, rooms=0, sizes={})
+
+
+ ("members", "before", "after"),
+ [
+ (1, "1", None),
+ (2, "2-5", "1"),
+ (5, "2-5", "2-5"),
+ (6, "6-10", "2-5"),
+ (10, "6-10", "6-10"),
+ (11, "11-20", "6-10"),
+ (20, "11-20", "11-20"),
+ (21, "21+", "11-20"),
+ ],
+)
+def test_room_distribution_crosses_fixed_bucket_boundaries(
+ observed_metrics: ObservedMetrics, members: int, before: str, after: str | None
+) -> None:
+ """Room-size buckets describe current rooms and move immediately on member departure."""
+ metrics, reader = observed_metrics
+ for index in range(members):
+ sid = f"sid-{index}"
+ metrics.connected(sid)
+ metrics.update_rooms(sid, [sid, "workflow-a"])
+ _assert_population(reader, connections=members, rooms=1, sizes={before: 1})
+
+ metrics.disconnected("sid-0")
+ _assert_population(reader, connections=members - 1, rooms=int(members > 1), sizes={after: 1} if after else {})
+
+
+def test_record_event_exports_seconds_and_matching_counter(observed_metrics: ObservedMetrics) -> None:
+ """Counter and histogram retain the same bounded dimensions and seconds unit."""
+ metrics, reader = observed_metrics
+ metrics.record_event("graph_event", "none", "success", 0.025)
+ _assert_event(reader, "graph_event", "none", "success")
+ duration = _metrics(reader)["dify.collaboration.event_duration"]
+ assert duration.unit == "s"
+ assert isinstance(duration.data, Histogram)
+ assert duration.data.data_points[0].sum == pytest.approx(0.025)
+
+
+ ("result", "outcome"),
+ [
+ (None, "success"),
+ (True, "success"),
+ (False, "rejected"),
+ (0, "success"),
+ ({"status": 500}, "success"),
+ (({"msg": "ok"}, 200), "success"),
+ (({"msg": "denied"}, 403), "rejected"),
+ (({"msg": "failed"}, 503), "error"),
+ (("ack", "opaque"), "success"),
+ ],
+)
+def test_event_outcome_preserves_exact_handler_return(
+ observed_metrics: ObservedMetrics, socket_server: socketio.Server, result: object, outcome: str
+) -> None:
+ """Only explicit rejection or HTTP-like status tuples classify a normal return as failure."""
+ assert isinstance(socket_server, instrumentation._SocketIO)
+ _, reader = observed_metrics
+
+ @instrumentation.observe_workflow_event("graph_event", socket_server)
+ def handler(_sid: str, _data: object) -> object:
+ return result
+
+ assert handler("sid-a", b"graph-data") is result
+ assert handler.__name__ == "handler"
+ _assert_event(reader, "graph_event", "none", outcome)
+
+
+ ("payload", "expected_type"),
+ [
+ ({"type": "mouse_move"}, "mouse_move"),
+ ({"type": "workflow_update"}, "workflow_update"),
+ ({"type": "untrusted-user-specific-event"}, "other"),
+ ({}, "other"),
+ ({"type": ["mouse_move"]}, "other"),
+ ({"type": None}, "other"),
+ (None, "other"),
+ (b"data", "other"),
+ ],
+)
+def test_event_types_are_bounded_without_changing_payload(
+ observed_metrics: ObservedMetrics, socket_server: socketio.Server, payload: object, expected_type: str
+) -> None:
+ """Unknown and malformed payloads stay handler-owned and cannot add metric series."""
+ assert isinstance(socket_server, instrumentation._SocketIO)
+ _, reader = observed_metrics
+
+ @instrumentation.observe_workflow_event("collaboration_event", socket_server)
+ def handler(_sid: str, data: object) -> object:
+ return data
+
+ assert handler("sid-a", payload) is payload
+ _assert_event(reader, "collaboration_event", expected_type, "success")
+
+
+def test_event_exception_is_recorded_and_reraised_unchanged(
+ observed_metrics: ObservedMetrics, socket_server: socketio.Server
+) -> None:
+ """Instrumentation preserves the original exception instance while reporting failure."""
+ assert isinstance(socket_server, instrumentation._SocketIO)
+ _, reader = observed_metrics
+ failure = ValueError("handler failed")
+
+ @instrumentation.observe_workflow_event("collaboration_event", socket_server)
+ def handler(_sid: str, _data: object) -> Never:
+ raise failure
+
+ with pytest.raises(ValueError, match="handler failed") as caught:
+ handler("sid-a", {"type": "mouse_move"})
+ assert caught.value is failure
+ _assert_event(reader, "collaboration_event", "mouse_move", "error")
+
+
[email protected]("accepted", [True, False])
+def test_connect_counts_only_authenticated_sockets(
+ observed_metrics: ObservedMetrics, socket_server: socketio.Server, accepted: bool
+) -> None:
+ """A provisional Socket.IO room exists before authentication, but rejected SIDs do not count."""
+ assert isinstance(socket_server, instrumentation._SocketIO)
+ assert isinstance(socket_server.manager, socketio.Manager)
+ _, reader = observed_metrics
+ sid = socket_server.manager.connect("engine-sid", "/")
+ assert isinstance(sid, str)
+
+ @instrumentation.observe_workflow_event("connect", socket_server)
+ def handler(_sid: str, _environ: object, _auth: object) -> bool:
+ return accepted
+
+ assert handler(sid, {}, None) is accepted
+ _assert_population(reader, connections=int(accepted), rooms=0, sizes={})
+ _assert_event(reader, "connect", "none", "success" if accepted else "rejected")
+
+
+def test_connect_completed_after_disconnect_does_not_restore_socket(
+ observed_metrics: ObservedMetrics, socket_server: socketio.Server
+) -> None:
+ """A successful handler cannot reintroduce a SID that disconnected during authentication."""
+ assert isinstance(socket_server, instrumentation._SocketIO)
+ assert isinstance(socket_server.manager, socketio.Manager)
+ _, reader = observed_metrics
+ sid = socket_server.manager.connect("engine-sid", "/")
+ assert isinstance(sid, str)
+
+ @instrumentation.observe_workflow_event("connect", socket_server)
+ def handler(sid: str, _environ: object, _auth: object) -> bool:
+ socket_server.manager.disconnect(sid, "/")
+ return True
+
+ assert handler(sid, {}, None) is True
+ _assert_population(reader, connections=0, rooms=0, sizes={})
+ _assert_event(reader, "connect", "none", "success")
+
+
+def test_partial_join_failure_reflects_real_membership(
+ observed_metrics: ObservedMetrics, socket_server: socketio.Server
+) -> None:
+ """An exception after enter_room retains actual local membership and the original failure."""
+ assert isinstance(socket_server, instrumentation._SocketIO)
+ assert isinstance(socket_server.manager, socketio.Manager)
+ metrics, reader = observed_metrics
+ sid = socket_server.manager.connect("engine-sid", "/")
+ assert isinstance(sid, str)
+ metrics.connected(sid)
+ failure = RuntimeError("broadcast failed after join")
+
+ @instrumentation.observe_workflow_event("user_connect", socket_server)
+ def handler(sid: str, data: dict[str, str]) -> Never:
+ socket_server.enter_room(sid, data["workflow_id"])
+ raise failure
+
+ with pytest.raises(RuntimeError) as caught:
+ handler(sid, {"workflow_id": "workflow-a"})
+ assert caught.value is failure
+ _assert_population(reader, connections=1, rooms=1, sizes={"1": 1})
+ _assert_event(reader, "user_connect", "none", "error")
+
+
+def test_disconnect_failure_still_removes_all_local_membership(
+ observed_metrics: ObservedMetrics, socket_server: socketio.Server
+) -> None:
+ """Population cleanup does not depend on the service's Redis cleanup succeeding."""
+ assert isinstance(socket_server, instrumentation._SocketIO)
+ assert isinstance(socket_server.manager, socketio.Manager)
+ metrics, reader = observed_metrics
+ sid = socket_server.manager.connect("engine-sid", "/")
+ assert isinstance(sid, str)
+ metrics.connected(sid)
+ socket_server.enter_room(sid, "workflow-a")
+ metrics.update_rooms(sid, socket_server.rooms(sid))
+ socket_server.manager.pre_disconnect(sid, "/")
+ failure = RuntimeError("service cleanup failed")
+
+ @instrumentation.observe_workflow_event("disconnect", socket_server)
+ def handler(_sid: str) -> Never:
+ raise failure
+
+ with pytest.raises(RuntimeError) as caught:
+ handler(sid)
+ assert caught.value is failure
+ _assert_population(reader, connections=0, rooms=0, sizes={})
+ _assert_event(reader, "disconnect", "none", "error")
+
+
+def test_room_lookup_failure_does_not_hide_return_or_event(
+ observed_metrics: ObservedMetrics, socket_server: socketio.Server, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ """A metrics-only membership lookup failure cannot change an ACK or suppress event accounting."""
+ assert isinstance(socket_server, instrumentation._SocketIO)
+ assert isinstance(socket_server.manager, socketio.Manager)
+ metrics, reader = observed_metrics
+ sid = socket_server.manager.connect("engine-sid", "/")
+ assert isinstance(sid, str)
+ metrics.connected(sid)
+ result = {"msg": "connected"}
+
+ def unavailable_rooms(*_args: object, **_kwargs: object) -> Never:
+ raise RuntimeError("room lookup failed")
+
+ monkeypatch.setattr(socket_server, "rooms", unavailable_rooms)
+
+ @instrumentation.observe_workflow_event("user_connect", socket_server)
+ def handler(_sid: str, _data: object) -> dict[str, str]:
+ return result
+
+ assert handler(sid, {}) is result
+ _assert_event(reader, "user_connect", "none", "success")
+
+
+def test_collecting_gauges_never_reads_live_socket_manager(
+ observed_metrics: ObservedMetrics, socket_server: socketio.Server, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ """SDK collection consumes the published snapshot even when every room accessor is unavailable."""
+ assert isinstance(socket_server, instrumentation._SocketIO)
+ assert isinstance(socket_server.manager, socketio.Manager)
+ metrics, reader = observed_metrics
+ sid = socket_server.manager.connect("engine-sid", "/")
+ assert isinstance(sid, str)
+ metrics.connected(sid)
+ socket_server.enter_room(sid, "workflow-a")
+
+ @instrumentation.observe_workflow_event("user_connect", socket_server)
+ def handler(_sid: str, _data: object) -> dict[str, str]:
+ return {"msg": "connected"}
+
+ handler(sid, {})
+
+ def forbidden_access(*_args: object, **_kwargs: object) -> Never:
+ raise AssertionError("observable callbacks must not touch live manager state")
+
+ monkeypatch.setattr(socket_server, "rooms", forbidden_access)
+ monkeypatch.setattr(socket_server.manager, "get_rooms", forbidden_access)
+ monkeypatch.setattr(socket_server.manager, "get_participants", forbidden_access)
+ _assert_population(reader, connections=1, rooms=1, sizes={"1": 1})
+
+
+def test_native_thread_collection_uses_complete_numeric_snapshots(observed_metrics: ObservedMetrics) -> None:
+ """Native SDK collection tolerates handler-side churn without traversing mutable membership."""
+ metrics, reader = observed_metrics
+ ready = Barrier(2)
+ handler_thread = get_ident()
+
+ def collect() -> int:
+ ready.wait(timeout=5)
+ for _ in range(50):
+ distribution = _metrics(reader)["dify.collaboration.room_sessions"].data
+ assert isinstance(distribution, Gauge)
+ assert len(distribution.data_points) == len(SIZE_RANGES)
+ counts = [point.value for point in distribution.data_points]
+ assert all(count >= 0 for count in counts)
+ assert sum(counts) in (0, 1)
+ return get_ident()
+
+ with ThreadPoolExecutor(max_workers=1) as executor:
+ collection = executor.submit(collect)
+ ready.wait(timeout=5)
+ for index in range(1000):
+ sid = f"sid-{index}"
+ metrics.connected(sid)
+ metrics.update_rooms(sid, [sid, "workflow-a"])
+ metrics.disconnected(sid)
+ assert collection.result(timeout=10) != handler_thread
+ _assert_population(reader, connections=0, rooms=0, sizes={})
+
+
[email protected]("handler_fails", [False, True])
+def test_sdk_failure_never_changes_handler_result_or_exception(
+ observed_metrics: ObservedMetrics,
+ socket_server: socketio.Server,
+ monkeypatch: pytest.MonkeyPatch,
+ handler_fails: bool,
+) -> None:
+ """An instrument failure stays isolated from the successful ACK or original business exception."""
+ assert isinstance(socket_server, instrumentation._SocketIO)
+ metrics, _ = observed_metrics
+ result = {"msg": "received"}
+ handler_failure = ValueError("original handler failure")
+
+ def unavailable_sdk(*_args: object, **_kwargs: object) -> Never:
+ raise RuntimeError("metric SDK failure")
+
+ monkeypatch.setattr(metrics._events, "add", unavailable_sdk)
+
+ @instrumentation.observe_workflow_event("graph_event", socket_server)
+ def handler(_sid: str, _data: object) -> dict[str, str]:
+ if handler_fails:
+ raise handler_failure
+ return result
+
+ if handler_fails:
+ with pytest.raises(ValueError, match="original handler failure") as caught:
+ handler("sid-a", b"graph-data")
+ assert caught.value is handler_failure
+ else:
+ assert handler("sid-a", b"graph-data") is result
+
+
+def test_startup_instance_is_shared_and_repeated_initialization_preserves_state(
+ observed_metrics: ObservedMetrics,
+ metrics_app: Flask,
+ socket_server: socketio.Server,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Handlers share startup-owned state; repeated initialization neither registers nor resets instruments."""
+ assert isinstance(socket_server, instrumentation._SocketIO)
+ assert isinstance(socket_server.manager, socketio.Manager)
+ metrics, reader = observed_metrics
+ registration_attempts: list[object] = []
+
+ def forbidden_registration(*args: object, **_kwargs: object) -> Never:
+ registration_attempts.extend(args)
+ raise AssertionError("event handling must use the initialized app instance")
+
+ monkeypatch.setattr(instrumentation, "get_meter", forbidden_registration)
+
+ @instrumentation.observe_workflow_event("connect", socket_server)
+ def connected(_sid: str, _environ: object, _auth: object) -> bool:
+ return True
+
+ @instrumentation.observe_workflow_event("user_connect", socket_server)
+ def joined(sid: str, _data: object) -> dict[str, str]:
+ socket_server.enter_room(sid, "workflow-a")
+ return {"msg": "connected"}
+
+ for index in range(2):
+ sid = socket_server.manager.connect(f"engine-{index}", "/")
+ assert isinstance(sid, str)
+ assert connected(sid, {}, None) is True
+ assert joined(sid, {}) == {"msg": "connected"}
+ instrumentation.init_workflow_collaboration_metrics(metrics_app)
+ assert metrics_app.extensions["workflow_collaboration_metrics"] is metrics
+
+ assert registration_attempts == []
+ _assert_population(reader, connections=2, rooms=1, sizes={"2-5": 1})
+ events = _metrics(reader)["dify.collaboration.events"].data
+ assert isinstance(events, Sum)
+ assert len(events.data_points) == 2
+ assert all(point.value == 2 for point in events.data_points)
+
+
[email protected]("initialized", [False, True])
[email protected]("handler_fails", [False, True])
+def test_disabled_metrics_bypass_all_instrumentation_with_or_without_app_instance(
+ observed_metrics: ObservedMetrics,
+ metrics_app: Flask,
+ socket_server: socketio.Server,
+ monkeypatch: pytest.MonkeyPatch,
+ config_overrides: Callable[..., None],
+ initialized: bool,
+ handler_fails: bool,
+) -> None:
+ """Disabled instrumentation preserves business behavior without timing, app lookup, membership, or SDK access."""
+ assert isinstance(socket_server, instrumentation._SocketIO)
+ metrics, reader = observed_metrics
+ if not initialized:
+ metrics_app.extensions.pop("workflow_collaboration_metrics")
+ accesses: list[str] = []
+ result = {"msg": "connected"}
+ failure = ValueError("business failure while metrics disabled")
+
+ def forbidden_access(*_args: object, **_kwargs: object) -> Never:
+ accesses.append("instrumentation")
+ raise AssertionError("disabled metrics must not access instrumentation")
+
+ monkeypatch.setattr(instrumentation, "perf_counter", forbidden_access)
+ monkeypatch.setattr(instrumentation, "get_meter", forbidden_access)
+ monkeypatch.setattr(type(socket_server), "app", property(forbidden_access), raising=False)
+ monkeypatch.setattr(socket_server, "rooms", forbidden_access)
+ monkeypatch.setattr(metrics._events, "add", forbidden_access)
+
+ @instrumentation.observe_workflow_event("user_connect", socket_server)
+ def handler(_sid: str, _data: object) -> dict[str, str]:
+ if handler_fails:
+ raise failure
+ return result
+
+ config_overrides(ENABLE_OTEL=False)
+ if handler_fails:
+ with pytest.raises(ValueError, match="business failure while metrics disabled") as caught:
+ handler("sid-a", {})
+ assert caught.value is failure
+ else:
+ assert handler("sid-a", {}) is result
+
+ assert accesses == []
+ _assert_population(reader, connections=0, rooms=0, sizes={})
+ assert "dify.collaboration.events" not in _metrics(reader)
+
+
[email protected]("initialization_fails", [False, True])
[email protected]("handler_fails", [False, True])
+def test_enabled_metrics_without_initialized_extension_leave_business_behavior_unchanged(
+ metrics_app: Flask,
+ socket_server: socketio.Server,
+ monkeypatch: pytest.MonkeyPatch,
+ config_overrides: Callable[..., None],
+ initialization_fails: bool,
+ handler_fails: bool,
+) -> None:
+ """Missing or failed startup instrumentation never registers lazily, times, or replaces handler results."""
+ assert isinstance(socket_server, instrumentation._SocketIO)
+ config_overrides(ENABLE_OTEL=True)
+ registration_attempts: list[str] = []
+ accesses: list[str] = []
+ result = {"msg": "connected"}
+ failure = ValueError("business failure without metrics instance")
+
+ def unavailable_meter(*_args: object, **_kwargs: object) -> Never:
+ registration_attempts.append("get_meter")
+ raise RuntimeError("SDK initialization failed")
+
+ def forbidden_access(*_args: object, **_kwargs: object) -> Never:
+ accesses.append("handler instrumentation")
+ raise AssertionError("missing instance must not trigger event instrumentation")
+
+ monkeypatch.setattr(instrumentation, "get_meter", unavailable_meter)
+ if initialization_fails:
+ instrumentation.init_workflow_collaboration_metrics(metrics_app)
+ assert "workflow_collaboration_metrics" not in metrics_app.extensions
+
+ monkeypatch.setattr(instrumentation, "perf_counter", forbidden_access)
+ monkeypatch.setattr(socket_server, "rooms", forbidden_access)
+
+ @instrumentation.observe_workflow_event("user_connect", socket_server)
+ def handler(_sid: str, _data: object) -> dict[str, str]:
+ if handler_fails:
+ raise failure
+ return result
+
+ if handler_fails:
+ with pytest.raises(ValueError, match="business failure without metrics instance") as caught:
+ handler("sid-a", {})
+ assert caught.value is failure
+ else:
+ assert handler("sid-a", {}) is result
+
+ assert accesses == []
+ assert registration_attempts == (["get_meter"] if initialization_fails else [])
+ assert "workflow_collaboration_metrics" not in metrics_app.extensions