Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
feat(api): add workflow collaboration metrics (#45446)
非法操作cf38e6ad67b1
diff --git a/api/controllers/console/socketio/workflow.py b/api/controllers/console/socketio/workflow.pyindex 13555b1a973..e062c0cd5a3 100644--- a/api/controllers/console/socketio/workflow.py+++ b/api/controllers/console/socketio/workflow.py@@ -7,6 +7,7 @@ from flask import Request as FlaskRequestfrom core.db.session_factory import session_factoryfrom extensions.ext_application_services import application_servicesfrom extensions.ext_socketio import sio+from extensions.otel.workflow_collaboration import observe_workflow_eventfrom libs.passport import PassportServicefrom libs.token import extract_access_tokenfrom repositories.workflow_collaboration_repository import WorkflowCollaborationRepository@@ -21,9 +22,10 @@ def _sio_on(event: str) -> Callable[[Callable[..., object]], Callable[..., objec@_sio_on("connect")-def socket_connect(sid, environ, auth):+@observe_workflow_event("connect", sio)+def socket_connect(sid, environ, auth=None):"""- WebSocket connect event, do authentication here.+ Authenticate a connect with optional Socket.IO auth data; preserve token-based admission."""try:request_environ = FlaskRequest(environ)@@ -61,6 +63,7 @@ def socket_connect(sid, environ, auth):@_sio_on("user_connect")+@observe_workflow_event("user_connect", sio)def handle_user_connect(sid, data):"""Handle user connect event. Each session (tab) is treated as an independent collaborator.@@ -78,15 +81,23 @@ def handle_user_connect(sid, data):return {"msg": "connected", "user_id": user_id, "sid": sid, "isLeader": is_leader}+@observe_workflow_event("disconnect", sio)+def _handle_disconnect(sid):+ """Remove the session from online users and observe the result."""+ collaboration_service.disconnect_session(sid)++@_sio_on("disconnect")def handle_disconnect(sid):+ """Let Socket.IO discard its reason before starting observable cleanup.++ A cleanup TypeError must propagate without Socket.IO retrying the cleanup."""- Handle session disconnect event. Remove the specific session from online users.- """- collaboration_service.disconnect_session(sid)+ _handle_disconnect(sid)@_sio_on("collaboration_event")+@observe_workflow_event("collaboration_event", sio)def handle_collaboration_event(sid, data):"""Handle general collaboration events, include:@@ -102,6 +113,7 @@ def handle_collaboration_event(sid, data):@_sio_on("graph_event")+@observe_workflow_event("graph_event", sio)def handle_graph_event(sid, data):"""Handle graph events - simple broadcast relay.diff --git a/api/extensions/otel/instrumentation.py b/api/extensions/otel/instrumentation.pyindex 1180e658ca9..023fa2e8c03 100644--- a/api/extensions/otel/instrumentation.py+++ b/api/extensions/otel/instrumentation.py@@ -19,6 +19,7 @@ from opentelemetry.trace.status import StatusCodefrom configs import dify_configfrom dify_app import DifyAppfrom extensions.otel.runtime import is_celery_worker+from extensions.otel.workflow_collaboration import init_workflow_collaboration_metricslogger = logging.getLogger(__name__)@@ -152,6 +153,7 @@ def init_httpx_instrumentor() -> None:def init_instruments(app: DifyApp) -> None:if not is_celery_worker():init_flask_instrumentor(app)+ init_workflow_collaboration_metrics(app)_new_celery_instrumentor().instrument(tracer_provider=get_tracer_provider(), meter_provider=get_meter_provider())diff --git a/api/extensions/otel/workflow_collaboration.py b/api/extensions/otel/workflow_collaboration.pynew file mode 100644index 00000000000..33df3cc8d00--- /dev/null+++ b/api/extensions/otel/workflow_collaboration.py@@ -0,0 +1,271 @@+"""Worker-local Socket.IO load metrics without persistence or per-message network I/O.++Lifecycle updates run on the Socket.IO gevent hub and never yield. Observable callbacks+only read immutable numeric snapshots, including when the SDK collects on a native thread.+Room counts describe local membership, not globally distinct rooms or distinct people.+No room, socket, tenant, user, payload, or exception values become metric attributes.+"""++from __future__ import annotations++import logging+from collections.abc import Callable, Iterable, Mapping, Sequence+from functools import wraps+from time import perf_counter+from typing import Literal, NamedTuple, Protocol, runtime_checkable++from flask import Flask+from opentelemetry.metrics import CallbackOptions, Meter, Observation, get_meter++from configs import dify_config++logger = logging.getLogger(__name__)++type SocketEvent = Literal["connect", "user_connect", "disconnect", "collaboration_event", "graph_event"]+type Outcome = Literal["success", "rejected", "error"]++_EVENT_TYPES = frozenset(+ {+ "mouse_move",+ "vars_and_features_update",+ "sync_request",+ "workflow_update",+ "comments_update",+ "node_panel_presence",+ "graph_view_state",+ "graph_resync_request",+ }+)+_SIZE_RANGES = ("1", "2-5", "6-10", "11-20", "21+")+_WORKER_ATTRIBUTES = {"aggregation_scope": "worker"}+_EXTENSION_KEY = "workflow_collaboration_metrics"+++class _SocketManager(Protocol):+ def is_connected(self, sid: str, namespace: str) -> bool: ...+++@runtime_checkable+class _SocketIO(Protocol):+ @property+ def app(self) -> Flask: ...++ @property+ def manager(self) -> _SocketManager: ...++ def rooms(self, sid: str, namespace: str) -> list[str]: ...+++class _Snapshot(NamedTuple):+ connections: int+ rooms: int+ size_ranges: tuple[int, ...]+++def _size_range(size: int) -> str:+ """Given a positive local room size, return one bounded attribute value without I/O."""+ if size == 1:+ return "1"+ if size <= 5:+ return "2-5"+ if size <= 10:+ return "6-10"+ if size <= 20:+ return "11-20"+ return "21+"+++class WorkflowCollaborationMetrics:+ """Own one worker's instruments and an idempotent mirror of accepted socket membership.++ Writers must run in Socket.IO handlers on the gevent hub; they perform no I/O or+ yielding operations. SDK collection never traverses the mutable membership maps.+ Unknown/late joins cannot recreate disconnected sessions, and duplicate lifecycle+ notifications cannot inflate counts. Instrument failures are isolated by the decorator.+ """++ def __init__(self, meter: Meter | None = None) -> None:+ """Register instruments with the supplied/default meter and start with zero load."""+ self._connections: dict[str, frozenset[str]] = {}+ self._room_sizes: dict[str, int] = {}+ self._size_counts = dict.fromkeys(_SIZE_RANGES, 0)+ self._snapshot = _Snapshot(0, 0, (0,) * len(_SIZE_RANGES))+ meter = meter if meter is not None else get_meter("dify.collaboration")+ meter.create_observable_gauge(+ "dify.collaboration.connections",+ callbacks=(self._observe_connections,),+ description="Authenticated Socket.IO sessions connected to this worker; not distinct users",+ unit="{connection}",+ )+ meter.create_observable_gauge(+ "dify.collaboration.rooms",+ callbacks=(self._observe_rooms,),+ description="Workflow rooms with local members; summing workers does not give globally distinct rooms",+ unit="{room}",+ )+ meter.create_observable_gauge(+ "dify.collaboration.room_sessions",+ callbacks=(self._observe_room_sessions,),+ description="Current local room count by local session-count range; not a join-weighted histogram",+ unit="{room}",+ )+ self._events = meter.create_counter(+ "dify.collaboration.events", description="Completed Socket.IO handlers by event and outcome", unit="{event}"+ )+ self._duration = meter.create_histogram(+ "dify.collaboration.event_duration",+ description="Socket.IO handler wall time, including dependency waits; not peer delivery or client ACK time",+ unit="s",+ explicit_bucket_boundaries_advisory=(0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 15, 30),+ )++ def _observe_connections(self, _: CallbackOptions) -> Iterable[Observation]:+ """Return a numeric snapshot for any collector thread without accessing live sockets."""+ return (Observation(self._snapshot.connections, attributes=_WORKER_ATTRIBUTES),)++ def _observe_rooms(self, _: CallbackOptions) -> Iterable[Observation]:+ """Return the local room snapshot, including zero after the last member leaves."""+ return (Observation(self._snapshot.rooms, attributes=_WORKER_ATTRIBUTES),)++ def _observe_room_sessions(self, _: CallbackOptions) -> Iterable[Observation]:+ """Return all fixed range counts from one snapshot without exposing room identifiers."""+ snapshot = self._snapshot+ return tuple(+ Observation(count, attributes={**_WORKER_ATTRIBUTES, "size_range": size_range})+ for size_range, count in zip(_SIZE_RANGES, snapshot.size_ranges, strict=True)+ )++ def _publish_snapshot(self) -> None:+ """After a lifecycle update, atomically replace the collector's immutable numeric view."""+ self._snapshot = _Snapshot(len(self._connections), len(self._room_sizes), tuple(self._size_counts.values()))++ def _change_room_size(self, room: str, delta: Literal[-1, 1]) -> None:+ """Apply one known member addition/removal and keep current room buckets consistent."""+ previous = self._room_sizes.get(room, 0)+ if previous:+ self._size_counts[_size_range(previous)] -= 1+ current = previous + delta+ if current:+ self._room_sizes[room] = current+ self._size_counts[_size_range(current)] += 1+ else:+ del self._room_sizes[room]++ def connected(self, sid: str) -> None:+ """Register a live, authenticated SID once; a duplicate preserves existing memberships."""+ if sid not in self._connections:+ self._connections[sid] = frozenset()+ self._publish_snapshot()++ def update_rooms(self, sid: str, rooms: Sequence[str]) -> None:+ """Mirror actual rooms for a registered SID; ignore late joins and exclude its private room."""+ previous = self._connections.get(sid)+ if previous is None:+ return+ current = frozenset(room for room in rooms if room != sid)+ for room in previous - current:+ self._change_room_size(room, -1)+ for room in current - previous:+ self._change_room_size(room, 1)+ self._connections[sid] = current+ self._publish_snapshot()++ def disconnected(self, sid: str) -> None:+ """Remove a SID and all local memberships, tolerating duplicate or rejected disconnects."""+ previous = self._connections.pop(sid, None)+ if previous is not None:+ for room in previous:+ self._change_room_size(room, -1)+ self._publish_snapshot()++ def record_event(self, event: SocketEvent, event_type: str, outcome: Outcome, duration_seconds: float) -> None:+ """Record one completed handler with bounded labels; propagate SDK failures to the isolation boundary."""+ bounded_type = event_type if event_type in _EVENT_TYPES else "other"+ attributes = {+ "event": event,+ "event_type": bounded_type if event == "collaboration_event" else "none",+ "outcome": outcome,+ }+ self._events.add(1, attributes=attributes)+ self._duration.record(duration_seconds, attributes=attributes)+++def init_workflow_collaboration_metrics(app: Flask) -> None:+ """Bind metrics during enabled OTel startup, after the provider and before serving events.++ The application owns one instance; repeated startup calls preserve its state.+ Registration failure leaves instrumentation absent and business handlers usable.+ """+ if _EXTENSION_KEY not in app.extensions:+ try:+ app.extensions[_EXTENSION_KEY] = WorkflowCollaborationMetrics()+ except Exception:+ logger.exception("Failed to initialize workflow collaboration metrics")+++def _outcome(result: object) -> Outcome:+ """Classify existing handler results without modifying their body, status, or identity."""+ if result is False:+ return "rejected"+ if isinstance(result, tuple) and len(result) == 2 and isinstance(result[1], int):+ if result[1] >= 500:+ return "error"+ if result[1] >= 400:+ return "rejected"+ return "success"+++def observe_workflow_event[**P, R](+ event: SocketEvent, socketio: _SocketIO+) -> Callable[[Callable[P, R]], Callable[P, R]]:+ """Observe an existing synchronous Socket.IO handler without changing returns or exceptions.++ The caller supplies the event and Socket.IO server bound to its initialized app.+ Disabled or unavailable instrumentation calls the handler directly. Otherwise capture+ membership in the handler thread, even when a join partially succeeds or Redis cleanup+ fails, without additional Redis/SQL queries. Telemetry failures never alter authentication,+ room membership, ACK payloads, or the handler's exception.+ """++ def decorate(handler: Callable[P, R]) -> Callable[P, R]:+ """Preserve the handler signature and metadata while observing its completion."""++ @wraps(handler)+ def observed(*args: P.args, **kwargs: P.kwargs) -> R:+ """Run the original handler once and isolate all metric-side failures in finally."""+ if not dify_config.ENABLE_OTEL:+ return handler(*args, **kwargs)+ metrics = socketio.app.extensions.get(_EXTENSION_KEY)+ if not isinstance(metrics, WorkflowCollaborationMetrics):+ return handler(*args, **kwargs)++ started = perf_counter()+ outcome: Outcome = "error"+ try:+ result = handler(*args, **kwargs)+ outcome = _outcome(result)+ return result+ finally:+ duration = perf_counter() - started+ try:+ sid = args[0] if args else kwargs.get("sid")+ try:+ if isinstance(sid, str):+ if event == "connect" and outcome == "success" and socketio.manager.is_connected(sid, "/"):+ metrics.connected(sid)+ elif event == "user_connect":+ metrics.update_rooms(sid, socketio.rooms(sid, "/"))+ elif event == "disconnect":+ metrics.disconnected(sid)+ finally:+ data = args[1] if len(args) > 1 else kwargs.get("data")+ event_type = data.get("type") if isinstance(data, Mapping) else None+ metrics.record_event(+ event, event_type if isinstance(event_type, str) else "other", outcome, duration+ )+ except Exception:+ logger.warning("Failed to record workflow collaboration metrics", exc_info=True)++ return observed++ return decoratediff --git a/api/tests/unit_tests/controllers/console/socketio/test_workflow.py b/api/tests/unit_tests/controllers/console/socketio/test_workflow.pynew file mode 100644index 00000000000..f4a4ecaa89c--- /dev/null+++ b/api/tests/unit_tests/controllers/console/socketio/test_workflow.py@@ -0,0 +1,157 @@+"""Exercise collaboration metrics through Socket.IO's real lifecycle dispatch."""++from collections.abc import Callable, Iterator+from unittest.mock import MagicMock++import pytest+import socketio+from engineio.socket import Socket+from flask import Flask+from opentelemetry.sdk.metrics import MeterProvider+from opentelemetry.sdk.metrics.export import Gauge, Histogram, InMemoryMetricReader, Metric, Sum+from socketio.packet import CONNECT_ERROR, Packet+from werkzeug.test import EnvironBuilder++from controllers.console.socketio import workflow+from extensions.ext_redis import RedisClientWrapper+from extensions.otel import workflow_collaboration+from extensions.otel.workflow_collaboration import WorkflowCollaborationMetrics+++def collaboration_server(+ monkeypatch: pytest.MonkeyPatch,+ redis_transport: tuple[RedisClientWrapper, MagicMock],+ config_overrides: Callable[..., None],+) -> Iterator[tuple[socketio.Server, WorkflowCollaborationMetrics, InMemoryMetricReader]]:+ """Initialize app-owned metrics for registered handlers while replacing only transports and metric export."""+ config_overrides(ENABLE_OTEL=True)+ app = Flask(__name__)+ server = workflow.sio+ manager = socketio.Manager()+ manager.set_server(server)+ monkeypatch.setattr(server, "manager", manager)+ monkeypatch.setattr(server, "environ", {})+ monkeypatch.setattr(server, "app", app, raising=False)+ monkeypatch.setattr(server.eio, "sockets", {})+ monkeypatch.setattr(workflow.repository, "_redis", redis_transport[0])++ reader = InMemoryMetricReader()+ provider = MeterProvider(metric_readers=[reader])+ monkeypatch.setattr(workflow_collaboration, "get_meter", provider.get_meter)+ workflow_collaboration.init_workflow_collaboration_metrics(app)+ metrics = app.extensions["workflow_collaboration_metrics"]+ assert isinstance(metrics, WorkflowCollaborationMetrics)+ try:+ yield server, metrics, reader+ finally:+ provider.shutdown()+++def _read_metrics(reader: InMemoryMetricReader) -> dict[str, Metric]:+ """Collect the real SDK instruments and return their exported values by metric name."""+ data = reader.get_metrics_data()+ assert data is not None+ return {+ metric.name: metric+ for resource in data.resource_metrics+ for scope in resource.scope_metrics+ for metric in scope.metrics+ }+++def _assert_single_event(metrics: dict[str, Metric], event: str, outcome: str) -> None:+ """Require exactly one completed handler observation and no compatibility-retry error series."""+ attributes = {"event": event, "event_type": "none", "outcome": outcome}+ events = metrics["dify.collaboration.events"].data+ assert isinstance(events, Sum)+ assert len(events.data_points) == 1+ assert events.data_points[0].attributes == attributes+ assert events.data_points[0].value == 1++ duration = metrics["dify.collaboration.event_duration"].data+ assert isinstance(duration, Histogram)+ assert len(duration.data_points) == 1+ assert duration.data_points[0].attributes == attributes+ assert duration.data_points[0].count == 1++ connections = metrics["dify.collaboration.connections"].data+ assert isinstance(connections, Gauge)+ assert connections.data_points[0].value == 0+++def test_connect_without_auth_payload_records_one_rejection(+ collaboration_server: tuple[socketio.Server, WorkflowCollaborationMetrics, InMemoryMetricReader],+) -> None:+ """A tokenless client omitting auth is rejected once without an argument-compatibility error."""+ server, _, reader = collaboration_server+ eio_sid = "connect-without-auth"+ transport = Socket(server.eio, eio_sid)+ server.eio.sockets[eio_sid] = transport+ builder = EnvironBuilder(path="/socket.io")+ try:+ server.environ[eio_sid] = builder.get_environ()+ finally:+ builder.close()++ assert server.handlers["/"]["connect"] is workflow.socket_connect+ server._handle_connect(eio_sid, "/", None)++ response = Packet(encoded_packet=transport.queue.get_nowait().data)+ assert response.packet_type == CONNECT_ERROR+ assert server.manager.sid_from_eio_sid(eio_sid, "/") is None+ _assert_single_event(_read_metrics(reader), "connect", "rejected")+++def test_disconnect_with_reason_records_one_success(+ collaboration_server: tuple[socketio.Server, WorkflowCollaborationMetrics, InMemoryMetricReader],+ redis_transport: tuple[RedisClientWrapper, MagicMock],+) -> None:+ """A reason-bearing disconnect executes real cleanup once and removes the worker's live connection."""+ server, metrics, reader = collaboration_server+ eio_sid = "disconnect-with-reason"+ sid = server.manager.connect(eio_sid, "/")+ assert isinstance(sid, str)+ metrics.connected(sid)++ assert server.handlers["/"]["disconnect"] is workflow.handle_disconnect+ server._handle_disconnect(eio_sid, "/", server.reason.PING_TIMEOUT)++ redis_transport[1].assert_called_once_with(+ "GET", workflow.repository.sid_key(sid), keys=[workflow.repository.sid_key(sid)]+ )+ assert not server.manager.is_connected(sid, "/")+ _assert_single_event(_read_metrics(reader), "disconnect", "success")++[email protected]("otel_enabled", [True, False])+def test_disconnect_type_error_is_not_retried(+ collaboration_server: tuple[socketio.Server, WorkflowCollaborationMetrics, InMemoryMetricReader],+ monkeypatch: pytest.MonkeyPatch,+ config_overrides: Callable[..., None],+ otel_enabled: bool,+) -> None:+ """A cleanup TypeError propagates after one call, regardless of instrumentation."""+ server, metrics, reader = collaboration_server+ config_overrides(ENABLE_OTEL=otel_enabled)+ eio_sid = "disconnect-cleanup-error"+ sid = server.manager.connect(eio_sid, "/")+ assert isinstance(sid, str)+ metrics.connected(sid)+ failure = TypeError("cleanup failed after partial work")+ calls: list[str] = []++ def cleanup(disconnected_sid: str) -> None:+ calls.append(disconnected_sid)+ if len(calls) == 1:+ raise failure++ monkeypatch.setattr(workflow.collaboration_service, "disconnect_session", cleanup)++ with pytest.raises(TypeError) as caught:+ server._handle_disconnect(eio_sid, "/", server.reason.CLIENT_DISCONNECT)++ assert caught.value is failure+ assert calls == [sid]+ if otel_enabled:+ _assert_single_event(_read_metrics(reader), "disconnect", "error")diff --git a/api/tests/unit_tests/extensions/otel/test_instrumentation.py b/api/tests/unit_tests/extensions/otel/test_instrumentation.pynew file mode 100644index 00000000000..350759830c7--- /dev/null+++ b/api/tests/unit_tests/extensions/otel/test_instrumentation.py@@ -0,0 +1,86 @@+"""Exercise collaboration metric startup through the existing OTel composition root."""++import os+import subprocess+import sys+from pathlib import Path+from textwrap import dedent+++def test_init_instruments_registers_app_owned_collaboration_metrics_before_events() -> None:+ """Real instrumentors bind one app-owned instance and collect zero gauges before traffic.++ A child interpreter isolates gevent patching, SDK providers, and global instrumentors+ from other tests. SQLite supplies a real SQLAlchemy engine without external services.+ """+ script = dedent(+ """+ from gevent import monkey+ monkey.patch_all()++ from flask_sqlalchemy import SQLAlchemy+ from opentelemetry.metrics import set_meter_provider+ from opentelemetry.sdk.metrics import MeterProvider+ from opentelemetry.sdk.metrics.export import Gauge, InMemoryMetricReader++ from dify_app import DifyApp+ from extensions.otel.instrumentation import init_instruments+ from extensions.otel.workflow_collaboration import WorkflowCollaborationMetrics++ app = DifyApp(__name__)+ app.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///:memory:"+ database = SQLAlchemy(app)+ reader = InMemoryMetricReader()+ provider = MeterProvider(metric_readers=[reader])+ set_meter_provider(provider)+ try:+ init_instruments(app)+ metrics = app.extensions["workflow_collaboration_metrics"]+ assert isinstance(metrics, WorkflowCollaborationMetrics)+ assert [+ extension for extension in app.extensions.values()+ if isinstance(extension, WorkflowCollaborationMetrics)+ ] == [metrics]++ data = reader.get_metrics_data()+ assert data is not None+ exported = {+ metric.name: metric+ for resource in data.resource_metrics+ for scope in resource.scope_metrics+ for metric in scope.metrics+ if metric.name.startswith("dify.collaboration.")+ }+ assert set(exported) == {+ "dify.collaboration.connections",+ "dify.collaboration.rooms",+ "dify.collaboration.room_sessions",+ }+ for name in ("connections", "rooms"):+ gauge = exported[f"dify.collaboration.{name}"].data+ assert isinstance(gauge, Gauge)+ assert len(gauge.data_points) == 1+ assert gauge.data_points[0].value == 0+ assert gauge.data_points[0].attributes == {"aggregation_scope": "worker"}+ distribution = exported["dify.collaboration.room_sessions"].data+ assert isinstance(distribution, Gauge)+ assert len(distribution.data_points) == 5+ assert {point.attributes["size_range"]: point.value for point in distribution.data_points} == {+ "1": 0, "2-5": 0, "6-10": 0, "11-20": 0, "21+": 0+ }+ finally:+ provider.shutdown()+ with app.app_context():+ database.engine.dispose()+ """+ )+ completed = subprocess.run(+ [sys.executable, "-B", "-c", script],+ cwd=Path(__file__).resolve().parents[4],+ env={**os.environ, "DEBUG": "false", "ENABLE_OTEL": "true"},+ capture_output=True,+ text=True,+ timeout=30,+ check=False,+ )+ assert completed.returncode == 0, completed.stdout + completed.stderrdiff --git a/api/tests/unit_tests/extensions/otel/test_workflow_collaboration.py b/api/tests/unit_tests/extensions/otel/test_workflow_collaboration.pynew file mode 100644index 00000000000..95232d972b6--- /dev/null+++ b/api/tests/unit_tests/extensions/otel/test_workflow_collaboration.py@@ -0,0 +1,605 @@+"""Exercise worker-local collaboration metrics with real SDK collection and room membership."""++from collections.abc import Callable, Iterator+from concurrent.futures import ThreadPoolExecutor+from threading import Barrier, get_ident+from typing import Never++import pytest+import socketio+from flask import Flask+from opentelemetry.sdk.metrics import MeterProvider+from opentelemetry.sdk.metrics.export import Gauge, Histogram, InMemoryMetricReader, Metric, Sum++from extensions.otel import workflow_collaboration as instrumentation++SIZE_RANGES = ("1", "2-5", "6-10", "11-20", "21+")+type ObservedMetrics = tuple[instrumentation.WorkflowCollaborationMetrics, InMemoryMetricReader]+++def metrics_app() -> Flask:+ """Create a real app with isolated extension ownership and no external services."""+ return Flask(__name__)+++def observed_metrics(+ monkeypatch: pytest.MonkeyPatch,+ config_overrides: Callable[..., None],+ metrics_app: Flask,+) -> Iterator[ObservedMetrics]:+ """Initialize app-owned instruments with a real SDK meter and isolate global configuration."""+ config_overrides(ENABLE_OTEL=True)+ reader = InMemoryMetricReader()+ provider = MeterProvider(metric_readers=[reader])+ monkeypatch.setattr(instrumentation, "get_meter", provider.get_meter)+ instrumentation.init_workflow_collaboration_metrics(metrics_app)+ metrics = metrics_app.extensions["workflow_collaboration_metrics"]+ assert isinstance(metrics, instrumentation.WorkflowCollaborationMetrics)+ yield metrics, reader+ provider.shutdown()+++def socket_server(metrics_app: Flask, monkeypatch: pytest.MonkeyPatch) -> socketio.Server:+ """Attach the real app to a network-free server as the application factory does."""+ server = socketio.Server(async_mode="threading")+ monkeypatch.setattr(server, "app", metrics_app, raising=False)+ return server+++def _metrics(reader: InMemoryMetricReader) -> dict[str, Metric]:+ """Collect actual SDK data, requiring a successful observable callback collection."""+ data = reader.get_metrics_data()+ assert data is not None+ return {+ metric.name: metric+ for resource in data.resource_metrics+ for scope in resource.scope_metrics+ for metric in scope.metrics+ }+++def _assert_population(reader: InMemoryMetricReader, connections: int, rooms: int, sizes: dict[str, int]) -> None:+ """Require exact worker-scope labels and all fixed buckets, including zeroes."""+ collected = _metrics(reader)+ for name, expected in (("connections", connections), ("rooms", rooms)):+ data = collected[f"dify.collaboration.{name}"].data+ assert isinstance(data, Gauge)+ assert len(data.data_points) == 1+ point = data.data_points[0]+ assert point.attributes == {"aggregation_scope": "worker"}+ assert point.value == expected++ distribution = collected["dify.collaboration.room_sessions"].data+ assert isinstance(distribution, Gauge)+ assert len(distribution.data_points) == len(SIZE_RANGES)+ actual: dict[str, int | float] = {}+ for point in distribution.data_points:+ assert point.attributes is not None+ assert set(point.attributes) == {"aggregation_scope", "size_range"}+ assert point.attributes["aggregation_scope"] == "worker"+ size_range = point.attributes["size_range"]+ assert isinstance(size_range, str)+ actual[size_range] = point.value+ assert actual == {size_range: sizes.get(size_range, 0) for size_range in SIZE_RANGES}+++def _assert_event(reader: InMemoryMetricReader, event: str, event_type: str, outcome: str) -> None:+ """Require one matching counter and duration sample without payload-derived labels."""+ collected = _metrics(reader)+ expected = {"event": event, "event_type": event_type, "outcome": outcome}+ events = collected["dify.collaboration.events"].data+ durations = collected["dify.collaboration.event_duration"].data+ assert isinstance(events, Sum)+ assert isinstance(durations, Histogram)+ assert len(events.data_points) == len(durations.data_points) == 1+ assert events.data_points[0].attributes == expected+ assert events.data_points[0].value == 1+ assert durations.data_points[0].attributes == expected+ assert durations.data_points[0].count == 1+ assert durations.data_points[0].sum >= 0+++def test_empty_worker_emits_all_zero_gauges(observed_metrics: ObservedMetrics) -> None:+ """An unused worker still exports zero-valued populations and every size bucket."""+ _, reader = observed_metrics+ _assert_population(reader, connections=0, rooms=0, sizes={})+++def test_duplicate_connections_and_joins_do_not_inflate_population(observed_metrics: ObservedMetrics) -> None:+ """Repeated lifecycle notifications preserve unique local SID and room membership."""+ metrics, reader = observed_metrics+ metrics.connected("sid-a")+ metrics.connected("sid-a")+ metrics.update_rooms("sid-a", ["sid-a", "workflow-a", "workflow-a"])+ metrics.update_rooms("sid-a", ["sid-a", "workflow-a"])+ _assert_population(reader, connections=1, rooms=1, sizes={"1": 1})++ metrics.connected("sid-b")+ metrics.update_rooms("sid-b", ["sid-b", "workflow-a"])+ _assert_population(reader, connections=2, rooms=1, sizes={"2-5": 1})+++def test_room_snapshot_replaces_membership_and_disconnect_clears_it(observed_metrics: ObservedMetrics) -> None:+ """Multiple actual rooms are counted until removed; late joins cannot resurrect a SID."""+ metrics, reader = observed_metrics+ metrics.connected("sid-a")+ metrics.connected("sid-b")+ metrics.update_rooms("sid-a", ["sid-a", "workflow-a", "workflow-b"])+ metrics.update_rooms("sid-b", ["sid-b", "workflow-a"])+ _assert_population(reader, connections=2, rooms=2, sizes={"1": 1, "2-5": 1})++ metrics.update_rooms("sid-a", ["sid-a", "workflow-b"])+ _assert_population(reader, connections=2, rooms=2, sizes={"1": 2})+ metrics.disconnected("sid-a")+ metrics.disconnected("sid-a")+ metrics.update_rooms("sid-a", ["workflow-b"])+ _assert_population(reader, connections=1, rooms=1, sizes={"1": 1})++ metrics.disconnected("sid-b")+ metrics.update_rooms("unknown-sid", ["new-workflow"])+ _assert_population(reader, connections=0, rooms=0, sizes={})+++ ("members", "before", "after"),+ [+ (1, "1", None),+ (2, "2-5", "1"),+ (5, "2-5", "2-5"),+ (6, "6-10", "2-5"),+ (10, "6-10", "6-10"),+ (11, "11-20", "6-10"),+ (20, "11-20", "11-20"),+ (21, "21+", "11-20"),+ ],+)+def test_room_distribution_crosses_fixed_bucket_boundaries(+ observed_metrics: ObservedMetrics, members: int, before: str, after: str | None+) -> None:+ """Room-size buckets describe current rooms and move immediately on member departure."""+ metrics, reader = observed_metrics+ for index in range(members):+ sid = f"sid-{index}"+ metrics.connected(sid)+ metrics.update_rooms(sid, [sid, "workflow-a"])+ _assert_population(reader, connections=members, rooms=1, sizes={before: 1})++ metrics.disconnected("sid-0")+ _assert_population(reader, connections=members - 1, rooms=int(members > 1), sizes={after: 1} if after else {})+++def test_record_event_exports_seconds_and_matching_counter(observed_metrics: ObservedMetrics) -> None:+ """Counter and histogram retain the same bounded dimensions and seconds unit."""+ metrics, reader = observed_metrics+ metrics.record_event("graph_event", "none", "success", 0.025)+ _assert_event(reader, "graph_event", "none", "success")+ duration = _metrics(reader)["dify.collaboration.event_duration"]+ assert duration.unit == "s"+ assert isinstance(duration.data, Histogram)+ assert duration.data.data_points[0].sum == pytest.approx(0.025)+++ ("result", "outcome"),+ [+ (None, "success"),+ (True, "success"),+ (False, "rejected"),+ (0, "success"),+ ({"status": 500}, "success"),+ (({"msg": "ok"}, 200), "success"),+ (({"msg": "denied"}, 403), "rejected"),+ (({"msg": "failed"}, 503), "error"),+ (("ack", "opaque"), "success"),+ ],+)+def test_event_outcome_preserves_exact_handler_return(+ observed_metrics: ObservedMetrics, socket_server: socketio.Server, result: object, outcome: str+) -> None:+ """Only explicit rejection or HTTP-like status tuples classify a normal return as failure."""+ assert isinstance(socket_server, instrumentation._SocketIO)+ _, reader = observed_metrics++ @instrumentation.observe_workflow_event("graph_event", socket_server)+ def handler(_sid: str, _data: object) -> object:+ return result++ assert handler("sid-a", b"graph-data") is result+ assert handler.__name__ == "handler"+ _assert_event(reader, "graph_event", "none", outcome)+++ ("payload", "expected_type"),+ [+ ({"type": "mouse_move"}, "mouse_move"),+ ({"type": "workflow_update"}, "workflow_update"),+ ({"type": "untrusted-user-specific-event"}, "other"),+ ({}, "other"),+ ({"type": ["mouse_move"]}, "other"),+ ({"type": None}, "other"),+ (None, "other"),+ (b"data", "other"),+ ],+)+def test_event_types_are_bounded_without_changing_payload(+ observed_metrics: ObservedMetrics, socket_server: socketio.Server, payload: object, expected_type: str+) -> None:+ """Unknown and malformed payloads stay handler-owned and cannot add metric series."""+ assert isinstance(socket_server, instrumentation._SocketIO)+ _, reader = observed_metrics++ @instrumentation.observe_workflow_event("collaboration_event", socket_server)+ def handler(_sid: str, data: object) -> object:+ return data++ assert handler("sid-a", payload) is payload+ _assert_event(reader, "collaboration_event", expected_type, "success")+++def test_event_exception_is_recorded_and_reraised_unchanged(+ observed_metrics: ObservedMetrics, socket_server: socketio.Server+) -> None:+ """Instrumentation preserves the original exception instance while reporting failure."""+ assert isinstance(socket_server, instrumentation._SocketIO)+ _, reader = observed_metrics+ failure = ValueError("handler failed")++ @instrumentation.observe_workflow_event("collaboration_event", socket_server)+ def handler(_sid: str, _data: object) -> Never:+ raise failure++ with pytest.raises(ValueError, match="handler failed") as caught:+ handler("sid-a", {"type": "mouse_move"})+ assert caught.value is failure+ _assert_event(reader, "collaboration_event", "mouse_move", "error")++[email protected]("accepted", [True, False])+def test_connect_counts_only_authenticated_sockets(+ observed_metrics: ObservedMetrics, socket_server: socketio.Server, accepted: bool+) -> None:+ """A provisional Socket.IO room exists before authentication, but rejected SIDs do not count."""+ assert isinstance(socket_server, instrumentation._SocketIO)+ assert isinstance(socket_server.manager, socketio.Manager)+ _, reader = observed_metrics+ sid = socket_server.manager.connect("engine-sid", "/")+ assert isinstance(sid, str)++ @instrumentation.observe_workflow_event("connect", socket_server)+ def handler(_sid: str, _environ: object, _auth: object) -> bool:+ return accepted++ assert handler(sid, {}, None) is accepted+ _assert_population(reader, connections=int(accepted), rooms=0, sizes={})+ _assert_event(reader, "connect", "none", "success" if accepted else "rejected")+++def test_connect_completed_after_disconnect_does_not_restore_socket(+ observed_metrics: ObservedMetrics, socket_server: socketio.Server+) -> None:+ """A successful handler cannot reintroduce a SID that disconnected during authentication."""+ assert isinstance(socket_server, instrumentation._SocketIO)+ assert isinstance(socket_server.manager, socketio.Manager)+ _, reader = observed_metrics+ sid = socket_server.manager.connect("engine-sid", "/")+ assert isinstance(sid, str)++ @instrumentation.observe_workflow_event("connect", socket_server)+ def handler(sid: str, _environ: object, _auth: object) -> bool:+ socket_server.manager.disconnect(sid, "/")+ return True++ assert handler(sid, {}, None) is True+ _assert_population(reader, connections=0, rooms=0, sizes={})+ _assert_event(reader, "connect", "none", "success")+++def test_partial_join_failure_reflects_real_membership(+ observed_metrics: ObservedMetrics, socket_server: socketio.Server+) -> None:+ """An exception after enter_room retains actual local membership and the original failure."""+ assert isinstance(socket_server, instrumentation._SocketIO)+ assert isinstance(socket_server.manager, socketio.Manager)+ metrics, reader = observed_metrics+ sid = socket_server.manager.connect("engine-sid", "/")+ assert isinstance(sid, str)+ metrics.connected(sid)+ failure = RuntimeError("broadcast failed after join")++ @instrumentation.observe_workflow_event("user_connect", socket_server)+ def handler(sid: str, data: dict[str, str]) -> Never:+ socket_server.enter_room(sid, data["workflow_id"])+ raise failure++ with pytest.raises(RuntimeError) as caught:+ handler(sid, {"workflow_id": "workflow-a"})+ assert caught.value is failure+ _assert_population(reader, connections=1, rooms=1, sizes={"1": 1})+ _assert_event(reader, "user_connect", "none", "error")+++def test_disconnect_failure_still_removes_all_local_membership(+ observed_metrics: ObservedMetrics, socket_server: socketio.Server+) -> None:+ """Population cleanup does not depend on the service's Redis cleanup succeeding."""+ assert isinstance(socket_server, instrumentation._SocketIO)+ assert isinstance(socket_server.manager, socketio.Manager)+ metrics, reader = observed_metrics+ sid = socket_server.manager.connect("engine-sid", "/")+ assert isinstance(sid, str)+ metrics.connected(sid)+ socket_server.enter_room(sid, "workflow-a")+ metrics.update_rooms(sid, socket_server.rooms(sid))+ socket_server.manager.pre_disconnect(sid, "/")+ failure = RuntimeError("service cleanup failed")++ @instrumentation.observe_workflow_event("disconnect", socket_server)+ def handler(_sid: str) -> Never:+ raise failure++ with pytest.raises(RuntimeError) as caught:+ handler(sid)+ assert caught.value is failure+ _assert_population(reader, connections=0, rooms=0, sizes={})+ _assert_event(reader, "disconnect", "none", "error")+++def test_room_lookup_failure_does_not_hide_return_or_event(+ observed_metrics: ObservedMetrics, socket_server: socketio.Server, monkeypatch: pytest.MonkeyPatch+) -> None:+ """A metrics-only membership lookup failure cannot change an ACK or suppress event accounting."""+ assert isinstance(socket_server, instrumentation._SocketIO)+ assert isinstance(socket_server.manager, socketio.Manager)+ metrics, reader = observed_metrics+ sid = socket_server.manager.connect("engine-sid", "/")+ assert isinstance(sid, str)+ metrics.connected(sid)+ result = {"msg": "connected"}++ def unavailable_rooms(*_args: object, **_kwargs: object) -> Never:+ raise RuntimeError("room lookup failed")++ monkeypatch.setattr(socket_server, "rooms", unavailable_rooms)++ @instrumentation.observe_workflow_event("user_connect", socket_server)+ def handler(_sid: str, _data: object) -> dict[str, str]:+ return result++ assert handler(sid, {}) is result+ _assert_event(reader, "user_connect", "none", "success")+++def test_collecting_gauges_never_reads_live_socket_manager(+ observed_metrics: ObservedMetrics, socket_server: socketio.Server, monkeypatch: pytest.MonkeyPatch+) -> None:+ """SDK collection consumes the published snapshot even when every room accessor is unavailable."""+ assert isinstance(socket_server, instrumentation._SocketIO)+ assert isinstance(socket_server.manager, socketio.Manager)+ metrics, reader = observed_metrics+ sid = socket_server.manager.connect("engine-sid", "/")+ assert isinstance(sid, str)+ metrics.connected(sid)+ socket_server.enter_room(sid, "workflow-a")++ @instrumentation.observe_workflow_event("user_connect", socket_server)+ def handler(_sid: str, _data: object) -> dict[str, str]:+ return {"msg": "connected"}++ handler(sid, {})++ def forbidden_access(*_args: object, **_kwargs: object) -> Never:+ raise AssertionError("observable callbacks must not touch live manager state")++ monkeypatch.setattr(socket_server, "rooms", forbidden_access)+ monkeypatch.setattr(socket_server.manager, "get_rooms", forbidden_access)+ monkeypatch.setattr(socket_server.manager, "get_participants", forbidden_access)+ _assert_population(reader, connections=1, rooms=1, sizes={"1": 1})+++def test_native_thread_collection_uses_complete_numeric_snapshots(observed_metrics: ObservedMetrics) -> None:+ """Native SDK collection tolerates handler-side churn without traversing mutable membership."""+ metrics, reader = observed_metrics+ ready = Barrier(2)+ handler_thread = get_ident()++ def collect() -> int:+ ready.wait(timeout=5)+ for _ in range(50):+ distribution = _metrics(reader)["dify.collaboration.room_sessions"].data+ assert isinstance(distribution, Gauge)+ assert len(distribution.data_points) == len(SIZE_RANGES)+ counts = [point.value for point in distribution.data_points]+ assert all(count >= 0 for count in counts)+ assert sum(counts) in (0, 1)+ return get_ident()++ with ThreadPoolExecutor(max_workers=1) as executor:+ collection = executor.submit(collect)+ ready.wait(timeout=5)+ for index in range(1000):+ sid = f"sid-{index}"+ metrics.connected(sid)+ metrics.update_rooms(sid, [sid, "workflow-a"])+ metrics.disconnected(sid)+ assert collection.result(timeout=10) != handler_thread+ _assert_population(reader, connections=0, rooms=0, sizes={})++[email protected]("handler_fails", [False, True])+def test_sdk_failure_never_changes_handler_result_or_exception(+ observed_metrics: ObservedMetrics,+ socket_server: socketio.Server,+ monkeypatch: pytest.MonkeyPatch,+ handler_fails: bool,+) -> None:+ """An instrument failure stays isolated from the successful ACK or original business exception."""+ assert isinstance(socket_server, instrumentation._SocketIO)+ metrics, _ = observed_metrics+ result = {"msg": "received"}+ handler_failure = ValueError("original handler failure")++ def unavailable_sdk(*_args: object, **_kwargs: object) -> Never:+ raise RuntimeError("metric SDK failure")++ monkeypatch.setattr(metrics._events, "add", unavailable_sdk)++ @instrumentation.observe_workflow_event("graph_event", socket_server)+ def handler(_sid: str, _data: object) -> dict[str, str]:+ if handler_fails:+ raise handler_failure+ return result++ if handler_fails:+ with pytest.raises(ValueError, match="original handler failure") as caught:+ handler("sid-a", b"graph-data")+ assert caught.value is handler_failure+ else:+ assert handler("sid-a", b"graph-data") is result+++def test_startup_instance_is_shared_and_repeated_initialization_preserves_state(+ observed_metrics: ObservedMetrics,+ metrics_app: Flask,+ socket_server: socketio.Server,+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Handlers share startup-owned state; repeated initialization neither registers nor resets instruments."""+ assert isinstance(socket_server, instrumentation._SocketIO)+ assert isinstance(socket_server.manager, socketio.Manager)+ metrics, reader = observed_metrics+ registration_attempts: list[object] = []++ def forbidden_registration(*args: object, **_kwargs: object) -> Never:+ registration_attempts.extend(args)+ raise AssertionError("event handling must use the initialized app instance")++ monkeypatch.setattr(instrumentation, "get_meter", forbidden_registration)++ @instrumentation.observe_workflow_event("connect", socket_server)+ def connected(_sid: str, _environ: object, _auth: object) -> bool:+ return True++ @instrumentation.observe_workflow_event("user_connect", socket_server)+ def joined(sid: str, _data: object) -> dict[str, str]:+ socket_server.enter_room(sid, "workflow-a")+ return {"msg": "connected"}++ for index in range(2):+ sid = socket_server.manager.connect(f"engine-{index}", "/")+ assert isinstance(sid, str)+ assert connected(sid, {}, None) is True+ assert joined(sid, {}) == {"msg": "connected"}+ instrumentation.init_workflow_collaboration_metrics(metrics_app)+ assert metrics_app.extensions["workflow_collaboration_metrics"] is metrics++ assert registration_attempts == []+ _assert_population(reader, connections=2, rooms=1, sizes={"2-5": 1})+ events = _metrics(reader)["dify.collaboration.events"].data+ assert isinstance(events, Sum)+ assert len(events.data_points) == 2+ assert all(point.value == 2 for point in events.data_points)++[email protected]("initialized", [False, True])[email protected]("handler_fails", [False, True])+def test_disabled_metrics_bypass_all_instrumentation_with_or_without_app_instance(+ observed_metrics: ObservedMetrics,+ metrics_app: Flask,+ socket_server: socketio.Server,+ monkeypatch: pytest.MonkeyPatch,+ config_overrides: Callable[..., None],+ initialized: bool,+ handler_fails: bool,+) -> None:+ """Disabled instrumentation preserves business behavior without timing, app lookup, membership, or SDK access."""+ assert isinstance(socket_server, instrumentation._SocketIO)+ metrics, reader = observed_metrics+ if not initialized:+ metrics_app.extensions.pop("workflow_collaboration_metrics")+ accesses: list[str] = []+ result = {"msg": "connected"}+ failure = ValueError("business failure while metrics disabled")++ def forbidden_access(*_args: object, **_kwargs: object) -> Never:+ accesses.append("instrumentation")+ raise AssertionError("disabled metrics must not access instrumentation")++ monkeypatch.setattr(instrumentation, "perf_counter", forbidden_access)+ monkeypatch.setattr(instrumentation, "get_meter", forbidden_access)+ monkeypatch.setattr(type(socket_server), "app", property(forbidden_access), raising=False)+ monkeypatch.setattr(socket_server, "rooms", forbidden_access)+ monkeypatch.setattr(metrics._events, "add", forbidden_access)++ @instrumentation.observe_workflow_event("user_connect", socket_server)+ def handler(_sid: str, _data: object) -> dict[str, str]:+ if handler_fails:+ raise failure+ return result++ config_overrides(ENABLE_OTEL=False)+ if handler_fails:+ with pytest.raises(ValueError, match="business failure while metrics disabled") as caught:+ handler("sid-a", {})+ assert caught.value is failure+ else:+ assert handler("sid-a", {}) is result++ assert accesses == []+ _assert_population(reader, connections=0, rooms=0, sizes={})+ assert "dify.collaboration.events" not in _metrics(reader)++[email protected]("initialization_fails", [False, True])[email protected]("handler_fails", [False, True])+def test_enabled_metrics_without_initialized_extension_leave_business_behavior_unchanged(+ metrics_app: Flask,+ socket_server: socketio.Server,+ monkeypatch: pytest.MonkeyPatch,+ config_overrides: Callable[..., None],+ initialization_fails: bool,+ handler_fails: bool,+) -> None:+ """Missing or failed startup instrumentation never registers lazily, times, or replaces handler results."""+ assert isinstance(socket_server, instrumentation._SocketIO)+ config_overrides(ENABLE_OTEL=True)+ registration_attempts: list[str] = []+ accesses: list[str] = []+ result = {"msg": "connected"}+ failure = ValueError("business failure without metrics instance")++ def unavailable_meter(*_args: object, **_kwargs: object) -> Never:+ registration_attempts.append("get_meter")+ raise RuntimeError("SDK initialization failed")++ def forbidden_access(*_args: object, **_kwargs: object) -> Never:+ accesses.append("handler instrumentation")+ raise AssertionError("missing instance must not trigger event instrumentation")++ monkeypatch.setattr(instrumentation, "get_meter", unavailable_meter)+ if initialization_fails:+ instrumentation.init_workflow_collaboration_metrics(metrics_app)+ assert "workflow_collaboration_metrics" not in metrics_app.extensions++ monkeypatch.setattr(instrumentation, "perf_counter", forbidden_access)+ monkeypatch.setattr(socket_server, "rooms", forbidden_access)++ @instrumentation.observe_workflow_event("user_connect", socket_server)+ def handler(_sid: str, _data: object) -> dict[str, str]:+ if handler_fails:+ raise failure+ return result++ if handler_fails:+ with pytest.raises(ValueError, match="business failure without metrics instance") as caught:+ handler("sid-a", {})+ assert caught.value is failure+ else:+ assert handler("sid-a", {}) is result++ assert accesses == []+ assert registration_attempts == (["get_meter"] if initialization_fails else [])+ assert "workflow_collaboration_metrics" not in metrics_app.extensions