Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.
Mock telemetry in pytest tests (#3340)
Greyson LaLondea17b93a7f8f2
diff --git a/pyproject.toml b/pyproject.tomlindex e19ec38e6..2c7c9770a 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -98,6 +98,11 @@ exclude = ["cli/templates"][tool.bandit]exclude_dirs = ["src/crewai/cli/templates"]+[tool.pytest.ini_options]+markers = [+ "telemetry: mark test as a telemetry test (don't mock telemetry)",+]+# PyTorch index configuration, since torch 2.5.0 is not compatible with python 3.13[[tool.uv.index]]name = "pytorch-nightly"diff --git a/tests/conftest.py b/tests/conftest.pyindex f67ad7222..6a03a163d 100644--- a/tests/conftest.py+++ b/tests/conftest.py@@ -2,6 +2,7 @@import osimport tempfilefrom pathlib import Path+from unittest.mock import Mock, patchimport pytestfrom dotenv import load_dotenv@@ -42,6 +43,122 @@ def setup_test_environment():# Cleanup is handled automatically when tempfile context exits+def pytest_configure(config):+ config.addinivalue_line(+ "markers", "telemetry: mark test as a telemetry test (don't mock telemetry)"+ )++[email protected](autouse=True)+def auto_mock_telemetry(request):+ if request.node.get_closest_marker("telemetry"):+ telemetry_env = {+ key: value+ for key, value in os.environ.items()+ if key not in ["CREWAI_DISABLE_TELEMETRY", "OTEL_SDK_DISABLED"]+ }+ with patch.dict(os.environ, telemetry_env, clear=True):+ yield+ return++ if "telemetry" in str(request.fspath):+ telemetry_env = {+ key: value+ for key, value in os.environ.items()+ if key not in ["CREWAI_DISABLE_TELEMETRY", "OTEL_SDK_DISABLED"]+ }+ with patch.dict(os.environ, telemetry_env, clear=True):+ yield+ return++ with patch.dict(+ os.environ, {"CREWAI_DISABLE_TELEMETRY": "true", "OTEL_SDK_DISABLED": "true"}+ ):+ with patch("crewai.telemetry.Telemetry") as mock_telemetry_class:+ mock_instance = create_mock_telemetry_instance()+ mock_telemetry_class.return_value = mock_instance++ with (+ patch(+ "crewai.utilities.events.event_listener.Telemetry",+ mock_telemetry_class,+ ),+ patch("crewai.tools.tool_usage.Telemetry", mock_telemetry_class),+ patch("crewai.cli.command.Telemetry", mock_telemetry_class),+ patch("crewai.cli.create_flow.Telemetry", mock_telemetry_class),+ ):+ yield mock_instance+++def create_mock_telemetry_instance():+ mock_instance = Mock()++ mock_instance.ready = False+ mock_instance.trace_set = False+ mock_instance._initialized = True++ mock_instance._is_telemetry_disabled.return_value = True+ mock_instance._should_execute_telemetry.return_value = False++ telemetry_methods = [+ "set_tracer",+ "crew_creation",+ "task_started",+ "task_ended",+ "tool_usage",+ "tool_repeated_usage",+ "tool_usage_error",+ "crew_execution_span",+ "end_crew",+ "flow_creation_span",+ "flow_execution_span",+ "individual_test_result_span",+ "test_execution_span",+ "deploy_signup_error_span",+ "start_deployment_span",+ "create_crew_deployment_span",+ "get_crew_logs_span",+ "remove_crew_span",+ "flow_plotting_span",+ "_add_attribute",+ "_safe_telemetry_operation",+ ]++ for method in telemetry_methods:+ setattr(mock_instance, method, Mock(return_value=None))++ mock_instance.task_started.return_value = None++ return mock_instance+++def mock_opentelemetry_components():+ with (+ patch("opentelemetry.trace.get_tracer") as mock_get_tracer,+ patch("opentelemetry.trace.set_tracer_provider") as mock_set_provider,+ patch("opentelemetry.baggage.set_baggage") as mock_set_baggage,+ patch("opentelemetry.baggage.get_baggage") as mock_get_baggage,+ patch("opentelemetry.context.attach") as mock_attach,+ patch("opentelemetry.context.detach") as mock_detach,+ ):+ mock_tracer = Mock()+ mock_span = Mock()+ mock_tracer.start_span.return_value = mock_span+ mock_get_tracer.return_value = mock_tracer++ yield {+ "get_tracer": mock_get_tracer,+ "set_tracer_provider": mock_set_provider,+ "tracer": mock_tracer,+ "span": mock_span,+ "set_baggage": mock_set_baggage,+ "get_baggage": mock_get_baggage,+ "attach": mock_attach,+ "detach": mock_detach,+ }++@pytest.fixture(scope="module")def vcr_config(request) -> dict:return {diff --git a/tests/telemetry/test_telemetry.py b/tests/telemetry/test_telemetry.pyindex 277578327..593f1711e 100644--- a/tests/telemetry/test_telemetry.py+++ b/tests/telemetry/test_telemetry.py@@ -1,4 +1,5 @@import os+import threadingfrom unittest.mock import patchimport pytest@@ -11,12 +12,16 @@ from opentelemetry import trace@pytest.fixture(autouse=True)def cleanup_telemetry():- """Automatically clean up Telemetry singleton between tests."""Telemetry._instance = None+ if hasattr(Telemetry, "_lock"):+ Telemetry._lock = threading.Lock()yieldTelemetry._instance = None+ if hasattr(Telemetry, "_lock"):+ Telemetry._lock = threading.Lock()@pytest.mark.parametrize("env_var,value,expected_ready",[@@ -36,6 +41,7 @@ def test_telemetry_environment_variables(env_var, value, expected_ready):assert telemetry.ready is expected_readydef test_telemetry_enabled_by_default():"""Test that telemetry is enabled by default."""with patch.dict(os.environ, {}, clear=True):@@ -44,6 +50,7 @@ def test_telemetry_enabled_by_default():assert telemetry.ready is True@patch("crewai.telemetry.telemetry.logger.error")@patch("opentelemetry.exporter.otlp.proto.http.trace_exporter.OTLPSpanExporter.export",@@ -76,6 +83,7 @@ def test_telemetry_fails_due_connect_timeout(export_mock, logger_mock):logger_mock.assert_called_once_with(error)def test_telemetry_singleton_pattern():"""Test that Telemetry uses the singleton pattern correctly."""Telemetry._instance = Nonediff --git a/tests/telemetry/test_telemetry_disable.py b/tests/telemetry/test_telemetry_disable.pyindex 96738ad5f..2168bc8c2 100644--- a/tests/telemetry/test_telemetry_disable.py+++ b/tests/telemetry/test_telemetry_disable.py@@ -14,14 +14,18 @@ def cleanup_telemetry():Telemetry._instance = None[email protected]("env_var,value,expected_ready", [- ("OTEL_SDK_DISABLED", "true", False),- ("OTEL_SDK_DISABLED", "TRUE", False),- ("CREWAI_DISABLE_TELEMETRY", "true", False),- ("CREWAI_DISABLE_TELEMETRY", "TRUE", False),- ("OTEL_SDK_DISABLED", "false", True),- ("CREWAI_DISABLE_TELEMETRY", "false", True),-])+ "env_var,value,expected_ready",+ [+ ("OTEL_SDK_DISABLED", "true", False),+ ("OTEL_SDK_DISABLED", "TRUE", False),+ ("CREWAI_DISABLE_TELEMETRY", "true", False),+ ("CREWAI_DISABLE_TELEMETRY", "TRUE", False),+ ("OTEL_SDK_DISABLED", "false", True),+ ("CREWAI_DISABLE_TELEMETRY", "false", True),+ ],+)def test_telemetry_environment_variables(env_var, value, expected_ready):"""Test telemetry state with different environment variable configurations."""with patch.dict(os.environ, {env_var: value}):@@ -30,6 +34,7 @@ def test_telemetry_environment_variables(env_var, value, expected_ready):assert telemetry.ready is expected_readydef test_telemetry_enabled_by_default():"""Test that telemetry is enabled by default."""with patch.dict(os.environ, {}, clear=True):@@ -38,57 +43,60 @@ def test_telemetry_enabled_by_default():assert telemetry.ready is Truedef test_telemetry_disable_after_singleton_creation():"""Test that telemetry operations are disabled when env var is set after singleton creation."""with patch.dict(os.environ, {}, clear=True):with patch("crewai.telemetry.telemetry.TracerProvider"):telemetry = Telemetry()assert telemetry.ready is True-+mock_operation = MagicMock()telemetry._safe_telemetry_operation(mock_operation)mock_operation.assert_called_once()-+mock_operation.reset_mock()-- os.environ['CREWAI_DISABLE_TELEMETRY'] = 'true'-++ os.environ["CREWAI_DISABLE_TELEMETRY"] = "true"+telemetry._safe_telemetry_operation(mock_operation)mock_operation.assert_not_called()def test_telemetry_disable_with_multiple_instances():"""Test that multiple telemetry instances respect dynamically changed env vars."""with patch.dict(os.environ, {}, clear=True):with patch("crewai.telemetry.telemetry.TracerProvider"):telemetry1 = Telemetry()assert telemetry1.ready is True-- os.environ['CREWAI_DISABLE_TELEMETRY'] = 'true'-++ os.environ["CREWAI_DISABLE_TELEMETRY"] = "true"+telemetry2 = Telemetry()assert telemetry2 is telemetry1assert telemetry2.ready is True-+mock_operation = MagicMock()telemetry2._safe_telemetry_operation(mock_operation)mock_operation.assert_not_called()def test_telemetry_otel_sdk_disabled_after_creation():"""Test that OTEL_SDK_DISABLED also works when set after singleton creation."""with patch.dict(os.environ, {}, clear=True):with patch("crewai.telemetry.telemetry.TracerProvider"):telemetry = Telemetry()assert telemetry.ready is True-+mock_operation = MagicMock()telemetry._safe_telemetry_operation(mock_operation)mock_operation.assert_called_once()-+mock_operation.reset_mock()-- os.environ['OTEL_SDK_DISABLED'] = 'true'-++ os.environ["OTEL_SDK_DISABLED"] = "true"+telemetry._safe_telemetry_operation(mock_operation)mock_operation.assert_not_called()