Build applications that make decisions (chatbots, agents, simulations, etc...). Monitor, trace, persist, and execute on your own infrastructure.
Python aiburrchatbot-frameworkdagsgenerative-aigraphshacktoberfestllmopsllmsmlopspersistent-data-structurestate-machinestate-managementvisibility
MVP for telemetry
elijahbenizzy4b438404b5e0
diff --git a/burr/cli/__main__.py b/burr/cli/__main__.pyindex cdca0091..c7344fe9 100644--- a/burr/cli/__main__.py+++ b/burr/cli/__main__.py@@ -8,6 +8,7 @@ import webbrowserfrom contextlib import contextmanagerfrom importlib.resources import files+from burr import telemetryfrom burr.integrations.base import require_plugintry:@@ -22,6 +23,12 @@ except ImportError as e:)+# TODO -- add this as a general callback to the CLI+def _telemetry_if_enabled(event: str):+ if telemetry.is_telemetry_enabled():+ telemetry.create_and_send_cli_event(event)++def _command(command: str, capture_output: bool) -> str:"""Runs a simple command"""logger.info(f"Running command: {command}")@@ -95,6 +102,7 @@ def build_ui():@click.option("--no-open", is_flag=True, help="Run the server without opening it")@click.option("--no-copy-demo_data", is_flag=True, help="Don't copy demo data over.")def run_server(port: int, dev_mode: bool, no_open: bool, no_copy_demo_data: bool):+ _telemetry_if_enabled("run_server")# TODO: Implement server running logic here# Example: Start a web server, configure ports, etc.logger.info(f"Starting server on port {port}")@@ -127,6 +135,7 @@ def run_server(port: int, dev_mode: bool, no_open: bool, no_copy_demo_data: bool@click.option("--prod", is_flag=True, help="Publish to pypi (rather than test pypi)")@click.option("--no-wipe-dist", is_flag=True, help="Wipe the dist/ directory before building")def build_and_publish(prod: bool, no_wipe_dist: bool):+ _telemetry_if_enabled("build_and_publish")git_root = _get_git_root()with cd(git_root):logger.info("Building UI -- this may take a bit...")@@ -143,6 +152,7 @@ def build_and_publish(prod: bool, no_wipe_dist: bool):@cli.command(help="generate demo data for the UI")def generate_demo_data():+ _telemetry_if_enabled("generate_demo_data")git_root = _get_git_root()# We need to add the examples directory to the path so we have all the imports# The GPT-one relies on a local importdiff --git a/burr/core/application.py b/burr/core/application.pyindex ef669537..5b03cd0a 100644--- a/burr/core/application.py+++ b/burr/core/application.py@@ -18,7 +18,7 @@ from typing import (cast,)-from burr import visibility+from burr import telemetry, visibilityfrom burr.core.action import (Action,Condition,@@ -255,6 +255,8 @@ class Application:)}+ # @telemetry.capture_function_usage # todo -- capture usage when we break this up into one that isn't called internally+ # This will be doable when we move sequence ID to the beginning of the function https://github.com/DAGWorks-Inc/burr/pull/73def step(self, inputs: Optional[Dict[str, Any]] = None) -> Optional[Tuple[Action, dict, State]]:"""Performs a single step, advancing the state machine along.This returns a tuple of the action that was run, the result of running@@ -358,6 +360,8 @@ class Application:)return processed_inputs+ # @telemetry.capture_function_usage+ # ditto with step()async def astep(self, inputs: Dict[str, Any] = None) -> Optional[Tuple[Action, dict, State]]:"""Asynchronous version of step.@@ -492,6 +496,7 @@ class Application:)return prior_action, result, self._state+ @telemetry.capture_function_usagedef iterate(self,*,@@ -528,6 +533,7 @@ class Application:breakreturn self._return_value_iterate(halt_before, halt_after, prior_action, result)+ @telemetry.capture_function_usageasync def aiterate(self,*,@@ -557,6 +563,7 @@ class Application:if self._should_halt_iterate(halt_before, halt_after, prior_action):break+ @telemetry.capture_function_usagedef run(self,*,@@ -580,6 +587,7 @@ class Application:except StopIteration as e:return e.value+ @telemetry.capture_function_usageasync def arun(self,*,@@ -615,6 +623,7 @@ class Application:f"Actions found: {[action.name for action in self._actions]}")+ @telemetry.capture_function_usagedef stream_result(self,halt_after: list[str],@@ -830,6 +839,7 @@ class Application:generator, self._state, process_result, callback)+ @telemetry.capture_function_usageasync def astream_result(self,halt_after: list[str],@@ -842,6 +852,7 @@ class Application:" for details. Please comment or vote to get it implemented quickly!")+ @telemetry.capture_function_usagedef visualize(self,output_file_path: Optional[str],@@ -1149,6 +1160,7 @@ class ApplicationBuilder:raise ValueError(f"Tracker {tracker} not supported")return self+ @telemetry.capture_function_usagedef build(self) -> Application:"""Builds the application.diff --git a/burr/telemetry.py b/burr/telemetry.pynew file mode 100644index 00000000..57879eed--- /dev/null+++ b/burr/telemetry.py@@ -0,0 +1,286 @@+"""+This module contains code that relates to sending Burr usage telemetry.++To disable sending telemetry there are three ways:++1. Set it to false programmatically in your driver:+ >>> from burr import telemetry+ >>> telemetry.disable_telemetry()+2. Set it to `false` in ~/.burr.conf under `DEFAULT`+ [DEFAULT]+ telemetry_enabled = False+3. Set BURR_TELEMETRY_ENABLED=false as an environment variable:+ BURR_TELEMETRY_ENABLED=false python run.py+ or:+ export BURR_TELEMETRY_ENABLED=false+"""++import configparser+import functools+import importlib.metadata+import json+import logging+import os+import platform+import threading+import uuid+from typing import TYPE_CHECKING, Callable, List+from urllib import request++if TYPE_CHECKING:+ from burr.lifecycle import internal++VERSION = importlib.metadata.version("burr")++logger = logging.getLogger(__name__)++STR_VERSION = ".".join([str(i) for i in VERSION])+HOST = "https://app.posthog.com"+TRACK_URL = f"{HOST}/capture/" # https://posthog.com/docs/api/post-only-endpoints+API_KEY = "phc_qMa4hWDdTruKaDb4Oa0tK0i1SKf69xf81OCFzjX6z4U"+APPLICATION_FUNCTION = "os_burr_application_function_call"+CLI_COMMAND = "os_burr_cli_command"+TIMEOUT = 2+MAX_COUNT_SESSION = 10 # max number of events collected per python process++DEFAULT_CONFIG_LOCATION = os.path.expanduser("~/.burr.conf")+++def _load_config(config_location: str) -> configparser.ConfigParser:+ """Pulls config. Gets/sets default anonymous ID.++ Creates the anonymous ID if it does not exist, writes it back if so.+ :param config_location: location of the config file.+ """+ config = configparser.ConfigParser()+ try:+ with open(config_location) as f:+ config.read_file(f)+ except Exception:+ config["DEFAULT"] = {}+ else:+ if "DEFAULT" not in config:+ config["DEFAULT"] = {}++ if "anonymous_id" not in config["DEFAULT"]:+ config["DEFAULT"]["anonymous_id"] = str(uuid.uuid4())+ try:+ with open(config_location, "w") as f:+ config.write(f)+ except Exception:+ pass+ return config+++def _check_config_and_environ_for_telemetry_flag(+ telemetry_default: bool, config_obj: configparser.ConfigParser+):+ """Checks the config and environment variables for the telemetry value.++ Note: the environment variable has greater precedence than the config value.+ """+ telemetry_enabled = telemetry_default+ if "telemetry_enabled" in config_obj["DEFAULT"]:+ try:+ telemetry_enabled = config_obj.getboolean("DEFAULT", "telemetry_enabled")+ except ValueError as e:+ logger.debug(+ "Unable to parse value for `telemetry_enabled` from config. " f"Encountered {e}"+ )+ if os.environ.get("BURR_TELEMETRY_ENABLED") is not None:+ env_value = os.environ.get("BURR_TELEMETRY_ENABLED")+ # set the value+ config_obj["DEFAULT"]["telemetry_enabled"] = env_value+ try:+ telemetry_enabled = config_obj.getboolean("DEFAULT", "telemetry_enabled")+ except ValueError as e:+ logger.debug(+ "Unable to parse value for `BURR_TELEMETRY_ENABLED` from environment. "+ f"Encountered {e}"+ )+ return telemetry_enabled+++config = _load_config(DEFAULT_CONFIG_LOCATION)+g_telemetry_enabled = _check_config_and_environ_for_telemetry_flag(True, config)+g_anonymous_id = config["DEFAULT"]["anonymous_id"]+call_counter = 0+++def disable_telemetry():+ """Disables telemetry tracking."""+ global g_telemetry_enabled+ g_telemetry_enabled = False+++def is_telemetry_enabled() -> bool:+ """Returns whether telemetry tracking is enabled or not.++ Increments a counter to stop sending telemetry after 1000 invocations.+ """+ if g_telemetry_enabled:+ global call_counter+ if call_counter == 0:+ # Log only the first time someone calls this function; don't want to spam them.+ logger.info(+ "Note: Burr collects completely anonymous data about usage. "+ "This will help us improve Burr over time. "+ "See https://github.com/dagworks-inc/burr#usage-analytics--data-privacy for details."+ )+ call_counter += 1+ if call_counter > MAX_COUNT_SESSION:+ # we have hit our limit -- disable telemetry.+ return False+ return True+ else:+ return False+++# base properties to instantiate on module load.+BASE_PROPERTIES = {+ "os_type": os.name,+ "os_version": platform.platform(),+ "python_version": f"{platform.python_version()}/{platform.python_implementation()}",+ "distinct_id": g_anonymous_id,+ "burr_version": VERSION,+ "telemetry_version": "0.0.1",+}+++def create_application_function_run_event(function_name: str) -> dict:+ """Function to create payload for tracking function name invocation.++ :param function_name: the name of the driver function+ :return: dict representing the JSON to send.+ """+ event = {+ "api_key": API_KEY,+ "event": APPLICATION_FUNCTION,+ "properties": {},+ }+ event["properties"].update(BASE_PROPERTIES)+ payload = {+ "function_name": function_name, # what was the name of the driver function?+ }+ event["properties"].update(payload)+ return event+++def _send_event_json(event_json: dict):+ """Internal function to send the event JSON to posthog.++ :param event_json: the dictionary of data to JSON serialize and send+ """+ headers = {+ "Content-Type": "application/json",+ "Authorization": "TODO",+ "User-Agent": f"burr/{STR_VERSION}",+ }+ try:+ data = json.dumps(event_json).encode()+ req = request.Request(TRACK_URL, data=data, headers=headers)+ with request.urlopen(req, timeout=TIMEOUT) as f:+ res = f.read()+ if f.code != 200:+ raise RuntimeError(res)+ except Exception as e:+ if logger.isEnabledFor(logging.DEBUG):+ logging.debug(f"Failed to send telemetry data: {e}")+ else:+ if logger.isEnabledFor(logging.DEBUG):+ logging.debug(f"Succeed in sending telemetry consisting of [{data}].")+++def send_event_json(event_json: dict):+ """Sends the event json in its own thread.++ :param event_json: the data to send+ """+ if not g_telemetry_enabled:+ raise RuntimeError("Won't send; tracking is disabled!")+ try:+ th = threading.Thread(target=_send_event_json, args=(event_json,))+ th.start()+ except Exception as e:+ # capture any exception!+ if logger.isEnabledFor(logging.DEBUG):+ logger.debug(f"Encountered error while sending event JSON via it's own thread:\n{e}")+++def get_all_adapters_names(adapter: "internal.LifecycleAdapterSet") -> List[str]:+ """Gives a list of all adapter names in the LifecycleAdapterSet.+ Simply a loop over the adapters it contains.++ :param adapter: LifecycleAdapterSet object.+ :return: list of adapter names.+ """+ adapters = adapter.adapters+ out = []+ for adapter in adapters:+ out.append(get_adapter_name(adapter))+ return out+++def get_adapter_name(adapter: "internal.LifecycleAdapter") -> str:+ """Get the class name of the ``burr`` adapter used.++ If we detect it's not a Burr one, we do not track it.++ :param adapter: lifecycle.internal.LifecycleAdapter object.+ :return: string module + class name of the adapter.+ """+ # Check whether it's a burr based adapter+ if adapter.__module__.startswith("burr."):+ adapter_name = f"{adapter.__module__}.{adapter.__class__.__name__}"+ else:+ adapter_name = "custom_adapter"+ return adapter_name+++def create_and_send_cli_event(command: str):+ """Function that creates JSON and sends to track CLI usage.++ :param command: the CLI command run.+ """+ event = {+ "api_key": API_KEY,+ "event": CLI_COMMAND,+ "properties": {},+ }+ event["properties"].update(BASE_PROPERTIES)++ payload = {+ "command": command,+ }+ event["properties"].update(payload)+ send_event_json(event)+++def capture_function_usage(call_fn: Callable) -> Callable:+ """Decorator to wrap some application functions for telemetry capture.++ We want to use this for non-execute functions.+ We don't capture information about the arguments at this stage,+ just the function name.++ :param call_fn: the Driver function to capture.+ :return: wrapped function.+ """++ @functools.wraps(call_fn)+ def wrapped_fn(*args, **kwargs):+ try:+ return call_fn(*args, **kwargs)+ finally:+ if is_telemetry_enabled():+ try:+ function_name = call_fn.__name__+ event_json = create_application_function_run_event(function_name)+ send_event_json(event_json)+ except Exception as e:+ if logger.isEnabledFor(logging.DEBUG):+ logger.error(+ f"Failed to send telemetry for function usage. Encountered: {e}\n"+ )++ return wrapped_fndiff --git a/docs/reference/index.rst b/docs/reference/index.rstindex dae21c46..0e35f4f1 100644--- a/docs/reference/index.rst+++ b/docs/reference/index.rst@@ -19,3 +19,4 @@ need functionality that is not publicly exposed, please open an issue and we canvisibilitylifecycleintegrations/index+ telemetrydiff --git a/docs/reference/telemetry.rst b/docs/reference/telemetry.rstnew file mode 100644index 00000000..7d7ed800--- /dev/null+++ b/docs/reference/telemetry.rst@@ -0,0 +1,47 @@+==============================+Usage analytics + data privacy+==============================++By default, when using Burr, it collects anonymous usage data to help improve Burr and know where to apply development efforts.++We capture events on the following occasions:++1. When an application is built+2. When one of the ``execution`` functions is run in ``Application``+3. When a CLI command is run++The captured data is limited to:++- Operating System and Python version+- A persistent UUID to indentify the session, stored in ~/.burr.conf.+- The name of the function/CLI command that was run++If you're worried, see ``telemetry.py`` for details.++If you do not wish to participate, one can opt-out with one of the following methods:++1. Set it to false programmatically in your code before creating a Burr application builder:++.. code-block:: python++ from burr import telemetry+ telemetry.disable_telemetry()++2. Set the key telemetry_enabled to false in ``~/.burr.conf`` under the DEFAULT section:++.. code-block:: ini++ [DEFAULT]+ telemetry_enabled = False++3. Set BURR_TELEMETRY_ENABLED=false as an environment variable. Either setting it for your shell session:++.. code-block:: bash++ export BURR_TELEMETRY_ENABLED=false++or passing it as part of the run command:++.. code-block:: bash++ BURR_TELEMETRY_ENABLED=false python NAME_OF_MY_DRIVER.pydiff --git a/tests/conftest.py b/tests/conftest.pynew file mode 100644index 00000000..f4997913--- /dev/null+++ b/tests/conftest.py@@ -0,0 +1,3 @@+from burr.telemetry import disable_telemetry++disable_telemetry()