Skip to content

armgit / burr

Public Imported from GitHub
0

Build applications that make decisions (chatbots, agents, simulations, etc...). Monitor, trace, persist, and execute on your own infrastructure.

MVP for telemetry

elijahbenizzy4b438404b5e0
diff --git a/burr/cli/__main__.py b/burr/cli/__main__.py
index cdca0091..c7344fe9 100644
--- a/burr/cli/__main__.py
+++ b/burr/cli/__main__.py
@@ -8,6 +8,7 @@ import webbrowser
from contextlib import contextmanager
from importlib.resources import files
+from burr import telemetry
from burr.integrations.base import require_plugin
try:
@@ -22,6 +23,12 @@ except ImportError as e:
)
+# TODO -- add this as a general callback to the CLI
+def _telemetry_if_enabled(event: str):
+ if telemetry.is_telemetry_enabled():
+ telemetry.create_and_send_cli_event(event)
+
+
def _command(command: str, capture_output: bool) -> str:
"""Runs a simple command"""
logger.info(f"Running command: {command}")
@@ -95,6 +102,7 @@ def build_ui():
@click.option("--no-open", is_flag=True, help="Run the server without opening it")
@click.option("--no-copy-demo_data", is_flag=True, help="Don't copy demo data over.")
def run_server(port: int, dev_mode: bool, no_open: bool, no_copy_demo_data: bool):
+ _telemetry_if_enabled("run_server")
# TODO: Implement server running logic here
# Example: Start a web server, configure ports, etc.
logger.info(f"Starting server on port {port}")
@@ -127,6 +135,7 @@ def run_server(port: int, dev_mode: bool, no_open: bool, no_copy_demo_data: bool
@click.option("--prod", is_flag=True, help="Publish to pypi (rather than test pypi)")
@click.option("--no-wipe-dist", is_flag=True, help="Wipe the dist/ directory before building")
def build_and_publish(prod: bool, no_wipe_dist: bool):
+ _telemetry_if_enabled("build_and_publish")
git_root = _get_git_root()
with cd(git_root):
logger.info("Building UI -- this may take a bit...")
@@ -143,6 +152,7 @@ def build_and_publish(prod: bool, no_wipe_dist: bool):
@cli.command(help="generate demo data for the UI")
def generate_demo_data():
+ _telemetry_if_enabled("generate_demo_data")
git_root = _get_git_root()
# We need to add the examples directory to the path so we have all the imports
# The GPT-one relies on a local import
diff --git a/burr/core/application.py b/burr/core/application.py
index ef669537..5b03cd0a 100644
--- a/burr/core/application.py
+++ b/burr/core/application.py
@@ -18,7 +18,7 @@ from typing import (
cast,
)
-from burr import visibility
+from burr import telemetry, visibility
from burr.core.action import (
Action,
Condition,
@@ -255,6 +255,8 @@ class Application:
)
}
+ # @telemetry.capture_function_usage # todo -- capture usage when we break this up into one that isn't called internally
+ # This will be doable when we move sequence ID to the beginning of the function https://github.com/DAGWorks-Inc/burr/pull/73
def step(self, inputs: Optional[Dict[str, Any]] = None) -> Optional[Tuple[Action, dict, State]]:
"""Performs a single step, advancing the state machine along.
This returns a tuple of the action that was run, the result of running
@@ -358,6 +360,8 @@ class Application:
)
return processed_inputs
+ # @telemetry.capture_function_usage
+ # ditto with step()
async def astep(self, inputs: Dict[str, Any] = None) -> Optional[Tuple[Action, dict, State]]:
"""Asynchronous version of step.
@@ -492,6 +496,7 @@ class Application:
)
return prior_action, result, self._state
+ @telemetry.capture_function_usage
def iterate(
self,
*,
@@ -528,6 +533,7 @@ class Application:
break
return self._return_value_iterate(halt_before, halt_after, prior_action, result)
+ @telemetry.capture_function_usage
async def aiterate(
self,
*,
@@ -557,6 +563,7 @@ class Application:
if self._should_halt_iterate(halt_before, halt_after, prior_action):
break
+ @telemetry.capture_function_usage
def run(
self,
*,
@@ -580,6 +587,7 @@ class Application:
except StopIteration as e:
return e.value
+ @telemetry.capture_function_usage
async def arun(
self,
*,
@@ -615,6 +623,7 @@ class Application:
f"Actions found: {[action.name for action in self._actions]}"
)
+ @telemetry.capture_function_usage
def stream_result(
self,
halt_after: list[str],
@@ -830,6 +839,7 @@ class Application:
generator, self._state, process_result, callback
)
+ @telemetry.capture_function_usage
async def astream_result(
self,
halt_after: list[str],
@@ -842,6 +852,7 @@ class Application:
" for details. Please comment or vote to get it implemented quickly!"
)
+ @telemetry.capture_function_usage
def visualize(
self,
output_file_path: Optional[str],
@@ -1149,6 +1160,7 @@ class ApplicationBuilder:
raise ValueError(f"Tracker {tracker} not supported")
return self
+ @telemetry.capture_function_usage
def build(self) -> Application:
"""Builds the application.
diff --git a/burr/telemetry.py b/burr/telemetry.py
new file mode 100644
index 00000000..57879eed
--- /dev/null
+++ b/burr/telemetry.py
@@ -0,0 +1,286 @@
+"""
+This module contains code that relates to sending Burr usage telemetry.
+
+To disable sending telemetry there are three ways:
+
+1. Set it to false programmatically in your driver:
+ >>> from burr import telemetry
+ >>> telemetry.disable_telemetry()
+2. Set it to `false` in ~/.burr.conf under `DEFAULT`
+ [DEFAULT]
+ telemetry_enabled = False
+3. Set BURR_TELEMETRY_ENABLED=false as an environment variable:
+ BURR_TELEMETRY_ENABLED=false python run.py
+ or:
+ export BURR_TELEMETRY_ENABLED=false
+"""
+
+import configparser
+import functools
+import importlib.metadata
+import json
+import logging
+import os
+import platform
+import threading
+import uuid
+from typing import TYPE_CHECKING, Callable, List
+from urllib import request
+
+if TYPE_CHECKING:
+ from burr.lifecycle import internal
+
+VERSION = importlib.metadata.version("burr")
+
+logger = logging.getLogger(__name__)
+
+STR_VERSION = ".".join([str(i) for i in VERSION])
+HOST = "https://app.posthog.com"
+TRACK_URL = f"{HOST}/capture/" # https://posthog.com/docs/api/post-only-endpoints
+API_KEY = "phc_qMa4hWDdTruKaDb4Oa0tK0i1SKf69xf81OCFzjX6z4U"
+APPLICATION_FUNCTION = "os_burr_application_function_call"
+CLI_COMMAND = "os_burr_cli_command"
+TIMEOUT = 2
+MAX_COUNT_SESSION = 10 # max number of events collected per python process
+
+DEFAULT_CONFIG_LOCATION = os.path.expanduser("~/.burr.conf")
+
+
+def _load_config(config_location: str) -> configparser.ConfigParser:
+ """Pulls config. Gets/sets default anonymous ID.
+
+ Creates the anonymous ID if it does not exist, writes it back if so.
+ :param config_location: location of the config file.
+ """
+ config = configparser.ConfigParser()
+ try:
+ with open(config_location) as f:
+ config.read_file(f)
+ except Exception:
+ config["DEFAULT"] = {}
+ else:
+ if "DEFAULT" not in config:
+ config["DEFAULT"] = {}
+
+ if "anonymous_id" not in config["DEFAULT"]:
+ config["DEFAULT"]["anonymous_id"] = str(uuid.uuid4())
+ try:
+ with open(config_location, "w") as f:
+ config.write(f)
+ except Exception:
+ pass
+ return config
+
+
+def _check_config_and_environ_for_telemetry_flag(
+ telemetry_default: bool, config_obj: configparser.ConfigParser
+):
+ """Checks the config and environment variables for the telemetry value.
+
+ Note: the environment variable has greater precedence than the config value.
+ """
+ telemetry_enabled = telemetry_default
+ if "telemetry_enabled" in config_obj["DEFAULT"]:
+ try:
+ telemetry_enabled = config_obj.getboolean("DEFAULT", "telemetry_enabled")
+ except ValueError as e:
+ logger.debug(
+ "Unable to parse value for `telemetry_enabled` from config. " f"Encountered {e}"
+ )
+ if os.environ.get("BURR_TELEMETRY_ENABLED") is not None:
+ env_value = os.environ.get("BURR_TELEMETRY_ENABLED")
+ # set the value
+ config_obj["DEFAULT"]["telemetry_enabled"] = env_value
+ try:
+ telemetry_enabled = config_obj.getboolean("DEFAULT", "telemetry_enabled")
+ except ValueError as e:
+ logger.debug(
+ "Unable to parse value for `BURR_TELEMETRY_ENABLED` from environment. "
+ f"Encountered {e}"
+ )
+ return telemetry_enabled
+
+
+config = _load_config(DEFAULT_CONFIG_LOCATION)
+g_telemetry_enabled = _check_config_and_environ_for_telemetry_flag(True, config)
+g_anonymous_id = config["DEFAULT"]["anonymous_id"]
+call_counter = 0
+
+
+def disable_telemetry():
+ """Disables telemetry tracking."""
+ global g_telemetry_enabled
+ g_telemetry_enabled = False
+
+
+def is_telemetry_enabled() -> bool:
+ """Returns whether telemetry tracking is enabled or not.
+
+ Increments a counter to stop sending telemetry after 1000 invocations.
+ """
+ if g_telemetry_enabled:
+ global call_counter
+ if call_counter == 0:
+ # Log only the first time someone calls this function; don't want to spam them.
+ logger.info(
+ "Note: Burr collects completely anonymous data about usage. "
+ "This will help us improve Burr over time. "
+ "See https://github.com/dagworks-inc/burr#usage-analytics--data-privacy for details."
+ )
+ call_counter += 1
+ if call_counter > MAX_COUNT_SESSION:
+ # we have hit our limit -- disable telemetry.
+ return False
+ return True
+ else:
+ return False
+
+
+# base properties to instantiate on module load.
+BASE_PROPERTIES = {
+ "os_type": os.name,
+ "os_version": platform.platform(),
+ "python_version": f"{platform.python_version()}/{platform.python_implementation()}",
+ "distinct_id": g_anonymous_id,
+ "burr_version": VERSION,
+ "telemetry_version": "0.0.1",
+}
+
+
+def create_application_function_run_event(function_name: str) -> dict:
+ """Function to create payload for tracking function name invocation.
+
+ :param function_name: the name of the driver function
+ :return: dict representing the JSON to send.
+ """
+ event = {
+ "api_key": API_KEY,
+ "event": APPLICATION_FUNCTION,
+ "properties": {},
+ }
+ event["properties"].update(BASE_PROPERTIES)
+ payload = {
+ "function_name": function_name, # what was the name of the driver function?
+ }
+ event["properties"].update(payload)
+ return event
+
+
+def _send_event_json(event_json: dict):
+ """Internal function to send the event JSON to posthog.
+
+ :param event_json: the dictionary of data to JSON serialize and send
+ """
+ headers = {
+ "Content-Type": "application/json",
+ "Authorization": "TODO",
+ "User-Agent": f"burr/{STR_VERSION}",
+ }
+ try:
+ data = json.dumps(event_json).encode()
+ req = request.Request(TRACK_URL, data=data, headers=headers)
+ with request.urlopen(req, timeout=TIMEOUT) as f:
+ res = f.read()
+ if f.code != 200:
+ raise RuntimeError(res)
+ except Exception as e:
+ if logger.isEnabledFor(logging.DEBUG):
+ logging.debug(f"Failed to send telemetry data: {e}")
+ else:
+ if logger.isEnabledFor(logging.DEBUG):
+ logging.debug(f"Succeed in sending telemetry consisting of [{data}].")
+
+
+def send_event_json(event_json: dict):
+ """Sends the event json in its own thread.
+
+ :param event_json: the data to send
+ """
+ if not g_telemetry_enabled:
+ raise RuntimeError("Won't send; tracking is disabled!")
+ try:
+ th = threading.Thread(target=_send_event_json, args=(event_json,))
+ th.start()
+ except Exception as e:
+ # capture any exception!
+ if logger.isEnabledFor(logging.DEBUG):
+ logger.debug(f"Encountered error while sending event JSON via it's own thread:\n{e}")
+
+
+def get_all_adapters_names(adapter: "internal.LifecycleAdapterSet") -> List[str]:
+ """Gives a list of all adapter names in the LifecycleAdapterSet.
+ Simply a loop over the adapters it contains.
+
+ :param adapter: LifecycleAdapterSet object.
+ :return: list of adapter names.
+ """
+ adapters = adapter.adapters
+ out = []
+ for adapter in adapters:
+ out.append(get_adapter_name(adapter))
+ return out
+
+
+def get_adapter_name(adapter: "internal.LifecycleAdapter") -> str:
+ """Get the class name of the ``burr`` adapter used.
+
+ If we detect it's not a Burr one, we do not track it.
+
+ :param adapter: lifecycle.internal.LifecycleAdapter object.
+ :return: string module + class name of the adapter.
+ """
+ # Check whether it's a burr based adapter
+ if adapter.__module__.startswith("burr."):
+ adapter_name = f"{adapter.__module__}.{adapter.__class__.__name__}"
+ else:
+ adapter_name = "custom_adapter"
+ return adapter_name
+
+
+def create_and_send_cli_event(command: str):
+ """Function that creates JSON and sends to track CLI usage.
+
+ :param command: the CLI command run.
+ """
+ event = {
+ "api_key": API_KEY,
+ "event": CLI_COMMAND,
+ "properties": {},
+ }
+ event["properties"].update(BASE_PROPERTIES)
+
+ payload = {
+ "command": command,
+ }
+ event["properties"].update(payload)
+ send_event_json(event)
+
+
+def capture_function_usage(call_fn: Callable) -> Callable:
+ """Decorator to wrap some application functions for telemetry capture.
+
+ We want to use this for non-execute functions.
+ We don't capture information about the arguments at this stage,
+ just the function name.
+
+ :param call_fn: the Driver function to capture.
+ :return: wrapped function.
+ """
+
+ @functools.wraps(call_fn)
+ def wrapped_fn(*args, **kwargs):
+ try:
+ return call_fn(*args, **kwargs)
+ finally:
+ if is_telemetry_enabled():
+ try:
+ function_name = call_fn.__name__
+ event_json = create_application_function_run_event(function_name)
+ send_event_json(event_json)
+ except Exception as e:
+ if logger.isEnabledFor(logging.DEBUG):
+ logger.error(
+ f"Failed to send telemetry for function usage. Encountered: {e}\n"
+ )
+
+ return wrapped_fn
diff --git a/docs/reference/index.rst b/docs/reference/index.rst
index dae21c46..0e35f4f1 100644
--- a/docs/reference/index.rst
+++ b/docs/reference/index.rst
@@ -19,3 +19,4 @@ need functionality that is not publicly exposed, please open an issue and we can
visibility
lifecycle
integrations/index
+ telemetry
diff --git a/docs/reference/telemetry.rst b/docs/reference/telemetry.rst
new file mode 100644
index 00000000..7d7ed800
--- /dev/null
+++ b/docs/reference/telemetry.rst
@@ -0,0 +1,47 @@
+==============================
+Usage analytics + data privacy
+==============================
+
+By default, when using Burr, it collects anonymous usage data to help improve Burr and know where to apply development efforts.
+
+We capture events on the following occasions:
+
+1. When an application is built
+2. When one of the ``execution`` functions is run in ``Application``
+3. When a CLI command is run
+
+The captured data is limited to:
+
+- Operating System and Python version
+- A persistent UUID to indentify the session, stored in ~/.burr.conf.
+- The name of the function/CLI command that was run
+
+If you're worried, see ``telemetry.py`` for details.
+
+If you do not wish to participate, one can opt-out with one of the following methods:
+
+1. Set it to false programmatically in your code before creating a Burr application builder:
+
+.. code-block:: python
+
+ from burr import telemetry
+ telemetry.disable_telemetry()
+
+2. Set the key telemetry_enabled to false in ``~/.burr.conf`` under the DEFAULT section:
+
+.. code-block:: ini
+
+ [DEFAULT]
+ telemetry_enabled = False
+
+3. Set BURR_TELEMETRY_ENABLED=false as an environment variable. Either setting it for your shell session:
+
+.. code-block:: bash
+
+ export BURR_TELEMETRY_ENABLED=false
+
+or passing it as part of the run command:
+
+.. code-block:: bash
+
+ BURR_TELEMETRY_ENABLED=false python NAME_OF_MY_DRIVER.py
diff --git a/tests/conftest.py b/tests/conftest.py
new file mode 100644
index 00000000..f4997913
--- /dev/null
+++ b/tests/conftest.py
@@ -0,0 +1,3 @@
+from burr.telemetry import disable_telemetry
+
+disable_telemetry()