Build applications that make decisions (chatbots, agents, simulations, etc...). Monitor, trace, persist, and execute on your own infrastructure.
Python aiburrchatbot-frameworkdagsgenerative-aigraphshacktoberfestllmopsllmsmlopspersistent-data-structurestate-machinestate-managementvisibility
(Fixes #208)Add MongoDB integration persister (#215)
Nandani Thakurd0d9d84603de
diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.ymlindex 5726a6c8..2ec73221 100644--- a/.github/workflows/python-package.yml+++ b/.github/workflows/python-package.yml@@ -18,6 +18,7 @@ jobs:- uses: actions/checkout@v3- uses: actions/setup-python@v3- uses: pre-commit/[email protected]+test:runs-on: ubuntu-lateststrategy:@@ -36,7 +37,7 @@ jobs:python -m pip install -e ".[tests,tracking-client]"- name: Run testsrun: |- python -m pytest tests+ python -m pytest tests --ignore=tests/integrations/persistersvalidate-examples:runs-on: ubuntu-latest@@ -50,3 +51,36 @@ jobs:working-directory: ./examplesrun: |python -m pytest validate_examples.py++ test-persister-dbs:+ runs-on: ubuntu-latest+ services:+ mongodb:+ image: mongo:7.0+ ports:+ - 27017:27017+ postgres:+ image: postgres:15+ ports:+ - 5432:5432+ redis:+ image: redis:7+ ports:+ - 6379:6379+ strategy:+ fail-fast: false+ matrix:+ python-version: ['3.9', '3.10', '3.11', '3.12']++ steps:+ - uses: actions/checkout@v3+ - name: Set up Python ${{ matrix.python-version }}+ uses: actions/setup-python@v3+ with:+ python-version: ${{ matrix.python-version }}+ - name: Install dependencies+ run: |+ python -m pip install -e ".[tests,tracking-client]"+ - name: Run tests+ run: |+ python -m pytest tests/integrations/persisters/diff --git a/burr/examples b/burr/examplesindex a6573af9..beeced1f 120000--- a/burr/examples+++ b/burr/examples@@ -1 +1 @@-../examples\ No newline at end of file+../examplesdiff --git a/burr/integrations/persisters/b_mongodb.py b/burr/integrations/persisters/b_mongodb.pynew file mode 100644index 00000000..19617f03--- /dev/null+++ b/burr/integrations/persisters/b_mongodb.py@@ -0,0 +1,92 @@+import json+import logging+from datetime import datetime, timezone+from typing import Literal, Optional++from pymongo import MongoClient++from burr.core import persistence, state++logger = logging.getLogger(__name__)+++class MongoDBPersister(persistence.BaseStatePersister):+ """A class used to represent a MongoDB Persister.+ Example usage:+ persister = MongoDBPersister(uri='mongodb://user:pass@localhost:27017', db_name='mydatabase', collection_name='mystates')+ persister.save(+ partition_key='example_partition',+ app_id='example_app',+ sequence_id=1,+ position='example_position',+ state=state.State({'key': 'value'}),+ status='completed'+ )+ loaded_state = persister.load(partition_key='example_partition', app_id='example_app', sequence_id=1)+ print(loaded_state)+ """++ def __init__(+ self, uri="mongodb://localhost:27017", db_name="mydatabase", collection_name="mystates"+ ):+ """Initializes the MongoDBPersister class."""+ self.client = MongoClient(uri)+ self.db = self.client[db_name]+ self.collection = self.db[collection_name]++ def list_app_ids(self, partition_key: str, **kwargs) -> list[str]:+ """List the app ids for a given partition key."""+ app_ids = self.collection.distinct("app_id", {"partition_key": partition_key})+ return app_ids++ def load(+ self, partition_key: str, app_id: str, sequence_id: int = None, **kwargs+ ) -> Optional[persistence.PersistedStateData]:+ """Load the state data for a given partition key, app id, and sequence id."""+ query = {"partition_key": partition_key, "app_id": app_id}+ if sequence_id is not None:+ query["sequence_id"] = sequence_id+ document = self.collection.find_one(query, sort=[("sequence_id", -1)])+ if not document:+ return None+ _state = state.State(json.loads(document["state"]))+ return {+ "partition_key": partition_key,+ "app_id": app_id,+ "sequence_id": document["sequence_id"],+ "position": document["position"],+ "state": _state,+ "created_at": document["created_at"],+ "status": document["status"],+ }++ def save(+ self,+ partition_key: str,+ app_id: str,+ sequence_id: int,+ position: str,+ state: state.State,+ status: Literal["completed", "failed"],+ **kwargs,+ ):+ """Save the state data to the MongoDB database."""+ key = {"partition_key": partition_key, "app_id": app_id, "sequence_id": sequence_id}+ if self.collection.find_one(key):+ raise ValueError(f"partition_key:app_id:sequence_id[{key}] already exists.")+ json_state = json.dumps(state.get_all())+ self.collection.insert_one(+ {+ "partition_key": partition_key,+ "app_id": app_id,+ "sequence_id": sequence_id,+ "position": position,+ "state": json_state,+ "status": status,+ "created_at": datetime.now(timezone.utc).isoformat(),+ # "created_at": datetime.datetime.utcnow().isoformat(),+ }+ )++ def __del__(self):+ self.client.close()diff --git a/docs/reference/persister.rst b/docs/reference/persister.rstindex 14d1c6eb..06cf1772 100644--- a/docs/reference/persister.rst+++ b/docs/reference/persister.rst@@ -54,6 +54,10 @@ Currently we support the following, although we highly recommend you contribute.. automethod:: __init__+.. autoclass:: burr.integrations.persisters.b_mongodb.MongoDBPersister+ :members:+ .. automethod:: __init__+Note that the :py:class:`LocalTrackingClient <burr.tracking.client.LocalTrackingClient>` leverages the :py:class:`BaseStateLoader <burr.core.persistence.BaseStateLoader>` to allow loading state,although it uses different mechanisms to save state (as it tracks more than just state).diff --git a/pyproject.toml b/pyproject.tomlindex 51cbd8fe..fafddd2a 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -51,7 +51,8 @@ redis = [tests = ["pytest","pytest-asyncio",- "burr[hamilton]"+ "burr[hamilton]",+ "pymongo"]documentation = [diff --git a/tests/integrations/persisters/test_b_mongodb.py b/tests/integrations/persisters/test_b_mongodb.pynew file mode 100644index 00000000..9cccf9c5--- /dev/null+++ b/tests/integrations/persisters/test_b_mongodb.py@@ -0,0 +1,32 @@+import pytest++from burr.core import state+from burr.integrations.persisters.b_mongodb import MongoDBPersister+++def mongodb_persister():+ persister = MongoDBPersister(+ uri="mongodb://localhost:27017", db_name="testdb", collection_name="testcollection"+ )+ yield persister+ persister.collection.drop()+++def test_save_and_load_state(mongodb_persister):+ mongodb_persister.save("pk", "app_id", 1, "pos", state.State({"a": 1, "b": 2}), "completed")+ data = mongodb_persister.load("pk", "app_id", 1)+ assert data["state"].get_all() == {"a": 1, "b": 2}+++def test_list_app_ids(mongodb_persister):+ mongodb_persister.save("pk", "app_id1", 1, "pos1", state.State({"a": 1}), "completed")+ mongodb_persister.save("pk", "app_id2", 2, "pos2", state.State({"b": 2}), "completed")+ app_ids = mongodb_persister.list_app_ids("pk")+ assert "app_id1" in app_ids+ assert "app_id2" in app_ids+++def test_load_nonexistent_key(mongodb_persister):+ state_data = mongodb_persister.load("pk", "nonexistent_key")+ assert state_data is None