Skip to content

armgit / burr

Public Imported from GitHub
0

Build applications that make decisions (chatbots, agents, simulations, etc...). Monitor, trace, persist, and execute on your own infrastructure.

(Fixes #208)Add MongoDB integration persister (#215)

Nandani Thakurd0d9d84603de
diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 5726a6c8..2ec73221 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -18,6 +18,7 @@ jobs:
- uses: actions/checkout@v3
- uses: actions/setup-python@v3
- uses: pre-commit/[email protected]
+
test:
runs-on: ubuntu-latest
strategy:
@@ -36,7 +37,7 @@ jobs:
python -m pip install -e ".[tests,tracking-client]"
- name: Run tests
run: |
- python -m pytest tests
+ python -m pytest tests --ignore=tests/integrations/persisters
validate-examples:
runs-on: ubuntu-latest
@@ -50,3 +51,36 @@ jobs:
working-directory: ./examples
run: |
python -m pytest validate_examples.py
+
+ test-persister-dbs:
+ runs-on: ubuntu-latest
+ services:
+ mongodb:
+ image: mongo:7.0
+ ports:
+ - 27017:27017
+ postgres:
+ image: postgres:15
+ ports:
+ - 5432:5432
+ redis:
+ image: redis:7
+ ports:
+ - 6379:6379
+ strategy:
+ fail-fast: false
+ matrix:
+ python-version: ['3.9', '3.10', '3.11', '3.12']
+
+ steps:
+ - uses: actions/checkout@v3
+ - name: Set up Python ${{ matrix.python-version }}
+ uses: actions/setup-python@v3
+ with:
+ python-version: ${{ matrix.python-version }}
+ - name: Install dependencies
+ run: |
+ python -m pip install -e ".[tests,tracking-client]"
+ - name: Run tests
+ run: |
+ python -m pytest tests/integrations/persisters/
diff --git a/burr/examples b/burr/examples
index a6573af9..beeced1f 120000
--- a/burr/examples
+++ b/burr/examples
@@ -1 +1 @@
-../examples
\ No newline at end of file
+../examples
diff --git a/burr/integrations/persisters/b_mongodb.py b/burr/integrations/persisters/b_mongodb.py
new file mode 100644
index 00000000..19617f03
--- /dev/null
+++ b/burr/integrations/persisters/b_mongodb.py
@@ -0,0 +1,92 @@
+import json
+import logging
+from datetime import datetime, timezone
+from typing import Literal, Optional
+
+from pymongo import MongoClient
+
+from burr.core import persistence, state
+
+logger = logging.getLogger(__name__)
+
+
+class MongoDBPersister(persistence.BaseStatePersister):
+ """A class used to represent a MongoDB Persister.
+ Example usage:
+ persister = MongoDBPersister(uri='mongodb://user:pass@localhost:27017', db_name='mydatabase', collection_name='mystates')
+ persister.save(
+ partition_key='example_partition',
+ app_id='example_app',
+ sequence_id=1,
+ position='example_position',
+ state=state.State({'key': 'value'}),
+ status='completed'
+ )
+ loaded_state = persister.load(partition_key='example_partition', app_id='example_app', sequence_id=1)
+ print(loaded_state)
+ """
+
+ def __init__(
+ self, uri="mongodb://localhost:27017", db_name="mydatabase", collection_name="mystates"
+ ):
+ """Initializes the MongoDBPersister class."""
+ self.client = MongoClient(uri)
+ self.db = self.client[db_name]
+ self.collection = self.db[collection_name]
+
+ def list_app_ids(self, partition_key: str, **kwargs) -> list[str]:
+ """List the app ids for a given partition key."""
+ app_ids = self.collection.distinct("app_id", {"partition_key": partition_key})
+ return app_ids
+
+ def load(
+ self, partition_key: str, app_id: str, sequence_id: int = None, **kwargs
+ ) -> Optional[persistence.PersistedStateData]:
+ """Load the state data for a given partition key, app id, and sequence id."""
+ query = {"partition_key": partition_key, "app_id": app_id}
+ if sequence_id is not None:
+ query["sequence_id"] = sequence_id
+ document = self.collection.find_one(query, sort=[("sequence_id", -1)])
+ if not document:
+ return None
+ _state = state.State(json.loads(document["state"]))
+ return {
+ "partition_key": partition_key,
+ "app_id": app_id,
+ "sequence_id": document["sequence_id"],
+ "position": document["position"],
+ "state": _state,
+ "created_at": document["created_at"],
+ "status": document["status"],
+ }
+
+ def save(
+ self,
+ partition_key: str,
+ app_id: str,
+ sequence_id: int,
+ position: str,
+ state: state.State,
+ status: Literal["completed", "failed"],
+ **kwargs,
+ ):
+ """Save the state data to the MongoDB database."""
+ key = {"partition_key": partition_key, "app_id": app_id, "sequence_id": sequence_id}
+ if self.collection.find_one(key):
+ raise ValueError(f"partition_key:app_id:sequence_id[{key}] already exists.")
+ json_state = json.dumps(state.get_all())
+ self.collection.insert_one(
+ {
+ "partition_key": partition_key,
+ "app_id": app_id,
+ "sequence_id": sequence_id,
+ "position": position,
+ "state": json_state,
+ "status": status,
+ "created_at": datetime.now(timezone.utc).isoformat(),
+ # "created_at": datetime.datetime.utcnow().isoformat(),
+ }
+ )
+
+ def __del__(self):
+ self.client.close()
diff --git a/docs/reference/persister.rst b/docs/reference/persister.rst
index 14d1c6eb..06cf1772 100644
--- a/docs/reference/persister.rst
+++ b/docs/reference/persister.rst
@@ -54,6 +54,10 @@ Currently we support the following, although we highly recommend you contribute
.. automethod:: __init__
+.. autoclass:: burr.integrations.persisters.b_mongodb.MongoDBPersister
+ :members:
+ .. automethod:: __init__
+
Note that the :py:class:`LocalTrackingClient <burr.tracking.client.LocalTrackingClient>` leverages the :py:class:`BaseStateLoader <burr.core.persistence.BaseStateLoader>` to allow loading state,
although it uses different mechanisms to save state (as it tracks more than just state).
diff --git a/pyproject.toml b/pyproject.toml
index 51cbd8fe..fafddd2a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -51,7 +51,8 @@ redis = [
tests = [
"pytest",
"pytest-asyncio",
- "burr[hamilton]"
+ "burr[hamilton]",
+ "pymongo"
]
documentation = [
diff --git a/tests/integrations/persisters/test_b_mongodb.py b/tests/integrations/persisters/test_b_mongodb.py
new file mode 100644
index 00000000..9cccf9c5
--- /dev/null
+++ b/tests/integrations/persisters/test_b_mongodb.py
@@ -0,0 +1,32 @@
+import pytest
+
+from burr.core import state
+from burr.integrations.persisters.b_mongodb import MongoDBPersister
+
+
+def mongodb_persister():
+ persister = MongoDBPersister(
+ uri="mongodb://localhost:27017", db_name="testdb", collection_name="testcollection"
+ )
+ yield persister
+ persister.collection.drop()
+
+
+def test_save_and_load_state(mongodb_persister):
+ mongodb_persister.save("pk", "app_id", 1, "pos", state.State({"a": 1, "b": 2}), "completed")
+ data = mongodb_persister.load("pk", "app_id", 1)
+ assert data["state"].get_all() == {"a": 1, "b": 2}
+
+
+def test_list_app_ids(mongodb_persister):
+ mongodb_persister.save("pk", "app_id1", 1, "pos1", state.State({"a": 1}), "completed")
+ mongodb_persister.save("pk", "app_id2", 2, "pos2", state.State({"b": 2}), "completed")
+ app_ids = mongodb_persister.list_app_ids("pk")
+ assert "app_id1" in app_ids
+ assert "app_id2" in app_ids
+
+
+def test_load_nonexistent_key(mongodb_persister):
+ state_data = mongodb_persister.load("pk", "nonexistent_key")
+ assert state_data is None