Skip to content

armgit / burr

Public Imported from GitHub
0

Build applications that make decisions (chatbots, agents, simulations, etc...). Monitor, trace, persist, and execute on your own infrastructure.

Adds namespace to redis persister (#284)

Stefan Krawczykb842eced91b2
diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 330d8e05..f0830a50 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -61,6 +61,8 @@ jobs:
- 27017:27017
postgres:
image: postgres:15
+ env:
+ POSTGRES_PASSWORD: postgres
ports:
- 5432:5432
redis:
diff --git a/burr/integrations/persisters/b_redis.py b/burr/integrations/persisters/b_redis.py
index 92bba950..3a5df962 100644
--- a/burr/integrations/persisters/b_redis.py
+++ b/burr/integrations/persisters/b_redis.py
@@ -31,6 +31,7 @@ class RedisPersister(persistence.BaseStatePersister):
password: str = None,
serde_kwargs: dict = None,
redis_client_kwargs: dict = None,
+ namespace: str = None,
):
"""Initializes the RedisPersister class.
@@ -40,6 +41,7 @@ class RedisPersister(persistence.BaseStatePersister):
:param password:
:param serde_kwargs:
:param redis_client_kwargs: Additional keyword arguments to pass to the redis.Redis client.
+ :param namespace: The name of the project to optionally use in the key prefix.
"""
if redis_client_kwargs is None:
redis_client_kwargs = {}
@@ -47,10 +49,14 @@ class RedisPersister(persistence.BaseStatePersister):
host=host, port=port, db=db, password=password, **redis_client_kwargs
)
self.serde_kwargs = serde_kwargs or {}
+ self.namespace = namespace if namespace else ""
def list_app_ids(self, partition_key: str, **kwargs) -> list[str]:
"""List the app ids for a given partition key."""
- app_ids = self.connection.zrevrange(partition_key, 0, -1)
+ namespaced_partition_key = (
+ f"{self.namespace}:{partition_key}" if self.namespace else partition_key
+ )
+ app_ids = self.connection.zrevrange(namespaced_partition_key, 0, -1)
return [app_id.decode() for app_id in app_ids]
def load(
@@ -66,8 +72,11 @@ class RedisPersister(persistence.BaseStatePersister):
:param kwargs:
:return: Value or None.
"""
+ namespaced_partition_key = (
+ f"{self.namespace}:{partition_key}" if self.namespace else partition_key
+ )
if sequence_id is None:
- sequence_id = self.connection.zscore(partition_key, app_id)
+ sequence_id = self.connection.zscore(namespaced_partition_key, app_id)
if sequence_id is None:
return None
sequence_id = int(sequence_id)
@@ -88,7 +97,10 @@ class RedisPersister(persistence.BaseStatePersister):
def create_key(self, app_id, partition_key, sequence_id):
"""Create a key for the Redis database."""
- key = f"{partition_key}:{app_id}:{sequence_id}"
+ if self.namespace:
+ key = f"{self.namespace}:{partition_key}:{app_id}:{sequence_id}"
+ else:
+ key = f"{partition_key}:{app_id}:{sequence_id}"
return key
def save(
@@ -128,7 +140,10 @@ class RedisPersister(persistence.BaseStatePersister):
"created_at": datetime.now(timezone.utc).isoformat(),
},
)
- self.connection.zadd(partition_key, {app_id: sequence_id})
+ namespaced_partition_key = (
+ f"{self.namespace}:{partition_key}" if self.namespace else partition_key
+ )
+ self.connection.zadd(namespaced_partition_key, {app_id: sequence_id})
def __del__(self):
self.connection.close()
diff --git a/burr/integrations/persisters/postgresql.py b/burr/integrations/persisters/postgresql.py
index f2f9bcf6..051aacab 100644
--- a/burr/integrations/persisters/postgresql.py
+++ b/burr/integrations/persisters/postgresql.py
@@ -95,7 +95,7 @@ class PostgreSQLPersister(persistence.BaseStatePersister):
cursor.execute(
f"""
CREATE TABLE IF NOT EXISTS {table_name} (
- partition_key TEXT DEFAULT {self.PARTITION_KEY_DEFAULT},
+ partition_key TEXT DEFAULT '{self.PARTITION_KEY_DEFAULT}',
app_id TEXT NOT NULL,
sequence_id INTEGER NOT NULL,
position TEXT NOT NULL,
diff --git a/pyproject.toml b/pyproject.toml
index e05cebd5..7fea900f 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -57,8 +57,10 @@ tests = [
"langchain_core",
"langchain_community",
"pandas",
+ "psycopg2",
"pydantic",
"pyarrow",
+ "redis",
]
documentation = [
diff --git a/tests/integrations/persisters/test_b_redis.py b/tests/integrations/persisters/test_b_redis.py
new file mode 100644
index 00000000..ddfe7355
--- /dev/null
+++ b/tests/integrations/persisters/test_b_redis.py
@@ -0,0 +1,63 @@
+import os
+
+import pytest
+
+from burr.core import state
+from burr.integrations.persisters.b_redis import RedisPersister
+
+if not os.environ.get("BURR_CI_INTEGRATION_TESTS") == "true":
+ pytest.skip("Skipping integration tests", allow_module_level=True)
+
+
+def redis_persister():
+ persister = RedisPersister(host="localhost", port=6379, db=0)
+ yield persister
+ persister.connection.close()
+
+
+def redis_persister_with_ns():
+ persister = RedisPersister(host="localhost", port=6379, db=0, namespace="test")
+ yield persister
+ persister.connection.close()
+
+
+def test_save_and_load_state(redis_persister):
+ redis_persister.save("pk", "app_id", 1, "pos", state.State({"a": 1, "b": 2}), "completed")
+ data = redis_persister.load("pk", "app_id", 1)
+ assert data["state"].get_all() == {"a": 1, "b": 2}
+
+
+def test_list_app_ids(redis_persister):
+ redis_persister.save("pk", "app_id1", 1, "pos1", state.State({"a": 1}), "completed")
+ redis_persister.save("pk", "app_id2", 2, "pos2", state.State({"b": 2}), "completed")
+ app_ids = redis_persister.list_app_ids("pk")
+ assert "app_id1" in app_ids
+ assert "app_id2" in app_ids
+
+
+def test_load_nonexistent_key(redis_persister):
+ state_data = redis_persister.load("pk", "nonexistent_key")
+ assert state_data is None
+
+
+def test_save_and_load_state_ns(redis_persister_with_ns):
+ redis_persister_with_ns.save(
+ "pk", "app_id", 1, "pos", state.State({"a": 1, "b": 2}), "completed"
+ )
+ data = redis_persister_with_ns.load("pk", "app_id", 1)
+ assert data["state"].get_all() == {"a": 1, "b": 2}
+
+
+def test_list_app_ids_with_ns(redis_persister_with_ns):
+ redis_persister_with_ns.save("pk", "app_id1", 1, "pos1", state.State({"a": 1}), "completed")
+ redis_persister_with_ns.save("pk", "app_id2", 2, "pos2", state.State({"b": 2}), "completed")
+ app_ids = redis_persister_with_ns.list_app_ids("pk")
+ assert "app_id1" in app_ids
+ assert "app_id2" in app_ids
+
+
+def test_load_nonexistent_key_with_ns(redis_persister_with_ns):
+ state_data = redis_persister_with_ns.load("pk", "nonexistent_key")
+ assert state_data is None
diff --git a/tests/integrations/persisters/test_postgresql.py b/tests/integrations/persisters/test_postgresql.py
new file mode 100644
index 00000000..345d7bac
--- /dev/null
+++ b/tests/integrations/persisters/test_postgresql.py
@@ -0,0 +1,43 @@
+import os
+
+import pytest
+
+from burr.core import state
+from burr.integrations.persisters.postgresql import PostgreSQLPersister
+
+if not os.environ.get("BURR_CI_INTEGRATION_TESTS") == "true":
+ pytest.skip("Skipping integration tests", allow_module_level=True)
+
+
+def postgresql_persister():
+ persister = PostgreSQLPersister.from_values(
+ db_name="postgres",
+ user="postgres",
+ password="postgres",
+ host="localhost",
+ port=5432,
+ table_name="testtable",
+ )
+ persister.initialize()
+ yield persister
+ persister.connection.close()
+
+
+def test_save_and_load_state(postgresql_persister):
+ postgresql_persister.save("pk", "app_id", 1, "pos", state.State({"a": 1, "b": 2}), "completed")
+ data = postgresql_persister.load("pk", "app_id", 1)
+ assert data["state"].get_all() == {"a": 1, "b": 2}
+
+
+def test_list_app_ids(postgresql_persister):
+ postgresql_persister.save("pk", "app_id1", 1, "pos1", state.State({"a": 1}), "completed")
+ postgresql_persister.save("pk", "app_id2", 2, "pos2", state.State({"b": 2}), "completed")
+ app_ids = postgresql_persister.list_app_ids("pk")
+ assert "app_id1" in app_ids
+ assert "app_id2" in app_ids
+
+
+def test_load_nonexistent_key(postgresql_persister):
+ state_data = postgresql_persister.load("pk", "nonexistent_key")
+ assert state_data is None