Build applications that make decisions (chatbots, agents, simulations, etc...). Monitor, trace, persist, and execute on your own infrastructure.
Python aiburrchatbot-frameworkdagsgenerative-aigraphshacktoberfestllmopsllmsmlopspersistent-data-structurestate-machinestate-managementvisibility
Adds namespace to redis persister (#284)
Stefan Krawczykb842eced91b2
diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.ymlindex 330d8e05..f0830a50 100644--- a/.github/workflows/python-package.yml+++ b/.github/workflows/python-package.yml@@ -61,6 +61,8 @@ jobs:- 27017:27017postgres:image: postgres:15+ env:+ POSTGRES_PASSWORD: postgresports:- 5432:5432redis:diff --git a/burr/integrations/persisters/b_redis.py b/burr/integrations/persisters/b_redis.pyindex 92bba950..3a5df962 100644--- a/burr/integrations/persisters/b_redis.py+++ b/burr/integrations/persisters/b_redis.py@@ -31,6 +31,7 @@ class RedisPersister(persistence.BaseStatePersister):password: str = None,serde_kwargs: dict = None,redis_client_kwargs: dict = None,+ namespace: str = None,):"""Initializes the RedisPersister class.@@ -40,6 +41,7 @@ class RedisPersister(persistence.BaseStatePersister)::param password::param serde_kwargs::param redis_client_kwargs: Additional keyword arguments to pass to the redis.Redis client.+ :param namespace: The name of the project to optionally use in the key prefix."""if redis_client_kwargs is None:redis_client_kwargs = {}@@ -47,10 +49,14 @@ class RedisPersister(persistence.BaseStatePersister):host=host, port=port, db=db, password=password, **redis_client_kwargs)self.serde_kwargs = serde_kwargs or {}+ self.namespace = namespace if namespace else ""def list_app_ids(self, partition_key: str, **kwargs) -> list[str]:"""List the app ids for a given partition key."""- app_ids = self.connection.zrevrange(partition_key, 0, -1)+ namespaced_partition_key = (+ f"{self.namespace}:{partition_key}" if self.namespace else partition_key+ )+ app_ids = self.connection.zrevrange(namespaced_partition_key, 0, -1)return [app_id.decode() for app_id in app_ids]def load(@@ -66,8 +72,11 @@ class RedisPersister(persistence.BaseStatePersister)::param kwargs::return: Value or None."""+ namespaced_partition_key = (+ f"{self.namespace}:{partition_key}" if self.namespace else partition_key+ )if sequence_id is None:- sequence_id = self.connection.zscore(partition_key, app_id)+ sequence_id = self.connection.zscore(namespaced_partition_key, app_id)if sequence_id is None:return Nonesequence_id = int(sequence_id)@@ -88,7 +97,10 @@ class RedisPersister(persistence.BaseStatePersister):def create_key(self, app_id, partition_key, sequence_id):"""Create a key for the Redis database."""- key = f"{partition_key}:{app_id}:{sequence_id}"+ if self.namespace:+ key = f"{self.namespace}:{partition_key}:{app_id}:{sequence_id}"+ else:+ key = f"{partition_key}:{app_id}:{sequence_id}"return keydef save(@@ -128,7 +140,10 @@ class RedisPersister(persistence.BaseStatePersister):"created_at": datetime.now(timezone.utc).isoformat(),},)- self.connection.zadd(partition_key, {app_id: sequence_id})+ namespaced_partition_key = (+ f"{self.namespace}:{partition_key}" if self.namespace else partition_key+ )+ self.connection.zadd(namespaced_partition_key, {app_id: sequence_id})def __del__(self):self.connection.close()diff --git a/burr/integrations/persisters/postgresql.py b/burr/integrations/persisters/postgresql.pyindex f2f9bcf6..051aacab 100644--- a/burr/integrations/persisters/postgresql.py+++ b/burr/integrations/persisters/postgresql.py@@ -95,7 +95,7 @@ class PostgreSQLPersister(persistence.BaseStatePersister):cursor.execute(f"""CREATE TABLE IF NOT EXISTS {table_name} (- partition_key TEXT DEFAULT {self.PARTITION_KEY_DEFAULT},+ partition_key TEXT DEFAULT '{self.PARTITION_KEY_DEFAULT}',app_id TEXT NOT NULL,sequence_id INTEGER NOT NULL,position TEXT NOT NULL,diff --git a/pyproject.toml b/pyproject.tomlindex e05cebd5..7fea900f 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -57,8 +57,10 @@ tests = ["langchain_core","langchain_community","pandas",+ "psycopg2","pydantic","pyarrow",+ "redis",]documentation = [diff --git a/tests/integrations/persisters/test_b_redis.py b/tests/integrations/persisters/test_b_redis.pynew file mode 100644index 00000000..ddfe7355--- /dev/null+++ b/tests/integrations/persisters/test_b_redis.py@@ -0,0 +1,63 @@+import os++import pytest++from burr.core import state+from burr.integrations.persisters.b_redis import RedisPersister++if not os.environ.get("BURR_CI_INTEGRATION_TESTS") == "true":+ pytest.skip("Skipping integration tests", allow_module_level=True)+++def redis_persister():+ persister = RedisPersister(host="localhost", port=6379, db=0)+ yield persister+ persister.connection.close()+++def redis_persister_with_ns():+ persister = RedisPersister(host="localhost", port=6379, db=0, namespace="test")+ yield persister+ persister.connection.close()+++def test_save_and_load_state(redis_persister):+ redis_persister.save("pk", "app_id", 1, "pos", state.State({"a": 1, "b": 2}), "completed")+ data = redis_persister.load("pk", "app_id", 1)+ assert data["state"].get_all() == {"a": 1, "b": 2}+++def test_list_app_ids(redis_persister):+ redis_persister.save("pk", "app_id1", 1, "pos1", state.State({"a": 1}), "completed")+ redis_persister.save("pk", "app_id2", 2, "pos2", state.State({"b": 2}), "completed")+ app_ids = redis_persister.list_app_ids("pk")+ assert "app_id1" in app_ids+ assert "app_id2" in app_ids+++def test_load_nonexistent_key(redis_persister):+ state_data = redis_persister.load("pk", "nonexistent_key")+ assert state_data is None+++def test_save_and_load_state_ns(redis_persister_with_ns):+ redis_persister_with_ns.save(+ "pk", "app_id", 1, "pos", state.State({"a": 1, "b": 2}), "completed"+ )+ data = redis_persister_with_ns.load("pk", "app_id", 1)+ assert data["state"].get_all() == {"a": 1, "b": 2}+++def test_list_app_ids_with_ns(redis_persister_with_ns):+ redis_persister_with_ns.save("pk", "app_id1", 1, "pos1", state.State({"a": 1}), "completed")+ redis_persister_with_ns.save("pk", "app_id2", 2, "pos2", state.State({"b": 2}), "completed")+ app_ids = redis_persister_with_ns.list_app_ids("pk")+ assert "app_id1" in app_ids+ assert "app_id2" in app_ids+++def test_load_nonexistent_key_with_ns(redis_persister_with_ns):+ state_data = redis_persister_with_ns.load("pk", "nonexistent_key")+ assert state_data is Nonediff --git a/tests/integrations/persisters/test_postgresql.py b/tests/integrations/persisters/test_postgresql.pynew file mode 100644index 00000000..345d7bac--- /dev/null+++ b/tests/integrations/persisters/test_postgresql.py@@ -0,0 +1,43 @@+import os++import pytest++from burr.core import state+from burr.integrations.persisters.postgresql import PostgreSQLPersister++if not os.environ.get("BURR_CI_INTEGRATION_TESTS") == "true":+ pytest.skip("Skipping integration tests", allow_module_level=True)+++def postgresql_persister():+ persister = PostgreSQLPersister.from_values(+ db_name="postgres",+ user="postgres",+ password="postgres",+ host="localhost",+ port=5432,+ table_name="testtable",+ )+ persister.initialize()+ yield persister+ persister.connection.close()+++def test_save_and_load_state(postgresql_persister):+ postgresql_persister.save("pk", "app_id", 1, "pos", state.State({"a": 1, "b": 2}), "completed")+ data = postgresql_persister.load("pk", "app_id", 1)+ assert data["state"].get_all() == {"a": 1, "b": 2}+++def test_list_app_ids(postgresql_persister):+ postgresql_persister.save("pk", "app_id1", 1, "pos1", state.State({"a": 1}), "completed")+ postgresql_persister.save("pk", "app_id2", 2, "pos2", state.State({"b": 2}), "completed")+ app_ids = postgresql_persister.list_app_ids("pk")+ assert "app_id1" in app_ids+ assert "app_id2" in app_ids+++def test_load_nonexistent_key(postgresql_persister):+ state_data = postgresql_persister.load("pk", "nonexistent_key")+ assert state_data is None