Skip to content

Build and run agents you can see, understand and trust.

feat(storage): support SQLAlchemy based storage (#2029)

DavdGaob49a26b94ccf
diff --git a/pyproject.toml b/pyproject.toml
index 6bd7fb563..cc0fe4425 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,4 +1,3 @@
-
[project]
name = "agentscope"
dynamic = ["version"]
@@ -67,6 +66,14 @@ service = [
storage = [
"redis",
]
+# ------------ SQL storage ------------
+# Async SQLAlchemy 2.0 + Alembic. The concrete driver
+# (aiosqlite / asyncpg / aiomysql / asyncmy) is left to the user
+# so the extra stays lightweight.
+sql = [
+ "sqlalchemy[asyncio]>=2.0",
+ "alembic>=1.13",
+]
# ------------ Workspace -----------
k8s = ["kubernetes-asyncio"]
workspace = [
@@ -120,6 +127,7 @@ full = [
"agentscope[models]",
"agentscope[service]",
"agentscope[storage]",
+ "agentscope[sql]",
"agentscope[workspace]",
"agentscope[tools]",
"agentscope[rag]",
@@ -141,6 +149,8 @@ dev = [
"myst_parser",
"matplotlib",
"fakeredis",
+ # SQL backend tests — SQLite via aiosqlite so no server is needed.
+ "aiosqlite",
# S3-backend testing — moto[server] spins up an in-process S3, the
# store-under-test (`S3BlobStore`) uses aioboto3, and boto3 is the
# sync client the test fixtures use to seed/verify buckets.
@@ -173,6 +183,17 @@ include-package-data = true
"_cosyvoice_models/*.yaml",
]
"agentscope.workspace._docker" = ["Dockerfile*.template"]
+# The Alembic scaffolding for the SQL storage backend lives in the
+# data-only ``_alembic/`` subtree (no ``__init__.py`` — Alembic loads
+# ``env.py`` / the version scripts by file path, not import). Ship them
+# explicitly so ``auto_migrate=True`` works from an installed wheel.
+"agentscope.app.storage._sql" = [
+ "_alembic/alembic.ini",
+ "_alembic/script.py.mako",
+ "_alembic/env.py",
+ "_alembic/versions/*.py",
+]
+
[build-system]
requires = ["setuptools>=45", "wheel"]
build-backend = "setuptools.build_meta"
diff --git a/src/agentscope/app/_router/_knowledge_base.py b/src/agentscope/app/_router/_knowledge_base.py
index 901acc4eb..f3951c430 100644
--- a/src/agentscope/app/_router/_knowledge_base.py
+++ b/src/agentscope/app/_router/_knowledge_base.py
@@ -501,7 +501,7 @@ async def upload_knowledge_document(
return UploadKnowledgeDocumentResponse(
document_id=record.id,
filename=record.data.filename,
- status=record.data.status,
+ status=record.status,
)
diff --git a/src/agentscope/app/_router/_schema/_knowledge_base.py b/src/agentscope/app/_router/_schema/_knowledge_base.py
index 5484b74e0..99c8effea 100644
--- a/src/agentscope/app/_router/_schema/_knowledge_base.py
+++ b/src/agentscope/app/_router/_schema/_knowledge_base.py
@@ -118,7 +118,7 @@ class KnowledgeDocumentView(BaseModel):
filename=record.data.filename,
size=record.data.size,
content_type=record.data.content_type,
- status=record.data.status,
+ status=record.status,
error=record.data.error,
chunk_count=record.data.chunk_count,
created_at=record.created_at,
diff --git a/src/agentscope/app/_service/_access.py b/src/agentscope/app/_service/_access.py
index 8026ba464..2c583f7ee 100644
--- a/src/agentscope/app/_service/_access.py
+++ b/src/agentscope/app/_service/_access.py
@@ -2,10 +2,11 @@
"""Resource access service — cross-owner reads with viewer-relative views."""
from __future__ import annotations
+from datetime import datetime
from typing import Any, Literal, TypeVar, overload
from fastapi import HTTPException, status
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, model_validator
from ..access import (
ResourceAccessPolicyBase,
@@ -16,6 +17,7 @@ from ..access import (
from ..storage import (
AgentRecord,
CredentialRecord,
+ EmbeddingModelConfig,
KnowledgeBaseRecord,
StorageBase,
)
@@ -58,20 +60,67 @@ class CredentialView(CredentialRecord):
)
-class KnowledgeBaseView(KnowledgeBaseRecord):
- """Knowledge base record + viewer-relative ``editable``.
+class KnowledgeBaseView(BaseModel):
+ """Flat, viewer-facing projection of a knowledge base.
- Subclasses :class:`KnowledgeBaseRecord` so the wire format is a
- strict superset of the record (one extra top-level field), matching
- :class:`AgentView` and :class:`CredentialView`.
+ Unlike :class:`AgentView` / :class:`CredentialView` — which subclass
+ their records and expose the nested ``data`` payload verbatim — the
+ knowledge base's public wire shape is intentionally **flat**: the
+ frontend and the documented API contract read ``name`` /
+ ``embedding_model_config`` at the top level. The storage-layer
+ nesting of those fields under :attr:`KnowledgeBaseRecord.data`
+ (introduced so the SQL backend can serialise a single JSON column)
+ must not leak into the HTTP response, so :meth:`_lift_data_payload`
+ lifts them back up. ``user_id`` and the opaque ``collection_name``
+ are dropped (pydantic ignores the surplus keys).
"""
+ id: str = Field(description="The knowledge base id.")
+ name: str = Field(description="Display name of the knowledge base.")
+ description: str = Field(
+ default="",
+ description="Free-form description of the knowledge base.",
+ )
+ embedding_model_config: EmbeddingModelConfig = Field(
+ description="Embedding model configuration pinned at creation.",
+ )
+ created_at: datetime = Field(description="Creation timestamp.")
+ updated_at: datetime = Field(description="Last-update timestamp.")
editable: bool = Field(
description=(
"Whether the current viewer may modify this knowledge base."
),
)
+ @model_validator(mode="before")
+ @classmethod
+ def _lift_data_payload(cls, obj: Any) -> Any:
+ """Flatten a record's nested ``data`` payload to the top level.
+
+ Lets the view be built with
+ ``model_validate({**record.model_dump(), "editable": ...})``
+ exactly like the other views while still presenting a flat wire
+ shape. Explicit top-level keys win over payload keys; inputs
+ already flat pass through unchanged.
+
+ Args:
+ obj (`Any`):
+ The raw validator input. Only ``dict`` inputs carrying a
+ ``data`` sub-dict are rewritten; anything else is
+ returned verbatim.
+
+ Returns:
+ `Any`:
+ The input with ``data``'s fields lifted to the top
+ level.
+ """
+ if not isinstance(obj, dict) or not isinstance(obj.get("data"), dict):
+ return obj
+ merged = {k: v for k, v in obj.items() if k != "data"}
+ for key, value in obj["data"].items():
+ merged.setdefault(key, value)
+ return merged
+
# ---------------------------------------------------------------------
# Service
diff --git a/src/agentscope/app/rag/knowledge_base_manager/_base.py b/src/agentscope/app/rag/knowledge_base_manager/_base.py
index d3557a248..d50efa42f 100644
--- a/src/agentscope/app/rag/knowledge_base_manager/_base.py
+++ b/src/agentscope/app/rag/knowledge_base_manager/_base.py
@@ -234,9 +234,9 @@ class KnowledgeBaseManagerBase(ABC):
if record is None:
return None
if name is not None:
- record.name = name
+ record.data.name = name
if description is not None:
- record.description = description
+ record.data.description = description
return await self._storage.upsert_knowledge_base(user_id, record)
@abstractmethod
diff --git a/src/agentscope/app/rag/knowledge_base_manager/_collection_per_kb.py b/src/agentscope/app/rag/knowledge_base_manager/_collection_per_kb.py
index 34f033b3b..aa0ca3280 100644
--- a/src/agentscope/app/rag/knowledge_base_manager/_collection_per_kb.py
+++ b/src/agentscope/app/rag/knowledge_base_manager/_collection_per_kb.py
@@ -22,7 +22,7 @@ from ._errors import KnowledgeBaseNotFoundError
from ...._logging import logger
from ....rag import KnowledgeBase
from ..._service._embedding import build_embedding_model
-from ...storage import KnowledgeBaseRecord
+from ...storage import KnowledgeBaseRecord, KnowledgeBaseData
if TYPE_CHECKING:
from ...storage import EmbeddingModelConfig
@@ -74,15 +74,17 @@ class CollectionPerKbManager(KnowledgeBaseManagerBase):
"""
record = KnowledgeBaseRecord(
user_id=user_id,
- name=name,
- description=description,
- embedding_model_config=embedding_model_config,
- collection_name="",
+ data=KnowledgeBaseData(
+ name=name,
+ description=description,
+ embedding_model_config=embedding_model_config,
+ collection_name="",
+ ),
)
- record.collection_name = f"kb_{record.id}"
+ record.data.collection_name = f"kb_{record.id}"
await self._vector_store.create_collection(
- name=record.collection_name,
+ name=record.data.collection_name,
dimensions=embedding_model_config.dimensions,
)
try:
@@ -94,13 +96,13 @@ class CollectionPerKbManager(KnowledgeBaseManagerBase):
# would leave no trace.
try:
await self._vector_store.delete_collection(
- record.collection_name,
+ record.data.collection_name,
)
except Exception: # noqa: BLE001 — best-effort cleanup
logger.exception(
"Failed to drop orphan collection %r after "
"upsert_knowledge_base failed; collection leaked.",
- record.collection_name,
+ record.data.collection_name,
)
raise
@@ -129,8 +131,12 @@ class CollectionPerKbManager(KnowledgeBaseManagerBase):
if record is None:
return False
- if await self._vector_store.has_collection(record.collection_name):
- await self._vector_store.delete_collection(record.collection_name)
+ if await self._vector_store.has_collection(
+ record.data.collection_name,
+ ):
+ await self._vector_store.delete_collection(
+ record.data.collection_name,
+ )
return await self._storage.delete_knowledge_base(
user_id,
knowledge_base_id,
@@ -180,23 +186,23 @@ class CollectionPerKbManager(KnowledgeBaseManagerBase):
# the owner).
credential_record = await self._storage.get_credential(
record.user_id,
- record.embedding_model_config.credential_id,
+ record.data.embedding_model_config.credential_id,
)
if credential_record is None:
raise KnowledgeBaseNotFoundError(
f"Credential "
- f"{record.embedding_model_config.credential_id!r} for "
+ f"{record.data.embedding_model_config.credential_id!r} for "
f"knowledge base {knowledge_base_id!r} not found.",
)
embedding_model = build_embedding_model(
credential_record=credential_record,
- config=record.embedding_model_config,
+ config=record.data.embedding_model_config,
)
return KnowledgeBase(
- name=record.name,
- description=record.description,
+ name=record.data.name,
+ description=record.data.description,
embedding_model=embedding_model,
vector_store=self._vector_store,
- collection=record.collection_name,
+ collection=record.data.collection_name,
metadata_filter=None,
)
diff --git a/src/agentscope/app/storage/__init__.py b/src/agentscope/app/storage/__init__.py
index 661203f8e..6bc2eb54f 100644
--- a/src/agentscope/app/storage/__init__.py
+++ b/src/agentscope/app/storage/__init__.py
@@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
"""The storage module in agentscope."""
+from typing import TYPE_CHECKING
from ._base import StorageBase
from ._redis_storage import RedisStorage
@@ -7,6 +8,7 @@ from ._model import (
AgentData,
AgentRecord,
CredentialRecord,
+ KnowledgeBaseData,
KnowledgeBaseRecord,
KnowledgeDocumentData,
KnowledgeDocumentRecord,
@@ -28,14 +30,43 @@ from ._model import (
InviteConfig,
)
+if TYPE_CHECKING:
+ # Re-exported for static type checkers only; at runtime the class is
+ # loaded lazily by ``__getattr__`` below so callers without the
+ # optional ``sql`` extra never trigger the SQLAlchemy import that the
+ # ``_sql`` package does at load time (its declarative-tables module
+ # cannot defer it).
+ from ._sql import AsyncSQLAlchemyStorage # noqa: F401
+
+
+def __getattr__(name: str) -> object:
+ """Lazily load the optional SQL backend on first attribute access.
+
+ Keeps ``import agentscope.app.storage`` cheap — ``StorageBase``,
+ ``RedisStorage`` and the record models never need SQLAlchemy — while
+ still exposing ``AsyncSQLAlchemyStorage`` from this package for
+ callers that do ``from agentscope.app.storage import
+ AsyncSQLAlchemyStorage``. SQLAlchemy is imported only at that point.
+ """
+ if name == "AsyncSQLAlchemyStorage":
+ from ._sql import AsyncSQLAlchemyStorage as _AsyncSQLAlchemyStorage
+
+ return _AsyncSQLAlchemyStorage
+ raise AttributeError(
+ f"module 'agentscope.app.storage' has no attribute {name!r}",
+ )
+
+
__all__ = [
"StorageBase",
"RedisStorage",
+ "AsyncSQLAlchemyStorage",
# The ORM models
"InviteConfig",
"AgentData",
"AgentRecord",
"CredentialRecord",
+ "KnowledgeBaseData",
"KnowledgeBaseRecord",
"KnowledgeDocumentData",
"KnowledgeDocumentRecord",
diff --git a/src/agentscope/app/storage/_model/__init__.py b/src/agentscope/app/storage/_model/__init__.py
index 14f25814a..9aac92b70 100644
--- a/src/agentscope/app/storage/_model/__init__.py
+++ b/src/agentscope/app/storage/_model/__init__.py
@@ -3,7 +3,7 @@
from ._agent import AgentRecord, AgentData, InviteConfig
from ._credential import CredentialRecord
-from ._knowledge_base import KnowledgeBaseRecord
+from ._knowledge_base import KnowledgeBaseData, KnowledgeBaseRecord
from ._knowledge_document import (
KnowledgeDocumentData,
KnowledgeDocumentRecord,
@@ -26,6 +26,7 @@ __all__ = [
"AgentData",
"AgentRecord",
"CredentialRecord",
+ "KnowledgeBaseData",
"KnowledgeBaseRecord",
"KnowledgeDocumentData",
"KnowledgeDocumentRecord",
diff --git a/src/agentscope/app/storage/_model/_knowledge_base.py b/src/agentscope/app/storage/_model/_knowledge_base.py
index cdeba3fba..2e9be619a 100644
--- a/src/agentscope/app/storage/_model/_knowledge_base.py
+++ b/src/agentscope/app/storage/_model/_knowledge_base.py
@@ -1,24 +1,25 @@
# -*- coding: utf-8 -*-
"""The knowledge base record."""
-from pydantic import Field
+from typing import Any
+
+from pydantic import BaseModel, Field, model_validator
from ._base import _RecordBase
from ._session import EmbeddingModelConfig
-class KnowledgeBaseRecord(_RecordBase):
- """A persisted knowledge base record.
+class KnowledgeBaseData(BaseModel):
+ """The mutable payload of a knowledge base record.
- Stores per-user metadata for a knowledge base; the actual chunks
- and vectors live in the configured ``VectorStoreBase`` backend.
- Each record is the canonical authorisation gate: HTTP handlers and
- middleware look the record up by ``(user_id, id)`` before talking
- to the vector store.
+ Groups every non-relational field of :class:`KnowledgeBaseRecord`
+ into a single nested model so the outer record only carries
+ relational keys (``user_id``) alongside the ``_RecordBase``
+ envelope. This matches the convention used by the other records
+ (``AgentRecord.data``, ``SessionRecord.config`` / ``.state``,
+ ``TeamRecord.data`` etc.) and lets the SQL backend serialise the
+ whole payload into a single JSON column without special-casing.
"""
- user_id: str = Field(description="The owner user id.")
- """The user id that owns this knowledge base."""
-
name: str = Field(description="Display name of the knowledge base.")
"""Display name shown in the UI."""
@@ -44,3 +45,62 @@ class KnowledgeBaseRecord(_RecordBase):
),
)
"""The vector store collection name (e.g. ``kb_<uuid_hex>``)."""
+
+
+class KnowledgeBaseRecord(_RecordBase):
+ """A persisted knowledge base record.
+
+ Stores per-user metadata for a knowledge base; the actual chunks
+ and vectors live in the configured ``VectorStoreBase`` backend.
+ Each record is the canonical authorisation gate: HTTP handlers and
+ middleware look the record up by ``(user_id, id)`` before talking
+ to the vector store.
+ """
+
+ user_id: str = Field(description="The owner user id.")
+ """The user id that owns this knowledge base."""
+
+ data: KnowledgeBaseData
+ """The knowledge base payload (name, description, embedding config,
+ vector-store collection name)."""
+
+ @model_validator(mode="before")
+ @classmethod
+ def _nest_legacy_flat_payload(cls, obj: Any) -> Any:
+ """Nest legacy flat-payload fields into :attr:`data`.
+
+ Pre-refactor records stored the payload fields (``name`` /
+ ``description`` / ``embedding_model_config`` /
+ ``collection_name``) at the record top level rather than
+ under ``data``. This validator runs in ``mode='before'`` so
+ it hooks every pydantic entry point —
+ :meth:`~pydantic.BaseModel.model_validate`,
+ :meth:`~pydantic.BaseModel.model_validate_json` (the path
+ the storage backend takes when reading from Redis / SQL),
+ and direct ``__init__`` construction — giving us one place
+ to migrate the legacy shape on read.
+
+ Args:
+ obj (`Any`):
+ The raw input to the validator. Only ``dict`` inputs
+ are inspected; other inputs are returned verbatim so
+ attribute-based construction still works.
+
+ Returns:
+ `Any`:
+ The input with legacy flat fields packed into
+ ``data`` when necessary.
+ """
+ if not isinstance(obj, dict) or "data" in obj:
+ return obj
+ legacy_keys = (
+ "name",
+ "description",
+ "embedding_model_config",
+ "collection_name",
+ )
+ if not any(key in obj for key in legacy_keys):
+ return obj
+ remainder = {k: v for k, v in obj.items() if k not in legacy_keys}
+ remainder["data"] = {k: obj[k] for k in legacy_keys if k in obj}
+ return remainder
diff --git a/src/agentscope/app/storage/_model/_knowledge_document.py b/src/agentscope/app/storage/_model/_knowledge_document.py
index f3c7b9fb1..29aa54312 100644
--- a/src/agentscope/app/storage/_model/_knowledge_document.py
+++ b/src/agentscope/app/storage/_model/_knowledge_document.py
@@ -9,14 +9,17 @@ the vector store cannot represent on its own (no chunks means no
``document_id`` to aggregate from).
Top-level fields are the relational keys (``user_id`` /
-``knowledge_base_id`` / ``processing_node``) that the storage backend
-indexes on; everything else lives inside ``data`` per the project
-record convention.
+``knowledge_base_id`` / ``processing_node``) **plus the lifecycle
+fields the storage backend must index on** (``status`` /
+``lease_expires_at``) — grouped at the top so a SQL backend can put
+them straight into columns and the sweeper can query them without
+deserialising :attr:`data`. Everything else lives inside ``data`` per
+the project record convention.
"""
from datetime import datetime
-from typing import Literal
+from typing import Any, Literal
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, model_validator
from ._base import _RecordBase
@@ -37,7 +40,13 @@ KnowledgeDocumentStatus = Literal[
class KnowledgeDocumentData(BaseModel):
- """The mutable payload of a knowledge document record."""
+ """The mutable payload of a knowledge document record.
+
+ Groups the non-relational, non-indexed fields of the document.
+ The lifecycle keys (``status`` / ``lease_expires_at``) live at
+ the :class:`KnowledgeDocumentRecord` top level so the sweeper can
+ query them via indexed columns without touching this payload.
+ """
filename: str = Field(
description="Original filename supplied by the uploader.",
@@ -70,13 +79,6 @@ class KnowledgeDocumentData(BaseModel):
:class:`~agentscope.app.rag.blob_store.BlobStoreBase`. The worker
streams bytes back through the same blob store."""
- status: KnowledgeDocumentStatus = Field(
- default="pending",
- description="Current lifecycle state.",
- )
- """Current lifecycle state. Read by the polling endpoint, written
- by the worker as it transitions phases."""
-
error: str | None = Field(
default=None,
description=(
@@ -96,23 +98,14 @@ class KnowledgeDocumentData(BaseModel):
"""The final chunk count, written by the worker when the document
reaches ``ready``."""
- lease_expires_at: datetime | None = Field(
- default=None,
- description=(
- "Wall-clock deadline for the worker that currently holds "
- "this document. ``None`` if no worker is processing it."
- ),
- )
- """Lease deadline. Together with ``processing_node`` lets the
- sweeper detect crashed workers and reassign their documents."""
-
class KnowledgeDocumentRecord(_RecordBase):
"""A persisted knowledge document record.
- Top-level fields are relational keys the storage backend needs to
- index on directly (per-user listing, per-KB listing, per-node lease
- sweeps); the mutable payload lives in :attr:`data`.
+ Top-level fields are relational keys AND the lifecycle keys the
+ storage backend must index on directly (per-user listing, per-KB
+ listing, per-node lease sweeps, status-based sweeps); the
+ remaining mutable payload lives in :attr:`data`.
"""
user_id: str = Field(description="The owner user id.")
@@ -135,5 +128,75 @@ class KnowledgeDocumentRecord(_RecordBase):
storage backend can look up "documents owned by node X" or
"documents with no owner" without deserialising every payload."""
+ status: KnowledgeDocumentStatus = Field(
+ default="pending",
+ description="Current lifecycle state.",
+ )
+ """Current lifecycle state. Read by the polling endpoint, written
+ by the worker as it transitions phases. Promoted to the top level
+ so the SQL backend can index it and the sweeper can filter on
+ non-terminal documents without payload deserialisation."""
+
+ lease_expires_at: datetime | None = Field(
+ default=None,
+ description=(
+ "Wall-clock deadline for the worker that currently holds "
+ "this document. ``None`` if no worker is processing it."
+ ),
+ )
+ """Lease deadline. Together with ``processing_node`` lets the
+ sweeper detect crashed workers and reassign their documents.
+ Promoted to the top level so it can be indexed alongside
+ ``status`` / ``processing_node``."""
+
data: KnowledgeDocumentData
- """The mutable document payload."""
+ """The mutable document payload (filename, size, blob_uri, error,
+ chunk_count)."""
+
+ @model_validator(mode="before")
+ @classmethod
+ def _lift_legacy_lifecycle_fields(cls, obj: Any) -> Any:
+ """Lift legacy in-``data`` lifecycle fields to the top level.
+
+ Pre-refactor payloads persisted ``status`` and
+ ``lease_expires_at`` inside :class:`KnowledgeDocumentData`.
+ The refactor promoted them to the record top level so the
+ storage backend can index / query them without deserialising
+ the payload; this validator makes the record accept both
+ shapes on read so records written before the refactor still
+ round-trip.
+
+ Runs in ``mode='before'`` so it hooks every entry point that
+ goes through the pydantic core validator —
+ :meth:`~pydantic.BaseModel.model_validate`,
+ :meth:`~pydantic.BaseModel.model_validate_json` (the path the
+ storage backend takes), and direct ``__init__`` construction
+ alike — giving us one place to keep the legacy compatibility
+ logic.
+
+ Args:
+ obj (`Any`):
+ The raw input passed to the validator. Only ``dict``
+ inputs are inspected; anything else is returned
+ verbatim so subclasses / attribute-based construction
+ still work.
+
+ Returns:
+ `Any`:
+ The input with legacy lifecycle fields lifted from
+ ``data`` to the top level when necessary.
+ """
+ if not isinstance(obj, dict):
+ return obj
+ data = obj.get("data")
+ if not isinstance(data, dict):
+ return obj
+ # Only lift when the top level does NOT already carry the
+ # field — a caller passing both wins at the top level.
+ lifted: dict = {}
+ for key in ("status", "lease_expires_at"):
+ if key in data and key not in obj:
+ lifted[key] = data.pop(key)
+ if lifted:
+ return {**obj, **lifted, "data": data}
+ return obj
diff --git a/src/agentscope/app/storage/_redis_storage.py b/src/agentscope/app/storage/_redis_storage.py
index a9cc45591..44dfa29dc 100644
--- a/src/agentscope/app/storage/_redis_storage.py
+++ b/src/agentscope/app/storage/_redis_storage.py
@@ -1634,7 +1634,7 @@ class RedisStorage(StorageBase):
if not raw:
return
record = KnowledgeDocumentRecord.model_validate_json(raw)
- record.data.status = status
+ record.status = status
if error is not None:
record.data.error = error
if chunk_count is not None:
@@ -1678,7 +1678,7 @@ class RedisStorage(StorageBase):
return False
record = KnowledgeDocumentRecord.model_validate_json(raw)
holder = record.processing_node
- deadline = record.data.lease_expires_at
+ deadline = record.lease_expires_at
if (
holder is not None
and deadline is not None
@@ -1687,7 +1687,7 @@ class RedisStorage(StorageBase):
await pipe.unwatch()
return False
record.processing_node = processing_node
- record.data.lease_expires_at = new_deadline
+ record.lease_expires_at = new_deadline
record.updated_at = now
pipe.multi()
pipe.set(key, record.model_dump_json())
@@ -1731,7 +1731,7 @@ class RedisStorage(StorageBase):
if record.processing_node != processing_node:
await pipe.unwatch()
return False
- record.data.lease_expires_at = new_deadline
+ record.lease_expires_at = new_deadline
record.updated_at = now
pipe.multi()
pipe.set(key, record.model_dump_json())
@@ -1769,7 +1769,7 @@ class RedisStorage(StorageBase):
await pipe.unwatch()
return
record.processing_node = None
- record.data.lease_expires_at = None
+ record.lease_expires_at = None
record.updated_at = datetime.now()
pipe.multi()
pipe.set(key, record.model_dump_json())
@@ -1811,13 +1811,13 @@ class RedisStorage(StorageBase):
if not raw:
continue
record = KnowledgeDocumentRecord.model_validate_json(raw)
- if record.data.status in terminal:
+ if record.status in terminal:
continue
if record.processing_node is None:
continue
if (
- record.data.lease_expires_at is not None
- and record.data.lease_expires_at < now
+ record.lease_expires_at is not None
+ and record.lease_expires_at < now
):
records.append(record)
return records
@@ -1842,7 +1842,7 @@ class RedisStorage(StorageBase):
if not raw:
continue
record = KnowledgeDocumentRecord.model_validate_json(raw)
- if record.data.status != "pending":
+ if record.status != "pending":
continue
if record.created_at < threshold:
records.append(record)
diff --git a/src/agentscope/app/storage/_sql/__init__.py b/src/agentscope/app/storage/_sql/__init__.py
new file mode 100644
index 000000000..0f48bf126
--- /dev/null
+++ b/src/agentscope/app/storage/_sql/__init__.py
@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+"""Async SQLAlchemy storage backend.
+
+The only public symbol is :class:`AsyncSQLAlchemyStorage`;
+every other module in this package is an implementation detail
+(tables, mappers, engine helpers, Alembic scaffolding) named
+with a leading underscore so :mod:`agentscope.app.storage`
+can re-export it without leaking internals.
+"""
+from ._storage import AsyncSQLAlchemyStorage
+
+__all__ = ["AsyncSQLAlchemyStorage"]
diff --git a/src/agentscope/app/storage/_sql/_alembic/alembic.ini b/src/agentscope/app/storage/_sql/_alembic/alembic.ini
new file mode 100644
index 000000000..81ba48e3b
--- /dev/null
+++ b/src/agentscope/app/storage/_sql/_alembic/alembic.ini
@@ -0,0 +1,50 @@
+# Alembic configuration for the AgentScope SQL storage backend.
+#
+# Pointed at the private env / template shipped inside the package so
+# users don't need to bootstrap Alembic themselves — just:
+#
+# AGENTSCOPE_SQL_URL="sqlite+aiosqlite:///./as.db" \
+# alembic -c src/agentscope/app/storage/_sql/_alembic/alembic.ini upgrade head
+#
+# In production you should copy this file next to your deployment
+# config and point ``script_location`` / ``sqlalchemy.url`` at your
+# checkout.
+[alembic]
+script_location = %(here)s
+# Empty by default — set via -x url=... or the AGENTSCOPE_SQL_URL env var.
+sqlalchemy.url =
+
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARNING
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARNING
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S
+
diff --git a/src/agentscope/app/storage/_sql/_alembic/env.py b/src/agentscope/app/storage/_sql/_alembic/env.py
new file mode 100644
index 000000000..dbd442c76
--- /dev/null
+++ b/src/agentscope/app/storage/_sql/_alembic/env.py
@@ -0,0 +1,89 @@
+# -*- coding: utf-8 -*-
+"""Alembic environment for :class:`AsyncSQLAlchemyStorage` schema migrations.
+
+Runs migrations in async mode against a SQLAlchemy URL. The URL is
+picked up (in order of precedence) from:
+
+1. the ``sqlalchemy.url`` key of the Alembic config (usually set via
+ ``alembic.ini`` or ``-x url=...``);
+2. the ``AGENTSCOPE_SQL_URL`` environment variable — convenient in
+ dev / CI when there is no ``alembic.ini`` around.
+
+The target metadata is :data:`_Base.metadata`, so every model added
+to :mod:`agentscope.app.storage._sql._tables` is picked up by
+``alembic revision --autogenerate`` without further wiring.
+"""
+from __future__ import with_statement
+
+import asyncio
+import os
+
+from alembic import context
+from sqlalchemy import pool
+from sqlalchemy.engine import Connection
+from sqlalchemy.ext.asyncio import async_engine_from_config
+
+from agentscope.app.storage._sql._tables import _Base
+
+# ---------------------------------------------------------------------
+# Config wiring
+# ---------------------------------------------------------------------
+config = context.config
+
+url_from_env = os.getenv("AGENTSCOPE_SQL_URL")
+if url_from_env and not config.get_main_option("sqlalchemy.url"):
+ config.set_main_option("sqlalchemy.url", url_from_env)
+
+target_metadata = _Base.metadata
+
+
+# ---------------------------------------------------------------------
+# Offline mode — emit SQL without a connection.
+# ---------------------------------------------------------------------
+def run_migrations_offline() -> None:
+ """Run migrations using URL-only context (no DBAPI connection)."""
+ url = config.get_main_option("sqlalchemy.url")
+ context.configure(
+ url=url,
+ target_metadata=target_metadata,
+ literal_binds=True,
+ dialect_opts={"paramstyle": "named"},
+ render_as_batch=(url or "").startswith("sqlite"),
+ )
+ with context.begin_transaction():
+ context.run_migrations()
+
+
+# ---------------------------------------------------------------------
+# Online mode — connect and run.
+# ---------------------------------------------------------------------
+def do_run_migrations(connection: Connection) -> None:
+ """Bind Alembic to *connection* and run every pending revision."""
+ is_sqlite = connection.dialect.name == "sqlite"
+ context.configure(
+ connection=connection,
+ target_metadata=target_metadata,
+ render_as_batch=is_sqlite,
+ )
+ with context.begin_transaction():
+ context.run_migrations()
+
+
+async def run_migrations_online() -> None:
+ """Create an :class:`AsyncEngine` and dispatch to
+ :func:`do_run_migrations`.
+ """
+ connectable = async_engine_from_config(
+ config.get_section(config.config_ini_section, {}),
+ prefix="sqlalchemy.",
+ poolclass=pool.NullPool,
+ )
+ async with connectable.connect() as connection:
+ await connection.run_sync(do_run_migrations)
+ await connectable.dispose()
+
+
+if context.is_offline_mode():
+ run_migrations_offline()
+else:
+ asyncio.run(run_migrations_online())
diff --git a/src/agentscope/app/storage/_sql/_alembic/script.py.mako b/src/agentscope/app/storage/_sql/_alembic/script.py.mako
new file mode 100644
index 000000000..3c2e787de
--- /dev/null
+++ b/src/agentscope/app/storage/_sql/_alembic/script.py.mako
@@ -0,0 +1,27 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision: str = ${repr(up_revision)}
+down_revision: Union[str, None] = ${repr(down_revision)}
+branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
+depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+ ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+ ${downgrades if downgrades else "pass"}
+
diff --git a/src/agentscope/app/storage/_sql/_alembic/versions/0001_initial.py b/src/agentscope/app/storage/_sql/_alembic/versions/0001_initial.py
new file mode 100644
index 000000000..ac9c9d040
--- /dev/null
+++ b/src/agentscope/app/storage/_sql/_alembic/versions/0001_initial.py
@@ -0,0 +1,384 @@
+# -*- coding: utf-8 -*-
+"""Initial schema — frozen snapshot of every table in ``_sql/_tables.py``.
+
+Revision ID: 0001_initial
+Revises:
+Create Date: 2026-07-20 10:24:57.000000
+
+This revision is a *static snapshot*: the ``op.create_table`` calls are
+frozen at the schema as it stood when the SQL backend first shipped.
+It deliberately does NOT delegate to ``_Base.metadata.create_all`` —
+doing so would make the initial revision track the *current* metadata,
+so a later column/table addition would be created here (on a fresh
+database) and then created *again* by its own autogenerated revision,
+which fails as a duplicate. Every future schema change must be added
+as a new ``alembic revision --autogenerate`` on top of this one.
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = "0001_initial"
+down_revision: Union[str, None] = None
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+ """Create every table at the first-shipped schema."""
+ op.create_table(
+ "agents",
+ sa.Column("user_id", sa.String(length=255), nullable=False),
+ sa.Column("source", sa.String(length=16), nullable=False),
+ sa.Column("id", sa.String(length=255), nullable=False),
+ sa.Column("created_at", sa.DateTime(), nullable=False),
+ sa.Column("updated_at", sa.DateTime(), nullable=False),
+ sa.Column("payload", sa.JSON(), nullable=False),
+ sa.PrimaryKeyConstraint("id"),
+ )
+ with op.batch_alter_table("agents", schema=None) as batch_op:
+ batch_op.create_index(
+ batch_op.f("ix_agents_created_at"),
+ ["created_at"],
+ unique=False,
+ )
+ batch_op.create_index(
+ batch_op.f("ix_agents_source"),
+ ["source"],
+ unique=False,
+ )
+ batch_op.create_index(
+ batch_op.f("ix_agents_updated_at"),
+ ["updated_at"],
+ unique=False,
+ )
+ batch_op.create_index(
+ batch_op.f("ix_agents_user_id"),
+ ["user_id"],
+ unique=False,
+ )
+ batch_op.create_index(
+ "ix_agents_user_source",
+ ["user_id", "source"],
+ unique=False,
+ )
+
+ op.create_table(
+ "credentials",
+ sa.Column("user_id", sa.String(length=255), nullable=False),
+ sa.Column("id", sa.String(length=255), nullable=False),
+ sa.Column("created_at", sa.DateTime(), nullable=False),
+ sa.Column("updated_at", sa.DateTime(), nullable=False),
+ sa.Column("payload", sa.JSON(), nullable=False),
+ sa.PrimaryKeyConstraint("id"),
+ )
+ with op.batch_alter_table("credentials", schema=None) as batch_op:
+ batch_op.create_index(
+ batch_op.f("ix_credentials_created_at"),
+ ["created_at"],
+ unique=False,
+ )
+ batch_op.create_index(
+ batch_op.f("ix_credentials_updated_at"),
+ ["updated_at"],
+ unique=False,
+ )
+ batch_op.create_index(
+ batch_op.f("ix_credentials_user_id"),
+ ["user_id"],
+ unique=False,
+ )
+
+ op.create_table(
+ "knowledge_bases",
+ sa.Column("user_id", sa.String(length=255), nullable=False),
+ sa.Column("id", sa.String(length=255), nullable=False),
+ sa.Column("created_at", sa.DateTime(), nullable=False),
+ sa.Column("updated_at", sa.DateTime(), nullable=False),
+ sa.Column("payload", sa.JSON(), nullable=False),
+ sa.PrimaryKeyConstraint("id"),
+ )
+ with op.batch_alter_table("knowledge_bases", schema=None) as batch_op:
+ batch_op.create_index(
+ batch_op.f("ix_knowledge_bases_created_at"),
+ ["created_at"],
+ unique=False,
+ )
+ batch_op.create_index(
+ batch_op.f("ix_knowledge_bases_updated_at"),
+ ["updated_at"],
+ unique=False,
+ )
+ batch_op.create_index(
+ batch_op.f("ix_knowledge_bases_user_id"),
+ ["user_id"],
+ unique=False,
+ )
+
+ op.create_table(
+ "messages",
+ sa.Column("session_id", sa.String(length=255), nullable=False),
+ sa.Column("msg_id", sa.String(length=255), nullable=False),
+ sa.Column("created_at", sa.DateTime(), nullable=False),
+ sa.Column("payload", sa.JSON(), nullable=False),
+ sa.PrimaryKeyConstraint("session_id", "msg_id"),
+ )
+ with op.batch_alter_table("messages", schema=None) as batch_op:
+ batch_op.create_index(
+ "ix_messages_session_created",
+ ["session_id", "created_at"],
+ unique=False,
+ )
+
+ op.create_table(
+ "schedules",
+ sa.Column("user_id", sa.String(length=255), nullable=False),
+ sa.Column("agent_id", sa.String(length=255), nullable=False),
+ sa.Column("id", sa.String(length=255), nullable=False),
+ sa.Column("created_at", sa.DateTime(), nullable=False),
+ sa.Column("updated_at", sa.DateTime(), nullable=False),
+ sa.Column("payload", sa.JSON(), nullable=False),
+ sa.PrimaryKeyConstraint("id"),
+ )
+ with op.batch_alter_table("schedules", schema=None) as batch_op:
+ batch_op.create_index(
+ batch_op.f("ix_schedules_agent_id"),
+ ["agent_id"],
+ unique=False,
+ )
+ batch_op.create_index(
+ batch_op.f("ix_schedules_created_at"),
+ ["created_at"],
+ unique=False,
+ )
+ batch_op.create_index(
+ batch_op.f("ix_schedules_updated_at"),
+ ["updated_at"],
+ unique=False,
+ )
+ batch_op.create_index(
+ batch_op.f("ix_schedules_user_id"),
+ ["user_id"],
+ unique=False,
+ )
+
+ op.create_table(
+ "sessions",
+ sa.Column("user_id", sa.String(length=255), nullable=False),
+ sa.Column("agent_id", sa.String(length=255), nullable=False),
+ sa.Column("source", sa.String(length=16), nullable=False),
+ sa.Column(
+ "source_schedule_id",
+ sa.String(length=255),
+ nullable=True,
+ ),
+ sa.Column("team_id", sa.String(length=255), nullable=True),
+ sa.Column("id", sa.String(length=255), nullable=False),
+ sa.Column("created_at", sa.DateTime(), nullable=False),
+ sa.Column("updated_at", sa.DateTime(), nullable=False),
+ sa.Column("payload", sa.JSON(), nullable=False),
+ sa.PrimaryKeyConstraint("id"),
+ )
+ with op.batch_alter_table("sessions", schema=None) as batch_op:
+ batch_op.create_index(
+ batch_op.f("ix_sessions_agent_id"),
+ ["agent_id"],
+ unique=False,
+ )
+ batch_op.create_index(
+ batch_op.f("ix_sessions_created_at"),
+ ["created_at"],
+ unique=False,
+ )
+ batch_op.create_index(
+ batch_op.f("ix_sessions_source"),
+ ["source"],
+ unique=False,
+ )
+ batch_op.create_index(
+ batch_op.f("ix_sessions_source_schedule_id"),
+ ["source_schedule_id"],
+ unique=False,
+ )
+ batch_op.create_index(
+ batch_op.f("ix_sessions_team_id"),
+ ["team_id"],
+ unique=False,
+ )
+ batch_op.create_index(
+ batch_op.f("ix_sessions_updated_at"),
+ ["updated_at"],
+ unique=False,
+ )
+ batch_op.create_index(
+ "ix_sessions_user_agent",
+ ["user_id", "agent_id"],
+ unique=False,
+ )
+ batch_op.create_index(
+ batch_op.f("ix_sessions_user_id"),
+ ["user_id"],
+ unique=False,
+ )
+
+ op.create_table(
+ "teams",
+ sa.Column("user_id", sa.String(length=255), nullable=False),
+ sa.Column("session_id", sa.String(length=255), nullable=False),
+ sa.Column("id", sa.String(length=255), nullable=False),
+ sa.Column("created_at", sa.DateTime(), nullable=False),
+ sa.Column("updated_at", sa.DateTime(), nullable=False),
+ sa.Column("payload", sa.JSON(), nullable=False),
+ sa.PrimaryKeyConstraint("id"),
+ )
+ with op.batch_alter_table("teams", schema=None) as batch_op:
+ batch_op.create_index(
+ batch_op.f("ix_teams_created_at"),
+ ["created_at"],
+ unique=False,
+ )
+ batch_op.create_index(
+ batch_op.f("ix_teams_updated_at"),
+ ["updated_at"],
+ unique=False,
+ )
+ batch_op.create_index(
+ batch_op.f("ix_teams_user_id"),
+ ["user_id"],
+ unique=False,
+ )
+
+ op.create_table(
+ "knowledge_documents",
+ sa.Column("user_id", sa.String(length=255), nullable=False),
+ sa.Column(
+ "knowledge_base_id",
+ sa.String(length=255),
+ nullable=False,
+ ),
+ sa.Column(
+ "processing_node",
+ sa.String(length=128),
+ nullable=True,
+ ),
+ sa.Column("status", sa.String(length=16), nullable=False),
+ sa.Column("lease_expires_at", sa.DateTime(), nullable=True),
+ sa.Column("id", sa.String(length=255), nullable=False),
+ sa.Column("created_at", sa.DateTime(), nullable=False),
+ sa.Column("updated_at", sa.DateTime(), nullable=False),
+ sa.Column("payload", sa.JSON(), nullable=False),
+ sa.ForeignKeyConstraint(
+ ["knowledge_base_id"],
+ ["knowledge_bases.id"],
+ ondelete="CASCADE",
+ ),
+ sa.PrimaryKeyConstraint("id"),
+ )
+ with op.batch_alter_table(
+ "knowledge_documents",
+ schema=None,
+ ) as batch_op:
+ batch_op.create_index(
+ "ix_kd_status_lease",
+ ["status", "lease_expires_at"],
+ unique=False,
+ )
+ batch_op.create_index(
+ "ix_kd_user_kb",
+ ["user_id", "knowledge_base_id"],
+ unique=False,
+ )
+ batch_op.create_index(
+ batch_op.f("ix_knowledge_documents_created_at"),
+ ["created_at"],
+ unique=False,
+ )
+ batch_op.create_index(
+ batch_op.f("ix_knowledge_documents_knowledge_base_id"),
+ ["knowledge_base_id"],
+ unique=False,
+ )
+ batch_op.create_index(
+ batch_op.f("ix_knowledge_documents_status"),
+ ["status"],
+ unique=False,
+ )
+ batch_op.create_index(
+ batch_op.f("ix_knowledge_documents_updated_at"),
+ ["updated_at"],
+ unique=False,
+ )
+ batch_op.create_index(
+ batch_op.f("ix_knowledge_documents_user_id"),
+ ["user_id"],
+ unique=False,
+ )
+
+
+def downgrade() -> None:
+ """Drop every table created by :func:`upgrade`."""
+ with op.batch_alter_table(
+ "knowledge_documents",
+ schema=None,
+ ) as batch_op:
+ batch_op.drop_index(batch_op.f("ix_knowledge_documents_user_id"))
+ batch_op.drop_index(batch_op.f("ix_knowledge_documents_updated_at"))
+ batch_op.drop_index(batch_op.f("ix_knowledge_documents_status"))
+ batch_op.drop_index(
+ batch_op.f("ix_knowledge_documents_knowledge_base_id"),
+ )
+ batch_op.drop_index(batch_op.f("ix_knowledge_documents_created_at"))
+ batch_op.drop_index("ix_kd_user_kb")
+ batch_op.drop_index("ix_kd_status_lease")
+ op.drop_table("knowledge_documents")
+
+ with op.batch_alter_table("teams", schema=None) as batch_op:
+ batch_op.drop_index(batch_op.f("ix_teams_user_id"))
+ batch_op.drop_index(batch_op.f("ix_teams_updated_at"))
+ batch_op.drop_index(batch_op.f("ix_teams_created_at"))
+ op.drop_table("teams")
+
+ with op.batch_alter_table("sessions", schema=None) as batch_op:
+ batch_op.drop_index(batch_op.f("ix_sessions_user_id"))
+ batch_op.drop_index("ix_sessions_user_agent")
+ batch_op.drop_index(batch_op.f("ix_sessions_updated_at"))
+ batch_op.drop_index(batch_op.f("ix_sessions_team_id"))
+ batch_op.drop_index(batch_op.f("ix_sessions_source_schedule_id"))
+ batch_op.drop_index(batch_op.f("ix_sessions_source"))
+ batch_op.drop_index(batch_op.f("ix_sessions_created_at"))
+ batch_op.drop_index(batch_op.f("ix_sessions_agent_id"))
+ op.drop_table("sessions")
+
+ with op.batch_alter_table("schedules", schema=None) as batch_op:
+ batch_op.drop_index(batch_op.f("ix_schedules_user_id"))
+ batch_op.drop_index(batch_op.f("ix_schedules_updated_at"))
+ batch_op.drop_index(batch_op.f("ix_schedules_created_at"))
+ batch_op.drop_index(batch_op.f("ix_schedules_agent_id"))
+ op.drop_table("schedules")
+
+ with op.batch_alter_table("messages", schema=None) as batch_op:
+ batch_op.drop_index("ix_messages_session_created")
+ op.drop_table("messages")
+
+ with op.batch_alter_table("knowledge_bases", schema=None) as batch_op:
+ batch_op.drop_index(batch_op.f("ix_knowledge_bases_user_id"))
+ batch_op.drop_index(batch_op.f("ix_knowledge_bases_updated_at"))
+ batch_op.drop_index(batch_op.f("ix_knowledge_bases_created_at"))
+ op.drop_table("knowledge_bases")
+
+ with op.batch_alter_table("credentials", schema=None) as batch_op:
+ batch_op.drop_index(batch_op.f("ix_credentials_user_id"))
+ batch_op.drop_index(batch_op.f("ix_credentials_updated_at"))
+ batch_op.drop_index(batch_op.f("ix_credentials_created_at"))
+ op.drop_table("credentials")
+
+ with op.batch_alter_table("agents", schema=None) as batch_op:
+ batch_op.drop_index("ix_agents_user_source")
+ batch_op.drop_index(batch_op.f("ix_agents_user_id"))
+ batch_op.drop_index(batch_op.f("ix_agents_updated_at"))
+ batch_op.drop_index(batch_op.f("ix_agents_source"))
+ batch_op.drop_index(batch_op.f("ix_agents_created_at"))
+ op.drop_table("agents")
diff --git a/src/agentscope/app/storage/_sql/_mappers.py b/src/agentscope/app/storage/_sql/_mappers.py
new file mode 100644
index 000000000..bea8dac4a
--- /dev/null
+++ b/src/agentscope/app/storage/_sql/_mappers.py
@@ -0,0 +1,100 @@
+# -*- coding: utf-8 -*-
+"""Pydantic ``_RecordBase`` ↔ SQLAlchemy row conversion helpers.
+
+Round-trip contract (the "no duplication" invariant enforced here):
+
+- On write (``_from_record``) the record is dumped once with
+ ``model_dump(mode="json")``, the envelope keys (``id`` /
+ ``created_at`` / ``updated_at``) and every field listed in the
+ row class' ``_indexed_fields`` are popped out into the promoted
+ columns, and whatever remains is stored verbatim in ``payload``.
+- On read (``_to_record``) the promoted columns are merged back into
+ the payload dict and the whole thing is fed through
+ ``model_validate`` — which, thanks to the ``mode="before"``
+ validators on records like :class:`KnowledgeDocumentRecord` /
+ :class:`KnowledgeBaseRecord`, also transparently absorbs any
+ legacy on-disk shapes.
+
+Both functions are dialect-agnostic — they only touch pydantic and
+plain Python dicts. All dialect specifics live in :mod:`_storage`.
+"""
+from typing import TYPE_CHECKING, TypeVar
+
+from .._model._base import _RecordBase
+
+if TYPE_CHECKING:
+ from ._tables import _JsonRecordMixin
+
+
+R = TypeVar("R", bound=_RecordBase)
+
+# Envelope fields promoted by :class:`_JsonRecordMixin`; also popped
+# from every record dump before it hits ``payload``.
+_ENVELOPE_FIELDS = ("id", "created_at", "updated_at")
+
+
+def _from_record(
+ row_cls: "type[_JsonRecordMixin]",
+ record: _RecordBase,
+) -> "_JsonRecordMixin":
+ """Project *record* onto a fresh instance of *row_cls*.
+
+ Args:
+ row_cls (`type[_JsonRecordMixin]`):
+ The concrete ``*Row`` class to instantiate. Must expose
+ ``_indexed_fields`` naming the promoted columns.
+ record (`_RecordBase`):
+ The pydantic record to persist. Its ``id``,
+ ``created_at`` and ``updated_at`` are taken as-is (the
+ caller is responsible for refreshing ``updated_at``
+ before the call, mirroring the Redis storage semantics).
+
+ Returns:
+ `_JsonRecordMixin`:
+ A row instance whose promoted columns and ``payload``
+ column together carry every field of the record with no
+ duplication.
+ """
+ dump = record.model_dump(mode="json")
+ for field in _ENVELOPE_FIELDS:
+ dump.pop(field, None)
+ column_values: dict = {}
+ for field in row_cls.get_indexed_fields():
+ column_values[field] = dump.pop(field, None)
+ return row_cls(
+ id=record.id,
+ created_at=record.created_at,
+ updated_at=record.updated_at,
+ payload=dump,
+ **column_values,
+ )
+
+
+def _to_record(
+ row: "_JsonRecordMixin",
+ record_cls: type[R],
+) -> R:
+ """Reconstruct a record from *row*.
+
+ Merges the promoted columns back into ``payload`` and delegates
+ to :meth:`~pydantic.BaseModel.model_validate` so any
+ ``mode='before'`` validators on the record (legacy-shape
+ migrators, defaults, etc.) still fire.
+
+ Args:
+ row (`_JsonRecordMixin`):
+ The SQLAlchemy row loaded from the database.
+ record_cls (`type[R]`):
+ The concrete :class:`_RecordBase` subclass to return.
+
+ Returns:
+ `R`:
+ The reconstituted record.
+ """
+ obj: dict = dict(row.payload or {})
+ obj["id"] = row.id
+ obj["created_at"] = row.created_at
+ obj["updated_at"] = row.updated_at
+ for field in row.__class__.get_indexed_fields():
+ obj[field] = getattr(row, field)
+ return record_cls.model_validate(obj)
diff --git a/src/agentscope/app/storage/_sql/_storage.py b/src/agentscope/app/storage/_sql/_storage.py
new file mode 100644
index 000000000..1cc558a12
--- /dev/null
+++ b/src/agentscope/app/storage/_sql/_storage.py
@@ -0,0 +1,1651 @@
+# -*- coding: utf-8 -*-
+# pylint: disable=too-many-public-methods
+"""SQLAlchemy 2.0 async implementation of :class:`StorageBase`.
+
+Talks to any dialect supported by SQLAlchemy's async engine
+(SQLite / Postgres / MySQL / …) — the caller only picks the URL and
+installs the matching driver. All schema and query building goes
+through dialect-neutral SA constructs; see the module doc of
+:mod:`_tables` for the portability constraints we hold ourselves to.
+
+Timestamps are stored as **naive UTC**: the backend generates every
+timestamp with :func:`_utcnow` (UTC, not the machine-local
+``datetime.now()``) and normalises any caller-supplied datetime with
+:func:`_to_naive_utc`. Using UTC keeps timestamps comparable across
+nodes in a distributed deployment regardless of each node's local
+timezone, while staying tz-naive so the plain ``DateTime`` columns
+need no dialect-specific timezone handling.
+"""
+from datetime import datetime, timedelta, timezone
+from typing import TYPE_CHECKING, Any, Self
+
+from .._base import StorageBase
+from .._model import (
+ AgentRecord,
+ CredentialRecord,
+ KnowledgeBaseRecord,
+ KnowledgeDocumentRecord,
+ KnowledgeDocumentStatus,
+ ScheduleRecord,
+ SessionRecord,
+ SessionConfig,
+ SessionSource,
+ TeamRecord,
+)
+from .._utils import _dump_with_secrets
+from ._mappers import _from_record, _to_record
+from ._tables import (
+ _Base,
+ AgentRow,
+ CredentialRow,
+ KnowledgeBaseRow,
+ KnowledgeDocumentRow,
+ MessageRow,
+ ScheduleRow,
+ SessionRow,
+ TeamRow,
+)
+from ....credential import CredentialBase
+from ....message import Msg
+from ....state import AgentState
+
+if TYPE_CHECKING:
+ from sqlalchemy.ext.asyncio import (
+ AsyncEngine,
+ AsyncSession,
+ async_sessionmaker,
+ )
+
+
+def _utcnow() -> datetime:
+ """Current time as a naive UTC timestamp.
+
+ Uses UTC rather than the machine-local ``datetime.now()`` so
+ timestamps stay comparable across nodes in a distributed
+ deployment; the ``tzinfo`` is stripped so the value round-trips
+ through the naive ``DateTime`` columns without mixing aware/naive
+ datetimes.
+ """
+ return datetime.now(timezone.utc).replace(tzinfo=None)
+
+
+def _to_naive_utc(dt: datetime) -> datetime:
+ """Normalise *dt* to a naive UTC timestamp.
+
+ Aware datetimes are converted to UTC; naive datetimes are assumed
+ to be in the machine-local zone — the project-wide convention for
+ ``datetime.now()`` — and re-anchored to UTC. Either way the result
+ is tz-naive so it matches the values produced by :func:`_utcnow`.
+ """
+ return dt.astimezone(timezone.utc).replace(tzinfo=None)
+
+
+class AsyncSQLAlchemyStorage(StorageBase):
+ """Async SQLAlchemy-backed :class:`StorageBase` implementation.
+
+ Instantiate with any SQLAlchemy async URL (the concrete driver is
+ a caller-installed dependency, not part of the ``sql`` extra):
+
+ .. code-block:: python
+
+ storage = AsyncSQLAlchemyStorage("sqlite+aiosqlite:///./as.db")
+ async with storage:
+ ...
+
+ ``__aenter__`` builds the engine and (optionally) provisions the
+ schema; ``aclose`` disposes the engine.
+ """
+
+ def __init__(
+ self,
+ url: str,
+ *,
+ create_tables: bool = True,
+ auto_migrate: bool = False,
+ engine: "AsyncEngine | None" = None,
+ engine_kwargs: dict[str, Any] | None = None,
+ ) -> None:
+ """Configure the backend; nothing is opened until
+ :meth:`__aenter__`.
+
+ Args:
+ url (`str`):
+ A SQLAlchemy async URL (e.g.
+ ``sqlite+aiosqlite:///./as.db``,
+ ``postgresql+asyncpg://user:pw@host/db``). Ignored
+ when ``engine`` is provided.
+ create_tables (`bool`, defaults to `True`):
+ When `True`, run ``Base.metadata.create_all`` on the
+ engine at ``__aenter__`` — convenient for tests and
+ single-node dev deployments where Alembic overhead is
+ unwanted.
+ auto_migrate (`bool`, defaults to `False`):
+ When `True`, run ``alembic upgrade head`` against the
+ packaged migration scripts at ``__aenter__``.
+ Convenient for single-node or dev deployments —
+ every process boot brings the schema up to date.
+ **Not recommended for multi-replica production**:
+ two replicas racing on the same migration is unsafe.
+ In that setup keep this `False` and run
+ ``alembic upgrade head`` as a discrete deploy step.
+ engine (`AsyncEngine | None`, optional):
+ An externally managed engine. When supplied the
+ engine is used as-is and **not** disposed by
+ :meth:`aclose` — the caller owns its lifecycle. When
+ omitted an engine is constructed from *url* on
+ :meth:`__aenter__` and disposed on :meth:`aclose`.
+ engine_kwargs (`dict[str, Any] | None`, optional):
+ Extra keyword arguments forwarded to
+ :func:`sqlalchemy.ext.asyncio.create_async_engine`
+ when the engine is created internally (e.g.
+ ``echo=True``, ``pool_size=20``). Ignored when
+ *engine* is supplied.
+ """
+ self._url = url
+ self._create_tables = create_tables
+ self._auto_migrate = auto_migrate
+ self._external_engine: "AsyncEngine | None" = engine
+ self._engine_kwargs = engine_kwargs or {}
+
+ # Populated in __aenter__; None until the context is entered.
+ self._engine: "AsyncEngine | None" = None
+ self._owns_engine: bool = False
+ self._session_factory: "async_sessionmaker[AsyncSession] | None" = None
+
+ async def __aenter__(self) -> Self:
+ """Build the engine (or adopt the external one) and optionally
+ provision the schema.
+ """
+ from sqlalchemy.ext.asyncio import (
+ async_sessionmaker,
+ create_async_engine,
+ )
+
+ if self._external_engine is not None:
+ self._engine = self._external_engine
+ self._owns_engine = False
+ else:
+ self._engine = create_async_engine(
+ self._url,
+ future=True,
+ **self._engine_kwargs,
+ )
+ self._owns_engine = True
+ # SQLite ignores foreign keys unless enabled per connection,
+ # so ``ON DELETE CASCADE`` would silently not fire. Turn it
+ # on for every pooled connection.
+ if self._engine.sync_engine.dialect.name == "sqlite":
+ from sqlalchemy import event
+
+ @event.listens_for(self._engine.sync_engine, "connect")
+ def _enable_sqlite_fk(dbapi_conn: Any, _rec: Any) -> None:
+ cursor = dbapi_conn.cursor()
+ cursor.execute("PRAGMA foreign_keys=ON")
+ cursor.close()
+
+ # ``expire_on_commit=False`` keeps mapper-returned rows readable
+ # after commit — :class:`AsyncSQLAlchemyStorage` immediately
+ # projects rows back into pydantic records outside the session
+ # scope, which would otherwise trigger a detached-load error.
+ self._session_factory = async_sessionmaker(
+ self._engine,
+ expire_on_commit=False,
+ )
+
+ if self._auto_migrate:
+ await self._run_alembic_upgrade()
+
+ if self._create_tables:
+ async with self._engine.begin() as conn:
+ await conn.run_sync(_Base.metadata.create_all)
+
+ return self
+
+ async def _run_alembic_upgrade(self) -> None:
+ """Bring the schema up to ``head`` via the packaged Alembic scripts.
+
+ Runs :func:`alembic.command.upgrade` against the ``_alembic/``
+ directory shipped alongside this module, using the current
+ engine URL. Executed inside :meth:`asyncio.to_thread` because
+ Alembic's runtime is synchronous (it drives an async engine
+ internally via :func:`asyncio.run` inside its own ``env.py``,
+ so calling it from the main event loop would nest loops and
+ deadlock).
+
+ Failures propagate: a broken migration MUST prevent
+ ``__aenter__`` from returning a half-configured storage.
+ """
+ import asyncio
+ from pathlib import Path
+
+ from alembic import command
+ from alembic.config import Config
+
+ alembic_dir = Path(__file__).resolve().parent / "_alembic"
+ cfg = Config(str(alembic_dir / "alembic.ini"))
+ cfg.set_main_option("script_location", str(alembic_dir))
+ cfg.set_main_option("sqlalchemy.url", self._url)
+
+ await asyncio.to_thread(command.upgrade, cfg, "head")
+
+ async def aclose(self) -> None:
+ """Dispose the engine (if owned) and drop the session factory."""
+ if self._engine is not None and self._owns_engine:
+ await self._engine.dispose()
+ self._engine = None
+ self._session_factory = None
+
+ # ------------------------------------------------------------------
+ # Small helpers shared by every method
+ # ------------------------------------------------------------------
+
+ def _session(self) -> "AsyncSession":
+ """Return a fresh :class:`AsyncSession` from the factory.
+
+ Raises:
+ `RuntimeError`:
+ If the storage has not been entered as an async
+ context manager yet.
+ """
+ if self._session_factory is None:
+ raise RuntimeError(
+ "AsyncSQLAlchemyStorage is not initialised — use "
+ "`async with storage:`.",
+ )
+ return self._session_factory()
+
+ def _upsert_stmt(
+ self,
+ row_cls: type,
+ values: dict[str, Any],
+ conflict_cols: list[str],
+ update_cols: tuple[str, ...],
+ ) -> Any:
+ """Build a dialect-native atomic upsert statement.
+
+ A single ``INSERT ... ON CONFLICT DO UPDATE`` (Postgres /
+ SQLite) or ``INSERT ... ON DUPLICATE KEY UPDATE`` (MySQL /
+ MariaDB) makes the insert-or-update decision **inside the
+ database**, so concurrent writers on the same key can no longer
+ race between a separate read and write (the failure mode of the
+ classic read-then-write upsert). ``conflict_cols`` name the
+ key that triggers the "update instead" branch; ``update_cols``
+ are the columns overwritten on conflict — deliberately
+ excluding ``id`` / ``created_at`` so an upsert refreshes the
+ mutable columns while keeping the original creation time.
+
+ Args:
+ row_cls (`type`):
+ The concrete ``*Row`` class whose table is targeted.
+ values (`dict[str, Any]`):
+ Full column → value mapping for the INSERT branch.
+ conflict_cols (`list[str]`):
+ Column names forming the conflict target (the primary
+ key, or the composite key for messages).
+ update_cols (`tuple[str, ...]`):
+ Column names to overwrite on conflict.
+
+ Returns:
+ `Any`:
+ A dialect-specific insert construct ready to execute.
+
+ Raises:
+ `NotImplementedError`:
+ For dialects without a supported native upsert
+ (e.g. Oracle / SQL Server, which would need ``MERGE``).
+ """
+ assert self._engine is not None
+ dialect = self._engine.sync_engine.dialect.name
+ table = row_cls.__table__
+
+ if dialect in ("postgresql", "sqlite"):
+ if dialect == "postgresql":
+ from sqlalchemy.dialects.postgresql import insert as _insert
+ else:
+ from sqlalchemy.dialects.sqlite import insert as _insert
+ stmt = _insert(table).values(**values)
+ return stmt.on_conflict_do_update(
+ index_elements=conflict_cols,
+ set_={c: getattr(stmt.excluded, c) for c in update_cols},
+ )
+
+ if dialect in ("mysql", "mariadb"):
+ from sqlalchemy.dialects.mysql import insert as _insert
+
+ stmt = _insert(table).values(**values)
+ return stmt.on_duplicate_key_update(
+ **{c: getattr(stmt.inserted, c) for c in update_cols},
+ )
+
+ raise NotImplementedError(
+ f"Atomic upsert is unsupported for the {dialect!r} dialect; "
+ "supported dialects are postgresql, sqlite, mysql, mariadb.",
+ )
+
+ async def _write_row(
+ self,
+ row_cls: type,
+ record: Any,
+ *,
+ preserve_created_at: bool = True,
+ ) -> Any:
+ """Atomically insert-or-update *record* via *row_cls*.
+
+ Issues a single dialect-native upsert (see :meth:`_upsert_stmt`)
+ so the insert-or-update decision is made inside the database —
+ an upsert refreshes ``updated_at`` / ``payload`` / the indexed
+ columns but keeps the original ``created_at`` (mirroring the
+ Redis backend's semantics). Because the statement is atomic
+ there is no read-then-write window for concurrent writers on
+ the same id to race through. Returns the (mutated) record so
+ callers can propagate the stamped timestamps.
+
+ Args:
+ row_cls (`type`):
+ Concrete ``*Row`` class from :mod:`_tables`.
+ record (`Any`):
+ The pydantic record to write. Mutated in place —
+ ``created_at`` / ``updated_at`` are refreshed.
+ preserve_created_at (`bool`, defaults to `True`):
+ When `True`, the DB-authoritative ``created_at`` (the
+ original one on an update) is read back into the
+ returned record. Set `False` on pure-create paths
+ where no prior row can exist, to skip that read.
+
+ Returns:
+ `Any`:
+ The (mutated) record.
+ """
+ from sqlalchemy import select
+
+ record.updated_at = _utcnow()
+ # The record default uses machine-local ``datetime.now()``;
+ # anchor it to UTC for the INSERT branch. On conflict the DB
+ # keeps its stored ``created_at`` and we read it back below.
+ record.created_at = _to_naive_utc(record.created_at)
+ new_row = _from_record(row_cls, record)
+ indexed = tuple(row_cls.get_indexed_fields())
+ values = {
+ col: getattr(new_row, col)
+ for col in ("id", "created_at", "updated_at", "payload") + indexed
+ }
+ update_cols = ("updated_at", "payload") + indexed
+
+ async with self._session() as sess:
+ await sess.execute(
+ self._upsert_stmt(row_cls, values, ["id"], update_cols),
+ )
+ if preserve_created_at:
+ # Reflect the creation time the DB kept on conflict
+ # back into the returned record.
+ record.created_at = (
+ await sess.execute(
+ select(row_cls.created_at).where(
+ row_cls.id == record.id,
+ ),
+ )
+ ).scalar_one()
+ await sess.commit()
+ return record
+
+ # ------------------------------------------------------------------
+ # Cascade-delete internals — every ``_delete_*_impl`` runs on the
+ # caller-supplied ``sess`` WITHOUT committing, so the public
+ # ``delete_*`` wrapper can run the whole (mutually recursive)
+ # cascade inside a single transaction and commit once. That makes
+ # the compound deletes atomic: a mid-cascade failure rolls the
+ # whole thing back instead of leaving a half-deleted graph.
+ #
+ # Reads use ``select`` (not ``session.get``) so they see the
+ # transaction's own uncommitted deletes instead of stale
+ # identity-map rows.
+ # ------------------------------------------------------------------
+
+ async def _delete_session_impl(
+ self,
+ sess: "AsyncSession",
+ user_id: str,
+ agent_id: str,
+ session_id: str,
+ ) -> bool:
+ """Delete a session (+ its messages, + leader-team) on *sess*."""
+ from sqlalchemy import delete, select
+
+ _ = agent_id # scoping enforced by caller
+ row = (
+ await sess.execute(
+ select(SessionRow).where(SessionRow.id == session_id),
+ )
+ ).scalar_one_or_none()
+ if row is None or row.user_id != user_id:
+ return False
+ record = _to_record(row, SessionRecord)
+
+ # If this session leads a team, dissolve the team first.
+ if record.team_id:
+ team_row = (
+ await sess.execute(
+ select(TeamRow).where(TeamRow.id == record.team_id),
+ )
+ ).scalar_one_or_none()
+ if (
+ team_row is not None
+ and _to_record(team_row, TeamRecord).session_id == session_id
+ ):
+ await self._delete_team_impl(sess, user_id, record.team_id)
+
+ await sess.execute(
+ delete(SessionRow).where(
+ SessionRow.id == session_id,
+ SessionRow.user_id == user_id,
+ ),
+ )
+ await sess.execute(
+ delete(MessageRow).where(MessageRow.session_id == session_id),
+ )
+ return True
+
+ async def _delete_schedule_impl(
+ self,
+ sess: "AsyncSession",
+ user_id: str,
+ schedule_id: str,
+ ) -> bool:
+ """Delete a schedule + cascade its execution sessions on *sess*."""
+ from sqlalchemy import delete, select
+
+ row = (
+ await sess.execute(
+ select(ScheduleRow).where(ScheduleRow.id == schedule_id),
+ )
+ ).scalar_one_or_none()
+ if row is None or row.user_id != user_id:
+ return False
+
+ sessions = (
+ (
+ await sess.execute(
+ select(SessionRow).where(
+ SessionRow.user_id == user_id,
+ SessionRow.source_schedule_id == schedule_id,
+ ),
+ )
+ )
+ .scalars()
+ .all()
+ )
+ for s in sessions:
+ await self._delete_session_impl(sess, user_id, s.agent_id, s.id)
+
+ await sess.execute(
+ delete(ScheduleRow).where(
+ ScheduleRow.id == schedule_id,
+ ScheduleRow.user_id == user_id,
+ ),
+ )
+ return True
+
+ async def _delete_agent_impl(
+ self,
+ sess: "AsyncSession",
+ user_id: str,
+ agent_id: str,
+ ) -> bool:
+ """Delete an agent + cascade sessions/schedules/team refs on *sess*."""
+ from sqlalchemy import delete, select, update
+
+ # Cascade: sessions owned by this agent.
+ sessions = (
+ (
+ await sess.execute(
+ select(SessionRow).where(
+ SessionRow.user_id == user_id,
+ SessionRow.agent_id == agent_id,
+ ),
+ )
+ )
+ .scalars()
+ .all()
+ )
+ for s in sessions:
+ await self._delete_session_impl(sess, user_id, agent_id, s.id)
+
+ # Cascade: schedules owned by this agent.
+ schedules = (
+ (
+ await sess.execute(
+ select(ScheduleRow).where(
+ ScheduleRow.user_id == user_id,
+ ScheduleRow.agent_id == agent_id,
+ ),
+ )
+ )
+ .scalars()
+ .all()
+ )
+ for sch in schedules:
+ await self._delete_schedule_impl(sess, user_id, sch.id)
+
+ # Defensive: scrub the agent from every surviving team roster
+ # (both the legacy ``member_ids`` list and the ``members`` list).
+ teams = (
+ (
+ await sess.execute(
+ select(TeamRow).where(TeamRow.user_id == user_id),
+ )
+ )
+ .scalars()
+ .all()
+ )
+ for team_row in teams:
+ team_rec = _to_record(team_row, TeamRecord)
+ dirty = False
+ if agent_id in team_rec.data.member_ids:
+ team_rec.data.member_ids = [
+ m for m in team_rec.data.member_ids if m != agent_id
+ ]
+ dirty = True
+ filtered = [
+ m for m in team_rec.data.members if m.agent_id != agent_id
+ ]
+ if len(filtered) != len(team_rec.data.members):
+ team_rec.data.members = filtered
+ dirty = True
+ if dirty:
+ team_rec.updated_at = _utcnow()
+ new_row = _from_record(TeamRow, team_rec)
+ await sess.execute(
+ update(TeamRow)
+ .where(TeamRow.id == team_rec.id)
+ .values(
+ payload=new_row.payload,
+ updated_at=new_row.updated_at,
+ ),
+ )
+
+ result = await sess.execute(
+ delete(AgentRow).where(
+ AgentRow.id == agent_id,
+ AgentRow.user_id == user_id,
+ ),
+ )
+ return result.rowcount > 0
+
+ async def _delete_team_impl(
+ self,
+ sess: "AsyncSession",
+ user_id: str,
+ team_id: str,
+ ) -> bool:
+ """Role-aware team cleanup + leader-detach + delete, on *sess*."""
+ from sqlalchemy import delete, select, update
+
+ from .._model._team import TeamMember
+
+ row = (
+ await sess.execute(
+ select(TeamRow).where(TeamRow.id == team_id),
+ )
+ ).scalar_one_or_none()
+ if row is None or row.user_id != user_id:
+ return False
+ team = _to_record(row, TeamRecord)
+
+ # Materialise the roster. Modern records carry ``members``;
+ # legacy ``member_ids``-only records are migrated inline on
+ # *sess* (no writeback — the team is being deleted anyway).
+ members = list(team.data.members)
+ if not members and team.data.member_ids:
+ for aid in team.data.member_ids:
+ worker = (
+ (
+ await sess.execute(
+ select(SessionRow).where(
+ SessionRow.user_id == user_id,
+ SessionRow.agent_id == aid,
+ ),
+ )
+ )
+ .scalars()
+ .first()
+ )
+ if worker is not None:
+ members.append(
+ TeamMember(
+ owner_id=user_id,
+ agent_id=aid,
+ session_id=worker.id,
+ role="created",
+ ),
+ )
+
+ for member in members:
+ if member.role == "created":
+ await self._delete_agent_impl(
+ sess,
+ member.owner_id,
+ member.agent_id,
+ )
+ else: # invited — only the borrowed session goes
+ await self._delete_session_impl(
+ sess,
+ member.owner_id,
+ member.agent_id,
+ member.session_id,
+ )
+
+ # Detach the leader session (idempotent — it may already be gone).
+ await sess.execute(
+ update(SessionRow)
+ .where(
+ SessionRow.id == team.session_id,
+ SessionRow.user_id == user_id,
+ )
+ .values(team_id=None, updated_at=_utcnow()),
+ )
+ await sess.execute(
+ delete(TeamRow).where(
+ TeamRow.id == team_id,
+ TeamRow.user_id == user_id,
+ ),
+ )
+ return True
+
+ # ------------------------------------------------------------------
+ # Credentials
+ # ------------------------------------------------------------------
+
+ async def _generate_credential_name(
+ self,
+ user_id: str,
+ credential_data: CredentialBase,
+ ) -> str:
+ """Mirror :meth:`RedisStorage._generate_credential_name`.
+
+ Auto-derive a unique per-user display name from the
+ credential type (``"OpenAI"``, ``"OpenAI (2)"`` …) so the
+ service layer does not have to.
+ """
+ cred_type = getattr(credential_data, "type", "")
+ base_name = (
+ cred_type.removesuffix("_credential").replace("_", " ").title()
+ )
+ if not base_name:
+ base_name = "Credential"
+
+ existing = await self.list_credentials(user_id)
+ same_type_names = [
+ c.data.get("name", "")
+ for c in existing
+ if c.data.get("type") == cred_type and c.id != credential_data.id
+ ]
+
+ if base_name not in same_type_names:
+ return base_name
+
+ idx = 2
+ while f"{base_name} ({idx})" in same_type_names:
+ idx += 1
+ return f"{base_name} ({idx})"
+
+ async def upsert_credential(
+ self,
+ user_id: str,
+ credential_data: CredentialBase,
+ ) -> str:
+ """Create or update a credential record for *user_id*.
+
+ Same contract as :meth:`RedisStorage.upsert_credential` — see
+ that method for the id / name handling rules.
+ """
+ if not credential_data.name:
+ credential_data.name = await self._generate_credential_name(
+ user_id,
+ credential_data,
+ )
+
+ data_dump = _dump_with_secrets(credential_data)
+
+ # Scope the create-or-update to *user_id*. The Redis backend
+ # namespaces its keys by user, so a preset id can only ever
+ # address the caller's own record; the SQL table keys on a
+ # global primary id, so an unscoped lookup — or the id-keyed
+ # upsert in ``_write_row`` — would let a caller read or clobber
+ # another tenant's credential. Only a row the caller already
+ # owns may be updated in place; any other preset id is created
+ # fresh under the caller via a plain INSERT that fails loudly on
+ # a global collision instead of overwriting the holder.
+ if credential_data.id:
+ from sqlalchemy import select
+
+ async with self._session() as sess:
+ existing = (
+ await sess.execute(
+ select(CredentialRow).where(
+ CredentialRow.id == credential_data.id,
+ CredentialRow.user_id == user_id,
+ ),
+ )
+ ).scalar_one_or_none()
+ if existing is not None:
+ # The caller's own credential: the id-keyed upsert can
+ # only touch this row, so update it in place.
+ record = _to_record(existing, CredentialRecord)
+ record.data = data_dump
+ await self._write_row(CredentialRow, record)
+ return record.id
+ record = CredentialRecord(
+ id=credential_data.id,
+ user_id=user_id,
+ data=data_dump,
+ )
+ else:
+ record = CredentialRecord(
+ user_id=user_id,
+ data=data_dump,
+ )
+
+ # A fresh record — either a server-generated id (never collides)
+ # or a caller-preset id proven not to belong to the caller. Use
+ # a plain INSERT rather than the id-conflict upsert so that a
+ # preset id already held by another tenant raises IntegrityError
+ # instead of silently overwriting them and stealing ownership.
+ record.created_at = _to_naive_utc(record.created_at)
+ record.updated_at = _utcnow()
+ async with self._session() as sess:
+ sess.add(_from_record(CredentialRow, record))
+ await sess.commit()
+ return record.id
+
+ async def list_credentials(self, user_id: str) -> list[CredentialRecord]:
+ """Return every credential for *user_id*."""
+ from sqlalchemy import select
+
+ async with self._session() as sess:
+ rows = (
+ (
+ await sess.execute(
+ select(CredentialRow).where(
+ CredentialRow.user_id == user_id,
+ ),
+ )
+ )
+ .scalars()
+ .all()
+ )
+ return [_to_record(r, CredentialRecord) for r in rows]
+
+ async def get_credential(
+ self,
+ user_id: str,
+ credential_id: str,
+ ) -> CredentialRecord | None:
+ """Fetch one credential record; enforces owner-scoping."""
+ async with self._session() as sess:
+ row = await sess.get(CredentialRow, credential_id)
+ if row is None or row.user_id != user_id:
+ return None
+ return _to_record(row, CredentialRecord)
+
+ async def delete_credential(
+ self,
+ user_id: str,
+ credential_id: str,
+ ) -> bool:
+ """Delete a credential; owner-scoped."""
+ from sqlalchemy import delete
+
+ async with self._session() as sess:
+ result = await sess.execute(
+ delete(CredentialRow).where(
+ CredentialRow.id == credential_id,
+ CredentialRow.user_id == user_id,
+ ),
+ )
+ await sess.commit()
+ return result.rowcount > 0
+
+ # ------------------------------------------------------------------
+ # Agents
+ # ------------------------------------------------------------------
+
+ async def upsert_agent(
+ self,
+ user_id: str,
+ agent_record: AgentRecord,
+ ) -> str:
+ """Persist an agent record (create or overwrite)."""
+ _ = user_id # scoping is enforced by the caller
+ await self._write_row(AgentRow, agent_record)
+ return agent_record.id
+
+ async def list_agents(self, user_id: str) -> list[AgentRecord]:
+ """Return the user's ``source='user'`` agents only.
+
+ Matches :meth:`RedisStorage.list_agents` — team-spawned
+ workers (``source='team'``) are addressable by id but are
+ never enumerated in the user's regular agent list.
+ """
+ from sqlalchemy import select
+
+ async with self._session() as sess:
+ rows = (
+ (
+ await sess.execute(
+ select(AgentRow).where(
+ AgentRow.user_id == user_id,
+ AgentRow.source == "user",
+ ),
+ )
+ )
+ .scalars()
+ .all()
+ )
+ return [_to_record(r, AgentRecord) for r in rows]
+
+ async def get_agent(
+ self,
+ user_id: str,
+ agent_id: str,
+ ) -> AgentRecord | None:
+ """Fetch one agent record; owner-scoped."""
+ async with self._session() as sess:
+ row = await sess.get(AgentRow, agent_id)
+ if row is None or row.user_id != user_id:
+ return None
+ return _to_record(row, AgentRecord)
+
+ async def delete_agent(self, user_id: str, agent_id: str) -> bool:
+ """Delete an agent + cascade sessions, schedules, team refs.
+
+ Mirrors the cascade order of :meth:`RedisStorage.delete_agent`.
+ The whole cascade runs in a **single transaction** (see
+ :meth:`_delete_agent_impl`) so it is atomic — a mid-cascade
+ failure rolls back rather than leaving orphans.
+ """
+ async with self._session() as sess:
+ ok = await self._delete_agent_impl(sess, user_id, agent_id)
+ await sess.commit()
+ return ok
+
+ # ------------------------------------------------------------------
+ # Sessions
+ # ------------------------------------------------------------------
+
+ async def upsert_session(
+ self,
+ user_id: str,
+ agent_id: str,
+ config: SessionConfig,
+ state: AgentState | None = None,
+ session_id: str | None = None,
+ source: SessionSource = SessionSource.USER,
+ source_schedule_id: str | None = None,
+ ) -> SessionRecord:
+ """Create or update a session — same shape as the Redis backend."""
+ if session_id:
+ async with self._session() as sess:
+ existing = await sess.get(SessionRow, session_id)
+ if existing is not None:
+ record = _to_record(existing, SessionRecord)
+ record.config = config
+ if state is not None:
+ record.state = state
+ await self._write_row(SessionRow, record)
+ return record
+
+ new_id_kwargs = {"id": session_id} if session_id else {}
+ record = SessionRecord(
+ user_id=user_id,
+ agent_id=agent_id,
+ config=config,
+ source=source,
+ source_schedule_id=source_schedule_id,
+ state=state if state is not None else AgentState(),
+ **new_id_kwargs,
+ )
+ await self._write_row(
+ SessionRow,
+ record,
+ preserve_created_at=False,
+ )
+ return record
+
+ async def set_session_team_id(
+ self,
+ user_id: str,
+ session_id: str,
+ team_id: str | None,
+ ) -> None:
+ """Single-column UPDATE — atomic on all supported dialects."""
+ from sqlalchemy import update
+
+ now = _utcnow()
+ async with self._session() as sess:
+ await sess.execute(
+ update(SessionRow)
+ .where(
+ SessionRow.id == session_id,
+ SessionRow.user_id == user_id,
+ )
+ .values(team_id=team_id, updated_at=now),
+ )
+ await sess.commit()
+
+ async def update_session_state(
+ self,
+ user_id: str,
+ agent_id: str,
+ session_id: str,
+ state: AgentState,
+ ) -> None:
+ """Read-modify-write on the payload; raises if absent."""
+ _ = user_id, agent_id # scoping enforced by caller
+ async with self._session() as sess:
+ row = await sess.get(SessionRow, session_id)
+ if row is None:
+ raise KeyError(f"Session {session_id!r} not found.")
+ record = _to_record(row, SessionRecord)
+ record.state = state
+ record.updated_at = _utcnow()
+ new_row = _from_record(SessionRow, record)
+ row.payload = new_row.payload
+ row.updated_at = new_row.updated_at
+ await sess.commit()
+
+ async def list_sessions(
+ self,
+ user_id: str,
+ agent_id: str,
+ ) -> list[SessionRecord]:
+ """Sessions for a (user, agent) pair — newest first."""
+ from sqlalchemy import select
+
+ async with self._session() as sess:
+ rows = (
+ (
+ await sess.execute(
+ select(SessionRow)
+ .where(
+ SessionRow.user_id == user_id,
+ SessionRow.agent_id == agent_id,
+ )
+ .order_by(SessionRow.created_at.desc()),
+ )
+ )
+ .scalars()
+ .all()
+ )
+ return [_to_record(r, SessionRecord) for r in rows]
+
+ async def get_session(
+ self,
+ user_id: str,
+ agent_id: str,
+ session_id: str,
+ ) -> SessionRecord | None:
+ """One session; owner-scoped."""
+ _ = agent_id
+ async with self._session() as sess:
+ row = await sess.get(SessionRow, session_id)
+ if row is None or row.user_id != user_id:
+ return None
+ return _to_record(row, SessionRecord)
+
+ async def delete_session(
+ self,
+ user_id: str,
+ agent_id: str,
+ session_id: str,
+ ) -> bool:
+ """Delete a session + cascade (message log, leader-team).
+
+ Atomic: the session row, its message log, and any team it leads
+ are removed inside one transaction (see
+ :meth:`_delete_session_impl`).
+ """
+ async with self._session() as sess:
+ ok = await self._delete_session_impl(
+ sess,
+ user_id,
+ agent_id,
+ session_id,
+ )
+ await sess.commit()
+ return ok
+
+ async def list_sessions_by_schedule(
+ self,
+ user_id: str,
+ schedule_id: str,
+ ) -> list[SessionRecord]:
+ """Sessions created by *schedule_id* — newest first."""
+ from sqlalchemy import select
+
+ async with self._session() as sess:
+ rows = (
+ (
+ await sess.execute(
+ select(SessionRow)
+ .where(
+ SessionRow.user_id == user_id,
+ SessionRow.source_schedule_id == schedule_id,
+ )
+ .order_by(SessionRow.created_at.desc()),
+ )
+ )
+ .scalars()
+ .all()
+ )
+ return [_to_record(r, SessionRecord) for r in rows]
+
+ # ------------------------------------------------------------------
+ # Schedules
+ # ------------------------------------------------------------------
+
+ async def upsert_schedule(
+ self,
+ user_id: str,
+ record: ScheduleRecord,
+ ) -> str:
+ """Persist a schedule record (create or overwrite)."""
+ _ = user_id
+ await self._write_row(ScheduleRow, record)
+ return record.id
+
+ async def get_schedule(
+ self,
+ user_id: str,
+ schedule_id: str,
+ ) -> ScheduleRecord | None:
+ """One schedule; owner-scoped."""
+ async with self._session() as sess:
+ row = await sess.get(ScheduleRow, schedule_id)
+ if row is None or row.user_id != user_id:
+ return None
+ return _to_record(row, ScheduleRecord)
+
+ async def list_schedules(
+ self,
+ user_id: str,
+ ) -> list[ScheduleRecord]:
+ """All schedules for a user."""
+ from sqlalchemy import select
+
+ async with self._session() as sess:
+ rows = (
+ (
+ await sess.execute(
+ select(ScheduleRow).where(
+ ScheduleRow.user_id == user_id,
+ ),
+ )
+ )
+ .scalars()
+ .all()
+ )
+ return [_to_record(r, ScheduleRecord) for r in rows]
+
+ async def delete_schedule(
+ self,
+ user_id: str,
+ schedule_id: str,
+ ) -> bool:
+ """Delete a schedule + cascade its execution sessions.
+
+ Atomic: the schedule and every session it spawned are removed
+ in one transaction (see :meth:`_delete_schedule_impl`).
+ """
+ async with self._session() as sess:
+ ok = await self._delete_schedule_impl(sess, user_id, schedule_id)
+ await sess.commit()
+ return ok
+
+ async def list_all_schedules(self) -> list[ScheduleRecord]:
+ """Every schedule across every user (used on startup)."""
+ from sqlalchemy import select
+
+ async with self._session() as sess:
+ rows = (await sess.execute(select(ScheduleRow))).scalars().all()
+ return [_to_record(r, ScheduleRecord) for r in rows]
+
+ # ------------------------------------------------------------------
+ # Messages
+ # ------------------------------------------------------------------
+
+ async def upsert_message(
+ self,
+ user_id: str,
+ session_id: str,
+ msg: Msg,
+ ) -> None:
+ """Insert-or-update by ``(session_id, msg_id)``.
+
+ Semantics mirror :meth:`RedisStorage.upsert_message` closely
+ enough for every caller in the codebase — the Redis version
+ only replaces when the *tail* message matches; ours replaces
+ on any matching id. The looser rule is safe because callers
+ never reuse a message id across turns.
+ """
+ _ = user_id # scoping enforced by caller
+ now = _utcnow()
+ payload = msg.model_dump(mode="json")
+
+ # Atomic upsert on the composite key: insert a new event, or
+ # replace the payload of an existing ``(session_id, msg_id)``
+ # while keeping its original ``created_at``.
+ values = {
+ "session_id": session_id,
+ "msg_id": msg.id,
+ "created_at": now,
+ "payload": payload,
+ }
+ async with self._session() as sess:
+ await sess.execute(
+ self._upsert_stmt(
+ MessageRow,
+ values,
+ ["session_id", "msg_id"],
+ ("payload",),
+ ),
+ )
+ await sess.commit()
+
+ async def get_message(
+ self,
+ user_id: str,
+ session_id: str,
+ message_id: str,
+ ) -> Msg | None:
+ """Fetch a message by ``(session_id, msg_id)``."""
+ _ = user_id
+ from sqlalchemy import select
+
+ async with self._session() as sess:
+ row = (
+ await sess.execute(
+ select(MessageRow).where(
+ MessageRow.session_id == session_id,
+ MessageRow.msg_id == message_id,
+ ),
+ )
+ ).scalar_one_or_none()
+ if row is None:
+ return None
+ return Msg.model_validate(row.payload)
+
+ async def list_messages(
+ self,
+ user_id: str,
+ session_id: str,
+ limit: int = 50,
+ before: str | None = None,
+ **kwargs: Any,
+ ) -> tuple[list[Msg], bool]:
+ """Most-recent page of messages with cursor-based pagination.
+
+ Mirrors :meth:`RedisStorage.list_messages`: returns up to
+ *limit* messages in chronological order together with a
+ ``has_more`` flag that is ``True`` when older messages exist
+ before the returned page. ``before`` is a message-id cursor —
+ when set, the page ends just before that message; when ``None``
+ the latest page is returned. An unknown ``before`` cursor
+ yields ``([], False)``. The legacy ``offset`` keyword is
+ accepted only to emit a ``DeprecationWarning`` and is ignored.
+ """
+ _ = user_id # scoping enforced by caller
+ if "offset" in kwargs:
+ import warnings
+
+ warnings.warn(
+ "The 'offset' parameter is deprecated and will be "
+ "removed in a future version. Use 'before' for "
+ "cursor-based pagination instead.",
+ DeprecationWarning,
+ stacklevel=2,
+ )
+
+ from sqlalchemy import and_, or_, select
+
+ async with self._session() as sess:
+ cond = MessageRow.session_id == session_id
+ if before is not None:
+ cursor = (
+ await sess.execute(
+ select(
+ MessageRow.created_at,
+ MessageRow.msg_id,
+ ).where(
+ MessageRow.session_id == session_id,
+ MessageRow.msg_id == before,
+ ),
+ )
+ ).one_or_none()
+ if cursor is None:
+ return [], False
+ c_created, c_msg = cursor
+ # Strictly older than the cursor in the
+ # ``(created_at, msg_id)`` total order (spelled out
+ # rather than a row-value comparison so every dialect
+ # can plan it).
+ cond = and_(
+ cond,
+ or_(
+ MessageRow.created_at < c_created,
+ and_(
+ MessageRow.created_at == c_created,
+ MessageRow.msg_id < c_msg,
+ ),
+ ),
+ )
+
+ # Fetch newest-first with one extra row to detect whether
+ # older messages remain, then flip back to chronological.
+ rows = list(
+ (
+ await sess.execute(
+ select(MessageRow)
+ .where(cond)
+ .order_by(
+ MessageRow.created_at.desc(),
+ MessageRow.msg_id.desc(),
+ )
+ .limit(limit + 1),
+ )
+ )
+ .scalars()
+ .all(),
+ )
+
+ has_more = len(rows) > limit
+ page = rows[:limit]
+ page.reverse()
+ return [Msg.model_validate(r.payload) for r in page], has_more
+
+ # ------------------------------------------------------------------
+ # Teams
+ # ------------------------------------------------------------------
+
+ async def upsert_team(
+ self,
+ user_id: str,
+ record: TeamRecord,
+ ) -> TeamRecord:
+ """Persist a team record."""
+ _ = user_id
+ await self._write_row(TeamRow, record)
+ return record
+
+ async def get_team(
+ self,
+ user_id: str,
+ team_id: str,
+ ) -> TeamRecord | None:
+ """One team; owner-scoped."""
+ async with self._session() as sess:
+ row = await sess.get(TeamRow, team_id)
+ if row is None or row.user_id != user_id:
+ return None
+ return _to_record(row, TeamRecord)
+
+ async def list_teams(self, user_id: str) -> list[TeamRecord]:
+ """All teams for a user."""
+ from sqlalchemy import select
+
+ async with self._session() as sess:
+ rows = (
+ (
+ await sess.execute(
+ select(TeamRow).where(TeamRow.user_id == user_id),
+ )
+ )
+ .scalars()
+ .all()
+ )
+ return [_to_record(r, TeamRecord) for r in rows]
+
+ async def delete_team(self, user_id: str, team_id: str) -> bool:
+ """Delete a team + role-aware member cleanup.
+
+ Atomic: the role-aware member teardown (``created`` members
+ deleted whole, ``invited`` members losing only their borrowed
+ session), the leader detach, and the team-record delete all run
+ in one transaction (see :meth:`_delete_team_impl`).
+ """
+ async with self._session() as sess:
+ ok = await self._delete_team_impl(sess, user_id, team_id)
+ await sess.commit()
+ return ok
+
+ # ------------------------------------------------------------------
+ # Knowledge bases
+ # ------------------------------------------------------------------
+
+ async def upsert_knowledge_base(
+ self,
+ user_id: str,
+ record: KnowledgeBaseRecord,
+ ) -> KnowledgeBaseRecord:
+ """Create or update a KB record; enforces ``record.user_id``."""
+ if record.user_id != user_id:
+ raise ValueError(
+ "record.user_id does not match the given user_id.",
+ )
+ await self._write_row(KnowledgeBaseRow, record)
+ return record
+
+ async def get_knowledge_base(
+ self,
+ user_id: str,
+ knowledge_base_id: str,
+ ) -> KnowledgeBaseRecord | None:
+ """One KB; owner-scoped."""
+ async with self._session() as sess:
+ row = await sess.get(KnowledgeBaseRow, knowledge_base_id)
+ if row is None or row.user_id != user_id:
+ return None
+ return _to_record(row, KnowledgeBaseRecord)
+
+ async def list_knowledge_bases(
+ self,
+ user_id: str,
+ ) -> list[KnowledgeBaseRecord]:
+ """All KBs for a user."""
+ from sqlalchemy import select
+
+ async with self._session() as sess:
+ rows = (
+ (
+ await sess.execute(
+ select(KnowledgeBaseRow).where(
+ KnowledgeBaseRow.user_id == user_id,
+ ),
+ )
+ )
+ .scalars()
+ .all()
+ )
+ return [_to_record(r, KnowledgeBaseRecord) for r in rows]
+
+ async def delete_knowledge_base(
+ self,
+ user_id: str,
+ knowledge_base_id: str,
+ ) -> bool:
+ """Delete a KB; its document rows cascade natively via the
+ ``knowledge_documents.knowledge_base_id`` foreign key
+ (``ON DELETE CASCADE``) — one atomic statement, no app loop.
+ """
+ from sqlalchemy import delete
+
+ async with self._session() as sess:
+ result = await sess.execute(
+ delete(KnowledgeBaseRow).where(
+ KnowledgeBaseRow.id == knowledge_base_id,
+ KnowledgeBaseRow.user_id == user_id,
+ ),
+ )
+ await sess.commit()
+ return result.rowcount > 0
+
+ # ------------------------------------------------------------------
+ # Knowledge documents
+ # ------------------------------------------------------------------
+
+ async def upsert_knowledge_document(
+ self,
+ user_id: str,
+ record: KnowledgeDocumentRecord,
+ ) -> KnowledgeDocumentRecord:
+ """Create or update a document record; enforces ``record.user_id``."""
+ if record.user_id != user_id:
+ raise ValueError(
+ "record.user_id does not match the given user_id.",
+ )
+ await self._write_row(KnowledgeDocumentRow, record)
+ return record
+
+ async def get_knowledge_document(
+ self,
+ user_id: str,
+ knowledge_base_id: str,
+ document_id: str,
+ ) -> KnowledgeDocumentRecord | None:
+ """One document; owner + KB scoped."""
+ async with self._session() as sess:
+ row = await sess.get(KnowledgeDocumentRow, document_id)
+ if (
+ row is None
+ or row.user_id != user_id
+ or row.knowledge_base_id != knowledge_base_id
+ ):
+ return None
+ return _to_record(row, KnowledgeDocumentRecord)
+
+ async def list_knowledge_documents(
+ self,
+ user_id: str,
+ knowledge_base_id: str,
+ ) -> list[KnowledgeDocumentRecord]:
+ """All documents inside a KB."""
+ from sqlalchemy import select
+
+ async with self._session() as sess:
+ rows = (
+ (
+ await sess.execute(
+ select(KnowledgeDocumentRow).where(
+ KnowledgeDocumentRow.user_id == user_id,
+ KnowledgeDocumentRow.knowledge_base_id
+ == knowledge_base_id,
+ ),
+ )
+ )
+ .scalars()
+ .all()
+ )
+ return [_to_record(r, KnowledgeDocumentRecord) for r in rows]
+
+ async def delete_knowledge_document(
+ self,
+ user_id: str,
+ knowledge_base_id: str,
+ document_id: str,
+ ) -> bool:
+ """Delete one document."""
+ from sqlalchemy import delete
+
+ async with self._session() as sess:
+ result = await sess.execute(
+ delete(KnowledgeDocumentRow).where(
+ KnowledgeDocumentRow.id == document_id,
+ KnowledgeDocumentRow.user_id == user_id,
+ KnowledgeDocumentRow.knowledge_base_id
+ == knowledge_base_id,
+ ),
+ )
+ await sess.commit()
+ return result.rowcount > 0
+
+ async def update_knowledge_document_status(
+ self,
+ user_id: str,
+ knowledge_base_id: str,
+ document_id: str,
+ status: KnowledgeDocumentStatus,
+ error: str | None = None,
+ chunk_count: int | None = None,
+ ) -> None:
+ """Fast-path column UPDATE + optional payload rewrite.
+
+ ``status`` is a column so it can always be updated with a
+ single ``UPDATE``; ``error`` / ``chunk_count`` live inside
+ :attr:`~KnowledgeDocumentRecord.data` (payload) and therefore
+ require the classic read-modify-write when they change.
+ """
+ async with self._session() as sess:
+ row = await sess.get(KnowledgeDocumentRow, document_id)
+ if (
+ row is None
+ or row.user_id != user_id
+ or row.knowledge_base_id != knowledge_base_id
+ ):
+ return
+ record = _to_record(row, KnowledgeDocumentRecord)
+ record.status = status
+ if error is not None:
+ record.data.error = error
+ if chunk_count is not None:
+ record.data.chunk_count = chunk_count
+ record.updated_at = _utcnow()
+ new_row = _from_record(KnowledgeDocumentRow, record)
+ row.status = new_row.status
+ row.payload = new_row.payload
+ row.updated_at = new_row.updated_at
+ await sess.commit()
+
+ async def acquire_knowledge_document_lease(
+ self,
+ user_id: str,
+ knowledge_base_id: str,
+ document_id: str,
+ processing_node: str,
+ lease_ttl: timedelta,
+ now: datetime | None = None,
+ ) -> bool:
+ """Conditional UPDATE — atomic on every supported dialect."""
+ from sqlalchemy import or_, update
+
+ now = _to_naive_utc(now) if now is not None else _utcnow()
+ deadline = now + lease_ttl
+
+ async with self._session() as sess:
+ result = await sess.execute(
+ update(KnowledgeDocumentRow)
+ .where(
+ KnowledgeDocumentRow.id == document_id,
+ KnowledgeDocumentRow.user_id == user_id,
+ KnowledgeDocumentRow.knowledge_base_id
+ == knowledge_base_id,
+ or_(
+ KnowledgeDocumentRow.processing_node.is_(None),
+ KnowledgeDocumentRow.lease_expires_at.is_(None),
+ KnowledgeDocumentRow.lease_expires_at < now,
+ ),
+ )
+ .values(
+ processing_node=processing_node,
+ lease_expires_at=deadline,
+ updated_at=now,
+ ),
+ )
+ await sess.commit()
+ return result.rowcount > 0
+
+ async def renew_knowledge_document_lease(
+ self,
+ user_id: str,
+ knowledge_base_id: str,
+ document_id: str,
+ processing_node: str,
+ lease_ttl: timedelta,
+ now: datetime | None = None,
+ ) -> bool:
+ """Conditional UPDATE constrained to the current holder."""
+ from sqlalchemy import update
+
+ now = _to_naive_utc(now) if now is not None else _utcnow()
+ deadline = now + lease_ttl
+
+ async with self._session() as sess:
+ result = await sess.execute(
+ update(KnowledgeDocumentRow)
+ .where(
+ KnowledgeDocumentRow.id == document_id,
+ KnowledgeDocumentRow.user_id == user_id,
+ KnowledgeDocumentRow.knowledge_base_id
+ == knowledge_base_id,
+ KnowledgeDocumentRow.processing_node == processing_node,
+ )
+ .values(lease_expires_at=deadline, updated_at=now),
+ )
+ await sess.commit()
+ return result.rowcount > 0
+
+ async def release_knowledge_document_lease(
+ self,
+ user_id: str,
+ knowledge_base_id: str,
+ document_id: str,
+ processing_node: str,
+ ) -> None:
+ """Conditional UPDATE constrained to the current holder."""
+ from sqlalchemy import update
+
+ now = _utcnow()
+ async with self._session() as sess:
+ await sess.execute(
+ update(KnowledgeDocumentRow)
+ .where(
+ KnowledgeDocumentRow.id == document_id,
+ KnowledgeDocumentRow.user_id == user_id,
+ KnowledgeDocumentRow.knowledge_base_id
+ == knowledge_base_id,
+ KnowledgeDocumentRow.processing_node == processing_node,
+ )
+ .values(
+ processing_node=None,
+ lease_expires_at=None,
+ updated_at=now,
+ ),
+ )
+ await sess.commit()
+
+ async def list_knowledge_documents_with_expired_lease(
+ self,
+ now: datetime | None = None,
+ ) -> list[KnowledgeDocumentRecord]:
+ """Documents past their lease deadline, non-terminal, held."""
+ from sqlalchemy import select
+
+ now = _to_naive_utc(now) if now is not None else _utcnow()
+ terminal = ("ready", "error")
+ async with self._session() as sess:
+ rows = (
+ (
+ await sess.execute(
+ select(KnowledgeDocumentRow).where(
+ KnowledgeDocumentRow.status.notin_(terminal),
+ KnowledgeDocumentRow.processing_node.is_not(None),
+ KnowledgeDocumentRow.lease_expires_at.is_not(None),
+ KnowledgeDocumentRow.lease_expires_at < now,
+ ),
+ )
+ )
+ .scalars()
+ .all()
+ )
+ return [_to_record(r, KnowledgeDocumentRecord) for r in rows]
+
+ async def list_knowledge_documents_pending_since(
+ self,
+ threshold: datetime,
+ ) -> list[KnowledgeDocumentRecord]:
+ """Documents stuck in ``pending`` before *threshold*."""
+ from sqlalchemy import select
+
+ threshold = _to_naive_utc(threshold)
+ async with self._session() as sess:
+ rows = (
+ (
+ await sess.execute(
+ select(KnowledgeDocumentRow).where(
+ KnowledgeDocumentRow.status == "pending",
+ KnowledgeDocumentRow.created_at < threshold,
+ ),
+ )
+ )
+ .scalars()
+ .all()
+ )
+ return [_to_record(r, KnowledgeDocumentRecord) for r in rows]
diff --git a/src/agentscope/app/storage/_sql/_tables.py b/src/agentscope/app/storage/_sql/_tables.py
new file mode 100644
index 000000000..4daa99e36
--- /dev/null
+++ b/src/agentscope/app/storage/_sql/_tables.py
@@ -0,0 +1,315 @@
+# -*- coding: utf-8 -*-
+"""SQLAlchemy 2.0 declarative tables backing :class:`AsyncSQLAlchemyStorage`.
+
+Every record type maps to one table with the layout:
+
+- ``id`` primary key + ``created_at`` / ``updated_at`` timestamps;
+- one column per relational / indexed field promoted from the
+ record's top level;
+- a single ``payload`` JSON column carrying the remainder of
+ ``record.model_dump(mode="json")`` **minus** the promoted columns —
+ so no field is ever stored in both places (see the design doc in
+ ``_mappers.py`` for the round-trip contract).
+
+Portability constraints — the *schema* intentionally sticks to the
+subset of SQLAlchemy that works on every async-capable dialect we
+target (SQLite / Postgres / MySQL): plain :class:`~sqlalchemy.JSON`
+(never JSONB), no generated columns, no ``FOR UPDATE``. Atomic
+upserts *are* emitted with dialect-native ``ON CONFLICT`` /
+``ON DUPLICATE KEY UPDATE`` syntax, but through the explicit
+per-dialect dispatch in
+:meth:`~agentscope.app.storage.AsyncSQLAlchemyStorage._upsert_stmt`
+rather than leaking into the table definitions here. The messages
+table sidesteps the
+JSON-record shape because it is inherently list-like — see
+:class:`MessageRow` for the shape and the write path in
+:class:`~agentscope.app.storage.AsyncSQLAlchemyStorage.upsert_message`.
+"""
+from datetime import datetime
+from typing import Any, ClassVar
+
+from sqlalchemy import (
+ JSON,
+ DateTime,
+ ForeignKey,
+ Index,
+ String,
+)
+from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
+
+
+# Every id column is sized uniformly
+_ID_LEN = 255
+
+
+class _Base(DeclarativeBase):
+ """Declarative base shared by every table in :mod:`_sql`.
+
+ Kept private because it is an implementation detail: users of
+ :class:`~agentscope.app.storage.AsyncSQLAlchemyStorage` never see it.
+ """
+
+
+class _JsonRecordMixin(_Base):
+ """Column set common to every ``*Row`` that stores a record payload.
+
+ Concrete tables inherit this mixin plus :class:`_Base`, add their
+ own promoted columns, and set two class variables that drive the
+ generic mapper in :mod:`_mappers`:
+
+ - ``_record_cls``: the pydantic :class:`_RecordBase` subclass
+ this row represents.
+ - ``_indexed_fields``: the tuple of top-level record fields that
+ live in dedicated columns (and therefore MUST be popped out of
+ ``payload`` on write and merged back on read).
+
+ The three envelope columns (``id`` / ``created_at`` /
+ ``updated_at``) are always promoted and are handled by the mapper
+ unconditionally, so ``_indexed_fields`` should list **only** the
+ extra table-specific columns.
+ """
+
+ __abstract__ = True
+
+ id: Mapped[str] = mapped_column(String(_ID_LEN), primary_key=True)
+ created_at: Mapped[datetime] = mapped_column(
+ DateTime(),
+ index=True,
+ nullable=False,
+ )
+ updated_at: Mapped[datetime] = mapped_column(
+ DateTime(),
+ index=True,
+ nullable=False,
+ )
+ payload: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
+
+ # Populated by subclasses; see class docstring.
+ _record_cls: ClassVar[type]
+ _indexed_fields: ClassVar[tuple[str, ...]] = ()
+
+ @classmethod
+ def get_indexed_fields(cls) -> tuple[str, ...]:
+ """Return the tuple of record fields promoted to dedicated columns."""
+ return cls._indexed_fields
+
+
+class CredentialRow(_JsonRecordMixin):
+ """One row per :class:`~agentscope.app.storage.CredentialRecord`."""
+
+ __tablename__ = "credentials"
+
+ user_id: Mapped[str] = mapped_column(
+ String(_ID_LEN),
+ nullable=False,
+ index=True,
+ )
+
+ _indexed_fields = ("user_id",)
+
+
+class AgentRow(_JsonRecordMixin):
+ """One row per :class:`~agentscope.app.storage.AgentRecord`."""
+
+ __tablename__ = "agents"
+
+ user_id: Mapped[str] = mapped_column(
+ String(_ID_LEN),
+ nullable=False,
+ index=True,
+ )
+ source: Mapped[str] = mapped_column(
+ String(16),
+ nullable=False,
+ index=True,
+ )
+
+ __table_args__ = (Index("ix_agents_user_source", "user_id", "source"),)
+
+ _indexed_fields = ("user_id", "source")
+
+
+class SessionRow(_JsonRecordMixin):
+ """One row per :class:`~agentscope.app.storage.SessionRecord`."""
+
+ __tablename__ = "sessions"
+
+ user_id: Mapped[str] = mapped_column(
+ String(_ID_LEN),
+ nullable=False,
+ index=True,
+ )
+ agent_id: Mapped[str] = mapped_column(
+ String(_ID_LEN),
+ nullable=False,
+ index=True,
+ )
+ source: Mapped[str] = mapped_column(
+ String(16),
+ nullable=False,
+ index=True,
+ )
+ source_schedule_id: Mapped[str | None] = mapped_column(
+ String(_ID_LEN),
+ nullable=True,
+ index=True,
+ )
+ team_id: Mapped[str | None] = mapped_column(
+ String(_ID_LEN),
+ nullable=True,
+ index=True,
+ )
+
+ __table_args__ = (Index("ix_sessions_user_agent", "user_id", "agent_id"),)
+
+ _indexed_fields = (
+ "user_id",
+ "agent_id",
+ "source",
+ "source_schedule_id",
+ "team_id",
+ )
+
+
+class ScheduleRow(_JsonRecordMixin):
+ """One row per :class:`~agentscope.app.storage.ScheduleRecord`."""
+
+ __tablename__ = "schedules"
+
+ user_id: Mapped[str] = mapped_column(
+ String(_ID_LEN),
+ nullable=False,
+ index=True,
+ )
+ agent_id: Mapped[str] = mapped_column(
+ String(_ID_LEN),
+ nullable=False,
+ index=True,
+ )
+
+ _indexed_fields = ("user_id", "agent_id")
+
+
+class TeamRow(_JsonRecordMixin):
+ """One row per :class:`~agentscope.app.storage.TeamRecord`."""
+
+ __tablename__ = "teams"
+
+ user_id: Mapped[str] = mapped_column(
+ String(_ID_LEN),
+ nullable=False,
+ index=True,
+ )
+ session_id: Mapped[str] = mapped_column(
+ String(_ID_LEN),
+ nullable=False,
+ )
+
+ _indexed_fields = ("user_id", "session_id")
+
+
+class KnowledgeBaseRow(_JsonRecordMixin):
+ """One row per :class:`~agentscope.app.storage.KnowledgeBaseRecord`."""
+
+ __tablename__ = "knowledge_bases"
+
+ user_id: Mapped[str] = mapped_column(
+ String(_ID_LEN),
+ nullable=False,
+ index=True,
+ )
+
+ _indexed_fields = ("user_id",)
+
+
+class KnowledgeDocumentRow(_JsonRecordMixin):
+ """One row per :class:`KnowledgeDocumentRecord`.
+
+ Promotes every lifecycle / sweeper field to a dedicated column so
+ :meth:`AsyncSQLAlchemyStorage.list_knowledge_documents_with_expired_lease`
+ can filter without deserialising :attr:`payload`. The composite
+ ``(status, lease_expires_at)`` index serves both the expired-lease
+ sweep (``WHERE status NOT IN (…) AND lease_expires_at < :now``)
+ and the pending-orphan sweep (``WHERE status = 'pending' AND
+ created_at < :threshold`` — covered by the plain ``status`` index
+ combined with the mixin's ``created_at`` index).
+ """
+
+ __tablename__ = "knowledge_documents"
+
+ user_id: Mapped[str] = mapped_column(
+ String(_ID_LEN),
+ nullable=False,
+ index=True,
+ )
+ knowledge_base_id: Mapped[str] = mapped_column(
+ String(_ID_LEN),
+ # Documents are structurally owned by their KB — let the DB
+ # cascade-delete them natively (atomic, no application loop).
+ ForeignKey("knowledge_bases.id", ondelete="CASCADE"),
+ nullable=False,
+ index=True,
+ )
+ processing_node: Mapped[str | None] = mapped_column(
+ String(128),
+ nullable=True,
+ )
+ status: Mapped[str] = mapped_column(
+ String(16),
+ nullable=False,
+ index=True,
+ )
+ lease_expires_at: Mapped[datetime | None] = mapped_column(
+ DateTime(),
+ nullable=True,
+ )
+
+ __table_args__ = (
+ Index(
+ "ix_kd_status_lease",
+ "status",
+ "lease_expires_at",
+ ),
+ Index(
+ "ix_kd_user_kb",
+ "user_id",
+ "knowledge_base_id",
+ ),
+ )
+
+ _indexed_fields = (
+ "user_id",
+ "knowledge_base_id",
+ "processing_node",
+ "status",
+ "lease_expires_at",
+ )
+
+
+class MessageRow(_Base):
+ """One row per persisted :class:`~agentscope.message.Msg`.
+
+ Sits outside the ``_JsonRecordMixin`` family because messages are
+ not stand-alone records — they are per-session events. A message is
+ identified by the **composite** primary key ``(session_id, msg_id)``
+ rather than a synthetic concatenated string: ``Msg.id`` is only
+ unique *within* a session, and a composite key keeps the write path
+ ("same ``(session, msg_id)`` → replace", inheriting
+ :class:`RedisStorage.upsert_message`'s semantic) enforced by the DB
+ without having to bound the length of a ``session_id:msg_id`` blob
+ (both ids come from a user-overridable id factory).
+ """
+
+ __tablename__ = "messages"
+
+ session_id: Mapped[str] = mapped_column(String(_ID_LEN), primary_key=True)
+ msg_id: Mapped[str] = mapped_column(String(_ID_LEN), primary_key=True)
+ created_at: Mapped[datetime] = mapped_column(
+ DateTime(),
+ nullable=False,
+ )
+ payload: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
+
+ __table_args__ = (
+ Index("ix_messages_session_created", "session_id", "created_at"),
+ )
diff --git a/tests/app_lifespan_dedicated_test.py b/tests/app_lifespan_dedicated_test.py
index df3cf0394..2aa658da8 100644
--- a/tests/app_lifespan_dedicated_test.py
+++ b/tests/app_lifespan_dedicated_test.py
@@ -47,6 +47,7 @@ from agentscope.app.rag.knowledge_base_manager._dimension_policy import (
from agentscope.app.message_bus import RedisMessageBus
from agentscope.app.storage import (
EmbeddingModelConfig,
+ KnowledgeBaseData,
KnowledgeBaseRecord,
RedisStorage,
)
@@ -262,19 +263,21 @@ class DedicatedModeUploadFlowTest(IsolatedAsyncioTestCase):
# manager's create flow over HTTP.
kb_record = KnowledgeBaseRecord(
user_id="user-1",
- name="kb",
- description="",
- embedding_model_config=EmbeddingModelConfig(
- type="openai_credential",
- credential_id="cred-1",
- model="text-embedding-3-small",
- dimensions=1,
+ data=KnowledgeBaseData(
+ name="kb",
+ description="",
+ embedding_model_config=EmbeddingModelConfig(
+ type="openai_credential",
+ credential_id="cred-1",
+ model="text-embedding-3-small",
+ dimensions=1,
+ ),
+ collection_name="",
),
- collection_name="",
)
- kb_record.collection_name = f"kb_{kb_record.id}"
+ kb_record.data.collection_name = f"kb_{kb_record.id}"
await self._vector_store.create_collection(
- kb_record.collection_name,
+ kb_record.data.collection_name,
1,
)
storage._client = self._fr
diff --git a/tests/resource_access_policy_test.py b/tests/resource_access_policy_test.py
index cc95783ec..46bcdf77f 100644
--- a/tests/resource_access_policy_test.py
+++ b/tests/resource_access_policy_test.py
@@ -15,7 +15,14 @@ from agentscope.app.access import (
ResourceRef,
)
from agentscope.app._service import ResourceAccessService
-from agentscope.app.storage import AgentData, AgentRecord, CredentialRecord
+from agentscope.app.storage import (
+ AgentData,
+ AgentRecord,
+ CredentialRecord,
+ EmbeddingModelConfig,
+ KnowledgeBaseData,
+ KnowledgeBaseRecord,
+)
from agentscope.app.storage import StorageBase
from agentscope.agent import ContextConfig, ReActConfig
@@ -112,6 +119,13 @@ class _FakeStorage:
data=_make_agent_data("Own team"),
),
}
+ self.knowledge_bases: dict[tuple[str, str], KnowledgeBaseRecord] = {
+ ("viewer", "kb-1"): KnowledgeBaseRecord(
+ id="kb-1",
+ user_id="viewer",
+ data=_make_kb_data("My KB"),
+ ),
+ }
async def list_credentials(
self,
@@ -148,6 +162,25 @@ class _FakeStorage:
"""Get an agent."""
return self.agents.get((user_id, agent_id))
+ async def list_knowledge_bases(
+ self,
+ user_id: str,
+ ) -> list[KnowledgeBaseRecord]:
+ """List knowledge bases owned by ``user_id``."""
+ return [
+ record
+ for (owner_id, _), record in self.knowledge_bases.items()
+ if owner_id == user_id
+ ]
+
+ async def get_knowledge_base(
+ self,
+ user_id: str,
+ knowledge_base_id: str,
+ ) -> KnowledgeBaseRecord | None:
+ """Get a knowledge base."""
+ return self.knowledge_bases.get((user_id, knowledge_base_id))
+
def _make_agent_data(name: str) -> AgentData:
"""Create valid agent data for tests."""
@@ -159,6 +192,21 @@ def _make_agent_data(name: str) -> AgentData:
)
+def _make_kb_data(name: str) -> KnowledgeBaseData:
+ """Create valid knowledge base data for tests."""
+ return KnowledgeBaseData(
+ name=name,
+ description="A test knowledge base.",
+ embedding_model_config=EmbeddingModelConfig(
+ type="openai_credential",
+ credential_id="credential-1",
+ model="text-embedding-3-small",
+ dimensions=1536,
+ ),
+ collection_name="kb_deadbeef",
+ )
+
+
class ResourceAccessPolicyTest(IsolatedAsyncioTestCase):
"""Test resource access policy defaults and edit fallback."""
@@ -343,6 +391,67 @@ class ResourceAccessServiceTest(IsolatedAsyncioTestCase):
],
)
+ async def test_get_knowledge_base_view_is_flat(self) -> None:
+ """KB views must lift ``data.*`` to the top level for the wire.
+
+ The frontend reads ``kb.name`` and
+ ``kb.embedding_model_config.model`` flat; a regression that let
+ the storage-layer ``data`` nesting leak onto the wire broke the
+ sidebar name and crashed the panel. Lock the flat shape in.
+ """
+ view = await self.service.get_resource(
+ "viewer",
+ ResourceKind.KNOWLEDGE_BASE,
+ "kb-1",
+ )
+
+ self.assertEqual(
+ view.model_dump(),
+ {
+ "id": "kb-1",
+ "name": "My KB",
+ "description": "A test knowledge base.",
+ "embedding_model_config": {
+ "type": "openai_credential",
+ "credential_id": "credential-1",
+ "model": "text-embedding-3-small",
+ "dimensions": 1536,
+ "parameters": {},
+ },
+ "created_at": AnyValue(),
+ "updated_at": AnyValue(),
+ "editable": True,
+ },
+ )
+
+ async def test_list_knowledge_bases_view_is_flat(self) -> None:
+ """Listed KB views must be flat too — same wire contract."""
+ views = await self.service.list_resource(
+ "viewer",
+ ResourceKind.KNOWLEDGE_BASE,
+ )
+
+ self.assertEqual(
+ [v.model_dump() for v in views],
+ [
+ {
+ "id": "kb-1",
+ "name": "My KB",
+ "description": "A test knowledge base.",
+ "embedding_model_config": {
+ "type": "openai_credential",
+ "credential_id": "credential-1",
+ "model": "text-embedding-3-small",
+ "dimensions": 1536,
+ "parameters": {},
+ },
+ "created_at": AnyValue(),
+ "updated_at": AnyValue(),
+ "editable": True,
+ },
+ ],
+ )
+
async def test_missing_resource_raises_404(self) -> None:
"""Missing visible resources should raise a 404 HTTPException."""
with self.assertRaises(HTTPException) as ctx:
diff --git a/tests/service_knowledge_base_upload_test.py b/tests/service_knowledge_base_upload_test.py
index 44a410fb7..a01845ef2 100644
--- a/tests/service_knowledge_base_upload_test.py
+++ b/tests/service_knowledge_base_upload_test.py
@@ -36,6 +36,7 @@ from agentscope.app.rag.knowledge_base_manager._dimension_policy import (
from agentscope.app.message_bus import RedisMessageBus
from agentscope.app.storage import (
EmbeddingModelConfig,
+ KnowledgeBaseData,
KnowledgeBaseRecord,
RedisStorage,
)
@@ -208,14 +209,16 @@ class _FakeKbManager(KnowledgeBaseManagerBase):
) -> KnowledgeBaseRecord:
record = KnowledgeBaseRecord(
user_id=user_id,
- name=name,
- description=description,
- embedding_model_config=embedding_model_config,
- collection_name="",
+ data=KnowledgeBaseData(
+ name=name,
+ description=description,
+ embedding_model_config=embedding_model_config,
+ collection_name="",
+ ),
)
- record.collection_name = f"kb_{record.id}"
+ record.data.collection_name = f"kb_{record.id}"
await self._vector_store.create_collection(
- name=record.collection_name,
+ name=record.data.collection_name,
dimensions=embedding_model_config.dimensions,
)
return await self._storage.upsert_knowledge_base(user_id, record)
@@ -231,7 +234,9 @@ class _FakeKbManager(KnowledgeBaseManagerBase):
)
if record is None:
return False
- await self._vector_store.delete_collection(record.collection_name)
+ await self._vector_store.delete_collection(
+ record.data.collection_name,
+ )
return await self._storage.delete_knowledge_base(
user_id,
knowledge_base_id,
@@ -252,7 +257,7 @@ class _FakeKbManager(KnowledgeBaseManagerBase):
)
return _FakeKnowledge(
vector_store=self._vector_store,
- collection_name=record.collection_name,
+ collection_name=record.data.collection_name,
)
@@ -335,19 +340,21 @@ class KnowledgeBaseUploadFlowTest(IsolatedAsyncioTestCase):
# have to mock the manager's create flow over HTTP.
kb_record = KnowledgeBaseRecord(
user_id="user-1",
- name="kb",
- description="",
- embedding_model_config=EmbeddingModelConfig(
- type="openai_credential",
- credential_id="cred-1",
- model="text-embedding-3-small",
- dimensions=1,
+ data=KnowledgeBaseData(
+ name="kb",
+ description="",
+ embedding_model_config=EmbeddingModelConfig(
+ type="openai_credential",
+ credential_id="cred-1",
+ model="text-embedding-3-small",
+ dimensions=1,
+ ),
+ collection_name="",
),
- collection_name="",
)
- kb_record.collection_name = f"kb_{kb_record.id}"
+ kb_record.data.collection_name = f"kb_{kb_record.id}"
await self._vector_store.create_collection(
- kb_record.collection_name,
+ kb_record.data.collection_name,
1,
)
diff --git a/tests/storage_redis_knowledge_base_test.py b/tests/storage_redis_knowledge_base_test.py
index f828c73e6..d9601540d 100644
--- a/tests/storage_redis_knowledge_base_test.py
+++ b/tests/storage_redis_knowledge_base_test.py
@@ -7,6 +7,7 @@ import fakeredis.aioredis
from agentscope.app.storage import (
EmbeddingModelConfig,
+ KnowledgeBaseData,
KnowledgeBaseRecord,
RedisStorage,
)
@@ -25,15 +26,17 @@ def make_record(user_id: str, name: str = "kb") -> KnowledgeBaseRecord:
"""Build a KnowledgeBaseRecord with a default embedding config."""
return KnowledgeBaseRecord(
user_id=user_id,
- name=name,
- description="desc",
- embedding_model_config=EmbeddingModelConfig(
- type="openai_credential",
- credential_id="cred-1",
- model="text-embedding-3-small",
- dimensions=1536,
+ data=KnowledgeBaseData(
+ name=name,
+ description="desc",
+ embedding_model_config=EmbeddingModelConfig(
+ type="openai_credential",
+ credential_id="cred-1",
+ model="text-embedding-3-small",
+ dimensions=1536,
+ ),
+ collection_name="kb_abc",
),
- collection_name="kb_abc",
)
@@ -55,7 +58,7 @@ class KnowledgeBaseStorageTest(IsolatedAsyncioTestCase):
# get returns the persisted record for the right owner
fetched = await storage.get_knowledge_base("user-1", stored_a.id)
self.assertEqual(fetched.id, stored_a.id)
- self.assertEqual(fetched.name, "first")
+ self.assertEqual(fetched.data.name, "first")
# cross-user lookups return None
self.assertIsNone(
@@ -102,9 +105,9 @@ class KnowledgeBaseStorageTest(IsolatedAsyncioTestCase):
rec = make_record("user-1")
first = await storage.upsert_knowledge_base("user-1", rec)
- rec.name = "renamed"
+ rec.data.name = "renamed"
second = await storage.upsert_knowledge_base("user-1", rec)
self.assertEqual(second.id, first.id)
self.assertEqual(second.created_at, first.created_at)
- self.assertEqual(second.name, "renamed")
+ self.assertEqual(second.data.name, "renamed")
diff --git a/tests/storage_sql_test.py b/tests/storage_sql_test.py
new file mode 100644
index 000000000..b44998169
--- /dev/null
+++ b/tests/storage_sql_test.py
@@ -0,0 +1,856 @@
+# -*- coding: utf-8 -*-
+# pylint: disable=too-many-public-methods
+"""Round-trip and semantic tests for :class:`AsyncSQLAlchemyStorage`.
+
+Runs against an in-memory SQLite database via ``sqlite+aiosqlite``
+so the whole suite is self-contained — no external server needed.
+The intent is to exercise every ``StorageBase`` method the Redis
+backend implements, mirroring the shape (though not always the
+literal assertions) of the Redis backend's tests so both backends
+stay behavioural equivalents.
+"""
+from contextlib import AsyncExitStack
+from datetime import datetime, timedelta
+from unittest.async_case import IsolatedAsyncioTestCase
+
+from pydantic import SecretStr
+
+from agentscope.app.storage import (
+ AgentData,
+ AgentRecord,
+ ChatModelConfig,
+ EmbeddingModelConfig,
+ KnowledgeBaseData,
+ KnowledgeBaseRecord,
+ KnowledgeDocumentData,
+ KnowledgeDocumentRecord,
+ ScheduleData,
+ ScheduleRecord,
+ SessionConfig,
+ SessionSource,
+ AsyncSQLAlchemyStorage,
+ TeamData,
+ TeamMember,
+ TeamRecord,
+)
+from agentscope.agent import ContextConfig, ReActConfig
+from agentscope.credential import DashScopeCredential
+from agentscope.message import AssistantMsg, UserMsg
+
+
+def _agent_record(user_id: str, name: str = "agent-x") -> AgentRecord:
+ """Build a minimal but complete :class:`AgentRecord`."""
+ return AgentRecord(
+ user_id=user_id,
+ data=AgentData(
+ name=name,
+ context_config=ContextConfig(),
+ react_config=ReActConfig(),
+ ),
+ )
+
+
+def _session_config() -> SessionConfig:
+ """Build a minimal :class:`SessionConfig`."""
+ return SessionConfig(
+ workspace_id="ws-1",
+ name="s",
+ chat_model_config=ChatModelConfig(
+ type="openai",
+ credential_id="cred-1",
+ model="gpt-4o",
+ parameters={},
+ ),
+ )
+
+
+def _kb_record(user_id: str, name: str = "kb") -> KnowledgeBaseRecord:
+ """Build a KB record with a default embedding config."""
+ return KnowledgeBaseRecord(
+ user_id=user_id,
+ data=KnowledgeBaseData(
+ name=name,
+ description="",
+ embedding_model_config=EmbeddingModelConfig(
+ type="openai_credential",
+ credential_id="cred-1",
+ model="text-embedding-3-small",
+ dimensions=8,
+ ),
+ collection_name="kb-x",
+ ),
+ )
+
+
+def _kd_record(
+ user_id: str,
+ knowledge_base_id: str,
+ filename: str = "f.txt",
+) -> KnowledgeDocumentRecord:
+ """Build a fresh (``pending`` / unclaimed) document record."""
+ return KnowledgeDocumentRecord(
+ user_id=user_id,
+ knowledge_base_id=knowledge_base_id,
+ data=KnowledgeDocumentData(
+ filename=filename,
+ size=42,
+ blob_uri=f"local://{filename}",
+ ),
+ )
+
+
+def _schedule_record(user_id: str, agent_id: str) -> ScheduleRecord:
+ """Build a schedule record."""
+ return ScheduleRecord(
+ user_id=user_id,
+ agent_id=agent_id,
+ data=ScheduleData(
+ name="daily",
+ cron_expression="0 9 * * *",
+ chat_model_config=ChatModelConfig(
+ type="openai",
+ credential_id="cred-1",
+ model="gpt-4o",
+ parameters={},
+ ),
+ ),
+ )
+
+
+class AsyncSQLAlchemyStorageTest(IsolatedAsyncioTestCase):
+ """End-to-end tests for :class:`AsyncSQLAlchemyStorage` over
+ in-memory SQLite."""
+
+ async def asyncSetUp(self) -> None:
+ # ``:memory:`` gives a private DB per connection; using a
+ # shared-cache URI would too, but per-test isolation is what
+ # we want here.
+ self._stack = AsyncExitStack()
+ self.storage = await self._stack.enter_async_context(
+ AsyncSQLAlchemyStorage(
+ "sqlite+aiosqlite:///:memory:",
+ create_tables=True,
+ ),
+ )
+
+ async def asyncTearDown(self) -> None:
+ await self._stack.aclose()
+
+ # ------------------------------------------------------------------
+ # Credentials
+ # ------------------------------------------------------------------
+
+ async def test_credentials_round_trip(self) -> None:
+ """Upsert / list / get / delete + owner scoping."""
+ cred = DashScopeCredential(api_key=SecretStr("sk-1"))
+ cid = await self.storage.upsert_credential("user-1", cred)
+
+ listed = await self.storage.list_credentials("user-1")
+ self.assertEqual([c.id for c in listed], [cid])
+ self.assertEqual(listed[0].data["api_key"], "sk-1")
+
+ fetched = await self.storage.get_credential("user-1", cid)
+ self.assertEqual(fetched.id, cid)
+
+ # Cross-user isolation
+ self.assertEqual(await self.storage.list_credentials("user-2"), [])
+ self.assertIsNone(await self.storage.get_credential("user-2", cid))
+
+ # Delete + double-delete
+ self.assertTrue(await self.storage.delete_credential("user-1", cid))
+ self.assertFalse(await self.storage.delete_credential("user-1", cid))
+
+ async def test_upsert_credential_is_owner_scoped(self) -> None:
+ """A preset id owned by another user is never read or clobbered."""
+ import sqlalchemy.exc
+
+ victim = DashScopeCredential(api_key=SecretStr("victim-key"))
+ cid = await self.storage.upsert_credential("user-1", victim)
+ before = await self.storage.get_credential("user-1", cid)
+
+ # Same owner + preset id → in-place update, created_at preserved.
+ rotated = DashScopeCredential(api_key=SecretStr("rotated"), id=cid)
+ self.assertEqual(
+ await self.storage.upsert_credential("user-1", rotated),
+ cid,
+ )
+ after = await self.storage.get_credential("user-1", cid)
+ self.assertEqual(after.data["api_key"], "rotated")
+ self.assertEqual(after.created_at, before.created_at)
+
+ # Attacker (user-2) presenting the victim's id must not touch the
+ # victim's row: the global-id INSERT collides and raises, and the
+ # victim's data + ownership stay intact.
+ attack = DashScopeCredential(api_key=SecretStr("attacker"), id=cid)
+ with self.assertRaises(sqlalchemy.exc.IntegrityError):
+ await self.storage.upsert_credential("user-2", attack)
+
+ victim_now = await self.storage.get_credential("user-1", cid)
+ self.assertEqual(victim_now.data["api_key"], "rotated")
+ self.assertIsNone(await self.storage.get_credential("user-2", cid))
+ self.assertEqual(await self.storage.list_credentials("user-2"), [])
+
+ # ------------------------------------------------------------------
+ # Agents
+ # ------------------------------------------------------------------
+
+ async def test_agents_round_trip_and_source_filter(self) -> None:
+ """``list_agents`` filters out ``source='team'`` workers."""
+ user_agent = _agent_record("user-1", "usr")
+ team_agent = _agent_record("user-1", "team-worker")
+ team_agent.source = "team"
+
+ await self.storage.upsert_agent("user-1", user_agent)
+ await self.storage.upsert_agent("user-1", team_agent)
+
+ listed = await self.storage.list_agents("user-1")
+ self.assertEqual([a.id for a in listed], [user_agent.id])
+ # But direct get works for the team-spawned worker
+ self.assertEqual(
+ (await self.storage.get_agent("user-1", team_agent.id)).id,
+ team_agent.id,
+ )
+
+ async def test_delete_agent_cascades_sessions_and_schedules(self) -> None:
+ """Deleting an agent removes its sessions + schedules."""
+ agent = _agent_record("user-1")
+ await self.storage.upsert_agent("user-1", agent)
+ session = await self.storage.upsert_session(
+ user_id="user-1",
+ agent_id=agent.id,
+ config=_session_config(),
+ )
+ schedule = _schedule_record("user-1", agent.id)
+ await self.storage.upsert_schedule("user-1", schedule)
+
+ self.assertTrue(await self.storage.delete_agent("user-1", agent.id))
+ self.assertEqual(
+ await self.storage.list_sessions("user-1", agent.id),
+ [],
+ )
+ self.assertIsNone(
+ await self.storage.get_session(
+ "user-1",
+ agent.id,
+ session.id,
+ ),
+ )
+ self.assertIsNone(
+ await self.storage.get_schedule("user-1", schedule.id),
+ )
+
+ async def test_upsert_replaces_in_place_and_keeps_created_at(self) -> None:
+ """Re-upserting the same id updates via the atomic upsert path.
+
+ Exercises the ``ON CONFLICT DO UPDATE`` branch: the second
+ write must overwrite the mutable columns, preserve the original
+ ``created_at``, and never create a duplicate row.
+ """
+ agent = _agent_record("user-1", "v1")
+ await self.storage.upsert_agent("user-1", agent)
+ first = await self.storage.get_agent("user-1", agent.id)
+
+ agent.data.name = "v2"
+ await self.storage.upsert_agent("user-1", agent)
+ second = await self.storage.get_agent("user-1", agent.id)
+
+ self.assertEqual(second.data.name, "v2")
+ self.assertEqual(second.created_at, first.created_at)
+ self.assertGreaterEqual(second.updated_at, first.updated_at)
+ # Exactly one row — the conflict updated rather than inserted.
+ self.assertEqual(len(await self.storage.list_agents("user-1")), 1)
+
+ # ------------------------------------------------------------------
+ # Sessions
+ # ------------------------------------------------------------------
+
+ async def test_sessions_upsert_and_state_update(self) -> None:
+ """Create + update + state-only update."""
+ agent = _agent_record("user-1")
+ await self.storage.upsert_agent("user-1", agent)
+
+ # Create
+ session = await self.storage.upsert_session(
+ user_id="user-1",
+ agent_id=agent.id,
+ config=_session_config(),
+ source=SessionSource.SCHEDULE,
+ source_schedule_id="sch-1",
+ )
+ self.assertEqual(session.source, SessionSource.SCHEDULE)
+
+ # Update (same session_id) — config swap
+ new_config = _session_config()
+ new_config.name = "renamed"
+ updated = await self.storage.upsert_session(
+ user_id="user-1",
+ agent_id=agent.id,
+ config=new_config,
+ session_id=session.id,
+ )
+ self.assertEqual(updated.id, session.id)
+ self.assertEqual(updated.config.name, "renamed")
+ self.assertEqual(updated.created_at, session.created_at)
+
+ # State-only update
+ from agentscope.state import AgentState
+
+ new_state = AgentState()
+ await self.storage.update_session_state(
+ "user-1",
+ agent.id,
+ session.id,
+ new_state,
+ )
+ fetched = await self.storage.get_session(
+ "user-1",
+ agent.id,
+ session.id,
+ )
+ self.assertEqual(fetched.state.model_dump(), new_state.model_dump())
+
+ # By-schedule listing
+ listed = await self.storage.list_sessions_by_schedule(
+ "user-1",
+ "sch-1",
+ )
+ self.assertEqual([s.id for s in listed], [session.id])
+
+ # set_session_team_id
+ await self.storage.set_session_team_id(
+ "user-1",
+ session.id,
+ "team-9",
+ )
+ fetched = await self.storage.get_session(
+ "user-1",
+ agent.id,
+ session.id,
+ )
+ self.assertEqual(fetched.team_id, "team-9")
+
+ async def test_update_session_state_missing_raises(self) -> None:
+ """Updating an absent session raises :class:`KeyError`."""
+ from agentscope.state import AgentState
+
+ with self.assertRaises(KeyError):
+ await self.storage.update_session_state(
+ "user-1",
+ "agent-x",
+ "no-such-session",
+ AgentState(),
+ )
+
+ # ------------------------------------------------------------------
+ # Messages
+ # ------------------------------------------------------------------
+
+ async def test_messages_upsert_and_pagination(self) -> None:
+ """Same-id upsert replaces; distinct ids append; pagination
+ yields chronological order."""
+ m1 = UserMsg(name="u", content="hello")
+ m2 = AssistantMsg(name="a", content="hi")
+
+ await self.storage.upsert_message("user-1", "sess-1", m1)
+ await self.storage.upsert_message("user-1", "sess-1", m2)
+
+ # Replace m1 in place (build a fresh Msg with the same id so
+ # Pydantic re-validates the content list rather than accepting
+ # a raw-string assignment on the model).
+ m1_updated = UserMsg(id=m1.id, name="u", content="hola")
+ await self.storage.upsert_message("user-1", "sess-1", m1_updated)
+
+ listed, has_more = await self.storage.list_messages(
+ "user-1",
+ "sess-1",
+ )
+ self.assertEqual([m.id for m in listed], [m1.id, m2.id])
+ self.assertFalse(has_more)
+ self.assertEqual(listed[0].content, m1_updated.content)
+
+ fetched = await self.storage.get_message("user-1", "sess-1", m2.id)
+ self.assertEqual(fetched.id, m2.id)
+
+ # Cursor-based pagination: the latest page of one message is the
+ # newest (m2), with older messages still available.
+ latest, has_more = await self.storage.list_messages(
+ "user-1",
+ "sess-1",
+ limit=1,
+ )
+ self.assertEqual([m.id for m in latest], [m2.id])
+ self.assertTrue(has_more)
+
+ # Walking backwards with ``before`` yields the previous page.
+ older, has_more = await self.storage.list_messages(
+ "user-1",
+ "sess-1",
+ limit=1,
+ before=m2.id,
+ )
+ self.assertEqual([m.id for m in older], [m1.id])
+ self.assertFalse(has_more)
+
+ # An unknown cursor yields an empty page.
+ self.assertEqual(
+ await self.storage.list_messages(
+ "user-1",
+ "sess-1",
+ before="does-not-exist",
+ ),
+ ([], False),
+ )
+
+ # The legacy ``offset`` keyword is ignored but warns.
+ with self.assertWarns(DeprecationWarning):
+ await self.storage.list_messages(
+ "user-1",
+ "sess-1",
+ offset=1,
+ )
+
+ async def test_messages_max_width_ids(self) -> None:
+ """Composite key stores ids a concatenated key couldn't hold.
+
+ Two 64-char ids would be 129 chars once joined as
+ ``session_id:msg_id`` — overflowing the old ``String(96)``
+ synthetic primary key. The composite ``(session_id, msg_id)``
+ key round-trips them without truncation.
+ """
+ long_session = "s" * 64
+ long_msg_id = "m" * 64
+ msg = UserMsg(id=long_msg_id, name="u", content="hi")
+ await self.storage.upsert_message("user-1", long_session, msg)
+
+ fetched = await self.storage.get_message(
+ "user-1",
+ long_session,
+ long_msg_id,
+ )
+ self.assertIsNotNone(fetched)
+ self.assertEqual(fetched.id, long_msg_id)
+ listed, _has_more = await self.storage.list_messages(
+ "user-1",
+ long_session,
+ )
+ self.assertEqual([m.id for m in listed], [long_msg_id])
+
+ # ------------------------------------------------------------------
+ # Teams
+ # ------------------------------------------------------------------
+
+ async def test_teams_upsert_get_list_delete(self) -> None:
+ """Team cascade: created-role member is fully deleted, invited-role
+ keeps their agent."""
+ # Two agents — one that will be "created" for the team,
+ # another that will be "invited".
+ created_agent = _agent_record("user-1", "created")
+ created_agent.source = "team"
+ invited_agent = _agent_record("user-1", "invited")
+
+ await self.storage.upsert_agent("user-1", created_agent)
+ await self.storage.upsert_agent("user-1", invited_agent)
+
+ # Sessions for both, plus the leader session.
+ leader = await self.storage.upsert_session(
+ user_id="user-1",
+ agent_id=created_agent.id,
+ config=_session_config(),
+ )
+ created_session = await self.storage.upsert_session(
+ user_id="user-1",
+ agent_id=created_agent.id,
+ config=_session_config(),
+ )
+ invited_session = await self.storage.upsert_session(
+ user_id="user-1",
+ agent_id=invited_agent.id,
+ config=_session_config(),
+ )
+ surviving_session = await self.storage.upsert_session(
+ user_id="user-1",
+ agent_id=invited_agent.id,
+ config=_session_config(),
+ )
+
+ team = TeamRecord(
+ user_id="user-1",
+ session_id=leader.id,
+ data=TeamData(
+ name="team",
+ members=[
+ TeamMember(
+ owner_id="user-1",
+ agent_id=created_agent.id,
+ session_id=created_session.id,
+ role="created",
+ ),
+ TeamMember(
+ owner_id="user-1",
+ agent_id=invited_agent.id,
+ session_id=invited_session.id,
+ role="invited",
+ ),
+ ],
+ ),
+ )
+ await self.storage.upsert_team("user-1", team)
+
+ self.assertEqual(
+ [t.id for t in await self.storage.list_teams("user-1")],
+ [team.id],
+ )
+
+ # Delete team.
+ self.assertTrue(await self.storage.delete_team("user-1", team.id))
+ self.assertIsNone(await self.storage.get_team("user-1", team.id))
+
+ # Created member: agent + session are gone.
+ self.assertIsNone(
+ await self.storage.get_agent("user-1", created_agent.id),
+ )
+ # Invited member: agent survives, only the invited session is gone.
+ self.assertIsNotNone(
+ await self.storage.get_agent("user-1", invited_agent.id),
+ )
+ self.assertIsNone(
+ await self.storage.get_session(
+ "user-1",
+ invited_agent.id,
+ invited_session.id,
+ ),
+ )
+ self.assertIsNotNone(
+ await self.storage.get_session(
+ "user-1",
+ invited_agent.id,
+ surviving_session.id,
+ ),
+ )
+
+ # ------------------------------------------------------------------
+ # Knowledge base + documents + lease CAS
+ # ------------------------------------------------------------------
+
+ async def test_kb_and_documents_round_trip(self) -> None:
+ """KB CRUD; document CRUD; cascading KB delete."""
+ kb = _kb_record("user-1")
+ await self.storage.upsert_knowledge_base("user-1", kb)
+
+ self.assertEqual(
+ [k.id for k in await self.storage.list_knowledge_bases("user-1")],
+ [kb.id],
+ )
+
+ doc_a = _kd_record("user-1", kb.id, "a.txt")
+ doc_b = _kd_record("user-1", kb.id, "b.txt")
+ await self.storage.upsert_knowledge_document("user-1", doc_a)
+ await self.storage.upsert_knowledge_document("user-1", doc_b)
+
+ listed = await self.storage.list_knowledge_documents("user-1", kb.id)
+ self.assertEqual(
+ sorted(d.id for d in listed),
+ sorted([doc_a.id, doc_b.id]),
+ )
+
+ # Cascade delete removes documents.
+ self.assertTrue(
+ await self.storage.delete_knowledge_base("user-1", kb.id),
+ )
+ self.assertEqual(
+ await self.storage.list_knowledge_documents("user-1", kb.id),
+ [],
+ )
+
+ async def test_document_kb_foreign_key_is_enforced(self) -> None:
+ """The document→KB FK is live (PRAGMA on): cascade + rejection.
+
+ ``delete_knowledge_base`` no longer deletes documents in Python
+ — it relies on ``ON DELETE CASCADE``. This only passes if SQLite
+ foreign-key enforcement is actually enabled, so the test doubles
+ as a guard that the ``PRAGMA foreign_keys=ON`` wiring works.
+ """
+ from sqlalchemy.exc import IntegrityError
+
+ kb = _kb_record("user-1")
+ await self.storage.upsert_knowledge_base("user-1", kb)
+ doc = _kd_record("user-1", kb.id)
+ await self.storage.upsert_knowledge_document("user-1", doc)
+
+ # Native cascade: dropping the KB row removes the document row.
+ await self.storage.delete_knowledge_base("user-1", kb.id)
+ self.assertEqual(
+ await self.storage.list_knowledge_documents("user-1", kb.id),
+ [],
+ )
+
+ # Enforcement: a document pointing at a missing KB is rejected.
+ orphan = _kd_record("user-1", "no-such-kb")
+ with self.assertRaises(IntegrityError):
+ await self.storage.upsert_knowledge_document("user-1", orphan)
+
+ async def test_document_status_and_lease_cas(self) -> None:
+ """``update_knowledge_document_status`` + lease CAS semantics."""
+ kb = _kb_record("user-1")
+ await self.storage.upsert_knowledge_base("user-1", kb)
+ doc = _kd_record("user-1", kb.id)
+ await self.storage.upsert_knowledge_document("user-1", doc)
+
+ # Status transition + error/chunk_count payload update
+ await self.storage.update_knowledge_document_status(
+ "user-1",
+ kb.id,
+ doc.id,
+ "error",
+ error="boom",
+ chunk_count=0,
+ )
+ fetched = await self.storage.get_knowledge_document(
+ "user-1",
+ kb.id,
+ doc.id,
+ )
+ self.assertEqual(fetched.status, "error")
+ self.assertEqual(fetched.data.error, "boom")
+
+ # Reset to pending for the lease dance.
+ await self.storage.update_knowledge_document_status(
+ "user-1",
+ kb.id,
+ doc.id,
+ "pending",
+ )
+
+ now = datetime.now()
+ ttl = timedelta(minutes=5)
+ # First worker wins.
+ self.assertTrue(
+ await self.storage.acquire_knowledge_document_lease(
+ "user-1",
+ kb.id,
+ doc.id,
+ "worker-A",
+ ttl,
+ now,
+ ),
+ )
+ # Second worker loses because the lease is fresh.
+ self.assertFalse(
+ await self.storage.acquire_knowledge_document_lease(
+ "user-1",
+ kb.id,
+ doc.id,
+ "worker-B",
+ ttl,
+ now,
+ ),
+ )
+ # Renew from the holder works, from a stranger fails.
+ self.assertTrue(
+ await self.storage.renew_knowledge_document_lease(
+ "user-1",
+ kb.id,
+ doc.id,
+ "worker-A",
+ ttl,
+ now,
+ ),
+ )
+ self.assertFalse(
+ await self.storage.renew_knowledge_document_lease(
+ "user-1",
+ kb.id,
+ doc.id,
+ "worker-B",
+ ttl,
+ now,
+ ),
+ )
+ # Release from a stranger is a no-op; from the holder clears it.
+ await self.storage.release_knowledge_document_lease(
+ "user-1",
+ kb.id,
+ doc.id,
+ "worker-B",
+ )
+ fetched = await self.storage.get_knowledge_document(
+ "user-1",
+ kb.id,
+ doc.id,
+ )
+ self.assertEqual(fetched.processing_node, "worker-A")
+
+ await self.storage.release_knowledge_document_lease(
+ "user-1",
+ kb.id,
+ doc.id,
+ "worker-A",
+ )
+ fetched = await self.storage.get_knowledge_document(
+ "user-1",
+ kb.id,
+ doc.id,
+ )
+ self.assertIsNone(fetched.processing_node)
+ self.assertIsNone(fetched.lease_expires_at)
+
+ async def test_expired_lease_and_pending_sweep(self) -> None:
+ """``list_..._with_expired_lease`` + ``..._pending_since`` filters."""
+ kb = _kb_record("user-1")
+ await self.storage.upsert_knowledge_base("user-1", kb)
+
+ # Doc 1: expired lease, non-terminal → should show up
+ d1 = _kd_record("user-1", kb.id, "d1.txt")
+ await self.storage.upsert_knowledge_document("user-1", d1)
+ now = datetime.now()
+ await self.storage.acquire_knowledge_document_lease(
+ "user-1",
+ kb.id,
+ d1.id,
+ "worker-A",
+ timedelta(seconds=1),
+ now - timedelta(hours=1), # ancient
+ )
+
+ # Doc 2: still pending, no lease → not caught by expired filter
+ d2 = _kd_record("user-1", kb.id, "d2.txt")
+ await self.storage.upsert_knowledge_document("user-1", d2)
+
+ # Doc 3: terminal → never returned
+ d3 = _kd_record("user-1", kb.id, "d3.txt")
+ await self.storage.upsert_knowledge_document("user-1", d3)
+ await self.storage.acquire_knowledge_document_lease(
+ "user-1",
+ kb.id,
+ d3.id,
+ "worker-B",
+ timedelta(seconds=1),
+ now - timedelta(hours=1),
+ )
+ await self.storage.update_knowledge_document_status(
+ "user-1",
+ kb.id,
+ d3.id,
+ "ready",
+ )
+
+ expired = (
+ await self.storage.list_knowledge_documents_with_expired_lease(
+ now,
+ )
+ )
+ self.assertEqual([d.id for d in expired], [d1.id])
+
+ pending = await self.storage.list_knowledge_documents_pending_since(
+ now + timedelta(minutes=1),
+ )
+ # d1 and d2 are both still 'pending' (acquiring a lease does
+ # not transition status by itself); d3 is 'ready' so excluded.
+ self.assertEqual(
+ sorted(d.id for d in pending),
+ sorted([d1.id, d2.id]),
+ )
+
+ # ------------------------------------------------------------------
+ # Schedules
+ # ------------------------------------------------------------------
+
+ async def test_schedules_round_trip(self) -> None:
+ """Basic schedule CRUD + list_all across users."""
+ s1 = _schedule_record("user-1", "agent-1")
+ s2 = _schedule_record("user-2", "agent-2")
+ await self.storage.upsert_schedule("user-1", s1)
+ await self.storage.upsert_schedule("user-2", s2)
+
+ self.assertEqual(
+ sorted(x.id for x in await self.storage.list_schedules("user-1")),
+ [s1.id],
+ )
+ self.assertEqual(
+ sorted(x.id for x in await self.storage.list_all_schedules()),
+ sorted([s1.id, s2.id]),
+ )
+
+ self.assertTrue(
+ await self.storage.delete_schedule("user-1", s1.id),
+ )
+ self.assertFalse(
+ await self.storage.delete_schedule("user-1", s1.id),
+ )
+
+
+class AsyncSQLAlchemyStorageAutoMigrateTest(IsolatedAsyncioTestCase):
+ """Boot via ``auto_migrate=True`` and confirm the schema is live.
+
+ Uses a file-backed SQLite database (not ``:memory:``) so the
+ Alembic-driven ``upgrade head`` run inside
+ :meth:`AsyncSQLAlchemyStorage.__aenter__`
+ and the subsequent record-write use the same physical database —
+ ``:memory:`` gives a private DB per connection, which would let
+ Alembic build tables that the storage's engine can't see.
+ """
+
+ async def test_auto_migrate_creates_schema(self) -> None:
+ """After ``auto_migrate=True`` the tables exist and CRUD works."""
+ import os
+ import tempfile
+
+ with tempfile.TemporaryDirectory() as tmp:
+ db_path = os.path.join(tmp, "as.db")
+ url = f"sqlite+aiosqlite:///{db_path}"
+
+ async with AsyncSQLAlchemyStorage(
+ url,
+ auto_migrate=True,
+ ) as storage:
+ agent = _agent_record("user-1")
+ await storage.upsert_agent("user-1", agent)
+ fetched = await storage.get_agent("user-1", agent.id)
+ self.assertEqual(fetched.id, agent.id)
+
+
+class LegacyRecordShapeTest(IsolatedAsyncioTestCase):
+ """The ``mode='before'`` validators absorb pre-refactor payloads.
+
+ Records written before the KB-``data`` nesting / KD-lifecycle
+ promotion refactor must still deserialise, so a database populated
+ by an older build keeps round-tripping after an upgrade.
+ """
+
+ async def test_knowledge_base_flat_payload_migrates(self) -> None:
+ """Flat KB fields fold into :attr:`KnowledgeBaseRecord.data`."""
+ record = KnowledgeBaseRecord.model_validate(
+ {
+ "user_id": "u1",
+ "name": "kb",
+ "description": "d",
+ "embedding_model_config": {
+ "type": "openai_credential",
+ "credential_id": "cred-1",
+ "model": "text-embedding-3-small",
+ "dimensions": 8,
+ },
+ "collection_name": "c",
+ },
+ )
+ self.assertEqual(record.data.name, "kb")
+ self.assertEqual(record.data.collection_name, "c")
+
+ async def test_knowledge_document_lifecycle_fields_lift(self) -> None:
+ """Legacy in-``data`` ``status`` / ``lease_expires_at`` lift up."""
+ record = KnowledgeDocumentRecord.model_validate(
+ {
+ "user_id": "u1",
+ "knowledge_base_id": "kb1",
+ "data": {
+ "filename": "f.txt",
+ "size": 1,
+ "blob_uri": "local://f.txt",
+ "status": "ready",
+ "lease_expires_at": None,
+ },
+ },
+ )
+ self.assertEqual(record.status, "ready")
+ # The fields moved to the top level and no longer shadow ``data``.
+ self.assertNotIn("status", record.data.model_dump())