Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
feat(storage): support SQLAlchemy based storage (#2029)
DavdGaob49a26b94ccf
diff --git a/pyproject.toml b/pyproject.tomlindex 6bd7fb563..cc0fe4425 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -1,4 +1,3 @@-[project]name = "agentscope"dynamic = ["version"]@@ -67,6 +66,14 @@ service = [storage = ["redis",]+# ------------ SQL storage ------------+# Async SQLAlchemy 2.0 + Alembic. The concrete driver+# (aiosqlite / asyncpg / aiomysql / asyncmy) is left to the user+# so the extra stays lightweight.+sql = [+ "sqlalchemy[asyncio]>=2.0",+ "alembic>=1.13",+]# ------------ Workspace -----------k8s = ["kubernetes-asyncio"]workspace = [@@ -120,6 +127,7 @@ full = ["agentscope[models]","agentscope[service]","agentscope[storage]",+ "agentscope[sql]","agentscope[workspace]","agentscope[tools]","agentscope[rag]",@@ -141,6 +149,8 @@ dev = ["myst_parser","matplotlib","fakeredis",+ # SQL backend tests — SQLite via aiosqlite so no server is needed.+ "aiosqlite",# S3-backend testing — moto[server] spins up an in-process S3, the# store-under-test (`S3BlobStore`) uses aioboto3, and boto3 is the# sync client the test fixtures use to seed/verify buckets.@@ -173,6 +183,17 @@ include-package-data = true"_cosyvoice_models/*.yaml",]"agentscope.workspace._docker" = ["Dockerfile*.template"]+# The Alembic scaffolding for the SQL storage backend lives in the+# data-only ``_alembic/`` subtree (no ``__init__.py`` — Alembic loads+# ``env.py`` / the version scripts by file path, not import). Ship them+# explicitly so ``auto_migrate=True`` works from an installed wheel.+"agentscope.app.storage._sql" = [+ "_alembic/alembic.ini",+ "_alembic/script.py.mako",+ "_alembic/env.py",+ "_alembic/versions/*.py",+]+[build-system]requires = ["setuptools>=45", "wheel"]build-backend = "setuptools.build_meta"diff --git a/src/agentscope/app/_router/_knowledge_base.py b/src/agentscope/app/_router/_knowledge_base.pyindex 901acc4eb..f3951c430 100644--- a/src/agentscope/app/_router/_knowledge_base.py+++ b/src/agentscope/app/_router/_knowledge_base.py@@ -501,7 +501,7 @@ async def upload_knowledge_document(return UploadKnowledgeDocumentResponse(document_id=record.id,filename=record.data.filename,- status=record.data.status,+ status=record.status,)diff --git a/src/agentscope/app/_router/_schema/_knowledge_base.py b/src/agentscope/app/_router/_schema/_knowledge_base.pyindex 5484b74e0..99c8effea 100644--- a/src/agentscope/app/_router/_schema/_knowledge_base.py+++ b/src/agentscope/app/_router/_schema/_knowledge_base.py@@ -118,7 +118,7 @@ class KnowledgeDocumentView(BaseModel):filename=record.data.filename,size=record.data.size,content_type=record.data.content_type,- status=record.data.status,+ status=record.status,error=record.data.error,chunk_count=record.data.chunk_count,created_at=record.created_at,diff --git a/src/agentscope/app/_service/_access.py b/src/agentscope/app/_service/_access.pyindex 8026ba464..2c583f7ee 100644--- a/src/agentscope/app/_service/_access.py+++ b/src/agentscope/app/_service/_access.py@@ -2,10 +2,11 @@"""Resource access service — cross-owner reads with viewer-relative views."""from __future__ import annotations+from datetime import datetimefrom typing import Any, Literal, TypeVar, overloadfrom fastapi import HTTPException, status-from pydantic import BaseModel, Field+from pydantic import BaseModel, Field, model_validatorfrom ..access import (ResourceAccessPolicyBase,@@ -16,6 +17,7 @@ from ..access import (from ..storage import (AgentRecord,CredentialRecord,+ EmbeddingModelConfig,KnowledgeBaseRecord,StorageBase,)@@ -58,20 +60,67 @@ class CredentialView(CredentialRecord):)-class KnowledgeBaseView(KnowledgeBaseRecord):- """Knowledge base record + viewer-relative ``editable``.+class KnowledgeBaseView(BaseModel):+ """Flat, viewer-facing projection of a knowledge base.- Subclasses :class:`KnowledgeBaseRecord` so the wire format is a- strict superset of the record (one extra top-level field), matching- :class:`AgentView` and :class:`CredentialView`.+ Unlike :class:`AgentView` / :class:`CredentialView` — which subclass+ their records and expose the nested ``data`` payload verbatim — the+ knowledge base's public wire shape is intentionally **flat**: the+ frontend and the documented API contract read ``name`` /+ ``embedding_model_config`` at the top level. The storage-layer+ nesting of those fields under :attr:`KnowledgeBaseRecord.data`+ (introduced so the SQL backend can serialise a single JSON column)+ must not leak into the HTTP response, so :meth:`_lift_data_payload`+ lifts them back up. ``user_id`` and the opaque ``collection_name``+ are dropped (pydantic ignores the surplus keys)."""+ id: str = Field(description="The knowledge base id.")+ name: str = Field(description="Display name of the knowledge base.")+ description: str = Field(+ default="",+ description="Free-form description of the knowledge base.",+ )+ embedding_model_config: EmbeddingModelConfig = Field(+ description="Embedding model configuration pinned at creation.",+ )+ created_at: datetime = Field(description="Creation timestamp.")+ updated_at: datetime = Field(description="Last-update timestamp.")editable: bool = Field(description=("Whether the current viewer may modify this knowledge base."),)+ @model_validator(mode="before")+ @classmethod+ def _lift_data_payload(cls, obj: Any) -> Any:+ """Flatten a record's nested ``data`` payload to the top level.++ Lets the view be built with+ ``model_validate({**record.model_dump(), "editable": ...})``+ exactly like the other views while still presenting a flat wire+ shape. Explicit top-level keys win over payload keys; inputs+ already flat pass through unchanged.++ Args:+ obj (`Any`):+ The raw validator input. Only ``dict`` inputs carrying a+ ``data`` sub-dict are rewritten; anything else is+ returned verbatim.++ Returns:+ `Any`:+ The input with ``data``'s fields lifted to the top+ level.+ """+ if not isinstance(obj, dict) or not isinstance(obj.get("data"), dict):+ return obj+ merged = {k: v for k, v in obj.items() if k != "data"}+ for key, value in obj["data"].items():+ merged.setdefault(key, value)+ return merged+# ---------------------------------------------------------------------# Servicediff --git a/src/agentscope/app/rag/knowledge_base_manager/_base.py b/src/agentscope/app/rag/knowledge_base_manager/_base.pyindex d3557a248..d50efa42f 100644--- a/src/agentscope/app/rag/knowledge_base_manager/_base.py+++ b/src/agentscope/app/rag/knowledge_base_manager/_base.py@@ -234,9 +234,9 @@ class KnowledgeBaseManagerBase(ABC):if record is None:return Noneif name is not None:- record.name = name+ record.data.name = nameif description is not None:- record.description = description+ record.data.description = descriptionreturn await self._storage.upsert_knowledge_base(user_id, record)@abstractmethoddiff --git a/src/agentscope/app/rag/knowledge_base_manager/_collection_per_kb.py b/src/agentscope/app/rag/knowledge_base_manager/_collection_per_kb.pyindex 34f033b3b..aa0ca3280 100644--- a/src/agentscope/app/rag/knowledge_base_manager/_collection_per_kb.py+++ b/src/agentscope/app/rag/knowledge_base_manager/_collection_per_kb.py@@ -22,7 +22,7 @@ from ._errors import KnowledgeBaseNotFoundErrorfrom ...._logging import loggerfrom ....rag import KnowledgeBasefrom ..._service._embedding import build_embedding_model-from ...storage import KnowledgeBaseRecord+from ...storage import KnowledgeBaseRecord, KnowledgeBaseDataif TYPE_CHECKING:from ...storage import EmbeddingModelConfig@@ -74,15 +74,17 @@ class CollectionPerKbManager(KnowledgeBaseManagerBase):"""record = KnowledgeBaseRecord(user_id=user_id,- name=name,- description=description,- embedding_model_config=embedding_model_config,- collection_name="",+ data=KnowledgeBaseData(+ name=name,+ description=description,+ embedding_model_config=embedding_model_config,+ collection_name="",+ ),)- record.collection_name = f"kb_{record.id}"+ record.data.collection_name = f"kb_{record.id}"await self._vector_store.create_collection(- name=record.collection_name,+ name=record.data.collection_name,dimensions=embedding_model_config.dimensions,)try:@@ -94,13 +96,13 @@ class CollectionPerKbManager(KnowledgeBaseManagerBase):# would leave no trace.try:await self._vector_store.delete_collection(- record.collection_name,+ record.data.collection_name,)except Exception: # noqa: BLE001 — best-effort cleanuplogger.exception("Failed to drop orphan collection %r after ""upsert_knowledge_base failed; collection leaked.",- record.collection_name,+ record.data.collection_name,)raise@@ -129,8 +131,12 @@ class CollectionPerKbManager(KnowledgeBaseManagerBase):if record is None:return False- if await self._vector_store.has_collection(record.collection_name):- await self._vector_store.delete_collection(record.collection_name)+ if await self._vector_store.has_collection(+ record.data.collection_name,+ ):+ await self._vector_store.delete_collection(+ record.data.collection_name,+ )return await self._storage.delete_knowledge_base(user_id,knowledge_base_id,@@ -180,23 +186,23 @@ class CollectionPerKbManager(KnowledgeBaseManagerBase):# the owner).credential_record = await self._storage.get_credential(record.user_id,- record.embedding_model_config.credential_id,+ record.data.embedding_model_config.credential_id,)if credential_record is None:raise KnowledgeBaseNotFoundError(f"Credential "- f"{record.embedding_model_config.credential_id!r} for "+ f"{record.data.embedding_model_config.credential_id!r} for "f"knowledge base {knowledge_base_id!r} not found.",)embedding_model = build_embedding_model(credential_record=credential_record,- config=record.embedding_model_config,+ config=record.data.embedding_model_config,)return KnowledgeBase(- name=record.name,- description=record.description,+ name=record.data.name,+ description=record.data.description,embedding_model=embedding_model,vector_store=self._vector_store,- collection=record.collection_name,+ collection=record.data.collection_name,metadata_filter=None,)diff --git a/src/agentscope/app/storage/__init__.py b/src/agentscope/app/storage/__init__.pyindex 661203f8e..6bc2eb54f 100644--- a/src/agentscope/app/storage/__init__.py+++ b/src/agentscope/app/storage/__init__.py@@ -1,5 +1,6 @@# -*- coding: utf-8 -*-"""The storage module in agentscope."""+from typing import TYPE_CHECKINGfrom ._base import StorageBasefrom ._redis_storage import RedisStorage@@ -7,6 +8,7 @@ from ._model import (AgentData,AgentRecord,CredentialRecord,+ KnowledgeBaseData,KnowledgeBaseRecord,KnowledgeDocumentData,KnowledgeDocumentRecord,@@ -28,14 +30,43 @@ from ._model import (InviteConfig,)+if TYPE_CHECKING:+ # Re-exported for static type checkers only; at runtime the class is+ # loaded lazily by ``__getattr__`` below so callers without the+ # optional ``sql`` extra never trigger the SQLAlchemy import that the+ # ``_sql`` package does at load time (its declarative-tables module+ # cannot defer it).+ from ._sql import AsyncSQLAlchemyStorage # noqa: F401+++def __getattr__(name: str) -> object:+ """Lazily load the optional SQL backend on first attribute access.++ Keeps ``import agentscope.app.storage`` cheap — ``StorageBase``,+ ``RedisStorage`` and the record models never need SQLAlchemy — while+ still exposing ``AsyncSQLAlchemyStorage`` from this package for+ callers that do ``from agentscope.app.storage import+ AsyncSQLAlchemyStorage``. SQLAlchemy is imported only at that point.+ """+ if name == "AsyncSQLAlchemyStorage":+ from ._sql import AsyncSQLAlchemyStorage as _AsyncSQLAlchemyStorage++ return _AsyncSQLAlchemyStorage+ raise AttributeError(+ f"module 'agentscope.app.storage' has no attribute {name!r}",+ )++__all__ = ["StorageBase","RedisStorage",+ "AsyncSQLAlchemyStorage",# The ORM models"InviteConfig","AgentData","AgentRecord","CredentialRecord",+ "KnowledgeBaseData","KnowledgeBaseRecord","KnowledgeDocumentData","KnowledgeDocumentRecord",diff --git a/src/agentscope/app/storage/_model/__init__.py b/src/agentscope/app/storage/_model/__init__.pyindex 14f25814a..9aac92b70 100644--- a/src/agentscope/app/storage/_model/__init__.py+++ b/src/agentscope/app/storage/_model/__init__.py@@ -3,7 +3,7 @@from ._agent import AgentRecord, AgentData, InviteConfigfrom ._credential import CredentialRecord-from ._knowledge_base import KnowledgeBaseRecord+from ._knowledge_base import KnowledgeBaseData, KnowledgeBaseRecordfrom ._knowledge_document import (KnowledgeDocumentData,KnowledgeDocumentRecord,@@ -26,6 +26,7 @@ __all__ = ["AgentData","AgentRecord","CredentialRecord",+ "KnowledgeBaseData","KnowledgeBaseRecord","KnowledgeDocumentData","KnowledgeDocumentRecord",diff --git a/src/agentscope/app/storage/_model/_knowledge_base.py b/src/agentscope/app/storage/_model/_knowledge_base.pyindex cdeba3fba..2e9be619a 100644--- a/src/agentscope/app/storage/_model/_knowledge_base.py+++ b/src/agentscope/app/storage/_model/_knowledge_base.py@@ -1,24 +1,25 @@# -*- coding: utf-8 -*-"""The knowledge base record."""-from pydantic import Field+from typing import Any++from pydantic import BaseModel, Field, model_validatorfrom ._base import _RecordBasefrom ._session import EmbeddingModelConfig-class KnowledgeBaseRecord(_RecordBase):- """A persisted knowledge base record.+class KnowledgeBaseData(BaseModel):+ """The mutable payload of a knowledge base record.- Stores per-user metadata for a knowledge base; the actual chunks- and vectors live in the configured ``VectorStoreBase`` backend.- Each record is the canonical authorisation gate: HTTP handlers and- middleware look the record up by ``(user_id, id)`` before talking- to the vector store.+ Groups every non-relational field of :class:`KnowledgeBaseRecord`+ into a single nested model so the outer record only carries+ relational keys (``user_id``) alongside the ``_RecordBase``+ envelope. This matches the convention used by the other records+ (``AgentRecord.data``, ``SessionRecord.config`` / ``.state``,+ ``TeamRecord.data`` etc.) and lets the SQL backend serialise the+ whole payload into a single JSON column without special-casing."""- user_id: str = Field(description="The owner user id.")- """The user id that owns this knowledge base."""-name: str = Field(description="Display name of the knowledge base.")"""Display name shown in the UI."""@@ -44,3 +45,62 @@ class KnowledgeBaseRecord(_RecordBase):),)"""The vector store collection name (e.g. ``kb_<uuid_hex>``)."""+++class KnowledgeBaseRecord(_RecordBase):+ """A persisted knowledge base record.++ Stores per-user metadata for a knowledge base; the actual chunks+ and vectors live in the configured ``VectorStoreBase`` backend.+ Each record is the canonical authorisation gate: HTTP handlers and+ middleware look the record up by ``(user_id, id)`` before talking+ to the vector store.+ """++ user_id: str = Field(description="The owner user id.")+ """The user id that owns this knowledge base."""++ data: KnowledgeBaseData+ """The knowledge base payload (name, description, embedding config,+ vector-store collection name)."""++ @model_validator(mode="before")+ @classmethod+ def _nest_legacy_flat_payload(cls, obj: Any) -> Any:+ """Nest legacy flat-payload fields into :attr:`data`.++ Pre-refactor records stored the payload fields (``name`` /+ ``description`` / ``embedding_model_config`` /+ ``collection_name``) at the record top level rather than+ under ``data``. This validator runs in ``mode='before'`` so+ it hooks every pydantic entry point —+ :meth:`~pydantic.BaseModel.model_validate`,+ :meth:`~pydantic.BaseModel.model_validate_json` (the path+ the storage backend takes when reading from Redis / SQL),+ and direct ``__init__`` construction — giving us one place+ to migrate the legacy shape on read.++ Args:+ obj (`Any`):+ The raw input to the validator. Only ``dict`` inputs+ are inspected; other inputs are returned verbatim so+ attribute-based construction still works.++ Returns:+ `Any`:+ The input with legacy flat fields packed into+ ``data`` when necessary.+ """+ if not isinstance(obj, dict) or "data" in obj:+ return obj+ legacy_keys = (+ "name",+ "description",+ "embedding_model_config",+ "collection_name",+ )+ if not any(key in obj for key in legacy_keys):+ return obj+ remainder = {k: v for k, v in obj.items() if k not in legacy_keys}+ remainder["data"] = {k: obj[k] for k in legacy_keys if k in obj}+ return remainderdiff --git a/src/agentscope/app/storage/_model/_knowledge_document.py b/src/agentscope/app/storage/_model/_knowledge_document.pyindex f3c7b9fb1..29aa54312 100644--- a/src/agentscope/app/storage/_model/_knowledge_document.py+++ b/src/agentscope/app/storage/_model/_knowledge_document.py@@ -9,14 +9,17 @@ the vector store cannot represent on its own (no chunks means no``document_id`` to aggregate from).Top-level fields are the relational keys (``user_id`` /-``knowledge_base_id`` / ``processing_node``) that the storage backend-indexes on; everything else lives inside ``data`` per the project-record convention.+``knowledge_base_id`` / ``processing_node``) **plus the lifecycle+fields the storage backend must index on** (``status`` /+``lease_expires_at``) — grouped at the top so a SQL backend can put+them straight into columns and the sweeper can query them without+deserialising :attr:`data`. Everything else lives inside ``data`` per+the project record convention."""from datetime import datetime-from typing import Literal+from typing import Any, Literal-from pydantic import BaseModel, Field+from pydantic import BaseModel, Field, model_validatorfrom ._base import _RecordBase@@ -37,7 +40,13 @@ KnowledgeDocumentStatus = Literal[class KnowledgeDocumentData(BaseModel):- """The mutable payload of a knowledge document record."""+ """The mutable payload of a knowledge document record.++ Groups the non-relational, non-indexed fields of the document.+ The lifecycle keys (``status`` / ``lease_expires_at``) live at+ the :class:`KnowledgeDocumentRecord` top level so the sweeper can+ query them via indexed columns without touching this payload.+ """filename: str = Field(description="Original filename supplied by the uploader.",@@ -70,13 +79,6 @@ class KnowledgeDocumentData(BaseModel)::class:`~agentscope.app.rag.blob_store.BlobStoreBase`. The workerstreams bytes back through the same blob store."""- status: KnowledgeDocumentStatus = Field(- default="pending",- description="Current lifecycle state.",- )- """Current lifecycle state. Read by the polling endpoint, written- by the worker as it transitions phases."""-error: str | None = Field(default=None,description=(@@ -96,23 +98,14 @@ class KnowledgeDocumentData(BaseModel):"""The final chunk count, written by the worker when the documentreaches ``ready``."""- lease_expires_at: datetime | None = Field(- default=None,- description=(- "Wall-clock deadline for the worker that currently holds "- "this document. ``None`` if no worker is processing it."- ),- )- """Lease deadline. Together with ``processing_node`` lets the- sweeper detect crashed workers and reassign their documents."""-class KnowledgeDocumentRecord(_RecordBase):"""A persisted knowledge document record.- Top-level fields are relational keys the storage backend needs to- index on directly (per-user listing, per-KB listing, per-node lease- sweeps); the mutable payload lives in :attr:`data`.+ Top-level fields are relational keys AND the lifecycle keys the+ storage backend must index on directly (per-user listing, per-KB+ listing, per-node lease sweeps, status-based sweeps); the+ remaining mutable payload lives in :attr:`data`."""user_id: str = Field(description="The owner user id.")@@ -135,5 +128,75 @@ class KnowledgeDocumentRecord(_RecordBase):storage backend can look up "documents owned by node X" or"documents with no owner" without deserialising every payload."""+ status: KnowledgeDocumentStatus = Field(+ default="pending",+ description="Current lifecycle state.",+ )+ """Current lifecycle state. Read by the polling endpoint, written+ by the worker as it transitions phases. Promoted to the top level+ so the SQL backend can index it and the sweeper can filter on+ non-terminal documents without payload deserialisation."""++ lease_expires_at: datetime | None = Field(+ default=None,+ description=(+ "Wall-clock deadline for the worker that currently holds "+ "this document. ``None`` if no worker is processing it."+ ),+ )+ """Lease deadline. Together with ``processing_node`` lets the+ sweeper detect crashed workers and reassign their documents.+ Promoted to the top level so it can be indexed alongside+ ``status`` / ``processing_node``."""+data: KnowledgeDocumentData- """The mutable document payload."""+ """The mutable document payload (filename, size, blob_uri, error,+ chunk_count)."""++ @model_validator(mode="before")+ @classmethod+ def _lift_legacy_lifecycle_fields(cls, obj: Any) -> Any:+ """Lift legacy in-``data`` lifecycle fields to the top level.++ Pre-refactor payloads persisted ``status`` and+ ``lease_expires_at`` inside :class:`KnowledgeDocumentData`.+ The refactor promoted them to the record top level so the+ storage backend can index / query them without deserialising+ the payload; this validator makes the record accept both+ shapes on read so records written before the refactor still+ round-trip.++ Runs in ``mode='before'`` so it hooks every entry point that+ goes through the pydantic core validator —+ :meth:`~pydantic.BaseModel.model_validate`,+ :meth:`~pydantic.BaseModel.model_validate_json` (the path the+ storage backend takes), and direct ``__init__`` construction+ alike — giving us one place to keep the legacy compatibility+ logic.++ Args:+ obj (`Any`):+ The raw input passed to the validator. Only ``dict``+ inputs are inspected; anything else is returned+ verbatim so subclasses / attribute-based construction+ still work.++ Returns:+ `Any`:+ The input with legacy lifecycle fields lifted from+ ``data`` to the top level when necessary.+ """+ if not isinstance(obj, dict):+ return obj+ data = obj.get("data")+ if not isinstance(data, dict):+ return obj+ # Only lift when the top level does NOT already carry the+ # field — a caller passing both wins at the top level.+ lifted: dict = {}+ for key in ("status", "lease_expires_at"):+ if key in data and key not in obj:+ lifted[key] = data.pop(key)+ if lifted:+ return {**obj, **lifted, "data": data}+ return objdiff --git a/src/agentscope/app/storage/_redis_storage.py b/src/agentscope/app/storage/_redis_storage.pyindex a9cc45591..44dfa29dc 100644--- a/src/agentscope/app/storage/_redis_storage.py+++ b/src/agentscope/app/storage/_redis_storage.py@@ -1634,7 +1634,7 @@ class RedisStorage(StorageBase):if not raw:returnrecord = KnowledgeDocumentRecord.model_validate_json(raw)- record.data.status = status+ record.status = statusif error is not None:record.data.error = errorif chunk_count is not None:@@ -1678,7 +1678,7 @@ class RedisStorage(StorageBase):return Falserecord = KnowledgeDocumentRecord.model_validate_json(raw)holder = record.processing_node- deadline = record.data.lease_expires_at+ deadline = record.lease_expires_atif (holder is not Noneand deadline is not None@@ -1687,7 +1687,7 @@ class RedisStorage(StorageBase):await pipe.unwatch()return Falserecord.processing_node = processing_node- record.data.lease_expires_at = new_deadline+ record.lease_expires_at = new_deadlinerecord.updated_at = nowpipe.multi()pipe.set(key, record.model_dump_json())@@ -1731,7 +1731,7 @@ class RedisStorage(StorageBase):if record.processing_node != processing_node:await pipe.unwatch()return False- record.data.lease_expires_at = new_deadline+ record.lease_expires_at = new_deadlinerecord.updated_at = nowpipe.multi()pipe.set(key, record.model_dump_json())@@ -1769,7 +1769,7 @@ class RedisStorage(StorageBase):await pipe.unwatch()returnrecord.processing_node = None- record.data.lease_expires_at = None+ record.lease_expires_at = Nonerecord.updated_at = datetime.now()pipe.multi()pipe.set(key, record.model_dump_json())@@ -1811,13 +1811,13 @@ class RedisStorage(StorageBase):if not raw:continuerecord = KnowledgeDocumentRecord.model_validate_json(raw)- if record.data.status in terminal:+ if record.status in terminal:continueif record.processing_node is None:continueif (- record.data.lease_expires_at is not None- and record.data.lease_expires_at < now+ record.lease_expires_at is not None+ and record.lease_expires_at < now):records.append(record)return records@@ -1842,7 +1842,7 @@ class RedisStorage(StorageBase):if not raw:continuerecord = KnowledgeDocumentRecord.model_validate_json(raw)- if record.data.status != "pending":+ if record.status != "pending":continueif record.created_at < threshold:records.append(record)diff --git a/src/agentscope/app/storage/_sql/__init__.py b/src/agentscope/app/storage/_sql/__init__.pynew file mode 100644index 000000000..0f48bf126--- /dev/null+++ b/src/agentscope/app/storage/_sql/__init__.py@@ -0,0 +1,12 @@+# -*- coding: utf-8 -*-+"""Async SQLAlchemy storage backend.++The only public symbol is :class:`AsyncSQLAlchemyStorage`;+every other module in this package is an implementation detail+(tables, mappers, engine helpers, Alembic scaffolding) named+with a leading underscore so :mod:`agentscope.app.storage`+can re-export it without leaking internals.+"""+from ._storage import AsyncSQLAlchemyStorage++__all__ = ["AsyncSQLAlchemyStorage"]diff --git a/src/agentscope/app/storage/_sql/_alembic/alembic.ini b/src/agentscope/app/storage/_sql/_alembic/alembic.ininew file mode 100644index 000000000..81ba48e3b--- /dev/null+++ b/src/agentscope/app/storage/_sql/_alembic/alembic.ini@@ -0,0 +1,50 @@+# Alembic configuration for the AgentScope SQL storage backend.+#+# Pointed at the private env / template shipped inside the package so+# users don't need to bootstrap Alembic themselves — just:+#+# AGENTSCOPE_SQL_URL="sqlite+aiosqlite:///./as.db" \+# alembic -c src/agentscope/app/storage/_sql/_alembic/alembic.ini upgrade head+#+# In production you should copy this file next to your deployment+# config and point ``script_location`` / ``sqlalchemy.url`` at your+# checkout.+[alembic]+script_location = %(here)s+# Empty by default — set via -x url=... or the AGENTSCOPE_SQL_URL env var.+sqlalchemy.url =++[loggers]+keys = root,sqlalchemy,alembic++[handlers]+keys = console++[formatters]+keys = generic++[logger_root]+level = WARNING+handlers = console+qualname =++[logger_sqlalchemy]+level = WARNING+handlers =+qualname = sqlalchemy.engine++[logger_alembic]+level = INFO+handlers =+qualname = alembic++[handler_console]+class = StreamHandler+args = (sys.stderr,)+level = NOTSET+formatter = generic++[formatter_generic]+format = %(levelname)-5.5s [%(name)s] %(message)s+datefmt = %H:%M:%S+diff --git a/src/agentscope/app/storage/_sql/_alembic/env.py b/src/agentscope/app/storage/_sql/_alembic/env.pynew file mode 100644index 000000000..dbd442c76--- /dev/null+++ b/src/agentscope/app/storage/_sql/_alembic/env.py@@ -0,0 +1,89 @@+# -*- coding: utf-8 -*-+"""Alembic environment for :class:`AsyncSQLAlchemyStorage` schema migrations.++Runs migrations in async mode against a SQLAlchemy URL. The URL is+picked up (in order of precedence) from:++1. the ``sqlalchemy.url`` key of the Alembic config (usually set via+ ``alembic.ini`` or ``-x url=...``);+2. the ``AGENTSCOPE_SQL_URL`` environment variable — convenient in+ dev / CI when there is no ``alembic.ini`` around.++The target metadata is :data:`_Base.metadata`, so every model added+to :mod:`agentscope.app.storage._sql._tables` is picked up by+``alembic revision --autogenerate`` without further wiring.+"""+from __future__ import with_statement++import asyncio+import os++from alembic import context+from sqlalchemy import pool+from sqlalchemy.engine import Connection+from sqlalchemy.ext.asyncio import async_engine_from_config++from agentscope.app.storage._sql._tables import _Base++# ---------------------------------------------------------------------+# Config wiring+# ---------------------------------------------------------------------+config = context.config++url_from_env = os.getenv("AGENTSCOPE_SQL_URL")+if url_from_env and not config.get_main_option("sqlalchemy.url"):+ config.set_main_option("sqlalchemy.url", url_from_env)++target_metadata = _Base.metadata+++# ---------------------------------------------------------------------+# Offline mode — emit SQL without a connection.+# ---------------------------------------------------------------------+def run_migrations_offline() -> None:+ """Run migrations using URL-only context (no DBAPI connection)."""+ url = config.get_main_option("sqlalchemy.url")+ context.configure(+ url=url,+ target_metadata=target_metadata,+ literal_binds=True,+ dialect_opts={"paramstyle": "named"},+ render_as_batch=(url or "").startswith("sqlite"),+ )+ with context.begin_transaction():+ context.run_migrations()+++# ---------------------------------------------------------------------+# Online mode — connect and run.+# ---------------------------------------------------------------------+def do_run_migrations(connection: Connection) -> None:+ """Bind Alembic to *connection* and run every pending revision."""+ is_sqlite = connection.dialect.name == "sqlite"+ context.configure(+ connection=connection,+ target_metadata=target_metadata,+ render_as_batch=is_sqlite,+ )+ with context.begin_transaction():+ context.run_migrations()+++async def run_migrations_online() -> None:+ """Create an :class:`AsyncEngine` and dispatch to+ :func:`do_run_migrations`.+ """+ connectable = async_engine_from_config(+ config.get_section(config.config_ini_section, {}),+ prefix="sqlalchemy.",+ poolclass=pool.NullPool,+ )+ async with connectable.connect() as connection:+ await connection.run_sync(do_run_migrations)+ await connectable.dispose()+++if context.is_offline_mode():+ run_migrations_offline()+else:+ asyncio.run(run_migrations_online())diff --git a/src/agentscope/app/storage/_sql/_alembic/script.py.mako b/src/agentscope/app/storage/_sql/_alembic/script.py.makonew file mode 100644index 000000000..3c2e787de--- /dev/null+++ b/src/agentscope/app/storage/_sql/_alembic/script.py.mako@@ -0,0 +1,27 @@+"""${message}++Revision ID: ${up_revision}+Revises: ${down_revision | comma,n}+Create Date: ${create_date}++"""+from typing import Sequence, Union++from alembic import op+import sqlalchemy as sa+${imports if imports else ""}++# revision identifiers, used by Alembic.+revision: str = ${repr(up_revision)}+down_revision: Union[str, None] = ${repr(down_revision)}+branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}+depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}+++def upgrade() -> None:+ ${upgrades if upgrades else "pass"}+++def downgrade() -> None:+ ${downgrades if downgrades else "pass"}+diff --git a/src/agentscope/app/storage/_sql/_alembic/versions/0001_initial.py b/src/agentscope/app/storage/_sql/_alembic/versions/0001_initial.pynew file mode 100644index 000000000..ac9c9d040--- /dev/null+++ b/src/agentscope/app/storage/_sql/_alembic/versions/0001_initial.py@@ -0,0 +1,384 @@+# -*- coding: utf-8 -*-+"""Initial schema — frozen snapshot of every table in ``_sql/_tables.py``.++Revision ID: 0001_initial+Revises:+Create Date: 2026-07-20 10:24:57.000000++This revision is a *static snapshot*: the ``op.create_table`` calls are+frozen at the schema as it stood when the SQL backend first shipped.+It deliberately does NOT delegate to ``_Base.metadata.create_all`` —+doing so would make the initial revision track the *current* metadata,+so a later column/table addition would be created here (on a fresh+database) and then created *again* by its own autogenerated revision,+which fails as a duplicate. Every future schema change must be added+as a new ``alembic revision --autogenerate`` on top of this one.+"""+from typing import Sequence, Union++from alembic import op+import sqlalchemy as sa+++# revision identifiers, used by Alembic.+revision: str = "0001_initial"+down_revision: Union[str, None] = None+branch_labels: Union[str, Sequence[str], None] = None+depends_on: Union[str, Sequence[str], None] = None+++def upgrade() -> None:+ """Create every table at the first-shipped schema."""+ op.create_table(+ "agents",+ sa.Column("user_id", sa.String(length=255), nullable=False),+ sa.Column("source", sa.String(length=16), nullable=False),+ sa.Column("id", sa.String(length=255), nullable=False),+ sa.Column("created_at", sa.DateTime(), nullable=False),+ sa.Column("updated_at", sa.DateTime(), nullable=False),+ sa.Column("payload", sa.JSON(), nullable=False),+ sa.PrimaryKeyConstraint("id"),+ )+ with op.batch_alter_table("agents", schema=None) as batch_op:+ batch_op.create_index(+ batch_op.f("ix_agents_created_at"),+ ["created_at"],+ unique=False,+ )+ batch_op.create_index(+ batch_op.f("ix_agents_source"),+ ["source"],+ unique=False,+ )+ batch_op.create_index(+ batch_op.f("ix_agents_updated_at"),+ ["updated_at"],+ unique=False,+ )+ batch_op.create_index(+ batch_op.f("ix_agents_user_id"),+ ["user_id"],+ unique=False,+ )+ batch_op.create_index(+ "ix_agents_user_source",+ ["user_id", "source"],+ unique=False,+ )++ op.create_table(+ "credentials",+ sa.Column("user_id", sa.String(length=255), nullable=False),+ sa.Column("id", sa.String(length=255), nullable=False),+ sa.Column("created_at", sa.DateTime(), nullable=False),+ sa.Column("updated_at", sa.DateTime(), nullable=False),+ sa.Column("payload", sa.JSON(), nullable=False),+ sa.PrimaryKeyConstraint("id"),+ )+ with op.batch_alter_table("credentials", schema=None) as batch_op:+ batch_op.create_index(+ batch_op.f("ix_credentials_created_at"),+ ["created_at"],+ unique=False,+ )+ batch_op.create_index(+ batch_op.f("ix_credentials_updated_at"),+ ["updated_at"],+ unique=False,+ )+ batch_op.create_index(+ batch_op.f("ix_credentials_user_id"),+ ["user_id"],+ unique=False,+ )++ op.create_table(+ "knowledge_bases",+ sa.Column("user_id", sa.String(length=255), nullable=False),+ sa.Column("id", sa.String(length=255), nullable=False),+ sa.Column("created_at", sa.DateTime(), nullable=False),+ sa.Column("updated_at", sa.DateTime(), nullable=False),+ sa.Column("payload", sa.JSON(), nullable=False),+ sa.PrimaryKeyConstraint("id"),+ )+ with op.batch_alter_table("knowledge_bases", schema=None) as batch_op:+ batch_op.create_index(+ batch_op.f("ix_knowledge_bases_created_at"),+ ["created_at"],+ unique=False,+ )+ batch_op.create_index(+ batch_op.f("ix_knowledge_bases_updated_at"),+ ["updated_at"],+ unique=False,+ )+ batch_op.create_index(+ batch_op.f("ix_knowledge_bases_user_id"),+ ["user_id"],+ unique=False,+ )++ op.create_table(+ "messages",+ sa.Column("session_id", sa.String(length=255), nullable=False),+ sa.Column("msg_id", sa.String(length=255), nullable=False),+ sa.Column("created_at", sa.DateTime(), nullable=False),+ sa.Column("payload", sa.JSON(), nullable=False),+ sa.PrimaryKeyConstraint("session_id", "msg_id"),+ )+ with op.batch_alter_table("messages", schema=None) as batch_op:+ batch_op.create_index(+ "ix_messages_session_created",+ ["session_id", "created_at"],+ unique=False,+ )++ op.create_table(+ "schedules",+ sa.Column("user_id", sa.String(length=255), nullable=False),+ sa.Column("agent_id", sa.String(length=255), nullable=False),+ sa.Column("id", sa.String(length=255), nullable=False),+ sa.Column("created_at", sa.DateTime(), nullable=False),+ sa.Column("updated_at", sa.DateTime(), nullable=False),+ sa.Column("payload", sa.JSON(), nullable=False),+ sa.PrimaryKeyConstraint("id"),+ )+ with op.batch_alter_table("schedules", schema=None) as batch_op:+ batch_op.create_index(+ batch_op.f("ix_schedules_agent_id"),+ ["agent_id"],+ unique=False,+ )+ batch_op.create_index(+ batch_op.f("ix_schedules_created_at"),+ ["created_at"],+ unique=False,+ )+ batch_op.create_index(+ batch_op.f("ix_schedules_updated_at"),+ ["updated_at"],+ unique=False,+ )+ batch_op.create_index(+ batch_op.f("ix_schedules_user_id"),+ ["user_id"],+ unique=False,+ )++ op.create_table(+ "sessions",+ sa.Column("user_id", sa.String(length=255), nullable=False),+ sa.Column("agent_id", sa.String(length=255), nullable=False),+ sa.Column("source", sa.String(length=16), nullable=False),+ sa.Column(+ "source_schedule_id",+ sa.String(length=255),+ nullable=True,+ ),+ sa.Column("team_id", sa.String(length=255), nullable=True),+ sa.Column("id", sa.String(length=255), nullable=False),+ sa.Column("created_at", sa.DateTime(), nullable=False),+ sa.Column("updated_at", sa.DateTime(), nullable=False),+ sa.Column("payload", sa.JSON(), nullable=False),+ sa.PrimaryKeyConstraint("id"),+ )+ with op.batch_alter_table("sessions", schema=None) as batch_op:+ batch_op.create_index(+ batch_op.f("ix_sessions_agent_id"),+ ["agent_id"],+ unique=False,+ )+ batch_op.create_index(+ batch_op.f("ix_sessions_created_at"),+ ["created_at"],+ unique=False,+ )+ batch_op.create_index(+ batch_op.f("ix_sessions_source"),+ ["source"],+ unique=False,+ )+ batch_op.create_index(+ batch_op.f("ix_sessions_source_schedule_id"),+ ["source_schedule_id"],+ unique=False,+ )+ batch_op.create_index(+ batch_op.f("ix_sessions_team_id"),+ ["team_id"],+ unique=False,+ )+ batch_op.create_index(+ batch_op.f("ix_sessions_updated_at"),+ ["updated_at"],+ unique=False,+ )+ batch_op.create_index(+ "ix_sessions_user_agent",+ ["user_id", "agent_id"],+ unique=False,+ )+ batch_op.create_index(+ batch_op.f("ix_sessions_user_id"),+ ["user_id"],+ unique=False,+ )++ op.create_table(+ "teams",+ sa.Column("user_id", sa.String(length=255), nullable=False),+ sa.Column("session_id", sa.String(length=255), nullable=False),+ sa.Column("id", sa.String(length=255), nullable=False),+ sa.Column("created_at", sa.DateTime(), nullable=False),+ sa.Column("updated_at", sa.DateTime(), nullable=False),+ sa.Column("payload", sa.JSON(), nullable=False),+ sa.PrimaryKeyConstraint("id"),+ )+ with op.batch_alter_table("teams", schema=None) as batch_op:+ batch_op.create_index(+ batch_op.f("ix_teams_created_at"),+ ["created_at"],+ unique=False,+ )+ batch_op.create_index(+ batch_op.f("ix_teams_updated_at"),+ ["updated_at"],+ unique=False,+ )+ batch_op.create_index(+ batch_op.f("ix_teams_user_id"),+ ["user_id"],+ unique=False,+ )++ op.create_table(+ "knowledge_documents",+ sa.Column("user_id", sa.String(length=255), nullable=False),+ sa.Column(+ "knowledge_base_id",+ sa.String(length=255),+ nullable=False,+ ),+ sa.Column(+ "processing_node",+ sa.String(length=128),+ nullable=True,+ ),+ sa.Column("status", sa.String(length=16), nullable=False),+ sa.Column("lease_expires_at", sa.DateTime(), nullable=True),+ sa.Column("id", sa.String(length=255), nullable=False),+ sa.Column("created_at", sa.DateTime(), nullable=False),+ sa.Column("updated_at", sa.DateTime(), nullable=False),+ sa.Column("payload", sa.JSON(), nullable=False),+ sa.ForeignKeyConstraint(+ ["knowledge_base_id"],+ ["knowledge_bases.id"],+ ondelete="CASCADE",+ ),+ sa.PrimaryKeyConstraint("id"),+ )+ with op.batch_alter_table(+ "knowledge_documents",+ schema=None,+ ) as batch_op:+ batch_op.create_index(+ "ix_kd_status_lease",+ ["status", "lease_expires_at"],+ unique=False,+ )+ batch_op.create_index(+ "ix_kd_user_kb",+ ["user_id", "knowledge_base_id"],+ unique=False,+ )+ batch_op.create_index(+ batch_op.f("ix_knowledge_documents_created_at"),+ ["created_at"],+ unique=False,+ )+ batch_op.create_index(+ batch_op.f("ix_knowledge_documents_knowledge_base_id"),+ ["knowledge_base_id"],+ unique=False,+ )+ batch_op.create_index(+ batch_op.f("ix_knowledge_documents_status"),+ ["status"],+ unique=False,+ )+ batch_op.create_index(+ batch_op.f("ix_knowledge_documents_updated_at"),+ ["updated_at"],+ unique=False,+ )+ batch_op.create_index(+ batch_op.f("ix_knowledge_documents_user_id"),+ ["user_id"],+ unique=False,+ )+++def downgrade() -> None:+ """Drop every table created by :func:`upgrade`."""+ with op.batch_alter_table(+ "knowledge_documents",+ schema=None,+ ) as batch_op:+ batch_op.drop_index(batch_op.f("ix_knowledge_documents_user_id"))+ batch_op.drop_index(batch_op.f("ix_knowledge_documents_updated_at"))+ batch_op.drop_index(batch_op.f("ix_knowledge_documents_status"))+ batch_op.drop_index(+ batch_op.f("ix_knowledge_documents_knowledge_base_id"),+ )+ batch_op.drop_index(batch_op.f("ix_knowledge_documents_created_at"))+ batch_op.drop_index("ix_kd_user_kb")+ batch_op.drop_index("ix_kd_status_lease")+ op.drop_table("knowledge_documents")++ with op.batch_alter_table("teams", schema=None) as batch_op:+ batch_op.drop_index(batch_op.f("ix_teams_user_id"))+ batch_op.drop_index(batch_op.f("ix_teams_updated_at"))+ batch_op.drop_index(batch_op.f("ix_teams_created_at"))+ op.drop_table("teams")++ with op.batch_alter_table("sessions", schema=None) as batch_op:+ batch_op.drop_index(batch_op.f("ix_sessions_user_id"))+ batch_op.drop_index("ix_sessions_user_agent")+ batch_op.drop_index(batch_op.f("ix_sessions_updated_at"))+ batch_op.drop_index(batch_op.f("ix_sessions_team_id"))+ batch_op.drop_index(batch_op.f("ix_sessions_source_schedule_id"))+ batch_op.drop_index(batch_op.f("ix_sessions_source"))+ batch_op.drop_index(batch_op.f("ix_sessions_created_at"))+ batch_op.drop_index(batch_op.f("ix_sessions_agent_id"))+ op.drop_table("sessions")++ with op.batch_alter_table("schedules", schema=None) as batch_op:+ batch_op.drop_index(batch_op.f("ix_schedules_user_id"))+ batch_op.drop_index(batch_op.f("ix_schedules_updated_at"))+ batch_op.drop_index(batch_op.f("ix_schedules_created_at"))+ batch_op.drop_index(batch_op.f("ix_schedules_agent_id"))+ op.drop_table("schedules")++ with op.batch_alter_table("messages", schema=None) as batch_op:+ batch_op.drop_index("ix_messages_session_created")+ op.drop_table("messages")++ with op.batch_alter_table("knowledge_bases", schema=None) as batch_op:+ batch_op.drop_index(batch_op.f("ix_knowledge_bases_user_id"))+ batch_op.drop_index(batch_op.f("ix_knowledge_bases_updated_at"))+ batch_op.drop_index(batch_op.f("ix_knowledge_bases_created_at"))+ op.drop_table("knowledge_bases")++ with op.batch_alter_table("credentials", schema=None) as batch_op:+ batch_op.drop_index(batch_op.f("ix_credentials_user_id"))+ batch_op.drop_index(batch_op.f("ix_credentials_updated_at"))+ batch_op.drop_index(batch_op.f("ix_credentials_created_at"))+ op.drop_table("credentials")++ with op.batch_alter_table("agents", schema=None) as batch_op:+ batch_op.drop_index("ix_agents_user_source")+ batch_op.drop_index(batch_op.f("ix_agents_user_id"))+ batch_op.drop_index(batch_op.f("ix_agents_updated_at"))+ batch_op.drop_index(batch_op.f("ix_agents_source"))+ batch_op.drop_index(batch_op.f("ix_agents_created_at"))+ op.drop_table("agents")diff --git a/src/agentscope/app/storage/_sql/_mappers.py b/src/agentscope/app/storage/_sql/_mappers.pynew file mode 100644index 000000000..bea8dac4a--- /dev/null+++ b/src/agentscope/app/storage/_sql/_mappers.py@@ -0,0 +1,100 @@+# -*- coding: utf-8 -*-+"""Pydantic ``_RecordBase`` ↔ SQLAlchemy row conversion helpers.++Round-trip contract (the "no duplication" invariant enforced here):++- On write (``_from_record``) the record is dumped once with+ ``model_dump(mode="json")``, the envelope keys (``id`` /+ ``created_at`` / ``updated_at``) and every field listed in the+ row class' ``_indexed_fields`` are popped out into the promoted+ columns, and whatever remains is stored verbatim in ``payload``.+- On read (``_to_record``) the promoted columns are merged back into+ the payload dict and the whole thing is fed through+ ``model_validate`` — which, thanks to the ``mode="before"``+ validators on records like :class:`KnowledgeDocumentRecord` /+ :class:`KnowledgeBaseRecord`, also transparently absorbs any+ legacy on-disk shapes.++Both functions are dialect-agnostic — they only touch pydantic and+plain Python dicts. All dialect specifics live in :mod:`_storage`.+"""+from typing import TYPE_CHECKING, TypeVar++from .._model._base import _RecordBase++if TYPE_CHECKING:+ from ._tables import _JsonRecordMixin+++R = TypeVar("R", bound=_RecordBase)++# Envelope fields promoted by :class:`_JsonRecordMixin`; also popped+# from every record dump before it hits ``payload``.+_ENVELOPE_FIELDS = ("id", "created_at", "updated_at")+++def _from_record(+ row_cls: "type[_JsonRecordMixin]",+ record: _RecordBase,+) -> "_JsonRecordMixin":+ """Project *record* onto a fresh instance of *row_cls*.++ Args:+ row_cls (`type[_JsonRecordMixin]`):+ The concrete ``*Row`` class to instantiate. Must expose+ ``_indexed_fields`` naming the promoted columns.+ record (`_RecordBase`):+ The pydantic record to persist. Its ``id``,+ ``created_at`` and ``updated_at`` are taken as-is (the+ caller is responsible for refreshing ``updated_at``+ before the call, mirroring the Redis storage semantics).++ Returns:+ `_JsonRecordMixin`:+ A row instance whose promoted columns and ``payload``+ column together carry every field of the record with no+ duplication.+ """+ dump = record.model_dump(mode="json")+ for field in _ENVELOPE_FIELDS:+ dump.pop(field, None)+ column_values: dict = {}+ for field in row_cls.get_indexed_fields():+ column_values[field] = dump.pop(field, None)+ return row_cls(+ id=record.id,+ created_at=record.created_at,+ updated_at=record.updated_at,+ payload=dump,+ **column_values,+ )+++def _to_record(+ row: "_JsonRecordMixin",+ record_cls: type[R],+) -> R:+ """Reconstruct a record from *row*.++ Merges the promoted columns back into ``payload`` and delegates+ to :meth:`~pydantic.BaseModel.model_validate` so any+ ``mode='before'`` validators on the record (legacy-shape+ migrators, defaults, etc.) still fire.++ Args:+ row (`_JsonRecordMixin`):+ The SQLAlchemy row loaded from the database.+ record_cls (`type[R]`):+ The concrete :class:`_RecordBase` subclass to return.++ Returns:+ `R`:+ The reconstituted record.+ """+ obj: dict = dict(row.payload or {})+ obj["id"] = row.id+ obj["created_at"] = row.created_at+ obj["updated_at"] = row.updated_at+ for field in row.__class__.get_indexed_fields():+ obj[field] = getattr(row, field)+ return record_cls.model_validate(obj)diff --git a/src/agentscope/app/storage/_sql/_storage.py b/src/agentscope/app/storage/_sql/_storage.pynew file mode 100644index 000000000..1cc558a12--- /dev/null+++ b/src/agentscope/app/storage/_sql/_storage.py@@ -0,0 +1,1651 @@+# -*- coding: utf-8 -*-+# pylint: disable=too-many-public-methods+"""SQLAlchemy 2.0 async implementation of :class:`StorageBase`.++Talks to any dialect supported by SQLAlchemy's async engine+(SQLite / Postgres / MySQL / …) — the caller only picks the URL and+installs the matching driver. All schema and query building goes+through dialect-neutral SA constructs; see the module doc of+:mod:`_tables` for the portability constraints we hold ourselves to.++Timestamps are stored as **naive UTC**: the backend generates every+timestamp with :func:`_utcnow` (UTC, not the machine-local+``datetime.now()``) and normalises any caller-supplied datetime with+:func:`_to_naive_utc`. Using UTC keeps timestamps comparable across+nodes in a distributed deployment regardless of each node's local+timezone, while staying tz-naive so the plain ``DateTime`` columns+need no dialect-specific timezone handling.+"""+from datetime import datetime, timedelta, timezone+from typing import TYPE_CHECKING, Any, Self++from .._base import StorageBase+from .._model import (+ AgentRecord,+ CredentialRecord,+ KnowledgeBaseRecord,+ KnowledgeDocumentRecord,+ KnowledgeDocumentStatus,+ ScheduleRecord,+ SessionRecord,+ SessionConfig,+ SessionSource,+ TeamRecord,+)+from .._utils import _dump_with_secrets+from ._mappers import _from_record, _to_record+from ._tables import (+ _Base,+ AgentRow,+ CredentialRow,+ KnowledgeBaseRow,+ KnowledgeDocumentRow,+ MessageRow,+ ScheduleRow,+ SessionRow,+ TeamRow,+)+from ....credential import CredentialBase+from ....message import Msg+from ....state import AgentState++if TYPE_CHECKING:+ from sqlalchemy.ext.asyncio import (+ AsyncEngine,+ AsyncSession,+ async_sessionmaker,+ )+++def _utcnow() -> datetime:+ """Current time as a naive UTC timestamp.++ Uses UTC rather than the machine-local ``datetime.now()`` so+ timestamps stay comparable across nodes in a distributed+ deployment; the ``tzinfo`` is stripped so the value round-trips+ through the naive ``DateTime`` columns without mixing aware/naive+ datetimes.+ """+ return datetime.now(timezone.utc).replace(tzinfo=None)+++def _to_naive_utc(dt: datetime) -> datetime:+ """Normalise *dt* to a naive UTC timestamp.++ Aware datetimes are converted to UTC; naive datetimes are assumed+ to be in the machine-local zone — the project-wide convention for+ ``datetime.now()`` — and re-anchored to UTC. Either way the result+ is tz-naive so it matches the values produced by :func:`_utcnow`.+ """+ return dt.astimezone(timezone.utc).replace(tzinfo=None)+++class AsyncSQLAlchemyStorage(StorageBase):+ """Async SQLAlchemy-backed :class:`StorageBase` implementation.++ Instantiate with any SQLAlchemy async URL (the concrete driver is+ a caller-installed dependency, not part of the ``sql`` extra):++ .. code-block:: python++ storage = AsyncSQLAlchemyStorage("sqlite+aiosqlite:///./as.db")+ async with storage:+ ...++ ``__aenter__`` builds the engine and (optionally) provisions the+ schema; ``aclose`` disposes the engine.+ """++ def __init__(+ self,+ url: str,+ *,+ create_tables: bool = True,+ auto_migrate: bool = False,+ engine: "AsyncEngine | None" = None,+ engine_kwargs: dict[str, Any] | None = None,+ ) -> None:+ """Configure the backend; nothing is opened until+ :meth:`__aenter__`.++ Args:+ url (`str`):+ A SQLAlchemy async URL (e.g.+ ``sqlite+aiosqlite:///./as.db``,+ ``postgresql+asyncpg://user:pw@host/db``). Ignored+ when ``engine`` is provided.+ create_tables (`bool`, defaults to `True`):+ When `True`, run ``Base.metadata.create_all`` on the+ engine at ``__aenter__`` — convenient for tests and+ single-node dev deployments where Alembic overhead is+ unwanted.+ auto_migrate (`bool`, defaults to `False`):+ When `True`, run ``alembic upgrade head`` against the+ packaged migration scripts at ``__aenter__``.+ Convenient for single-node or dev deployments —+ every process boot brings the schema up to date.+ **Not recommended for multi-replica production**:+ two replicas racing on the same migration is unsafe.+ In that setup keep this `False` and run+ ``alembic upgrade head`` as a discrete deploy step.+ engine (`AsyncEngine | None`, optional):+ An externally managed engine. When supplied the+ engine is used as-is and **not** disposed by+ :meth:`aclose` — the caller owns its lifecycle. When+ omitted an engine is constructed from *url* on+ :meth:`__aenter__` and disposed on :meth:`aclose`.+ engine_kwargs (`dict[str, Any] | None`, optional):+ Extra keyword arguments forwarded to+ :func:`sqlalchemy.ext.asyncio.create_async_engine`+ when the engine is created internally (e.g.+ ``echo=True``, ``pool_size=20``). Ignored when+ *engine* is supplied.+ """+ self._url = url+ self._create_tables = create_tables+ self._auto_migrate = auto_migrate+ self._external_engine: "AsyncEngine | None" = engine+ self._engine_kwargs = engine_kwargs or {}++ # Populated in __aenter__; None until the context is entered.+ self._engine: "AsyncEngine | None" = None+ self._owns_engine: bool = False+ self._session_factory: "async_sessionmaker[AsyncSession] | None" = None++ async def __aenter__(self) -> Self:+ """Build the engine (or adopt the external one) and optionally+ provision the schema.+ """+ from sqlalchemy.ext.asyncio import (+ async_sessionmaker,+ create_async_engine,+ )++ if self._external_engine is not None:+ self._engine = self._external_engine+ self._owns_engine = False+ else:+ self._engine = create_async_engine(+ self._url,+ future=True,+ **self._engine_kwargs,+ )+ self._owns_engine = True+ # SQLite ignores foreign keys unless enabled per connection,+ # so ``ON DELETE CASCADE`` would silently not fire. Turn it+ # on for every pooled connection.+ if self._engine.sync_engine.dialect.name == "sqlite":+ from sqlalchemy import event++ @event.listens_for(self._engine.sync_engine, "connect")+ def _enable_sqlite_fk(dbapi_conn: Any, _rec: Any) -> None:+ cursor = dbapi_conn.cursor()+ cursor.execute("PRAGMA foreign_keys=ON")+ cursor.close()++ # ``expire_on_commit=False`` keeps mapper-returned rows readable+ # after commit — :class:`AsyncSQLAlchemyStorage` immediately+ # projects rows back into pydantic records outside the session+ # scope, which would otherwise trigger a detached-load error.+ self._session_factory = async_sessionmaker(+ self._engine,+ expire_on_commit=False,+ )++ if self._auto_migrate:+ await self._run_alembic_upgrade()++ if self._create_tables:+ async with self._engine.begin() as conn:+ await conn.run_sync(_Base.metadata.create_all)++ return self++ async def _run_alembic_upgrade(self) -> None:+ """Bring the schema up to ``head`` via the packaged Alembic scripts.++ Runs :func:`alembic.command.upgrade` against the ``_alembic/``+ directory shipped alongside this module, using the current+ engine URL. Executed inside :meth:`asyncio.to_thread` because+ Alembic's runtime is synchronous (it drives an async engine+ internally via :func:`asyncio.run` inside its own ``env.py``,+ so calling it from the main event loop would nest loops and+ deadlock).++ Failures propagate: a broken migration MUST prevent+ ``__aenter__`` from returning a half-configured storage.+ """+ import asyncio+ from pathlib import Path++ from alembic import command+ from alembic.config import Config++ alembic_dir = Path(__file__).resolve().parent / "_alembic"+ cfg = Config(str(alembic_dir / "alembic.ini"))+ cfg.set_main_option("script_location", str(alembic_dir))+ cfg.set_main_option("sqlalchemy.url", self._url)++ await asyncio.to_thread(command.upgrade, cfg, "head")++ async def aclose(self) -> None:+ """Dispose the engine (if owned) and drop the session factory."""+ if self._engine is not None and self._owns_engine:+ await self._engine.dispose()+ self._engine = None+ self._session_factory = None++ # ------------------------------------------------------------------+ # Small helpers shared by every method+ # ------------------------------------------------------------------++ def _session(self) -> "AsyncSession":+ """Return a fresh :class:`AsyncSession` from the factory.++ Raises:+ `RuntimeError`:+ If the storage has not been entered as an async+ context manager yet.+ """+ if self._session_factory is None:+ raise RuntimeError(+ "AsyncSQLAlchemyStorage is not initialised — use "+ "`async with storage:`.",+ )+ return self._session_factory()++ def _upsert_stmt(+ self,+ row_cls: type,+ values: dict[str, Any],+ conflict_cols: list[str],+ update_cols: tuple[str, ...],+ ) -> Any:+ """Build a dialect-native atomic upsert statement.++ A single ``INSERT ... ON CONFLICT DO UPDATE`` (Postgres /+ SQLite) or ``INSERT ... ON DUPLICATE KEY UPDATE`` (MySQL /+ MariaDB) makes the insert-or-update decision **inside the+ database**, so concurrent writers on the same key can no longer+ race between a separate read and write (the failure mode of the+ classic read-then-write upsert). ``conflict_cols`` name the+ key that triggers the "update instead" branch; ``update_cols``+ are the columns overwritten on conflict — deliberately+ excluding ``id`` / ``created_at`` so an upsert refreshes the+ mutable columns while keeping the original creation time.++ Args:+ row_cls (`type`):+ The concrete ``*Row`` class whose table is targeted.+ values (`dict[str, Any]`):+ Full column → value mapping for the INSERT branch.+ conflict_cols (`list[str]`):+ Column names forming the conflict target (the primary+ key, or the composite key for messages).+ update_cols (`tuple[str, ...]`):+ Column names to overwrite on conflict.++ Returns:+ `Any`:+ A dialect-specific insert construct ready to execute.++ Raises:+ `NotImplementedError`:+ For dialects without a supported native upsert+ (e.g. Oracle / SQL Server, which would need ``MERGE``).+ """+ assert self._engine is not None+ dialect = self._engine.sync_engine.dialect.name+ table = row_cls.__table__++ if dialect in ("postgresql", "sqlite"):+ if dialect == "postgresql":+ from sqlalchemy.dialects.postgresql import insert as _insert+ else:+ from sqlalchemy.dialects.sqlite import insert as _insert+ stmt = _insert(table).values(**values)+ return stmt.on_conflict_do_update(+ index_elements=conflict_cols,+ set_={c: getattr(stmt.excluded, c) for c in update_cols},+ )++ if dialect in ("mysql", "mariadb"):+ from sqlalchemy.dialects.mysql import insert as _insert++ stmt = _insert(table).values(**values)+ return stmt.on_duplicate_key_update(+ **{c: getattr(stmt.inserted, c) for c in update_cols},+ )++ raise NotImplementedError(+ f"Atomic upsert is unsupported for the {dialect!r} dialect; "+ "supported dialects are postgresql, sqlite, mysql, mariadb.",+ )++ async def _write_row(+ self,+ row_cls: type,+ record: Any,+ *,+ preserve_created_at: bool = True,+ ) -> Any:+ """Atomically insert-or-update *record* via *row_cls*.++ Issues a single dialect-native upsert (see :meth:`_upsert_stmt`)+ so the insert-or-update decision is made inside the database —+ an upsert refreshes ``updated_at`` / ``payload`` / the indexed+ columns but keeps the original ``created_at`` (mirroring the+ Redis backend's semantics). Because the statement is atomic+ there is no read-then-write window for concurrent writers on+ the same id to race through. Returns the (mutated) record so+ callers can propagate the stamped timestamps.++ Args:+ row_cls (`type`):+ Concrete ``*Row`` class from :mod:`_tables`.+ record (`Any`):+ The pydantic record to write. Mutated in place —+ ``created_at`` / ``updated_at`` are refreshed.+ preserve_created_at (`bool`, defaults to `True`):+ When `True`, the DB-authoritative ``created_at`` (the+ original one on an update) is read back into the+ returned record. Set `False` on pure-create paths+ where no prior row can exist, to skip that read.++ Returns:+ `Any`:+ The (mutated) record.+ """+ from sqlalchemy import select++ record.updated_at = _utcnow()+ # The record default uses machine-local ``datetime.now()``;+ # anchor it to UTC for the INSERT branch. On conflict the DB+ # keeps its stored ``created_at`` and we read it back below.+ record.created_at = _to_naive_utc(record.created_at)+ new_row = _from_record(row_cls, record)+ indexed = tuple(row_cls.get_indexed_fields())+ values = {+ col: getattr(new_row, col)+ for col in ("id", "created_at", "updated_at", "payload") + indexed+ }+ update_cols = ("updated_at", "payload") + indexed++ async with self._session() as sess:+ await sess.execute(+ self._upsert_stmt(row_cls, values, ["id"], update_cols),+ )+ if preserve_created_at:+ # Reflect the creation time the DB kept on conflict+ # back into the returned record.+ record.created_at = (+ await sess.execute(+ select(row_cls.created_at).where(+ row_cls.id == record.id,+ ),+ )+ ).scalar_one()+ await sess.commit()+ return record++ # ------------------------------------------------------------------+ # Cascade-delete internals — every ``_delete_*_impl`` runs on the+ # caller-supplied ``sess`` WITHOUT committing, so the public+ # ``delete_*`` wrapper can run the whole (mutually recursive)+ # cascade inside a single transaction and commit once. That makes+ # the compound deletes atomic: a mid-cascade failure rolls the+ # whole thing back instead of leaving a half-deleted graph.+ #+ # Reads use ``select`` (not ``session.get``) so they see the+ # transaction's own uncommitted deletes instead of stale+ # identity-map rows.+ # ------------------------------------------------------------------++ async def _delete_session_impl(+ self,+ sess: "AsyncSession",+ user_id: str,+ agent_id: str,+ session_id: str,+ ) -> bool:+ """Delete a session (+ its messages, + leader-team) on *sess*."""+ from sqlalchemy import delete, select++ _ = agent_id # scoping enforced by caller+ row = (+ await sess.execute(+ select(SessionRow).where(SessionRow.id == session_id),+ )+ ).scalar_one_or_none()+ if row is None or row.user_id != user_id:+ return False+ record = _to_record(row, SessionRecord)++ # If this session leads a team, dissolve the team first.+ if record.team_id:+ team_row = (+ await sess.execute(+ select(TeamRow).where(TeamRow.id == record.team_id),+ )+ ).scalar_one_or_none()+ if (+ team_row is not None+ and _to_record(team_row, TeamRecord).session_id == session_id+ ):+ await self._delete_team_impl(sess, user_id, record.team_id)++ await sess.execute(+ delete(SessionRow).where(+ SessionRow.id == session_id,+ SessionRow.user_id == user_id,+ ),+ )+ await sess.execute(+ delete(MessageRow).where(MessageRow.session_id == session_id),+ )+ return True++ async def _delete_schedule_impl(+ self,+ sess: "AsyncSession",+ user_id: str,+ schedule_id: str,+ ) -> bool:+ """Delete a schedule + cascade its execution sessions on *sess*."""+ from sqlalchemy import delete, select++ row = (+ await sess.execute(+ select(ScheduleRow).where(ScheduleRow.id == schedule_id),+ )+ ).scalar_one_or_none()+ if row is None or row.user_id != user_id:+ return False++ sessions = (+ (+ await sess.execute(+ select(SessionRow).where(+ SessionRow.user_id == user_id,+ SessionRow.source_schedule_id == schedule_id,+ ),+ )+ )+ .scalars()+ .all()+ )+ for s in sessions:+ await self._delete_session_impl(sess, user_id, s.agent_id, s.id)++ await sess.execute(+ delete(ScheduleRow).where(+ ScheduleRow.id == schedule_id,+ ScheduleRow.user_id == user_id,+ ),+ )+ return True++ async def _delete_agent_impl(+ self,+ sess: "AsyncSession",+ user_id: str,+ agent_id: str,+ ) -> bool:+ """Delete an agent + cascade sessions/schedules/team refs on *sess*."""+ from sqlalchemy import delete, select, update++ # Cascade: sessions owned by this agent.+ sessions = (+ (+ await sess.execute(+ select(SessionRow).where(+ SessionRow.user_id == user_id,+ SessionRow.agent_id == agent_id,+ ),+ )+ )+ .scalars()+ .all()+ )+ for s in sessions:+ await self._delete_session_impl(sess, user_id, agent_id, s.id)++ # Cascade: schedules owned by this agent.+ schedules = (+ (+ await sess.execute(+ select(ScheduleRow).where(+ ScheduleRow.user_id == user_id,+ ScheduleRow.agent_id == agent_id,+ ),+ )+ )+ .scalars()+ .all()+ )+ for sch in schedules:+ await self._delete_schedule_impl(sess, user_id, sch.id)++ # Defensive: scrub the agent from every surviving team roster+ # (both the legacy ``member_ids`` list and the ``members`` list).+ teams = (+ (+ await sess.execute(+ select(TeamRow).where(TeamRow.user_id == user_id),+ )+ )+ .scalars()+ .all()+ )+ for team_row in teams:+ team_rec = _to_record(team_row, TeamRecord)+ dirty = False+ if agent_id in team_rec.data.member_ids:+ team_rec.data.member_ids = [+ m for m in team_rec.data.member_ids if m != agent_id+ ]+ dirty = True+ filtered = [+ m for m in team_rec.data.members if m.agent_id != agent_id+ ]+ if len(filtered) != len(team_rec.data.members):+ team_rec.data.members = filtered+ dirty = True+ if dirty:+ team_rec.updated_at = _utcnow()+ new_row = _from_record(TeamRow, team_rec)+ await sess.execute(+ update(TeamRow)+ .where(TeamRow.id == team_rec.id)+ .values(+ payload=new_row.payload,+ updated_at=new_row.updated_at,+ ),+ )++ result = await sess.execute(+ delete(AgentRow).where(+ AgentRow.id == agent_id,+ AgentRow.user_id == user_id,+ ),+ )+ return result.rowcount > 0++ async def _delete_team_impl(+ self,+ sess: "AsyncSession",+ user_id: str,+ team_id: str,+ ) -> bool:+ """Role-aware team cleanup + leader-detach + delete, on *sess*."""+ from sqlalchemy import delete, select, update++ from .._model._team import TeamMember++ row = (+ await sess.execute(+ select(TeamRow).where(TeamRow.id == team_id),+ )+ ).scalar_one_or_none()+ if row is None or row.user_id != user_id:+ return False+ team = _to_record(row, TeamRecord)++ # Materialise the roster. Modern records carry ``members``;+ # legacy ``member_ids``-only records are migrated inline on+ # *sess* (no writeback — the team is being deleted anyway).+ members = list(team.data.members)+ if not members and team.data.member_ids:+ for aid in team.data.member_ids:+ worker = (+ (+ await sess.execute(+ select(SessionRow).where(+ SessionRow.user_id == user_id,+ SessionRow.agent_id == aid,+ ),+ )+ )+ .scalars()+ .first()+ )+ if worker is not None:+ members.append(+ TeamMember(+ owner_id=user_id,+ agent_id=aid,+ session_id=worker.id,+ role="created",+ ),+ )++ for member in members:+ if member.role == "created":+ await self._delete_agent_impl(+ sess,+ member.owner_id,+ member.agent_id,+ )+ else: # invited — only the borrowed session goes+ await self._delete_session_impl(+ sess,+ member.owner_id,+ member.agent_id,+ member.session_id,+ )++ # Detach the leader session (idempotent — it may already be gone).+ await sess.execute(+ update(SessionRow)+ .where(+ SessionRow.id == team.session_id,+ SessionRow.user_id == user_id,+ )+ .values(team_id=None, updated_at=_utcnow()),+ )+ await sess.execute(+ delete(TeamRow).where(+ TeamRow.id == team_id,+ TeamRow.user_id == user_id,+ ),+ )+ return True++ # ------------------------------------------------------------------+ # Credentials+ # ------------------------------------------------------------------++ async def _generate_credential_name(+ self,+ user_id: str,+ credential_data: CredentialBase,+ ) -> str:+ """Mirror :meth:`RedisStorage._generate_credential_name`.++ Auto-derive a unique per-user display name from the+ credential type (``"OpenAI"``, ``"OpenAI (2)"`` …) so the+ service layer does not have to.+ """+ cred_type = getattr(credential_data, "type", "")+ base_name = (+ cred_type.removesuffix("_credential").replace("_", " ").title()+ )+ if not base_name:+ base_name = "Credential"++ existing = await self.list_credentials(user_id)+ same_type_names = [+ c.data.get("name", "")+ for c in existing+ if c.data.get("type") == cred_type and c.id != credential_data.id+ ]++ if base_name not in same_type_names:+ return base_name++ idx = 2+ while f"{base_name} ({idx})" in same_type_names:+ idx += 1+ return f"{base_name} ({idx})"++ async def upsert_credential(+ self,+ user_id: str,+ credential_data: CredentialBase,+ ) -> str:+ """Create or update a credential record for *user_id*.++ Same contract as :meth:`RedisStorage.upsert_credential` — see+ that method for the id / name handling rules.+ """+ if not credential_data.name:+ credential_data.name = await self._generate_credential_name(+ user_id,+ credential_data,+ )++ data_dump = _dump_with_secrets(credential_data)++ # Scope the create-or-update to *user_id*. The Redis backend+ # namespaces its keys by user, so a preset id can only ever+ # address the caller's own record; the SQL table keys on a+ # global primary id, so an unscoped lookup — or the id-keyed+ # upsert in ``_write_row`` — would let a caller read or clobber+ # another tenant's credential. Only a row the caller already+ # owns may be updated in place; any other preset id is created+ # fresh under the caller via a plain INSERT that fails loudly on+ # a global collision instead of overwriting the holder.+ if credential_data.id:+ from sqlalchemy import select++ async with self._session() as sess:+ existing = (+ await sess.execute(+ select(CredentialRow).where(+ CredentialRow.id == credential_data.id,+ CredentialRow.user_id == user_id,+ ),+ )+ ).scalar_one_or_none()+ if existing is not None:+ # The caller's own credential: the id-keyed upsert can+ # only touch this row, so update it in place.+ record = _to_record(existing, CredentialRecord)+ record.data = data_dump+ await self._write_row(CredentialRow, record)+ return record.id+ record = CredentialRecord(+ id=credential_data.id,+ user_id=user_id,+ data=data_dump,+ )+ else:+ record = CredentialRecord(+ user_id=user_id,+ data=data_dump,+ )++ # A fresh record — either a server-generated id (never collides)+ # or a caller-preset id proven not to belong to the caller. Use+ # a plain INSERT rather than the id-conflict upsert so that a+ # preset id already held by another tenant raises IntegrityError+ # instead of silently overwriting them and stealing ownership.+ record.created_at = _to_naive_utc(record.created_at)+ record.updated_at = _utcnow()+ async with self._session() as sess:+ sess.add(_from_record(CredentialRow, record))+ await sess.commit()+ return record.id++ async def list_credentials(self, user_id: str) -> list[CredentialRecord]:+ """Return every credential for *user_id*."""+ from sqlalchemy import select++ async with self._session() as sess:+ rows = (+ (+ await sess.execute(+ select(CredentialRow).where(+ CredentialRow.user_id == user_id,+ ),+ )+ )+ .scalars()+ .all()+ )+ return [_to_record(r, CredentialRecord) for r in rows]++ async def get_credential(+ self,+ user_id: str,+ credential_id: str,+ ) -> CredentialRecord | None:+ """Fetch one credential record; enforces owner-scoping."""+ async with self._session() as sess:+ row = await sess.get(CredentialRow, credential_id)+ if row is None or row.user_id != user_id:+ return None+ return _to_record(row, CredentialRecord)++ async def delete_credential(+ self,+ user_id: str,+ credential_id: str,+ ) -> bool:+ """Delete a credential; owner-scoped."""+ from sqlalchemy import delete++ async with self._session() as sess:+ result = await sess.execute(+ delete(CredentialRow).where(+ CredentialRow.id == credential_id,+ CredentialRow.user_id == user_id,+ ),+ )+ await sess.commit()+ return result.rowcount > 0++ # ------------------------------------------------------------------+ # Agents+ # ------------------------------------------------------------------++ async def upsert_agent(+ self,+ user_id: str,+ agent_record: AgentRecord,+ ) -> str:+ """Persist an agent record (create or overwrite)."""+ _ = user_id # scoping is enforced by the caller+ await self._write_row(AgentRow, agent_record)+ return agent_record.id++ async def list_agents(self, user_id: str) -> list[AgentRecord]:+ """Return the user's ``source='user'`` agents only.++ Matches :meth:`RedisStorage.list_agents` — team-spawned+ workers (``source='team'``) are addressable by id but are+ never enumerated in the user's regular agent list.+ """+ from sqlalchemy import select++ async with self._session() as sess:+ rows = (+ (+ await sess.execute(+ select(AgentRow).where(+ AgentRow.user_id == user_id,+ AgentRow.source == "user",+ ),+ )+ )+ .scalars()+ .all()+ )+ return [_to_record(r, AgentRecord) for r in rows]++ async def get_agent(+ self,+ user_id: str,+ agent_id: str,+ ) -> AgentRecord | None:+ """Fetch one agent record; owner-scoped."""+ async with self._session() as sess:+ row = await sess.get(AgentRow, agent_id)+ if row is None or row.user_id != user_id:+ return None+ return _to_record(row, AgentRecord)++ async def delete_agent(self, user_id: str, agent_id: str) -> bool:+ """Delete an agent + cascade sessions, schedules, team refs.++ Mirrors the cascade order of :meth:`RedisStorage.delete_agent`.+ The whole cascade runs in a **single transaction** (see+ :meth:`_delete_agent_impl`) so it is atomic — a mid-cascade+ failure rolls back rather than leaving orphans.+ """+ async with self._session() as sess:+ ok = await self._delete_agent_impl(sess, user_id, agent_id)+ await sess.commit()+ return ok++ # ------------------------------------------------------------------+ # Sessions+ # ------------------------------------------------------------------++ async def upsert_session(+ self,+ user_id: str,+ agent_id: str,+ config: SessionConfig,+ state: AgentState | None = None,+ session_id: str | None = None,+ source: SessionSource = SessionSource.USER,+ source_schedule_id: str | None = None,+ ) -> SessionRecord:+ """Create or update a session — same shape as the Redis backend."""+ if session_id:+ async with self._session() as sess:+ existing = await sess.get(SessionRow, session_id)+ if existing is not None:+ record = _to_record(existing, SessionRecord)+ record.config = config+ if state is not None:+ record.state = state+ await self._write_row(SessionRow, record)+ return record++ new_id_kwargs = {"id": session_id} if session_id else {}+ record = SessionRecord(+ user_id=user_id,+ agent_id=agent_id,+ config=config,+ source=source,+ source_schedule_id=source_schedule_id,+ state=state if state is not None else AgentState(),+ **new_id_kwargs,+ )+ await self._write_row(+ SessionRow,+ record,+ preserve_created_at=False,+ )+ return record++ async def set_session_team_id(+ self,+ user_id: str,+ session_id: str,+ team_id: str | None,+ ) -> None:+ """Single-column UPDATE — atomic on all supported dialects."""+ from sqlalchemy import update++ now = _utcnow()+ async with self._session() as sess:+ await sess.execute(+ update(SessionRow)+ .where(+ SessionRow.id == session_id,+ SessionRow.user_id == user_id,+ )+ .values(team_id=team_id, updated_at=now),+ )+ await sess.commit()++ async def update_session_state(+ self,+ user_id: str,+ agent_id: str,+ session_id: str,+ state: AgentState,+ ) -> None:+ """Read-modify-write on the payload; raises if absent."""+ _ = user_id, agent_id # scoping enforced by caller+ async with self._session() as sess:+ row = await sess.get(SessionRow, session_id)+ if row is None:+ raise KeyError(f"Session {session_id!r} not found.")+ record = _to_record(row, SessionRecord)+ record.state = state+ record.updated_at = _utcnow()+ new_row = _from_record(SessionRow, record)+ row.payload = new_row.payload+ row.updated_at = new_row.updated_at+ await sess.commit()++ async def list_sessions(+ self,+ user_id: str,+ agent_id: str,+ ) -> list[SessionRecord]:+ """Sessions for a (user, agent) pair — newest first."""+ from sqlalchemy import select++ async with self._session() as sess:+ rows = (+ (+ await sess.execute(+ select(SessionRow)+ .where(+ SessionRow.user_id == user_id,+ SessionRow.agent_id == agent_id,+ )+ .order_by(SessionRow.created_at.desc()),+ )+ )+ .scalars()+ .all()+ )+ return [_to_record(r, SessionRecord) for r in rows]++ async def get_session(+ self,+ user_id: str,+ agent_id: str,+ session_id: str,+ ) -> SessionRecord | None:+ """One session; owner-scoped."""+ _ = agent_id+ async with self._session() as sess:+ row = await sess.get(SessionRow, session_id)+ if row is None or row.user_id != user_id:+ return None+ return _to_record(row, SessionRecord)++ async def delete_session(+ self,+ user_id: str,+ agent_id: str,+ session_id: str,+ ) -> bool:+ """Delete a session + cascade (message log, leader-team).++ Atomic: the session row, its message log, and any team it leads+ are removed inside one transaction (see+ :meth:`_delete_session_impl`).+ """+ async with self._session() as sess:+ ok = await self._delete_session_impl(+ sess,+ user_id,+ agent_id,+ session_id,+ )+ await sess.commit()+ return ok++ async def list_sessions_by_schedule(+ self,+ user_id: str,+ schedule_id: str,+ ) -> list[SessionRecord]:+ """Sessions created by *schedule_id* — newest first."""+ from sqlalchemy import select++ async with self._session() as sess:+ rows = (+ (+ await sess.execute(+ select(SessionRow)+ .where(+ SessionRow.user_id == user_id,+ SessionRow.source_schedule_id == schedule_id,+ )+ .order_by(SessionRow.created_at.desc()),+ )+ )+ .scalars()+ .all()+ )+ return [_to_record(r, SessionRecord) for r in rows]++ # ------------------------------------------------------------------+ # Schedules+ # ------------------------------------------------------------------++ async def upsert_schedule(+ self,+ user_id: str,+ record: ScheduleRecord,+ ) -> str:+ """Persist a schedule record (create or overwrite)."""+ _ = user_id+ await self._write_row(ScheduleRow, record)+ return record.id++ async def get_schedule(+ self,+ user_id: str,+ schedule_id: str,+ ) -> ScheduleRecord | None:+ """One schedule; owner-scoped."""+ async with self._session() as sess:+ row = await sess.get(ScheduleRow, schedule_id)+ if row is None or row.user_id != user_id:+ return None+ return _to_record(row, ScheduleRecord)++ async def list_schedules(+ self,+ user_id: str,+ ) -> list[ScheduleRecord]:+ """All schedules for a user."""+ from sqlalchemy import select++ async with self._session() as sess:+ rows = (+ (+ await sess.execute(+ select(ScheduleRow).where(+ ScheduleRow.user_id == user_id,+ ),+ )+ )+ .scalars()+ .all()+ )+ return [_to_record(r, ScheduleRecord) for r in rows]++ async def delete_schedule(+ self,+ user_id: str,+ schedule_id: str,+ ) -> bool:+ """Delete a schedule + cascade its execution sessions.++ Atomic: the schedule and every session it spawned are removed+ in one transaction (see :meth:`_delete_schedule_impl`).+ """+ async with self._session() as sess:+ ok = await self._delete_schedule_impl(sess, user_id, schedule_id)+ await sess.commit()+ return ok++ async def list_all_schedules(self) -> list[ScheduleRecord]:+ """Every schedule across every user (used on startup)."""+ from sqlalchemy import select++ async with self._session() as sess:+ rows = (await sess.execute(select(ScheduleRow))).scalars().all()+ return [_to_record(r, ScheduleRecord) for r in rows]++ # ------------------------------------------------------------------+ # Messages+ # ------------------------------------------------------------------++ async def upsert_message(+ self,+ user_id: str,+ session_id: str,+ msg: Msg,+ ) -> None:+ """Insert-or-update by ``(session_id, msg_id)``.++ Semantics mirror :meth:`RedisStorage.upsert_message` closely+ enough for every caller in the codebase — the Redis version+ only replaces when the *tail* message matches; ours replaces+ on any matching id. The looser rule is safe because callers+ never reuse a message id across turns.+ """+ _ = user_id # scoping enforced by caller+ now = _utcnow()+ payload = msg.model_dump(mode="json")++ # Atomic upsert on the composite key: insert a new event, or+ # replace the payload of an existing ``(session_id, msg_id)``+ # while keeping its original ``created_at``.+ values = {+ "session_id": session_id,+ "msg_id": msg.id,+ "created_at": now,+ "payload": payload,+ }+ async with self._session() as sess:+ await sess.execute(+ self._upsert_stmt(+ MessageRow,+ values,+ ["session_id", "msg_id"],+ ("payload",),+ ),+ )+ await sess.commit()++ async def get_message(+ self,+ user_id: str,+ session_id: str,+ message_id: str,+ ) -> Msg | None:+ """Fetch a message by ``(session_id, msg_id)``."""+ _ = user_id+ from sqlalchemy import select++ async with self._session() as sess:+ row = (+ await sess.execute(+ select(MessageRow).where(+ MessageRow.session_id == session_id,+ MessageRow.msg_id == message_id,+ ),+ )+ ).scalar_one_or_none()+ if row is None:+ return None+ return Msg.model_validate(row.payload)++ async def list_messages(+ self,+ user_id: str,+ session_id: str,+ limit: int = 50,+ before: str | None = None,+ **kwargs: Any,+ ) -> tuple[list[Msg], bool]:+ """Most-recent page of messages with cursor-based pagination.++ Mirrors :meth:`RedisStorage.list_messages`: returns up to+ *limit* messages in chronological order together with a+ ``has_more`` flag that is ``True`` when older messages exist+ before the returned page. ``before`` is a message-id cursor —+ when set, the page ends just before that message; when ``None``+ the latest page is returned. An unknown ``before`` cursor+ yields ``([], False)``. The legacy ``offset`` keyword is+ accepted only to emit a ``DeprecationWarning`` and is ignored.+ """+ _ = user_id # scoping enforced by caller+ if "offset" in kwargs:+ import warnings++ warnings.warn(+ "The 'offset' parameter is deprecated and will be "+ "removed in a future version. Use 'before' for "+ "cursor-based pagination instead.",+ DeprecationWarning,+ stacklevel=2,+ )++ from sqlalchemy import and_, or_, select++ async with self._session() as sess:+ cond = MessageRow.session_id == session_id+ if before is not None:+ cursor = (+ await sess.execute(+ select(+ MessageRow.created_at,+ MessageRow.msg_id,+ ).where(+ MessageRow.session_id == session_id,+ MessageRow.msg_id == before,+ ),+ )+ ).one_or_none()+ if cursor is None:+ return [], False+ c_created, c_msg = cursor+ # Strictly older than the cursor in the+ # ``(created_at, msg_id)`` total order (spelled out+ # rather than a row-value comparison so every dialect+ # can plan it).+ cond = and_(+ cond,+ or_(+ MessageRow.created_at < c_created,+ and_(+ MessageRow.created_at == c_created,+ MessageRow.msg_id < c_msg,+ ),+ ),+ )++ # Fetch newest-first with one extra row to detect whether+ # older messages remain, then flip back to chronological.+ rows = list(+ (+ await sess.execute(+ select(MessageRow)+ .where(cond)+ .order_by(+ MessageRow.created_at.desc(),+ MessageRow.msg_id.desc(),+ )+ .limit(limit + 1),+ )+ )+ .scalars()+ .all(),+ )++ has_more = len(rows) > limit+ page = rows[:limit]+ page.reverse()+ return [Msg.model_validate(r.payload) for r in page], has_more++ # ------------------------------------------------------------------+ # Teams+ # ------------------------------------------------------------------++ async def upsert_team(+ self,+ user_id: str,+ record: TeamRecord,+ ) -> TeamRecord:+ """Persist a team record."""+ _ = user_id+ await self._write_row(TeamRow, record)+ return record++ async def get_team(+ self,+ user_id: str,+ team_id: str,+ ) -> TeamRecord | None:+ """One team; owner-scoped."""+ async with self._session() as sess:+ row = await sess.get(TeamRow, team_id)+ if row is None or row.user_id != user_id:+ return None+ return _to_record(row, TeamRecord)++ async def list_teams(self, user_id: str) -> list[TeamRecord]:+ """All teams for a user."""+ from sqlalchemy import select++ async with self._session() as sess:+ rows = (+ (+ await sess.execute(+ select(TeamRow).where(TeamRow.user_id == user_id),+ )+ )+ .scalars()+ .all()+ )+ return [_to_record(r, TeamRecord) for r in rows]++ async def delete_team(self, user_id: str, team_id: str) -> bool:+ """Delete a team + role-aware member cleanup.++ Atomic: the role-aware member teardown (``created`` members+ deleted whole, ``invited`` members losing only their borrowed+ session), the leader detach, and the team-record delete all run+ in one transaction (see :meth:`_delete_team_impl`).+ """+ async with self._session() as sess:+ ok = await self._delete_team_impl(sess, user_id, team_id)+ await sess.commit()+ return ok++ # ------------------------------------------------------------------+ # Knowledge bases+ # ------------------------------------------------------------------++ async def upsert_knowledge_base(+ self,+ user_id: str,+ record: KnowledgeBaseRecord,+ ) -> KnowledgeBaseRecord:+ """Create or update a KB record; enforces ``record.user_id``."""+ if record.user_id != user_id:+ raise ValueError(+ "record.user_id does not match the given user_id.",+ )+ await self._write_row(KnowledgeBaseRow, record)+ return record++ async def get_knowledge_base(+ self,+ user_id: str,+ knowledge_base_id: str,+ ) -> KnowledgeBaseRecord | None:+ """One KB; owner-scoped."""+ async with self._session() as sess:+ row = await sess.get(KnowledgeBaseRow, knowledge_base_id)+ if row is None or row.user_id != user_id:+ return None+ return _to_record(row, KnowledgeBaseRecord)++ async def list_knowledge_bases(+ self,+ user_id: str,+ ) -> list[KnowledgeBaseRecord]:+ """All KBs for a user."""+ from sqlalchemy import select++ async with self._session() as sess:+ rows = (+ (+ await sess.execute(+ select(KnowledgeBaseRow).where(+ KnowledgeBaseRow.user_id == user_id,+ ),+ )+ )+ .scalars()+ .all()+ )+ return [_to_record(r, KnowledgeBaseRecord) for r in rows]++ async def delete_knowledge_base(+ self,+ user_id: str,+ knowledge_base_id: str,+ ) -> bool:+ """Delete a KB; its document rows cascade natively via the+ ``knowledge_documents.knowledge_base_id`` foreign key+ (``ON DELETE CASCADE``) — one atomic statement, no app loop.+ """+ from sqlalchemy import delete++ async with self._session() as sess:+ result = await sess.execute(+ delete(KnowledgeBaseRow).where(+ KnowledgeBaseRow.id == knowledge_base_id,+ KnowledgeBaseRow.user_id == user_id,+ ),+ )+ await sess.commit()+ return result.rowcount > 0++ # ------------------------------------------------------------------+ # Knowledge documents+ # ------------------------------------------------------------------++ async def upsert_knowledge_document(+ self,+ user_id: str,+ record: KnowledgeDocumentRecord,+ ) -> KnowledgeDocumentRecord:+ """Create or update a document record; enforces ``record.user_id``."""+ if record.user_id != user_id:+ raise ValueError(+ "record.user_id does not match the given user_id.",+ )+ await self._write_row(KnowledgeDocumentRow, record)+ return record++ async def get_knowledge_document(+ self,+ user_id: str,+ knowledge_base_id: str,+ document_id: str,+ ) -> KnowledgeDocumentRecord | None:+ """One document; owner + KB scoped."""+ async with self._session() as sess:+ row = await sess.get(KnowledgeDocumentRow, document_id)+ if (+ row is None+ or row.user_id != user_id+ or row.knowledge_base_id != knowledge_base_id+ ):+ return None+ return _to_record(row, KnowledgeDocumentRecord)++ async def list_knowledge_documents(+ self,+ user_id: str,+ knowledge_base_id: str,+ ) -> list[KnowledgeDocumentRecord]:+ """All documents inside a KB."""+ from sqlalchemy import select++ async with self._session() as sess:+ rows = (+ (+ await sess.execute(+ select(KnowledgeDocumentRow).where(+ KnowledgeDocumentRow.user_id == user_id,+ KnowledgeDocumentRow.knowledge_base_id+ == knowledge_base_id,+ ),+ )+ )+ .scalars()+ .all()+ )+ return [_to_record(r, KnowledgeDocumentRecord) for r in rows]++ async def delete_knowledge_document(+ self,+ user_id: str,+ knowledge_base_id: str,+ document_id: str,+ ) -> bool:+ """Delete one document."""+ from sqlalchemy import delete++ async with self._session() as sess:+ result = await sess.execute(+ delete(KnowledgeDocumentRow).where(+ KnowledgeDocumentRow.id == document_id,+ KnowledgeDocumentRow.user_id == user_id,+ KnowledgeDocumentRow.knowledge_base_id+ == knowledge_base_id,+ ),+ )+ await sess.commit()+ return result.rowcount > 0++ async def update_knowledge_document_status(+ self,+ user_id: str,+ knowledge_base_id: str,+ document_id: str,+ status: KnowledgeDocumentStatus,+ error: str | None = None,+ chunk_count: int | None = None,+ ) -> None:+ """Fast-path column UPDATE + optional payload rewrite.++ ``status`` is a column so it can always be updated with a+ single ``UPDATE``; ``error`` / ``chunk_count`` live inside+ :attr:`~KnowledgeDocumentRecord.data` (payload) and therefore+ require the classic read-modify-write when they change.+ """+ async with self._session() as sess:+ row = await sess.get(KnowledgeDocumentRow, document_id)+ if (+ row is None+ or row.user_id != user_id+ or row.knowledge_base_id != knowledge_base_id+ ):+ return+ record = _to_record(row, KnowledgeDocumentRecord)+ record.status = status+ if error is not None:+ record.data.error = error+ if chunk_count is not None:+ record.data.chunk_count = chunk_count+ record.updated_at = _utcnow()+ new_row = _from_record(KnowledgeDocumentRow, record)+ row.status = new_row.status+ row.payload = new_row.payload+ row.updated_at = new_row.updated_at+ await sess.commit()++ async def acquire_knowledge_document_lease(+ self,+ user_id: str,+ knowledge_base_id: str,+ document_id: str,+ processing_node: str,+ lease_ttl: timedelta,+ now: datetime | None = None,+ ) -> bool:+ """Conditional UPDATE — atomic on every supported dialect."""+ from sqlalchemy import or_, update++ now = _to_naive_utc(now) if now is not None else _utcnow()+ deadline = now + lease_ttl++ async with self._session() as sess:+ result = await sess.execute(+ update(KnowledgeDocumentRow)+ .where(+ KnowledgeDocumentRow.id == document_id,+ KnowledgeDocumentRow.user_id == user_id,+ KnowledgeDocumentRow.knowledge_base_id+ == knowledge_base_id,+ or_(+ KnowledgeDocumentRow.processing_node.is_(None),+ KnowledgeDocumentRow.lease_expires_at.is_(None),+ KnowledgeDocumentRow.lease_expires_at < now,+ ),+ )+ .values(+ processing_node=processing_node,+ lease_expires_at=deadline,+ updated_at=now,+ ),+ )+ await sess.commit()+ return result.rowcount > 0++ async def renew_knowledge_document_lease(+ self,+ user_id: str,+ knowledge_base_id: str,+ document_id: str,+ processing_node: str,+ lease_ttl: timedelta,+ now: datetime | None = None,+ ) -> bool:+ """Conditional UPDATE constrained to the current holder."""+ from sqlalchemy import update++ now = _to_naive_utc(now) if now is not None else _utcnow()+ deadline = now + lease_ttl++ async with self._session() as sess:+ result = await sess.execute(+ update(KnowledgeDocumentRow)+ .where(+ KnowledgeDocumentRow.id == document_id,+ KnowledgeDocumentRow.user_id == user_id,+ KnowledgeDocumentRow.knowledge_base_id+ == knowledge_base_id,+ KnowledgeDocumentRow.processing_node == processing_node,+ )+ .values(lease_expires_at=deadline, updated_at=now),+ )+ await sess.commit()+ return result.rowcount > 0++ async def release_knowledge_document_lease(+ self,+ user_id: str,+ knowledge_base_id: str,+ document_id: str,+ processing_node: str,+ ) -> None:+ """Conditional UPDATE constrained to the current holder."""+ from sqlalchemy import update++ now = _utcnow()+ async with self._session() as sess:+ await sess.execute(+ update(KnowledgeDocumentRow)+ .where(+ KnowledgeDocumentRow.id == document_id,+ KnowledgeDocumentRow.user_id == user_id,+ KnowledgeDocumentRow.knowledge_base_id+ == knowledge_base_id,+ KnowledgeDocumentRow.processing_node == processing_node,+ )+ .values(+ processing_node=None,+ lease_expires_at=None,+ updated_at=now,+ ),+ )+ await sess.commit()++ async def list_knowledge_documents_with_expired_lease(+ self,+ now: datetime | None = None,+ ) -> list[KnowledgeDocumentRecord]:+ """Documents past their lease deadline, non-terminal, held."""+ from sqlalchemy import select++ now = _to_naive_utc(now) if now is not None else _utcnow()+ terminal = ("ready", "error")+ async with self._session() as sess:+ rows = (+ (+ await sess.execute(+ select(KnowledgeDocumentRow).where(+ KnowledgeDocumentRow.status.notin_(terminal),+ KnowledgeDocumentRow.processing_node.is_not(None),+ KnowledgeDocumentRow.lease_expires_at.is_not(None),+ KnowledgeDocumentRow.lease_expires_at < now,+ ),+ )+ )+ .scalars()+ .all()+ )+ return [_to_record(r, KnowledgeDocumentRecord) for r in rows]++ async def list_knowledge_documents_pending_since(+ self,+ threshold: datetime,+ ) -> list[KnowledgeDocumentRecord]:+ """Documents stuck in ``pending`` before *threshold*."""+ from sqlalchemy import select++ threshold = _to_naive_utc(threshold)+ async with self._session() as sess:+ rows = (+ (+ await sess.execute(+ select(KnowledgeDocumentRow).where(+ KnowledgeDocumentRow.status == "pending",+ KnowledgeDocumentRow.created_at < threshold,+ ),+ )+ )+ .scalars()+ .all()+ )+ return [_to_record(r, KnowledgeDocumentRecord) for r in rows]diff --git a/src/agentscope/app/storage/_sql/_tables.py b/src/agentscope/app/storage/_sql/_tables.pynew file mode 100644index 000000000..4daa99e36--- /dev/null+++ b/src/agentscope/app/storage/_sql/_tables.py@@ -0,0 +1,315 @@+# -*- coding: utf-8 -*-+"""SQLAlchemy 2.0 declarative tables backing :class:`AsyncSQLAlchemyStorage`.++Every record type maps to one table with the layout:++- ``id`` primary key + ``created_at`` / ``updated_at`` timestamps;+- one column per relational / indexed field promoted from the+ record's top level;+- a single ``payload`` JSON column carrying the remainder of+ ``record.model_dump(mode="json")`` **minus** the promoted columns —+ so no field is ever stored in both places (see the design doc in+ ``_mappers.py`` for the round-trip contract).++Portability constraints — the *schema* intentionally sticks to the+subset of SQLAlchemy that works on every async-capable dialect we+target (SQLite / Postgres / MySQL): plain :class:`~sqlalchemy.JSON`+(never JSONB), no generated columns, no ``FOR UPDATE``. Atomic+upserts *are* emitted with dialect-native ``ON CONFLICT`` /+``ON DUPLICATE KEY UPDATE`` syntax, but through the explicit+per-dialect dispatch in+:meth:`~agentscope.app.storage.AsyncSQLAlchemyStorage._upsert_stmt`+rather than leaking into the table definitions here. The messages+table sidesteps the+JSON-record shape because it is inherently list-like — see+:class:`MessageRow` for the shape and the write path in+:class:`~agentscope.app.storage.AsyncSQLAlchemyStorage.upsert_message`.+"""+from datetime import datetime+from typing import Any, ClassVar++from sqlalchemy import (+ JSON,+ DateTime,+ ForeignKey,+ Index,+ String,+)+from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column+++# Every id column is sized uniformly+_ID_LEN = 255+++class _Base(DeclarativeBase):+ """Declarative base shared by every table in :mod:`_sql`.++ Kept private because it is an implementation detail: users of+ :class:`~agentscope.app.storage.AsyncSQLAlchemyStorage` never see it.+ """+++class _JsonRecordMixin(_Base):+ """Column set common to every ``*Row`` that stores a record payload.++ Concrete tables inherit this mixin plus :class:`_Base`, add their+ own promoted columns, and set two class variables that drive the+ generic mapper in :mod:`_mappers`:++ - ``_record_cls``: the pydantic :class:`_RecordBase` subclass+ this row represents.+ - ``_indexed_fields``: the tuple of top-level record fields that+ live in dedicated columns (and therefore MUST be popped out of+ ``payload`` on write and merged back on read).++ The three envelope columns (``id`` / ``created_at`` /+ ``updated_at``) are always promoted and are handled by the mapper+ unconditionally, so ``_indexed_fields`` should list **only** the+ extra table-specific columns.+ """++ __abstract__ = True++ id: Mapped[str] = mapped_column(String(_ID_LEN), primary_key=True)+ created_at: Mapped[datetime] = mapped_column(+ DateTime(),+ index=True,+ nullable=False,+ )+ updated_at: Mapped[datetime] = mapped_column(+ DateTime(),+ index=True,+ nullable=False,+ )+ payload: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)++ # Populated by subclasses; see class docstring.+ _record_cls: ClassVar[type]+ _indexed_fields: ClassVar[tuple[str, ...]] = ()++ @classmethod+ def get_indexed_fields(cls) -> tuple[str, ...]:+ """Return the tuple of record fields promoted to dedicated columns."""+ return cls._indexed_fields+++class CredentialRow(_JsonRecordMixin):+ """One row per :class:`~agentscope.app.storage.CredentialRecord`."""++ __tablename__ = "credentials"++ user_id: Mapped[str] = mapped_column(+ String(_ID_LEN),+ nullable=False,+ index=True,+ )++ _indexed_fields = ("user_id",)+++class AgentRow(_JsonRecordMixin):+ """One row per :class:`~agentscope.app.storage.AgentRecord`."""++ __tablename__ = "agents"++ user_id: Mapped[str] = mapped_column(+ String(_ID_LEN),+ nullable=False,+ index=True,+ )+ source: Mapped[str] = mapped_column(+ String(16),+ nullable=False,+ index=True,+ )++ __table_args__ = (Index("ix_agents_user_source", "user_id", "source"),)++ _indexed_fields = ("user_id", "source")+++class SessionRow(_JsonRecordMixin):+ """One row per :class:`~agentscope.app.storage.SessionRecord`."""++ __tablename__ = "sessions"++ user_id: Mapped[str] = mapped_column(+ String(_ID_LEN),+ nullable=False,+ index=True,+ )+ agent_id: Mapped[str] = mapped_column(+ String(_ID_LEN),+ nullable=False,+ index=True,+ )+ source: Mapped[str] = mapped_column(+ String(16),+ nullable=False,+ index=True,+ )+ source_schedule_id: Mapped[str | None] = mapped_column(+ String(_ID_LEN),+ nullable=True,+ index=True,+ )+ team_id: Mapped[str | None] = mapped_column(+ String(_ID_LEN),+ nullable=True,+ index=True,+ )++ __table_args__ = (Index("ix_sessions_user_agent", "user_id", "agent_id"),)++ _indexed_fields = (+ "user_id",+ "agent_id",+ "source",+ "source_schedule_id",+ "team_id",+ )+++class ScheduleRow(_JsonRecordMixin):+ """One row per :class:`~agentscope.app.storage.ScheduleRecord`."""++ __tablename__ = "schedules"++ user_id: Mapped[str] = mapped_column(+ String(_ID_LEN),+ nullable=False,+ index=True,+ )+ agent_id: Mapped[str] = mapped_column(+ String(_ID_LEN),+ nullable=False,+ index=True,+ )++ _indexed_fields = ("user_id", "agent_id")+++class TeamRow(_JsonRecordMixin):+ """One row per :class:`~agentscope.app.storage.TeamRecord`."""++ __tablename__ = "teams"++ user_id: Mapped[str] = mapped_column(+ String(_ID_LEN),+ nullable=False,+ index=True,+ )+ session_id: Mapped[str] = mapped_column(+ String(_ID_LEN),+ nullable=False,+ )++ _indexed_fields = ("user_id", "session_id")+++class KnowledgeBaseRow(_JsonRecordMixin):+ """One row per :class:`~agentscope.app.storage.KnowledgeBaseRecord`."""++ __tablename__ = "knowledge_bases"++ user_id: Mapped[str] = mapped_column(+ String(_ID_LEN),+ nullable=False,+ index=True,+ )++ _indexed_fields = ("user_id",)+++class KnowledgeDocumentRow(_JsonRecordMixin):+ """One row per :class:`KnowledgeDocumentRecord`.++ Promotes every lifecycle / sweeper field to a dedicated column so+ :meth:`AsyncSQLAlchemyStorage.list_knowledge_documents_with_expired_lease`+ can filter without deserialising :attr:`payload`. The composite+ ``(status, lease_expires_at)`` index serves both the expired-lease+ sweep (``WHERE status NOT IN (…) AND lease_expires_at < :now``)+ and the pending-orphan sweep (``WHERE status = 'pending' AND+ created_at < :threshold`` — covered by the plain ``status`` index+ combined with the mixin's ``created_at`` index).+ """++ __tablename__ = "knowledge_documents"++ user_id: Mapped[str] = mapped_column(+ String(_ID_LEN),+ nullable=False,+ index=True,+ )+ knowledge_base_id: Mapped[str] = mapped_column(+ String(_ID_LEN),+ # Documents are structurally owned by their KB — let the DB+ # cascade-delete them natively (atomic, no application loop).+ ForeignKey("knowledge_bases.id", ondelete="CASCADE"),+ nullable=False,+ index=True,+ )+ processing_node: Mapped[str | None] = mapped_column(+ String(128),+ nullable=True,+ )+ status: Mapped[str] = mapped_column(+ String(16),+ nullable=False,+ index=True,+ )+ lease_expires_at: Mapped[datetime | None] = mapped_column(+ DateTime(),+ nullable=True,+ )++ __table_args__ = (+ Index(+ "ix_kd_status_lease",+ "status",+ "lease_expires_at",+ ),+ Index(+ "ix_kd_user_kb",+ "user_id",+ "knowledge_base_id",+ ),+ )++ _indexed_fields = (+ "user_id",+ "knowledge_base_id",+ "processing_node",+ "status",+ "lease_expires_at",+ )+++class MessageRow(_Base):+ """One row per persisted :class:`~agentscope.message.Msg`.++ Sits outside the ``_JsonRecordMixin`` family because messages are+ not stand-alone records — they are per-session events. A message is+ identified by the **composite** primary key ``(session_id, msg_id)``+ rather than a synthetic concatenated string: ``Msg.id`` is only+ unique *within* a session, and a composite key keeps the write path+ ("same ``(session, msg_id)`` → replace", inheriting+ :class:`RedisStorage.upsert_message`'s semantic) enforced by the DB+ without having to bound the length of a ``session_id:msg_id`` blob+ (both ids come from a user-overridable id factory).+ """++ __tablename__ = "messages"++ session_id: Mapped[str] = mapped_column(String(_ID_LEN), primary_key=True)+ msg_id: Mapped[str] = mapped_column(String(_ID_LEN), primary_key=True)+ created_at: Mapped[datetime] = mapped_column(+ DateTime(),+ nullable=False,+ )+ payload: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)++ __table_args__ = (+ Index("ix_messages_session_created", "session_id", "created_at"),+ )diff --git a/tests/app_lifespan_dedicated_test.py b/tests/app_lifespan_dedicated_test.pyindex df3cf0394..2aa658da8 100644--- a/tests/app_lifespan_dedicated_test.py+++ b/tests/app_lifespan_dedicated_test.py@@ -47,6 +47,7 @@ from agentscope.app.rag.knowledge_base_manager._dimension_policy import (from agentscope.app.message_bus import RedisMessageBusfrom agentscope.app.storage import (EmbeddingModelConfig,+ KnowledgeBaseData,KnowledgeBaseRecord,RedisStorage,)@@ -262,19 +263,21 @@ class DedicatedModeUploadFlowTest(IsolatedAsyncioTestCase):# manager's create flow over HTTP.kb_record = KnowledgeBaseRecord(user_id="user-1",- name="kb",- description="",- embedding_model_config=EmbeddingModelConfig(- type="openai_credential",- credential_id="cred-1",- model="text-embedding-3-small",- dimensions=1,+ data=KnowledgeBaseData(+ name="kb",+ description="",+ embedding_model_config=EmbeddingModelConfig(+ type="openai_credential",+ credential_id="cred-1",+ model="text-embedding-3-small",+ dimensions=1,+ ),+ collection_name="",),- collection_name="",)- kb_record.collection_name = f"kb_{kb_record.id}"+ kb_record.data.collection_name = f"kb_{kb_record.id}"await self._vector_store.create_collection(- kb_record.collection_name,+ kb_record.data.collection_name,1,)storage._client = self._frdiff --git a/tests/resource_access_policy_test.py b/tests/resource_access_policy_test.pyindex cc95783ec..46bcdf77f 100644--- a/tests/resource_access_policy_test.py+++ b/tests/resource_access_policy_test.py@@ -15,7 +15,14 @@ from agentscope.app.access import (ResourceRef,)from agentscope.app._service import ResourceAccessService-from agentscope.app.storage import AgentData, AgentRecord, CredentialRecord+from agentscope.app.storage import (+ AgentData,+ AgentRecord,+ CredentialRecord,+ EmbeddingModelConfig,+ KnowledgeBaseData,+ KnowledgeBaseRecord,+)from agentscope.app.storage import StorageBasefrom agentscope.agent import ContextConfig, ReActConfig@@ -112,6 +119,13 @@ class _FakeStorage:data=_make_agent_data("Own team"),),}+ self.knowledge_bases: dict[tuple[str, str], KnowledgeBaseRecord] = {+ ("viewer", "kb-1"): KnowledgeBaseRecord(+ id="kb-1",+ user_id="viewer",+ data=_make_kb_data("My KB"),+ ),+ }async def list_credentials(self,@@ -148,6 +162,25 @@ class _FakeStorage:"""Get an agent."""return self.agents.get((user_id, agent_id))+ async def list_knowledge_bases(+ self,+ user_id: str,+ ) -> list[KnowledgeBaseRecord]:+ """List knowledge bases owned by ``user_id``."""+ return [+ record+ for (owner_id, _), record in self.knowledge_bases.items()+ if owner_id == user_id+ ]++ async def get_knowledge_base(+ self,+ user_id: str,+ knowledge_base_id: str,+ ) -> KnowledgeBaseRecord | None:+ """Get a knowledge base."""+ return self.knowledge_bases.get((user_id, knowledge_base_id))+def _make_agent_data(name: str) -> AgentData:"""Create valid agent data for tests."""@@ -159,6 +192,21 @@ def _make_agent_data(name: str) -> AgentData:)+def _make_kb_data(name: str) -> KnowledgeBaseData:+ """Create valid knowledge base data for tests."""+ return KnowledgeBaseData(+ name=name,+ description="A test knowledge base.",+ embedding_model_config=EmbeddingModelConfig(+ type="openai_credential",+ credential_id="credential-1",+ model="text-embedding-3-small",+ dimensions=1536,+ ),+ collection_name="kb_deadbeef",+ )++class ResourceAccessPolicyTest(IsolatedAsyncioTestCase):"""Test resource access policy defaults and edit fallback."""@@ -343,6 +391,67 @@ class ResourceAccessServiceTest(IsolatedAsyncioTestCase):],)+ async def test_get_knowledge_base_view_is_flat(self) -> None:+ """KB views must lift ``data.*`` to the top level for the wire.++ The frontend reads ``kb.name`` and+ ``kb.embedding_model_config.model`` flat; a regression that let+ the storage-layer ``data`` nesting leak onto the wire broke the+ sidebar name and crashed the panel. Lock the flat shape in.+ """+ view = await self.service.get_resource(+ "viewer",+ ResourceKind.KNOWLEDGE_BASE,+ "kb-1",+ )++ self.assertEqual(+ view.model_dump(),+ {+ "id": "kb-1",+ "name": "My KB",+ "description": "A test knowledge base.",+ "embedding_model_config": {+ "type": "openai_credential",+ "credential_id": "credential-1",+ "model": "text-embedding-3-small",+ "dimensions": 1536,+ "parameters": {},+ },+ "created_at": AnyValue(),+ "updated_at": AnyValue(),+ "editable": True,+ },+ )++ async def test_list_knowledge_bases_view_is_flat(self) -> None:+ """Listed KB views must be flat too — same wire contract."""+ views = await self.service.list_resource(+ "viewer",+ ResourceKind.KNOWLEDGE_BASE,+ )++ self.assertEqual(+ [v.model_dump() for v in views],+ [+ {+ "id": "kb-1",+ "name": "My KB",+ "description": "A test knowledge base.",+ "embedding_model_config": {+ "type": "openai_credential",+ "credential_id": "credential-1",+ "model": "text-embedding-3-small",+ "dimensions": 1536,+ "parameters": {},+ },+ "created_at": AnyValue(),+ "updated_at": AnyValue(),+ "editable": True,+ },+ ],+ )+async def test_missing_resource_raises_404(self) -> None:"""Missing visible resources should raise a 404 HTTPException."""with self.assertRaises(HTTPException) as ctx:diff --git a/tests/service_knowledge_base_upload_test.py b/tests/service_knowledge_base_upload_test.pyindex 44a410fb7..a01845ef2 100644--- a/tests/service_knowledge_base_upload_test.py+++ b/tests/service_knowledge_base_upload_test.py@@ -36,6 +36,7 @@ from agentscope.app.rag.knowledge_base_manager._dimension_policy import (from agentscope.app.message_bus import RedisMessageBusfrom agentscope.app.storage import (EmbeddingModelConfig,+ KnowledgeBaseData,KnowledgeBaseRecord,RedisStorage,)@@ -208,14 +209,16 @@ class _FakeKbManager(KnowledgeBaseManagerBase):) -> KnowledgeBaseRecord:record = KnowledgeBaseRecord(user_id=user_id,- name=name,- description=description,- embedding_model_config=embedding_model_config,- collection_name="",+ data=KnowledgeBaseData(+ name=name,+ description=description,+ embedding_model_config=embedding_model_config,+ collection_name="",+ ),)- record.collection_name = f"kb_{record.id}"+ record.data.collection_name = f"kb_{record.id}"await self._vector_store.create_collection(- name=record.collection_name,+ name=record.data.collection_name,dimensions=embedding_model_config.dimensions,)return await self._storage.upsert_knowledge_base(user_id, record)@@ -231,7 +234,9 @@ class _FakeKbManager(KnowledgeBaseManagerBase):)if record is None:return False- await self._vector_store.delete_collection(record.collection_name)+ await self._vector_store.delete_collection(+ record.data.collection_name,+ )return await self._storage.delete_knowledge_base(user_id,knowledge_base_id,@@ -252,7 +257,7 @@ class _FakeKbManager(KnowledgeBaseManagerBase):)return _FakeKnowledge(vector_store=self._vector_store,- collection_name=record.collection_name,+ collection_name=record.data.collection_name,)@@ -335,19 +340,21 @@ class KnowledgeBaseUploadFlowTest(IsolatedAsyncioTestCase):# have to mock the manager's create flow over HTTP.kb_record = KnowledgeBaseRecord(user_id="user-1",- name="kb",- description="",- embedding_model_config=EmbeddingModelConfig(- type="openai_credential",- credential_id="cred-1",- model="text-embedding-3-small",- dimensions=1,+ data=KnowledgeBaseData(+ name="kb",+ description="",+ embedding_model_config=EmbeddingModelConfig(+ type="openai_credential",+ credential_id="cred-1",+ model="text-embedding-3-small",+ dimensions=1,+ ),+ collection_name="",),- collection_name="",)- kb_record.collection_name = f"kb_{kb_record.id}"+ kb_record.data.collection_name = f"kb_{kb_record.id}"await self._vector_store.create_collection(- kb_record.collection_name,+ kb_record.data.collection_name,1,)diff --git a/tests/storage_redis_knowledge_base_test.py b/tests/storage_redis_knowledge_base_test.pyindex f828c73e6..d9601540d 100644--- a/tests/storage_redis_knowledge_base_test.py+++ b/tests/storage_redis_knowledge_base_test.py@@ -7,6 +7,7 @@ import fakeredis.aioredisfrom agentscope.app.storage import (EmbeddingModelConfig,+ KnowledgeBaseData,KnowledgeBaseRecord,RedisStorage,)@@ -25,15 +26,17 @@ def make_record(user_id: str, name: str = "kb") -> KnowledgeBaseRecord:"""Build a KnowledgeBaseRecord with a default embedding config."""return KnowledgeBaseRecord(user_id=user_id,- name=name,- description="desc",- embedding_model_config=EmbeddingModelConfig(- type="openai_credential",- credential_id="cred-1",- model="text-embedding-3-small",- dimensions=1536,+ data=KnowledgeBaseData(+ name=name,+ description="desc",+ embedding_model_config=EmbeddingModelConfig(+ type="openai_credential",+ credential_id="cred-1",+ model="text-embedding-3-small",+ dimensions=1536,+ ),+ collection_name="kb_abc",),- collection_name="kb_abc",)@@ -55,7 +58,7 @@ class KnowledgeBaseStorageTest(IsolatedAsyncioTestCase):# get returns the persisted record for the right ownerfetched = await storage.get_knowledge_base("user-1", stored_a.id)self.assertEqual(fetched.id, stored_a.id)- self.assertEqual(fetched.name, "first")+ self.assertEqual(fetched.data.name, "first")# cross-user lookups return Noneself.assertIsNone(@@ -102,9 +105,9 @@ class KnowledgeBaseStorageTest(IsolatedAsyncioTestCase):rec = make_record("user-1")first = await storage.upsert_knowledge_base("user-1", rec)- rec.name = "renamed"+ rec.data.name = "renamed"second = await storage.upsert_knowledge_base("user-1", rec)self.assertEqual(second.id, first.id)self.assertEqual(second.created_at, first.created_at)- self.assertEqual(second.name, "renamed")+ self.assertEqual(second.data.name, "renamed")diff --git a/tests/storage_sql_test.py b/tests/storage_sql_test.pynew file mode 100644index 000000000..b44998169--- /dev/null+++ b/tests/storage_sql_test.py@@ -0,0 +1,856 @@+# -*- coding: utf-8 -*-+# pylint: disable=too-many-public-methods+"""Round-trip and semantic tests for :class:`AsyncSQLAlchemyStorage`.++Runs against an in-memory SQLite database via ``sqlite+aiosqlite``+so the whole suite is self-contained — no external server needed.+The intent is to exercise every ``StorageBase`` method the Redis+backend implements, mirroring the shape (though not always the+literal assertions) of the Redis backend's tests so both backends+stay behavioural equivalents.+"""+from contextlib import AsyncExitStack+from datetime import datetime, timedelta+from unittest.async_case import IsolatedAsyncioTestCase++from pydantic import SecretStr++from agentscope.app.storage import (+ AgentData,+ AgentRecord,+ ChatModelConfig,+ EmbeddingModelConfig,+ KnowledgeBaseData,+ KnowledgeBaseRecord,+ KnowledgeDocumentData,+ KnowledgeDocumentRecord,+ ScheduleData,+ ScheduleRecord,+ SessionConfig,+ SessionSource,+ AsyncSQLAlchemyStorage,+ TeamData,+ TeamMember,+ TeamRecord,+)+from agentscope.agent import ContextConfig, ReActConfig+from agentscope.credential import DashScopeCredential+from agentscope.message import AssistantMsg, UserMsg+++def _agent_record(user_id: str, name: str = "agent-x") -> AgentRecord:+ """Build a minimal but complete :class:`AgentRecord`."""+ return AgentRecord(+ user_id=user_id,+ data=AgentData(+ name=name,+ context_config=ContextConfig(),+ react_config=ReActConfig(),+ ),+ )+++def _session_config() -> SessionConfig:+ """Build a minimal :class:`SessionConfig`."""+ return SessionConfig(+ workspace_id="ws-1",+ name="s",+ chat_model_config=ChatModelConfig(+ type="openai",+ credential_id="cred-1",+ model="gpt-4o",+ parameters={},+ ),+ )+++def _kb_record(user_id: str, name: str = "kb") -> KnowledgeBaseRecord:+ """Build a KB record with a default embedding config."""+ return KnowledgeBaseRecord(+ user_id=user_id,+ data=KnowledgeBaseData(+ name=name,+ description="",+ embedding_model_config=EmbeddingModelConfig(+ type="openai_credential",+ credential_id="cred-1",+ model="text-embedding-3-small",+ dimensions=8,+ ),+ collection_name="kb-x",+ ),+ )+++def _kd_record(+ user_id: str,+ knowledge_base_id: str,+ filename: str = "f.txt",+) -> KnowledgeDocumentRecord:+ """Build a fresh (``pending`` / unclaimed) document record."""+ return KnowledgeDocumentRecord(+ user_id=user_id,+ knowledge_base_id=knowledge_base_id,+ data=KnowledgeDocumentData(+ filename=filename,+ size=42,+ blob_uri=f"local://{filename}",+ ),+ )+++def _schedule_record(user_id: str, agent_id: str) -> ScheduleRecord:+ """Build a schedule record."""+ return ScheduleRecord(+ user_id=user_id,+ agent_id=agent_id,+ data=ScheduleData(+ name="daily",+ cron_expression="0 9 * * *",+ chat_model_config=ChatModelConfig(+ type="openai",+ credential_id="cred-1",+ model="gpt-4o",+ parameters={},+ ),+ ),+ )+++class AsyncSQLAlchemyStorageTest(IsolatedAsyncioTestCase):+ """End-to-end tests for :class:`AsyncSQLAlchemyStorage` over+ in-memory SQLite."""++ async def asyncSetUp(self) -> None:+ # ``:memory:`` gives a private DB per connection; using a+ # shared-cache URI would too, but per-test isolation is what+ # we want here.+ self._stack = AsyncExitStack()+ self.storage = await self._stack.enter_async_context(+ AsyncSQLAlchemyStorage(+ "sqlite+aiosqlite:///:memory:",+ create_tables=True,+ ),+ )++ async def asyncTearDown(self) -> None:+ await self._stack.aclose()++ # ------------------------------------------------------------------+ # Credentials+ # ------------------------------------------------------------------++ async def test_credentials_round_trip(self) -> None:+ """Upsert / list / get / delete + owner scoping."""+ cred = DashScopeCredential(api_key=SecretStr("sk-1"))+ cid = await self.storage.upsert_credential("user-1", cred)++ listed = await self.storage.list_credentials("user-1")+ self.assertEqual([c.id for c in listed], [cid])+ self.assertEqual(listed[0].data["api_key"], "sk-1")++ fetched = await self.storage.get_credential("user-1", cid)+ self.assertEqual(fetched.id, cid)++ # Cross-user isolation+ self.assertEqual(await self.storage.list_credentials("user-2"), [])+ self.assertIsNone(await self.storage.get_credential("user-2", cid))++ # Delete + double-delete+ self.assertTrue(await self.storage.delete_credential("user-1", cid))+ self.assertFalse(await self.storage.delete_credential("user-1", cid))++ async def test_upsert_credential_is_owner_scoped(self) -> None:+ """A preset id owned by another user is never read or clobbered."""+ import sqlalchemy.exc++ victim = DashScopeCredential(api_key=SecretStr("victim-key"))+ cid = await self.storage.upsert_credential("user-1", victim)+ before = await self.storage.get_credential("user-1", cid)++ # Same owner + preset id → in-place update, created_at preserved.+ rotated = DashScopeCredential(api_key=SecretStr("rotated"), id=cid)+ self.assertEqual(+ await self.storage.upsert_credential("user-1", rotated),+ cid,+ )+ after = await self.storage.get_credential("user-1", cid)+ self.assertEqual(after.data["api_key"], "rotated")+ self.assertEqual(after.created_at, before.created_at)++ # Attacker (user-2) presenting the victim's id must not touch the+ # victim's row: the global-id INSERT collides and raises, and the+ # victim's data + ownership stay intact.+ attack = DashScopeCredential(api_key=SecretStr("attacker"), id=cid)+ with self.assertRaises(sqlalchemy.exc.IntegrityError):+ await self.storage.upsert_credential("user-2", attack)++ victim_now = await self.storage.get_credential("user-1", cid)+ self.assertEqual(victim_now.data["api_key"], "rotated")+ self.assertIsNone(await self.storage.get_credential("user-2", cid))+ self.assertEqual(await self.storage.list_credentials("user-2"), [])++ # ------------------------------------------------------------------+ # Agents+ # ------------------------------------------------------------------++ async def test_agents_round_trip_and_source_filter(self) -> None:+ """``list_agents`` filters out ``source='team'`` workers."""+ user_agent = _agent_record("user-1", "usr")+ team_agent = _agent_record("user-1", "team-worker")+ team_agent.source = "team"++ await self.storage.upsert_agent("user-1", user_agent)+ await self.storage.upsert_agent("user-1", team_agent)++ listed = await self.storage.list_agents("user-1")+ self.assertEqual([a.id for a in listed], [user_agent.id])+ # But direct get works for the team-spawned worker+ self.assertEqual(+ (await self.storage.get_agent("user-1", team_agent.id)).id,+ team_agent.id,+ )++ async def test_delete_agent_cascades_sessions_and_schedules(self) -> None:+ """Deleting an agent removes its sessions + schedules."""+ agent = _agent_record("user-1")+ await self.storage.upsert_agent("user-1", agent)+ session = await self.storage.upsert_session(+ user_id="user-1",+ agent_id=agent.id,+ config=_session_config(),+ )+ schedule = _schedule_record("user-1", agent.id)+ await self.storage.upsert_schedule("user-1", schedule)++ self.assertTrue(await self.storage.delete_agent("user-1", agent.id))+ self.assertEqual(+ await self.storage.list_sessions("user-1", agent.id),+ [],+ )+ self.assertIsNone(+ await self.storage.get_session(+ "user-1",+ agent.id,+ session.id,+ ),+ )+ self.assertIsNone(+ await self.storage.get_schedule("user-1", schedule.id),+ )++ async def test_upsert_replaces_in_place_and_keeps_created_at(self) -> None:+ """Re-upserting the same id updates via the atomic upsert path.++ Exercises the ``ON CONFLICT DO UPDATE`` branch: the second+ write must overwrite the mutable columns, preserve the original+ ``created_at``, and never create a duplicate row.+ """+ agent = _agent_record("user-1", "v1")+ await self.storage.upsert_agent("user-1", agent)+ first = await self.storage.get_agent("user-1", agent.id)++ agent.data.name = "v2"+ await self.storage.upsert_agent("user-1", agent)+ second = await self.storage.get_agent("user-1", agent.id)++ self.assertEqual(second.data.name, "v2")+ self.assertEqual(second.created_at, first.created_at)+ self.assertGreaterEqual(second.updated_at, first.updated_at)+ # Exactly one row — the conflict updated rather than inserted.+ self.assertEqual(len(await self.storage.list_agents("user-1")), 1)++ # ------------------------------------------------------------------+ # Sessions+ # ------------------------------------------------------------------++ async def test_sessions_upsert_and_state_update(self) -> None:+ """Create + update + state-only update."""+ agent = _agent_record("user-1")+ await self.storage.upsert_agent("user-1", agent)++ # Create+ session = await self.storage.upsert_session(+ user_id="user-1",+ agent_id=agent.id,+ config=_session_config(),+ source=SessionSource.SCHEDULE,+ source_schedule_id="sch-1",+ )+ self.assertEqual(session.source, SessionSource.SCHEDULE)++ # Update (same session_id) — config swap+ new_config = _session_config()+ new_config.name = "renamed"+ updated = await self.storage.upsert_session(+ user_id="user-1",+ agent_id=agent.id,+ config=new_config,+ session_id=session.id,+ )+ self.assertEqual(updated.id, session.id)+ self.assertEqual(updated.config.name, "renamed")+ self.assertEqual(updated.created_at, session.created_at)++ # State-only update+ from agentscope.state import AgentState++ new_state = AgentState()+ await self.storage.update_session_state(+ "user-1",+ agent.id,+ session.id,+ new_state,+ )+ fetched = await self.storage.get_session(+ "user-1",+ agent.id,+ session.id,+ )+ self.assertEqual(fetched.state.model_dump(), new_state.model_dump())++ # By-schedule listing+ listed = await self.storage.list_sessions_by_schedule(+ "user-1",+ "sch-1",+ )+ self.assertEqual([s.id for s in listed], [session.id])++ # set_session_team_id+ await self.storage.set_session_team_id(+ "user-1",+ session.id,+ "team-9",+ )+ fetched = await self.storage.get_session(+ "user-1",+ agent.id,+ session.id,+ )+ self.assertEqual(fetched.team_id, "team-9")++ async def test_update_session_state_missing_raises(self) -> None:+ """Updating an absent session raises :class:`KeyError`."""+ from agentscope.state import AgentState++ with self.assertRaises(KeyError):+ await self.storage.update_session_state(+ "user-1",+ "agent-x",+ "no-such-session",+ AgentState(),+ )++ # ------------------------------------------------------------------+ # Messages+ # ------------------------------------------------------------------++ async def test_messages_upsert_and_pagination(self) -> None:+ """Same-id upsert replaces; distinct ids append; pagination+ yields chronological order."""+ m1 = UserMsg(name="u", content="hello")+ m2 = AssistantMsg(name="a", content="hi")++ await self.storage.upsert_message("user-1", "sess-1", m1)+ await self.storage.upsert_message("user-1", "sess-1", m2)++ # Replace m1 in place (build a fresh Msg with the same id so+ # Pydantic re-validates the content list rather than accepting+ # a raw-string assignment on the model).+ m1_updated = UserMsg(id=m1.id, name="u", content="hola")+ await self.storage.upsert_message("user-1", "sess-1", m1_updated)++ listed, has_more = await self.storage.list_messages(+ "user-1",+ "sess-1",+ )+ self.assertEqual([m.id for m in listed], [m1.id, m2.id])+ self.assertFalse(has_more)+ self.assertEqual(listed[0].content, m1_updated.content)++ fetched = await self.storage.get_message("user-1", "sess-1", m2.id)+ self.assertEqual(fetched.id, m2.id)++ # Cursor-based pagination: the latest page of one message is the+ # newest (m2), with older messages still available.+ latest, has_more = await self.storage.list_messages(+ "user-1",+ "sess-1",+ limit=1,+ )+ self.assertEqual([m.id for m in latest], [m2.id])+ self.assertTrue(has_more)++ # Walking backwards with ``before`` yields the previous page.+ older, has_more = await self.storage.list_messages(+ "user-1",+ "sess-1",+ limit=1,+ before=m2.id,+ )+ self.assertEqual([m.id for m in older], [m1.id])+ self.assertFalse(has_more)++ # An unknown cursor yields an empty page.+ self.assertEqual(+ await self.storage.list_messages(+ "user-1",+ "sess-1",+ before="does-not-exist",+ ),+ ([], False),+ )++ # The legacy ``offset`` keyword is ignored but warns.+ with self.assertWarns(DeprecationWarning):+ await self.storage.list_messages(+ "user-1",+ "sess-1",+ offset=1,+ )++ async def test_messages_max_width_ids(self) -> None:+ """Composite key stores ids a concatenated key couldn't hold.++ Two 64-char ids would be 129 chars once joined as+ ``session_id:msg_id`` — overflowing the old ``String(96)``+ synthetic primary key. The composite ``(session_id, msg_id)``+ key round-trips them without truncation.+ """+ long_session = "s" * 64+ long_msg_id = "m" * 64+ msg = UserMsg(id=long_msg_id, name="u", content="hi")+ await self.storage.upsert_message("user-1", long_session, msg)++ fetched = await self.storage.get_message(+ "user-1",+ long_session,+ long_msg_id,+ )+ self.assertIsNotNone(fetched)+ self.assertEqual(fetched.id, long_msg_id)+ listed, _has_more = await self.storage.list_messages(+ "user-1",+ long_session,+ )+ self.assertEqual([m.id for m in listed], [long_msg_id])++ # ------------------------------------------------------------------+ # Teams+ # ------------------------------------------------------------------++ async def test_teams_upsert_get_list_delete(self) -> None:+ """Team cascade: created-role member is fully deleted, invited-role+ keeps their agent."""+ # Two agents — one that will be "created" for the team,+ # another that will be "invited".+ created_agent = _agent_record("user-1", "created")+ created_agent.source = "team"+ invited_agent = _agent_record("user-1", "invited")++ await self.storage.upsert_agent("user-1", created_agent)+ await self.storage.upsert_agent("user-1", invited_agent)++ # Sessions for both, plus the leader session.+ leader = await self.storage.upsert_session(+ user_id="user-1",+ agent_id=created_agent.id,+ config=_session_config(),+ )+ created_session = await self.storage.upsert_session(+ user_id="user-1",+ agent_id=created_agent.id,+ config=_session_config(),+ )+ invited_session = await self.storage.upsert_session(+ user_id="user-1",+ agent_id=invited_agent.id,+ config=_session_config(),+ )+ surviving_session = await self.storage.upsert_session(+ user_id="user-1",+ agent_id=invited_agent.id,+ config=_session_config(),+ )++ team = TeamRecord(+ user_id="user-1",+ session_id=leader.id,+ data=TeamData(+ name="team",+ members=[+ TeamMember(+ owner_id="user-1",+ agent_id=created_agent.id,+ session_id=created_session.id,+ role="created",+ ),+ TeamMember(+ owner_id="user-1",+ agent_id=invited_agent.id,+ session_id=invited_session.id,+ role="invited",+ ),+ ],+ ),+ )+ await self.storage.upsert_team("user-1", team)++ self.assertEqual(+ [t.id for t in await self.storage.list_teams("user-1")],+ [team.id],+ )++ # Delete team.+ self.assertTrue(await self.storage.delete_team("user-1", team.id))+ self.assertIsNone(await self.storage.get_team("user-1", team.id))++ # Created member: agent + session are gone.+ self.assertIsNone(+ await self.storage.get_agent("user-1", created_agent.id),+ )+ # Invited member: agent survives, only the invited session is gone.+ self.assertIsNotNone(+ await self.storage.get_agent("user-1", invited_agent.id),+ )+ self.assertIsNone(+ await self.storage.get_session(+ "user-1",+ invited_agent.id,+ invited_session.id,+ ),+ )+ self.assertIsNotNone(+ await self.storage.get_session(+ "user-1",+ invited_agent.id,+ surviving_session.id,+ ),+ )++ # ------------------------------------------------------------------+ # Knowledge base + documents + lease CAS+ # ------------------------------------------------------------------++ async def test_kb_and_documents_round_trip(self) -> None:+ """KB CRUD; document CRUD; cascading KB delete."""+ kb = _kb_record("user-1")+ await self.storage.upsert_knowledge_base("user-1", kb)++ self.assertEqual(+ [k.id for k in await self.storage.list_knowledge_bases("user-1")],+ [kb.id],+ )++ doc_a = _kd_record("user-1", kb.id, "a.txt")+ doc_b = _kd_record("user-1", kb.id, "b.txt")+ await self.storage.upsert_knowledge_document("user-1", doc_a)+ await self.storage.upsert_knowledge_document("user-1", doc_b)++ listed = await self.storage.list_knowledge_documents("user-1", kb.id)+ self.assertEqual(+ sorted(d.id for d in listed),+ sorted([doc_a.id, doc_b.id]),+ )++ # Cascade delete removes documents.+ self.assertTrue(+ await self.storage.delete_knowledge_base("user-1", kb.id),+ )+ self.assertEqual(+ await self.storage.list_knowledge_documents("user-1", kb.id),+ [],+ )++ async def test_document_kb_foreign_key_is_enforced(self) -> None:+ """The document→KB FK is live (PRAGMA on): cascade + rejection.++ ``delete_knowledge_base`` no longer deletes documents in Python+ — it relies on ``ON DELETE CASCADE``. This only passes if SQLite+ foreign-key enforcement is actually enabled, so the test doubles+ as a guard that the ``PRAGMA foreign_keys=ON`` wiring works.+ """+ from sqlalchemy.exc import IntegrityError++ kb = _kb_record("user-1")+ await self.storage.upsert_knowledge_base("user-1", kb)+ doc = _kd_record("user-1", kb.id)+ await self.storage.upsert_knowledge_document("user-1", doc)++ # Native cascade: dropping the KB row removes the document row.+ await self.storage.delete_knowledge_base("user-1", kb.id)+ self.assertEqual(+ await self.storage.list_knowledge_documents("user-1", kb.id),+ [],+ )++ # Enforcement: a document pointing at a missing KB is rejected.+ orphan = _kd_record("user-1", "no-such-kb")+ with self.assertRaises(IntegrityError):+ await self.storage.upsert_knowledge_document("user-1", orphan)++ async def test_document_status_and_lease_cas(self) -> None:+ """``update_knowledge_document_status`` + lease CAS semantics."""+ kb = _kb_record("user-1")+ await self.storage.upsert_knowledge_base("user-1", kb)+ doc = _kd_record("user-1", kb.id)+ await self.storage.upsert_knowledge_document("user-1", doc)++ # Status transition + error/chunk_count payload update+ await self.storage.update_knowledge_document_status(+ "user-1",+ kb.id,+ doc.id,+ "error",+ error="boom",+ chunk_count=0,+ )+ fetched = await self.storage.get_knowledge_document(+ "user-1",+ kb.id,+ doc.id,+ )+ self.assertEqual(fetched.status, "error")+ self.assertEqual(fetched.data.error, "boom")++ # Reset to pending for the lease dance.+ await self.storage.update_knowledge_document_status(+ "user-1",+ kb.id,+ doc.id,+ "pending",+ )++ now = datetime.now()+ ttl = timedelta(minutes=5)+ # First worker wins.+ self.assertTrue(+ await self.storage.acquire_knowledge_document_lease(+ "user-1",+ kb.id,+ doc.id,+ "worker-A",+ ttl,+ now,+ ),+ )+ # Second worker loses because the lease is fresh.+ self.assertFalse(+ await self.storage.acquire_knowledge_document_lease(+ "user-1",+ kb.id,+ doc.id,+ "worker-B",+ ttl,+ now,+ ),+ )+ # Renew from the holder works, from a stranger fails.+ self.assertTrue(+ await self.storage.renew_knowledge_document_lease(+ "user-1",+ kb.id,+ doc.id,+ "worker-A",+ ttl,+ now,+ ),+ )+ self.assertFalse(+ await self.storage.renew_knowledge_document_lease(+ "user-1",+ kb.id,+ doc.id,+ "worker-B",+ ttl,+ now,+ ),+ )+ # Release from a stranger is a no-op; from the holder clears it.+ await self.storage.release_knowledge_document_lease(+ "user-1",+ kb.id,+ doc.id,+ "worker-B",+ )+ fetched = await self.storage.get_knowledge_document(+ "user-1",+ kb.id,+ doc.id,+ )+ self.assertEqual(fetched.processing_node, "worker-A")++ await self.storage.release_knowledge_document_lease(+ "user-1",+ kb.id,+ doc.id,+ "worker-A",+ )+ fetched = await self.storage.get_knowledge_document(+ "user-1",+ kb.id,+ doc.id,+ )+ self.assertIsNone(fetched.processing_node)+ self.assertIsNone(fetched.lease_expires_at)++ async def test_expired_lease_and_pending_sweep(self) -> None:+ """``list_..._with_expired_lease`` + ``..._pending_since`` filters."""+ kb = _kb_record("user-1")+ await self.storage.upsert_knowledge_base("user-1", kb)++ # Doc 1: expired lease, non-terminal → should show up+ d1 = _kd_record("user-1", kb.id, "d1.txt")+ await self.storage.upsert_knowledge_document("user-1", d1)+ now = datetime.now()+ await self.storage.acquire_knowledge_document_lease(+ "user-1",+ kb.id,+ d1.id,+ "worker-A",+ timedelta(seconds=1),+ now - timedelta(hours=1), # ancient+ )++ # Doc 2: still pending, no lease → not caught by expired filter+ d2 = _kd_record("user-1", kb.id, "d2.txt")+ await self.storage.upsert_knowledge_document("user-1", d2)++ # Doc 3: terminal → never returned+ d3 = _kd_record("user-1", kb.id, "d3.txt")+ await self.storage.upsert_knowledge_document("user-1", d3)+ await self.storage.acquire_knowledge_document_lease(+ "user-1",+ kb.id,+ d3.id,+ "worker-B",+ timedelta(seconds=1),+ now - timedelta(hours=1),+ )+ await self.storage.update_knowledge_document_status(+ "user-1",+ kb.id,+ d3.id,+ "ready",+ )++ expired = (+ await self.storage.list_knowledge_documents_with_expired_lease(+ now,+ )+ )+ self.assertEqual([d.id for d in expired], [d1.id])++ pending = await self.storage.list_knowledge_documents_pending_since(+ now + timedelta(minutes=1),+ )+ # d1 and d2 are both still 'pending' (acquiring a lease does+ # not transition status by itself); d3 is 'ready' so excluded.+ self.assertEqual(+ sorted(d.id for d in pending),+ sorted([d1.id, d2.id]),+ )++ # ------------------------------------------------------------------+ # Schedules+ # ------------------------------------------------------------------++ async def test_schedules_round_trip(self) -> None:+ """Basic schedule CRUD + list_all across users."""+ s1 = _schedule_record("user-1", "agent-1")+ s2 = _schedule_record("user-2", "agent-2")+ await self.storage.upsert_schedule("user-1", s1)+ await self.storage.upsert_schedule("user-2", s2)++ self.assertEqual(+ sorted(x.id for x in await self.storage.list_schedules("user-1")),+ [s1.id],+ )+ self.assertEqual(+ sorted(x.id for x in await self.storage.list_all_schedules()),+ sorted([s1.id, s2.id]),+ )++ self.assertTrue(+ await self.storage.delete_schedule("user-1", s1.id),+ )+ self.assertFalse(+ await self.storage.delete_schedule("user-1", s1.id),+ )+++class AsyncSQLAlchemyStorageAutoMigrateTest(IsolatedAsyncioTestCase):+ """Boot via ``auto_migrate=True`` and confirm the schema is live.++ Uses a file-backed SQLite database (not ``:memory:``) so the+ Alembic-driven ``upgrade head`` run inside+ :meth:`AsyncSQLAlchemyStorage.__aenter__`+ and the subsequent record-write use the same physical database —+ ``:memory:`` gives a private DB per connection, which would let+ Alembic build tables that the storage's engine can't see.+ """++ async def test_auto_migrate_creates_schema(self) -> None:+ """After ``auto_migrate=True`` the tables exist and CRUD works."""+ import os+ import tempfile++ with tempfile.TemporaryDirectory() as tmp:+ db_path = os.path.join(tmp, "as.db")+ url = f"sqlite+aiosqlite:///{db_path}"++ async with AsyncSQLAlchemyStorage(+ url,+ auto_migrate=True,+ ) as storage:+ agent = _agent_record("user-1")+ await storage.upsert_agent("user-1", agent)+ fetched = await storage.get_agent("user-1", agent.id)+ self.assertEqual(fetched.id, agent.id)+++class LegacyRecordShapeTest(IsolatedAsyncioTestCase):+ """The ``mode='before'`` validators absorb pre-refactor payloads.++ Records written before the KB-``data`` nesting / KD-lifecycle+ promotion refactor must still deserialise, so a database populated+ by an older build keeps round-tripping after an upgrade.+ """++ async def test_knowledge_base_flat_payload_migrates(self) -> None:+ """Flat KB fields fold into :attr:`KnowledgeBaseRecord.data`."""+ record = KnowledgeBaseRecord.model_validate(+ {+ "user_id": "u1",+ "name": "kb",+ "description": "d",+ "embedding_model_config": {+ "type": "openai_credential",+ "credential_id": "cred-1",+ "model": "text-embedding-3-small",+ "dimensions": 8,+ },+ "collection_name": "c",+ },+ )+ self.assertEqual(record.data.name, "kb")+ self.assertEqual(record.data.collection_name, "c")++ async def test_knowledge_document_lifecycle_fields_lift(self) -> None:+ """Legacy in-``data`` ``status`` / ``lease_expires_at`` lift up."""+ record = KnowledgeDocumentRecord.model_validate(+ {+ "user_id": "u1",+ "knowledge_base_id": "kb1",+ "data": {+ "filename": "f.txt",+ "size": 1,+ "blob_uri": "local://f.txt",+ "status": "ready",+ "lease_expires_at": None,+ },+ },+ )+ self.assertEqual(record.status, "ready")+ # The fields moved to the top level and no longer shadow ``data``.+ self.assertNotIn("status", record.data.model_dump())