Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
"""Durable, action-specific messages for the embedding queue."""
from __future__ import annotations
import json
from dataclasses import dataclass, field
from typing import Any, Dict, List, Mapping, Optional, Union
from uuid import uuid4
from openviking.storage.index_action import FieldPatch, IndexAction
EmbeddingInput = Union[str, List[Dict[str, Any]]]
class IncompleteInitialRecordError(RuntimeError):
"""A missing record cannot be recreated from the queued seed."""
def __init__(self, record_id: str, missing_fields: List[str]):
self.record_id = record_id
self.missing_fields = list(missing_fields)
super().__init__(
"update_fields could not recreate missing vector record: "
f"record_id={record_id} missing_fields={self.missing_fields}"
)
def missing_initial_record_fields(record: Mapping[str, Any]) -> List[str]:
"""Return fields required to create a usable vector record."""
missing = [name for name in ("id", "uri", "account_id", "level") if record.get(name) is None]
if not record.get("vector") and not record.get("sparse_vector"):
missing.append("vector")
return missing
@dataclass
class EmbedPayload:
message: EmbeddingInput
context_data: Dict[str, Any]
field_patch: FieldPatch | None = None
def to_dict(self) -> dict[str, Any]:
data = {"type": "embed", "message": self.message, "context_data": self.context_data}
if self.field_patch is not None:
data["field_patch"] = self.field_patch.to_dict()
return data
@dataclass
class UpdateFieldsPayload:
record_id: str
context_data: Dict[str, Any]
field_patch: FieldPatch
def to_dict(self) -> dict[str, Any]:
return {
"type": "update_fields",
"record_id": self.record_id,
"context_data": self.context_data,
"field_patch": self.field_patch.to_dict(),
}
@dataclass
class DeletePayload:
record_ids: List[str]
context_data: Dict[str, Any]
def to_dict(self) -> dict[str, Any]:
return {
"type": "delete",
"record_ids": list(self.record_ids),
"context_data": self.context_data,
}
@dataclass
class NoopPayload:
context_data: Dict[str, Any]
def to_dict(self) -> dict[str, Any]:
return {"type": "noop", "context_data": self.context_data}
EmbeddingPayload = EmbedPayload | UpdateFieldsPayload | DeletePayload | NoopPayload
_PAYLOAD_FIELDS = {
"embed": frozenset({"type", "message", "context_data", "field_patch"}),
"update_fields": frozenset({"type", "record_id", "context_data", "field_patch"}),
"delete": frozenset({"type", "record_ids", "context_data"}),
"noop": frozenset({"type", "context_data"}),
}
@dataclass(init=False)
class EmbeddingMsg:
"""Stable queue envelope whose payload is determined by ``action``."""
action: IndexAction
payload: EmbeddingPayload
id: str = field(default_factory=lambda: str(uuid4()))
telemetry_id: str = ""
task_id: str = ""
_task_work_id: str = ""
queue_enqueued_at: float = 0.0
failure_requeue_count: int = 0
source_type: str = ""
def __init__(
self,
message: Optional[EmbeddingInput] = None,
context_data: Optional[Dict[str, Any]] = None,
telemetry_id: str = "",
task_id: str = "",
_task_work_id: str = "",
action: IndexAction | str = IndexAction.UPSERT,
record_ids: Optional[List[str]] = None,
update_fields: Optional[Dict[str, Any]] = None,
field_modes: Optional[Dict[str, str]] = None,
initial_fields: Optional[Dict[str, Any]] = None,
queue_enqueued_at: float = 0.0,
failure_requeue_count: int = 0,
source_type: str = "",
*,
payload: EmbeddingPayload | None = None,
) -> None:
self.id = str(uuid4())
self.telemetry_id = telemetry_id
self.task_id = task_id
self._task_work_id = _task_work_id
self.queue_enqueued_at = max(float(queue_enqueued_at or 0.0), 0.0)
self.failure_requeue_count = max(int(failure_requeue_count or 0), 0)
self.source_type = str(source_type or "")
self.action = IndexAction(action)
if payload is not None:
if any(
value is not None
for value in (
message,
context_data,
record_ids,
update_fields,
field_modes,
initial_fields,
)
):
raise ValueError("payload cannot be combined with legacy embedding fields")
self.payload = payload
else:
self.payload = self._legacy_payload(
message=message,
context_data=dict(context_data or {}),
record_ids=list(record_ids or []),
update_fields=dict(update_fields or {}),
field_modes=dict(field_modes or {}),
initial_fields=dict(initial_fields or {}),
)
self._validate_payload()
self._validate_account_context()
def _validate_account_context(self) -> None:
context_data = getattr(self.payload, "context_data", {})
account_id = context_data.get("account_id") if isinstance(context_data, dict) else None
if not isinstance(account_id, str) or not account_id.strip():
raise ValueError("Embedding message requires account_id")
def _legacy_payload(
self,
*,
message: Optional[EmbeddingInput],
context_data: Dict[str, Any],
record_ids: List[str],
update_fields: Dict[str, Any],
field_modes: Dict[str, str],
initial_fields: Dict[str, Any],
) -> EmbeddingPayload:
patch = (
FieldPatch(update_fields, field_modes, initial_fields)
if update_fields or field_modes or initial_fields
else None
)
if self.action is IndexAction.NONE:
if message is not None or record_ids or patch is not None:
raise ValueError("none action cannot carry work")
return NoopPayload(context_data)
if self.action is IndexAction.DELETE:
if message is not None or patch is not None:
raise ValueError("delete does not accept embedding or field-patch data")
return DeletePayload(record_ids, context_data)
if self.action is IndexAction.UPDATE_FIELDS:
if message is not None or len(record_ids) != 1 or patch is None:
raise ValueError("update_fields requires one record id and a field patch")
return UpdateFieldsPayload(record_ids[0], context_data, patch)
if not isinstance(message, (str, list)):
raise ValueError(f"{self.action.value} requires an embedding message")
if self.action is IndexAction.UPSERT:
if field_modes:
raise ValueError("upsert carries resolved fields and cannot use field modes")
# Historical flat UPSERT messages allowed these fields but never
# consumed them. Preserve queue-read compatibility while emitting
# only the strict action-specific payload for new messages.
return EmbedPayload(message, context_data)
return EmbedPayload(message, context_data, patch)
def _validate_payload(self) -> None:
expected = {
IndexAction.NONE: NoopPayload,
IndexAction.UPSERT: EmbedPayload,
IndexAction.MERGE: EmbedPayload,
IndexAction.UPDATE_FIELDS: UpdateFieldsPayload,
IndexAction.DELETE: DeletePayload,
}[self.action]
if not isinstance(self.payload, expected):
raise ValueError(
f"{self.action.value} requires {expected.__name__}, got {type(self.payload).__name__}"
)
if isinstance(self.payload, EmbedPayload) and not isinstance(
self.payload.message, (str, list)
):
raise ValueError(f"{self.action.value} requires an embedding message")
if isinstance(self.payload, DeletePayload) and not self.payload.record_ids:
raise ValueError("delete requires record_ids")
if isinstance(self.payload, UpdateFieldsPayload):
if not self.payload.record_id or not self.payload.field_patch.values:
raise ValueError("update_fields requires one record id and a non-empty field patch")
if (
self.action is IndexAction.UPSERT
and isinstance(self.payload, EmbedPayload)
and self.payload.field_patch is not None
):
raise ValueError("upsert does not accept a field patch")
@classmethod
def for_embed(
cls,
*,
message: EmbeddingInput,
context_data: Dict[str, Any],
action: IndexAction | str = IndexAction.UPSERT,
field_patch: FieldPatch | None = None,
telemetry_id: str = "",
source_type: str = "",
) -> "EmbeddingMsg":
action = IndexAction(action)
return cls(
action=action,
payload=EmbedPayload(message, context_data, field_patch),
telemetry_id=telemetry_id,
source_type=source_type,
)
@classmethod
def for_update_fields(
cls,
*,
record_id: str,
context_data: Dict[str, Any],
field_patch: FieldPatch | None = None,
fields: Optional[Dict[str, Any]] = None,
field_modes: Optional[Dict[str, str]] = None,
initial_fields: Optional[Dict[str, Any]] = None,
telemetry_id: str = "",
) -> "EmbeddingMsg":
if field_patch is not None and any(
value is not None for value in (fields, field_modes, initial_fields)
):
raise ValueError(
"field_patch cannot be combined with legacy fields, field_modes, or initial_fields"
)
resolved_patch = field_patch or FieldPatch(
fields or {}, field_modes or {}, initial_fields or {}
)
return cls(
action=IndexAction.UPDATE_FIELDS,
payload=UpdateFieldsPayload(record_id, context_data, resolved_patch),
telemetry_id=telemetry_id,
)
@classmethod
def for_delete(
cls,
*,
record_ids: List[str],
context_data: Dict[str, Any],
telemetry_id: str = "",
) -> "EmbeddingMsg":
return cls(
action=IndexAction.DELETE,
payload=DeletePayload(record_ids, context_data),
telemetry_id=telemetry_id,
)
@classmethod
def noop(cls, *, context_data: Dict[str, Any], telemetry_id: str = "") -> "EmbeddingMsg":
return cls(
action=IndexAction.NONE,
payload=NoopPayload(context_data),
telemetry_id=telemetry_id,
)
@property
def context_data(self) -> Dict[str, Any]:
return self.payload.context_data
@property
def message(self) -> Optional[EmbeddingInput]:
return self.payload.message if isinstance(self.payload, EmbedPayload) else None
@property
def record_ids(self) -> List[str]:
if isinstance(self.payload, DeletePayload):
return self.payload.record_ids
if isinstance(self.payload, UpdateFieldsPayload):
return [self.payload.record_id]
return []
@property
def field_patch(self) -> FieldPatch | None:
if isinstance(self.payload, (EmbedPayload, UpdateFieldsPayload)):
return self.payload.field_patch
return None
@property
def update_fields(self) -> Dict[str, Any]:
return dict(self.field_patch.values) if self.field_patch is not None else {}
@property
def field_modes(self) -> Dict[str, str]:
return dict(self.field_patch.modes) if self.field_patch is not None else {}
@property
def initial_fields(self) -> Dict[str, Any]:
return dict(self.field_patch.seed_fields) if self.field_patch is not None else {}
def to_dict(self) -> Dict[str, Any]:
data = {
"id": self.id,
"telemetry_id": self.telemetry_id,
"task_id": self.task_id,
"_task_work_id": self._task_work_id,
"queue_enqueued_at": self.queue_enqueued_at,
"failure_requeue_count": self.failure_requeue_count,
"action": self.action.value,
"payload": self.payload.to_dict(),
}
if self.source_type:
data["source_type"] = self.source_type
return data
def to_json(self) -> str:
return json.dumps(self.to_dict())
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "EmbeddingMsg":
if not isinstance(data, dict):
raise ValueError("embedding message must be an object")
action = IndexAction(data.get("action", IndexAction.UPSERT.value))
payload_data = data.get("payload")
if isinstance(payload_data, dict):
ambiguous = {
name
for name in (
"message",
"context_data",
"record_ids",
"update_fields",
"field_modes",
"initial_fields",
)
if name in data
}
if ambiguous:
raise ValueError(
f"payload cannot be combined with legacy embedding fields: {sorted(ambiguous)}"
)
payload_type = payload_data.get("type")
allowed_fields = _PAYLOAD_FIELDS.get(payload_type)
if allowed_fields is None:
raise ValueError(f"unsupported embedding payload type: {payload_type}")
unknown_fields = set(payload_data) - allowed_fields
if unknown_fields:
raise ValueError(
f"{payload_type} payload contains unsupported fields: {sorted(unknown_fields)}"
)
context_data = dict(payload_data.get("context_data") or {})
if payload_type == "embed":
raw_patch = payload_data.get("field_patch")
payload: EmbeddingPayload = EmbedPayload(
payload_data.get("message"),
context_data,
FieldPatch.from_dict(raw_patch) if isinstance(raw_patch, dict) else None,
)
elif payload_type == "update_fields":
payload = UpdateFieldsPayload(
str(payload_data.get("record_id") or ""),
context_data,
FieldPatch.from_dict(dict(payload_data.get("field_patch") or {})),
)
elif payload_type == "delete":
payload = DeletePayload(
[str(value) for value in payload_data.get("record_ids") or []],
context_data,
)
elif payload_type == "noop":
payload = NoopPayload(context_data)
obj = cls(
action=action,
payload=payload,
telemetry_id=str(data.get("telemetry_id") or ""),
task_id=str(data.get("task_id") or ""),
_task_work_id=str(data.get("_task_work_id") or ""),
queue_enqueued_at=data.get("queue_enqueued_at", 0.0),
failure_requeue_count=data.get("failure_requeue_count", 0),
source_type=str(data.get("source_type") or ""),
)
else:
obj = cls(
message=data.get("message"),
context_data=dict(data.get("context_data") or {}),
telemetry_id=str(data.get("telemetry_id") or ""),
task_id=str(data.get("task_id") or ""),
_task_work_id=str(data.get("_task_work_id") or ""),
action=action,
record_ids=data.get("record_ids"),
update_fields=data.get("update_fields"),
field_modes=data.get("field_modes"),
initial_fields=data.get("initial_fields"),
queue_enqueued_at=data.get("queue_enqueued_at", 0.0),
failure_requeue_count=data.get("failure_requeue_count", 0),
source_type=str(data.get("source_type") or ""),
)
obj.id = str(data.get("id") or obj.id)
return obj
@classmethod
def from_json(cls, json_str: str) -> "EmbeddingMsg":
try:
return cls.from_dict(json.loads(json_str))
except json.JSONDecodeError as exc:
raise ValueError(f"Invalid JSON string: {exc}") from exc
__all__ = [
"DeletePayload",
"EmbedPayload",
"EmbeddingMsg",
"IncompleteInitialRecordError",
"NoopPayload",
"UpdateFieldsPayload",
"missing_initial_record_fields",
]