Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
"""Memory compaction service: token tracking plus client-side summarization.
Every provider compacts the same way: :meth:`CompactionService.compact_context`
sends the text to summarize through ``run_native_llm_step`` (ChatUnifier).
``anthropic_config`` / ``anthropic_api_config`` / ``openai_config`` build
provider-managed compaction settings, but no execution path passes them to
an SDK; ``anthropic_config`` is used only to compute a numeric threshold.
Threshold strategy: per-session custom_threshold > model-aware threshold.
Model-aware threshold = compaction ratio x the model's context window, never
below 10,000 tokens. The ratio is the per-user ``compaction_ratio`` when it is
set and between 0.1 and 0.9, else ``COMPACTION_RATIO`` (default 0.8);
``agent.compaction.ratio`` in ``llm_defaults.json`` is read only if Settings
cannot load. See :meth:`CompactionService._get_compaction_ratio`.
"""
from dataclasses import asdict
from pydantic import BaseModel
from datetime import datetime, timezone
from typing import Dict, Any, Optional, TYPE_CHECKING
from core.logging import get_logger
from services.llm.protocol import LLMError
from services.pricing import get_pricing_service
if TYPE_CHECKING:
from core.database import Database
from core.config import Settings
logger = get_logger(__name__)
# The global compaction ratio comes from ``ModelRegistryService.get_agent_defaults``:
# env ``COMPACTION_RATIO`` (core.config.Settings, default 0.8) wins, and
# ``agent.compaction.ratio`` in server/config/llm_defaults.json is used only
# when Settings cannot load. A per-user override in ``user_settings`` wins
# over both via :meth:`CompactionService._get_compaction_ratio`.
def _extract_text_from_response(content) -> str:
"""Extract text content from various LLM response formats.
Handles:
- String content (OpenAI, Anthropic)
- List of content blocks (Gemini format: [{"type": "text", "text": "..."}])
- Other complex formats
"""
# Handle list content (Gemini format)
if isinstance(content, list):
text_parts = []
for block in content:
if isinstance(block, dict):
if block.get("type") == "text" and block.get("text"):
text_parts.append(block["text"])
elif "text" in block:
text_parts.append(str(block["text"]))
elif isinstance(block, str):
text_parts.append(block)
return "\n".join(text_parts)
# Handle string content
if isinstance(content, str):
return content
# Fallback
return str(content) if content else ""
class CompactionConfig(BaseModel):
"""Provider-agnostic compaction configuration."""
enabled: bool = True
class CompactionService:
"""Minimal compaction service using native provider APIs."""
def __init__(self, database: "Database", settings: "Settings"):
self._db = database
self._config = CompactionConfig(enabled=settings.compaction_enabled)
self._ai_service = None
def set_ai_service(self, ai_service) -> None:
"""Wire AI service for generating compaction summaries."""
self._ai_service = ai_service
async def _get_compaction_ratio(self) -> float:
"""Per-user override from ``user_settings.compaction_ratio``, falling
back to ``agent.compaction.ratio`` in ``llm_defaults.json``."""
from services.model_registry import get_model_registry
try:
settings = await self._db.get_user_settings("default")
if settings and "compaction_ratio" in settings:
ratio = float(settings["compaction_ratio"])
if 0.1 <= ratio <= 0.9:
return ratio
except Exception:
pass
return float(get_model_registry().get_agent_defaults()["compaction"]["ratio"])
def get_model_threshold(self, model: str, provider: str, ratio: Optional[float] = None) -> int:
"""Compute compaction threshold based on model's context window.
``ratio`` defaults to ``agent.compaction.ratio`` from
``llm_defaults.json``; pass an explicit value to override per call.
Per-session ``custom_threshold`` (checked in :meth:`track`) still
wins over the value returned here.
"""
from services.model_registry import get_model_registry
registry = get_model_registry()
if ratio is None:
ratio = float(registry.get_agent_defaults()["compaction"]["ratio"])
context_length = registry.get_context_length(model, provider)
model_threshold = int(context_length * ratio)
# Never go below the configured minimum (ge=10000 in Settings)
return max(model_threshold, 10000)
async def anthropic_config(self, threshold: Optional[int] = None, model: str = "", provider: str = "anthropic") -> Dict[str, Any]:
"""Anthropic SDK compaction_control for tool_runner."""
if not threshold:
ratio = await self._get_compaction_ratio()
threshold = self.get_model_threshold(model, provider, ratio=ratio)
return {"enabled": self._config.enabled, "context_token_threshold": threshold}
async def anthropic_api_config(self, threshold: Optional[int] = None, model: str = "", provider: str = "anthropic") -> Dict[str, Any]:
"""Anthropic Messages API context_management config."""
if not threshold:
ratio = await self._get_compaction_ratio()
threshold = self.get_model_threshold(model, provider, ratio=ratio)
return {
"betas": ["compact-2026-01-12"],
"context_management": {"edits": [{"type": "compact_20260112", "trigger": {"type": "input_tokens", "value": threshold}}]},
}
async def openai_config(self, threshold: Optional[int] = None, model: str = "", provider: str = "openai") -> Dict[str, Any]:
"""OpenAI context_management config."""
if not threshold:
ratio = await self._get_compaction_ratio()
threshold = self.get_model_threshold(model, provider, ratio=ratio)
return {"context_management": {"compact_threshold": threshold}}
async def track(self, session_id: str, node_id: str, provider: str, model: str, usage: Dict[str, int]) -> Dict[str, Any]:
"""Track token usage and cost, return compaction status."""
cost = await self._record_usage_metric(
session_id=session_id,
node_id=node_id,
provider=provider,
model=model,
usage=usage,
)
state = await self._db.get_or_create_session_token_state(session_id)
new_total = state["cumulative_total"] + usage.get("total_tokens", 0)
new_total_cost = state.get("cumulative_total_cost", 0.0) + cost["total_cost"]
# Update cumulative state with cost
await self._db.update_session_token_state(
session_id,
{
"cumulative_input_tokens": state["cumulative_input_tokens"] + usage.get("input_tokens", 0),
"cumulative_output_tokens": state["cumulative_output_tokens"] + usage.get("output_tokens", 0),
"cumulative_total": new_total,
"cumulative_input_cost": state.get("cumulative_input_cost", 0.0) + cost["input_cost"],
"cumulative_output_cost": state.get("cumulative_output_cost", 0.0) + cost["output_cost"],
"cumulative_total_cost": new_total_cost,
},
)
# Priority: per-session custom > model-aware (with user ratio) > global default
custom = state.get("custom_threshold")
if custom:
threshold = custom
else:
ratio = await self._get_compaction_ratio()
threshold = self.get_model_threshold(model, provider, ratio=ratio)
# Get the model's full context window size for frontend display
from services.model_registry import get_model_registry
registry = get_model_registry()
context_length = registry.get_context_length(model, provider)
return {
"total": new_total,
"total_cost": new_total_cost,
"cost": cost,
"threshold": threshold,
"context_length": context_length,
"needs_compaction": self._config.enabled and new_total >= threshold,
}
async def _record_usage_metric(
self,
*,
session_id: str,
node_id: str,
provider: str,
model: str,
usage: Dict[str, int],
iteration: int = 1,
) -> Dict[str, float]:
"""Persist billed usage without changing active-context counters."""
pricing_service = get_pricing_service()
cost = pricing_service.calculate_cost(
provider=provider,
model=model,
input_tokens=usage.get("input_tokens", 0),
output_tokens=usage.get("output_tokens", 0),
cache_read_tokens=usage.get("cache_read_tokens", 0),
cache_creation_tokens=usage.get("cache_creation_tokens", 0),
reasoning_tokens=usage.get("reasoning_tokens", 0),
)
# Save token metric with cost fields
await self._db.save_token_metric(
{
"session_id": session_id,
"node_id": node_id,
"provider": provider,
"model": model,
"iteration": iteration,
**usage,
"input_cost": cost["input_cost"],
"output_cost": cost["output_cost"],
"cache_cost": cost["cache_cost"],
"total_cost": cost["total_cost"],
}
)
return cost
async def record(
self, session_id: str, node_id: str, provider: str, model: str, tokens_before: int, tokens_after: int, summary: Optional[str] = None
) -> None:
"""Record compaction event after native API handles it."""
state = await self._db.get_or_create_session_token_state(session_id)
await self._db.save_compaction_event(
{
"session_id": session_id,
"node_id": node_id,
"trigger_reason": "native",
"tokens_before": tokens_before,
"tokens_after": tokens_after,
"summary_model": model,
"summary_provider": provider,
"success": True,
"summary_content": summary,
}
)
await self._db.update_session_token_state(
session_id,
{
"cumulative_total": tokens_after,
"last_compaction_at": datetime.now(timezone.utc),
"compaction_count": state["compaction_count"] + 1,
},
)
async def stats(self, session_id: str, model: str = "", provider: str = "") -> Dict[str, Any]:
"""Get session statistics.
When model/provider are provided, the threshold reflects the model's
context window. Otherwise falls back to the global default.
"""
state = await self._db.get_or_create_session_token_state(session_id)
custom = state.get("custom_threshold")
if custom:
threshold = custom
else:
ratio = await self._get_compaction_ratio()
threshold = self.get_model_threshold(model, provider, ratio=ratio)
# Get context window size for frontend display
context_length = 0
if model and provider:
from services.model_registry import get_model_registry
registry = get_model_registry()
context_length = registry.get_context_length(model, provider)
return {
"session_id": session_id,
"total": state["cumulative_total"],
"threshold": threshold,
"context_length": context_length,
"count": state.get("compaction_count", 0),
}
async def configure(self, session_id: str, threshold: Optional[int] = None, enabled: Optional[bool] = None) -> bool:
"""Configure session settings."""
updates = {}
if threshold is not None:
updates["custom_threshold"] = threshold
if enabled is not None:
updates["compaction_enabled"] = enabled
return await self._db.update_session_token_state(session_id, updates) if updates else True
async def compact_context(
self,
session_id: str,
node_id: str,
memory_content: str,
provider: str,
api_key: str,
model: str,
*,
explicit_max_retries: int = 2,
raise_llm_errors: bool = False,
) -> Dict[str, Any]:
"""Perform compaction by summarizing conversation history.
Uses the AI service to generate a structured summary following Claude Code pattern.
A failed summarizer call, or an empty summary, returns ``success:
False`` and records nothing, so an empty reply never replaces the
conversation. With ``raise_llm_errors`` a summarizer
:class:`LLMError` raises instead, for a caller that retries or stops
on it the way it does for an LLM step (the Temporal activity).
"""
if not self._ai_service:
logger.warning("[Compaction] AI service not wired")
return {"success": False, "error": "AI service not available"}
if not memory_content or len(memory_content.strip()) < 100:
return {"success": False, "error": "Memory content too short"}
state = await self._db.get_or_create_session_token_state(session_id)
tokens_before = state["cumulative_total"]
compaction_usage: Dict[str, int] = {}
try:
prompt = f"""Summarize this conversation into a structured format:
## Task Overview
What the user is trying to accomplish.
## Current State
What's been completed and what's in progress.
## Important Discoveries
Key findings, decisions, or problems encountered.
## Next Steps
What needs to happen next.
## Context to Preserve
Details that must be retained for continuity.
---
CONVERSATION:
{memory_content}
---
Provide a concise but complete summary."""
# Use a reasonable summary size: min(4096, model's max output)
from services.model_registry import get_model_registry
model_max = get_model_registry().get_max_output_tokens(model, provider)
summary_tokens = min(4096, model_max)
if self._ai_service.chat_unifier is None:
raise RuntimeError("ChatUnifier is required for compaction")
from services.agent_runtime import run_native_llm_step
from services.llm.protocol import Message
response = await run_native_llm_step(
self._ai_service.chat_unifier,
provider=provider,
api_key=api_key,
messages=[Message(role="user", content=prompt)],
model=model,
temperature=0.3,
max_tokens=summary_tokens,
sdk_max_retries=0,
explicit_max_retries=explicit_max_retries,
translate_errors=False,
)
compaction_usage = asdict(response.usage)
await self._record_usage_metric(
session_id=session_id,
node_id=node_id,
provider=provider,
model=model,
usage=compaction_usage,
# Iteration zero distinguishes the summarizer call in the
# existing metric schema without adding a public field.
iteration=0,
)
summary = _extract_text_from_response(response.content)
if not summary.strip():
# Saved behind the header below, an empty reply would replace
# the whole conversation.
logger.error("[Compaction] Failed: empty summary")
return {"success": False, "error": "empty summary", "usage": compaction_usage}
new_memory = f"# Conversation Summary (Compacted)\n*Generated: {datetime.now(timezone.utc).isoformat()}*\n\n{summary}"
await self.record(session_id, node_id, provider, model, tokens_before, 0, new_memory)
logger.info(f"[Compaction] Session {session_id}: {tokens_before} -> 0 tokens")
return {
"success": True,
"summary": new_memory,
"tokens_before": tokens_before,
"tokens_after": 0,
"usage": compaction_usage,
}
except LLMError as e:
if raise_llm_errors:
raise
# user_message: the raw SDK text can carry payload fragments.
logger.error(f"[Compaction] Failed: {e.user_message}")
return {
"success": False,
"error": e.user_message,
"usage": compaction_usage,
}
except Exception as e:
logger.error(f"[Compaction] Failed: {e}")
return {
"success": False,
"error": str(e),
"usage": compaction_usage,
}
_service: Optional[CompactionService] = None
def get_compaction_service() -> Optional[CompactionService]:
return _service
def init_compaction_service(database: "Database", settings: "Settings") -> CompactionService:
global _service
_service = CompactionService(database, settings)
logger.info("[Compaction] Initialized")
return _service