Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

feat(ai): connect task routing, guarded fallback, and scoped run accounting

Juan Denisbebc69f71c4f
diff --git a/backend/app/api/ai.py b/backend/app/api/ai.py
index 49c26b10..5e29eabc 100644
--- a/backend/app/api/ai.py
+++ b/backend/app/api/ai.py
@@ -42,7 +42,8 @@ from app.services.ai_attachment_registry import (
)
from app.services.ai_tool_registry import ai_tool_registry
from app.error_reporting import unexpected_response
-from app.exceptions import DependencyUnavailableError, ValidationError
+from app.exceptions import ApplicationError, DependencyUnavailableError, ValidationError
+from app.services import ai_management, ai_usage, ai_runtime
logger = logging.getLogger(__name__)
ai_bp = Blueprint('ai', __name__)
@@ -88,6 +89,10 @@ def _validate_chat_data(data):
return 'page_context must be a small object'
if data.get('mode', 'assistant') not in ('assistant', 'simple'):
return 'Invalid assistant mode'
+ if data.get('profile') is not None and data['profile'] not in ai_management.PROFILES:
+ return 'Invalid model role'
+ if not isinstance(data.get('workflow', 'chat'), str) or data.get('workflow', 'chat') not in ai_management.WORKFLOWS:
+ return 'Invalid task'
# ---------------------------------------------------------------------------
@@ -105,15 +110,21 @@ def _owned_conversation(conversation_id: str, user):
return row
-def _load_or_create(conversation_id, user, mode, connection_id=None, model=None):
+def _load_or_create(conversation_id, user, mode, connection_id=None, model=None, *, profile=None, workflow='chat', message=''):
if conversation_id:
row = _owned_conversation(conversation_id, user)
return row # may be None -> caller 404s
connection = None
+ selection = {}
if connection_id is not None and (not isinstance(connection_id, str) or len(connection_id) > 64):
raise ValidationError('Invalid connection ID')
if connection_id or ai_connections.default_id():
- connection = ai_connections.get_connection(connection_id)
+ selection = ai_management.resolve(profile=profile, workflow=workflow,
+ connection_id=connection_id, model=model, message=message)
+ connection = ai_connections.get_connection(selection['connection_id'])
+ model = selection['model']
+ elif workflow != 'chat' or profile:
+ raise ValidationError('Save a provider connection before choosing task models.')
if model is not None and (not isinstance(model, str) or not model.strip() or len(model) > 256):
raise ValidationError('Invalid model ID')
row = AiConversation(
@@ -122,6 +133,8 @@ def _load_or_create(conversation_id, user, mode, connection_id=None, model=None)
model_name=ai_connections.model_name(connection, model) if connection else ai_service.current_model_name(),
connection_id=connection.id if connection else None,
)
+ selection['workspace_id'] = ai_management.workspace_id(user, request.headers.get('X-Workspace-Id'))
+ row.management = selection
db.session.add(row)
db.session.commit()
return row
@@ -150,9 +163,39 @@ def status():
'mode_default': AiConversation.MODE_ASSISTANT,
'pii_redaction': bool(ai_service._setting('ai_pii_redaction', True)),
'injection_detection': bool(ai_service._setting('ai_injection_detection', True)),
+ 'management': ai_management.public_profiles(),
})
+@ai_bp.route('/management', methods=['GET'])
+@admin_required
+def get_management():
+ return jsonify(ai_management.settings())
+
+
+@ai_bp.route('/management', methods=['PUT'])
+@admin_required
+def update_management():
+ return jsonify(ai_management.save(request.get_json(silent=True)))
+
+
+@ai_bp.route('/management/preview', methods=['POST'])
+@admin_required
+def preview_management():
+ data = request.get_json(silent=True) or {}
+ if not isinstance(data, dict) or not isinstance(data.get('message', ''), str) or len(data.get('message', '')) > MAX_MESSAGE_CHARS:
+ raise ValidationError('Enter a task of at most 16000 characters.')
+ config = ai_management.validate(data['config']) if 'config' in data else None
+ return jsonify(ai_management.resolve(profile=data.get('profile'), workflow=data.get('workflow', 'chat'),
+ message=data.get('message', ''), config=config))
+
+
+@ai_bp.route('/usage', methods=['GET'])
+@admin_required
+def get_ai_usage():
+ return jsonify(ai_usage.report(request.args))
+
+
@ai_bp.route('/settings', methods=['GET'])
@admin_required
def get_settings():
@@ -163,7 +206,6 @@ def get_settings():
'model': SettingsService.get('ai_model', ''),
'endpoint': SettingsService.get('ai_endpoint', ''),
'max_cost_usd': SettingsService.get('ai_max_cost_usd', 0.5),
- 'fallback_models': SettingsService.get('ai_fallback_models', []),
'pii_redaction': bool(SettingsService.get('ai_pii_redaction', True)),
'injection_detection': bool(SettingsService.get('ai_injection_detection', True)),
# Never return the key itself — only whether one is configured.
@@ -183,6 +225,14 @@ def update_settings():
user = get_current_user()
uid = user.id if user else None
+ if not isinstance(data, dict):
+ raise ValidationError('AI settings must be an object.')
+ if 'fallback_models' in data:
+ raise ValidationError('Configure connection-bound fallbacks in AI management settings.')
+ for field in ('enabled', 'pii_redaction', 'injection_detection'):
+ if field in data and not isinstance(data[field], bool):
+ raise ValidationError(f'{field} must be a boolean.')
+
if 'max_cost_usd' in data:
import math
try:
@@ -204,9 +254,6 @@ def update_settings():
for field, (key, caster) in field_map.items():
if field in data:
SettingsService.set(key, caster(data[field]), user_id=uid)
- if 'fallback_models' in data and isinstance(data['fallback_models'], list):
- SettingsService.set('ai_fallback_models', data['fallback_models'], user_id=uid)
-
# API key: encrypt and store; empty string clears it.
if 'api_key' in data:
raw = (data['api_key'] or '').strip()
@@ -345,7 +392,8 @@ def create_conversation():
data = request.get_json(silent=True) or {}
mode = data.get('mode') or AiConversation.MODE_ASSISTANT
try:
- row = _load_or_create(None, user, mode, data.get('connection_id'), data.get('model'))
+ row = _load_or_create(None, user, mode, data.get('connection_id'), data.get('model'),
+ profile=data.get('profile'), workflow=data.get('workflow', 'chat'))
except ValueError as exc:
return jsonify({'error': str(exc)}), 400
row.title = data.get('title')
@@ -420,7 +468,8 @@ def chat():
attachment_refs = _attachment_references(data)
try:
- row = _load_or_create(data.get('conversation_id'), user, mode, data.get('connection_id'), data.get('model'))
+ row = _load_or_create(data.get('conversation_id'), user, mode, data.get('connection_id'), data.get('model'),
+ profile=data.get('profile'), workflow=data.get('workflow', 'chat'), message=message)
except ValueError as exc:
return jsonify({'error': str(exc)}), 400
if row is None:
@@ -443,6 +492,7 @@ def chat():
def _run_chat(row, user, mode, page_context, attachment_refs, message, safe_message):
attachment_result = resolve_attachments(user, attachment_refs)
+ recorder = ai_usage.start(row, user)
_persist_user_message(row, message, attachments=attachment_result['manifest'])
conv = None
try:
@@ -450,25 +500,31 @@ def _run_chat(row, user, mode, page_context, attachment_refs, message, safe_mess
conv = ai_service.build_conversation(
row, user, mode, page_context, gate=None,
attachment_context=attachment_result['context'],
+ recorder=recorder,
)
- reply = conv.ask(safe_message)
+ reply = ai_runtime.ask(conv, safe_message, row.management.get('workflow', 'chat'))
except ai_service.AIProtectionError as exc:
+ recorder.finish('error', conv)
+ if conv is not None:
+ ai_service.persist_conversation(row, conv, page_context=page_context)
raise DependencyUnavailableError(str(exc)) from exc
except Exception as exc: # noqa: BLE001 - reported, not swallowed
budget_message = ai_service.budget_error_message(exc)
+ usage = recorder.finish('budget_exceeded' if budget_message else 'error', conv)
+ if conv is not None:
+ ai_service.persist_conversation(row, conv, page_context=page_context)
if budget_message:
- if conv is not None:
- ai_service.persist_conversation(row, conv, page_context=page_context)
return jsonify({'error': budget_message, 'code': 'ai_budget_exceeded',
- 'conversation_id': row.id, 'usage': conv.usage if conv else {}}), 409
+ 'conversation_id': row.id, 'usage': usage}), 409
return unexpected_response(exc)
- _persist_assistant_message(row, reply, tool_calls=[], usage=conv.usage)
+ usage = recorder.finish('success', conv)
+ _persist_assistant_message(row, reply, tool_calls=[], usage=usage)
ai_service.persist_conversation(row, conv, page_context=page_context)
return jsonify({
'conversation_id': row.id,
'reply': reply,
- 'usage': conv.usage,
+ 'usage': usage,
'attachment_warnings': attachment_result['warnings'],
})
@@ -499,7 +555,8 @@ def chat_stream():
attachment_refs = _attachment_references(data)
try:
- row = _load_or_create(data.get('conversation_id'), user, mode, data.get('connection_id'), data.get('model'))
+ row = _load_or_create(data.get('conversation_id'), user, mode, data.get('connection_id'), data.get('model'),
+ profile=data.get('profile'), workflow=data.get('workflow', 'chat'), message=message)
except ValueError as exc:
return jsonify({'error': str(exc)}), 400
if row is None:
@@ -550,6 +607,8 @@ def chat_stream():
tool_order: list[str] = []
last_usage: dict = {}
conv = None
+ recorder = None
+ outcome = 'success'
try:
if flagged:
emit('error', {'message': 'Your message was flagged by the prompt-injection guardrail.'})
@@ -561,11 +620,14 @@ def chat_stream():
if conv_user is None or conv_row is None:
emit('error', {'message': 'Your session is no longer authorized.'})
return
+ recorder = ai_usage.start(conv_row, conv_user)
+ emit('run_start', {'run_id': recorder.run.id, **conv_row.management, 'model': conv_row.model_name})
conv = ai_service.build_conversation(
conv_row, conv_user, mode, page_context, gate,
attachment_context=attachment_result['context'],
+ recorder=recorder,
)
- for event in conv.ask_live(safe_message):
+ for event in ai_runtime.ask_live(conv, safe_message, conv_row.management.get('workflow', 'chat')):
if cancel_event.is_set():
break
name, payload = ai_service.live_event_to_frame(event)
@@ -596,8 +658,11 @@ def chat_stream():
emit(name, payload)
except Exception as exc:
budget_message = ai_service.budget_error_message(exc)
+ outcome = 'budget_exceeded' if budget_message else 'error'
if budget_message:
emit('error', {'message': budget_message, 'code': 'ai_budget_exceeded'})
+ elif isinstance(exc, ApplicationError):
+ emit('error', {'message': str(exc), 'code': exc.code})
else:
logger.exception("AI stream worker failed")
emit('error', {'message': str(exc) if isinstance(exc, ai_service.AIProtectionError)
@@ -608,6 +673,8 @@ def chat_stream():
text = ''.join(acc_text)
if conv is not None:
last_usage = conv.usage
+ if recorder is not None:
+ last_usage = recorder.finish('interrupted' if cancel_event.is_set() else outcome, conv)
if text or tool_order or conv is not None:
conv_row = db.session.get(AiConversation, conversation_id)
if conv_row is not None:
diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py
index 1fef9d75..48584216 100644
--- a/backend/app/models/__init__.py
+++ b/backend/app/models/__init__.py
@@ -60,7 +60,7 @@ from app.models.status_page import StatusPage, StatusComponent, HealthCheck, Sta
from app.models.cloud_server import CloudProvider, CloudServer, CloudSnapshot
from app.models.pending_agent import PendingAgent
from app.models.plugin import InstalledPlugin
-from app.models.ai import AiConversation, AiMessage, AiPendingAction
+from app.models.ai import AiConversation, AiMessage, AiPendingAction, AiRun
from app.models.image_scan import ImageVulnerabilityScan, SbomArtifact
from app.models.passkey import PasskeyCredential
from app.models.secret_vault import SecretVault, Secret
@@ -113,7 +113,7 @@ __all__ = [
'CloudProvider', 'CloudServer', 'CloudSnapshot',
'PendingAgent',
'InstalledPlugin',
- 'AiConversation', 'AiMessage', 'AiPendingAction',
+ 'AiConversation', 'AiMessage', 'AiPendingAction', 'AiRun',
'ImageVulnerabilityScan', 'SbomArtifact',
'PasskeyCredential',
'SecretVault', 'Secret',
diff --git a/backend/app/models/ai.py b/backend/app/models/ai.py
index 7bf8d72d..881e0033 100644
--- a/backend/app/models/ai.py
+++ b/backend/app/models/ai.py
@@ -43,6 +43,15 @@ class AiConversation(JsonColumnMixin, TimestampMixin, db.Model):
connection_id = db.Column(db.String(64), db.ForeignKey('ai_provider_connections.id'), index=True)
export_json = db.Column(db.Text) # Prompture conv.export(strip_images=True)
last_page = db.Column(db.String(256)) # last route the assistant saw (for resume context)
+ management_json = db.Column(db.Text)
+
+ @property
+ def management(self):
+ return self._json_read('management_json', {})
+
+ @management.setter
+ def management(self, value):
+ self.management_json = json.dumps(value)
messages = db.relationship(
'AiMessage', backref='conversation', cascade='all, delete-orphan',
@@ -73,6 +82,7 @@ class AiConversation(JsonColumnMixin, TimestampMixin, db.Model):
'model_name': self.model_name,
'connection_id': self.connection_id,
'last_page': self.last_page,
+ 'management': self.management,
'message_count': self.messages.count(),
'created_at': self.created_at.isoformat() if self.created_at else None,
'updated_at': self.updated_at.isoformat() if self.updated_at else None,
@@ -91,8 +101,17 @@ class AiProviderConnection(JsonColumnMixin, TimestampMixin, db.Model):
provider = db.Column(db.String(64), nullable=False)
model = db.Column(db.String(256), nullable=False)
config_encrypted = db.Column(db.Text, nullable=False)
+ model_catalog_json = db.Column(db.Text)
conversations = db.relationship('AiConversation', backref='connection')
+ @property
+ def model_catalog(self):
+ return self._json_read('model_catalog_json', [])
+
+ @model_catalog.setter
+ def model_catalog(self, value):
+ self.model_catalog_json = json.dumps(value)
+
@property
def config(self):
config = self._json_read('_decrypted_config_json', None, expect=dict)
@@ -167,6 +186,50 @@ class AiMessage(JsonColumnMixin, db.Model):
}
+class AiRun(JsonColumnMixin, db.Model):
+ """Durable per-turn accounting, retained when a transcript is deleted."""
+ __tablename__ = 'ai_runs'
+
+ id = db.Column(db.String(64), primary_key=True, default=_new_id)
+ conversation_id = db.Column(db.String(64), db.ForeignKey('ai_conversations.id', ondelete='SET NULL'), index=True)
+ user_id = db.Column(db.Integer, db.ForeignKey('users.id', ondelete='SET NULL'), index=True)
+ workspace_id = db.Column(db.Integer, db.ForeignKey('workspaces.id', ondelete='SET NULL'), index=True)
+ conversation = db.relationship('AiConversation', backref=db.backref('runs', lazy='dynamic'))
+ user = db.relationship('User', backref=db.backref('ai_runs', lazy='dynamic'))
+ workspace = db.relationship('Workspace', backref=db.backref('ai_runs', lazy='dynamic'))
+ connection_id = db.Column(db.String(64), index=True) # immutable reporting snapshot
+ model = db.Column(db.String(384))
+ profile = db.Column(db.String(16), nullable=False, default='standard')
+ workflow = db.Column(db.String(16), nullable=False, default='chat')
+ strategy = db.Column(db.String(32), nullable=False, default='explicit')
+ reason = db.Column(db.String(512))
+ status = db.Column(db.String(24), nullable=False, default='running', index=True)
+ cost = db.Column(db.Float, nullable=False, default=0)
+ budget_charge = db.Column(db.Float, nullable=False, default=0)
+ reserved_cost = db.Column(db.Float, nullable=False, default=0)
+ total_tokens = db.Column(db.Integer, nullable=False, default=0)
+ duration_ms = db.Column(db.Float, nullable=False, default=0)
+ usage_json = db.Column(db.Text)
+ created_at = db.Column(db.DateTime, default=datetime.utcnow, nullable=False, index=True)
+ completed_at = db.Column(db.DateTime)
+
+ @property
+ def usage(self):
+ return self._json_read('usage_json', {})
+
+ @usage.setter
+ def usage(self, value):
+ self.usage_json = json.dumps(value)
+
+ def to_dict(self):
+ return {**{key: getattr(self, key) for key in (
+ 'id', 'conversation_id', 'user_id', 'workspace_id', 'connection_id', 'model',
+ 'profile', 'workflow', 'strategy', 'reason', 'status', 'cost', 'budget_charge',
+ 'reserved_cost', 'total_tokens', 'duration_ms')},
+ 'usage': self.usage, 'created_at': self.created_at.isoformat() + 'Z',
+ 'completed_at': self.completed_at.isoformat() + 'Z' if self.completed_at else None}
+
+
class AiPendingAction(JsonColumnMixin, db.Model):
"""A guarded write-tool action awaiting human confirmation.
diff --git a/backend/app/plugins_sdk/ai.py b/backend/app/plugins_sdk/ai.py
index 92731ca0..17edc422 100644
--- a/backend/app/plugins_sdk/ai.py
+++ b/backend/app/plugins_sdk/ai.py
@@ -100,16 +100,18 @@ def register_context_provider(route_pattern: str, func: Callable[..., Any],
ai_tool_registry.register_context_provider(route_pattern, func, plugin_slug=plugin_slug)
-def ask(prompt: str, *, mode: str = "simple", page_context: Optional[dict] = None) -> str:
+def ask(prompt: str, *, mode: str = "simple", page_context: Optional[dict] = None,
+ profile: Optional[str] = None, workflow: str = 'chat') -> str:
"""Invoke the assistant in-process as the current JWT user (RBAC applies)."""
from app.plugins_sdk import current_user
from app.services import ai_service
- return ai_service.oneshot_ask(current_user(), prompt, mode=mode, page_context=page_context)
+ return ai_service.oneshot_ask(current_user(), prompt, mode=mode, page_context=page_context, profile=profile, workflow=workflow)
def ask_stream(prompt: str, *, mode: str = "simple",
- page_context: Optional[dict] = None) -> Iterator[str]:
+ page_context: Optional[dict] = None, profile: Optional[str] = None,
+ workflow: str = 'chat') -> Iterator[str]:
"""Streaming in-process assistant call; yields text chunks."""
from app.plugins_sdk import current_user
from app.services import ai_service
- return ai_service.oneshot_stream(current_user(), prompt, mode=mode, page_context=page_context)
+ return ai_service.oneshot_stream(current_user(), prompt, mode=mode, page_context=page_context, profile=profile, workflow=workflow)
diff --git a/backend/app/services/ai_connections.py b/backend/app/services/ai_connections.py
index ee7cb13d..78eed638 100644
--- a/backend/app/services/ai_connections.py
+++ b/backend/app/services/ai_connections.py
@@ -94,6 +94,7 @@ def provider_meta(provider):
def public_connection(row, *, details=False):
data = {'id': row.id, 'name': row.name, 'provider': row.provider, 'model': row.model}
+ data['model_catalog'] = row.model_catalog
if details:
config = row.config
data['config'] = {k: v for k, v in config.items() if not is_secret(k)}
@@ -118,7 +119,10 @@ def require_connection(connection_id):
def delete_connection(connection_id):
+ from app.services.ai_management import references
row = require_connection(connection_id)
+ if references(connection_id):
+ raise ConflictError('Remove this connection from task models, routing, and fallback before deleting it.')
if row.conversations or default_id() == row.id:
raise ConflictError('Choose another default and delete conversations using this connection before removing it.')
db.session.delete(row)
@@ -222,6 +226,8 @@ def save(data, existing=None):
if existing.provider != draft['provider'] or any(
(before.get(f['name']) or f['default']).rstrip('/') != draft['config'].get(f['name'], '') for f in urls):
raise ValueError('This connection has conversations. Add a new connection to change its provider or endpoint.')
+ if existing and (existing.provider != draft['provider'] or existing.config != draft['config']):
+ row.model_catalog = []
for key in ('name', 'provider', 'model', 'config'):
setattr(row, key, draft[key])
db.session.add(row)
@@ -320,7 +326,15 @@ def probe(data, existing=None, *, test=False):
if draft['model'] not in models:
raise ValueError('Model was not returned by this provider. Check the model ID or verify it in chat.')
return {'ok': True, 'message': 'Model discovery succeeded. Streaming and tool support depend on the selected model.'}
- return {'models': models, 'source': 'live'}
+ from app.services.ai_management import model_info
+ from types import SimpleNamespace
+ details = [model_info(SimpleNamespace(provider=provider), model) for model in models]
+ # Cache only an exact saved destination's catalog; a draft endpoint must
+ # not replace another connection's verified discovery results.
+ if existing and existing.provider == provider and existing.config == config:
+ existing.model_catalog = details
+ db.session.commit()
+ return {'models': models, 'source': 'live', 'model_details': details}
except requests.HTTPError as exc:
code = exc.response.status_code if exc.response is not None else 'error'
raise ValueError(f'Provider returned HTTP {code}. Check credentials, endpoint and model.') from None
diff --git a/backend/app/services/ai_management.py b/backend/app/services/ai_management.py
new file mode 100644
index 00000000..5ebfbc6c
--- /dev/null
+++ b/backend/app/services/ai_management.py
@@ -0,0 +1,210 @@
+"""Validated AI policy and connection-bound Prompture model selection."""
+import copy
+import math
+
+from app import db
+from app.exceptions import ValidationError, PermissionDeniedError
+from app.models.system_settings import SystemSettings
+from app.services import ai_connections
+
+PROFILES = ('utility', 'standard', 'advanced')
+WORKFLOWS = {'chat': 'standard', 'summarize': 'utility', 'extract': 'utility', 'diagnose': 'advanced'}
+STRATEGIES = ('cost_optimized', 'balanced', 'quality_first', 'fast')
+DEFAULTS = {
+ 'profiles': {key: None for key in PROFILES},
+ 'routing_enabled': False, 'strategy': 'balanced', 'pool': [],
+ 'fallback_enabled': False, 'fallbacks': [], 'budget_policy': 'hard_stop',
+ 'max_tokens': 0, 'max_output_tokens': 2048, 'max_tool_rounds': 6,
+ 'max_tool_result_length': 4000, 'max_history_messages': 40,
+ 'temperature': None, 'reasoning_effort': None,
+ 'monthly_limit_usd': 0, 'user_monthly_limit_usd': 0,
+ 'workspace_monthly_limit_usd': 0, 'reservation_usd': 0.5,
+}
+
+
+def settings():
+ stored = SystemSettings.get('ai_management', {})
+ return {**copy.deepcopy(DEFAULTS), **(stored if isinstance(stored, dict) else {})}
+
+
+def number(value, label, lower, upper, *, integer=False):
+ try:
+ n = float(value)
+ except (TypeError, ValueError, OverflowError):
+ raise ValidationError(f'{label} must be a number.') from None
+ if isinstance(value, bool) or not math.isfinite(n) or not lower <= n <= upper or (integer and n != int(n)):
+ raise ValidationError(f'{label} must be between {lower} and {upper}' + (' (whole number).' if integer else '.'))
+ return int(n) if integer else n
+
+
+def binding(data, *, tier=False):
+ if not isinstance(data, dict) or set(data) - {'connection_id', 'model', 'tier'}:
+ raise ValidationError('Select a saved connection and model.')
+ cid, model = data.get('connection_id'), data.get('model')
+ if not isinstance(cid, str) or len(cid) > 64 or not isinstance(model, str) or not model.strip() or len(model) > 256:
+ raise ValidationError('Select a saved connection and an exact model ID.')
+ ai_connections.require_connection(cid)
+ result = {'connection_id': cid, 'model': model.strip()}
+ if tier:
+ result['tier'] = data.get('tier', 'standard')
+ if result['tier'] not in ('budget', 'standard', 'premium'):
+ raise ValidationError('Invalid model capability tier.')
+ return result
+
+
+def validate(data):
+ if not isinstance(data, dict) or set(data) - set(DEFAULTS):
+ raise ValidationError('Unsupported AI management setting.')
+ result = {**settings(), **data}
+ profiles = result['profiles']
+ if not isinstance(profiles, dict) or set(profiles) != set(PROFILES):
+ raise ValidationError('Configure Utility, Standard, and Advanced task models.')
+ result['profiles'] = {key: binding(value) if value is not None else None for key, value in profiles.items()}
+ for key in ('routing_enabled', 'fallback_enabled'):
+ if not isinstance(result[key], bool):
+ raise ValidationError(f'{key} must be a boolean.')
+ for key in ('pool', 'fallbacks'):
+ values = result[key]
+ if not isinstance(values, list) or len(values) > 12:
+ raise ValidationError('Configure at most 12 candidates.')
+ result[key] = [binding(value, tier=key == 'pool') for value in values]
+ if len({(v['connection_id'], v['model']) for v in result[key]}) != len(values):
+ raise ValidationError('Remove duplicate model candidates.')
+ if result['strategy'] not in STRATEGIES or result['budget_policy'] not in ('hard_stop', 'warn_and_continue', 'degrade'):
+ raise ValidationError('Invalid routing strategy or budget policy.')
+ if result['routing_enabled'] and not result['pool']:
+ raise ValidationError('Add approved models before enabling routing.')
+ if (result['fallback_enabled'] or result['budget_policy'] == 'degrade') and not result['fallbacks']:
+ raise ValidationError('Add fallback models before enabling fallback or degradation.')
+ for key, bounds in {
+ 'max_tokens': (0, 10000000), 'max_output_tokens': (64, 65536),
+ 'max_tool_rounds': (1, 20), 'max_tool_result_length': (256, 32000),
+ 'max_history_messages': (4, 200),
+ }.items():
+ result[key] = number(result[key], key, *bounds, integer=True)
+ for key in ('monthly_limit_usd', 'user_monthly_limit_usd', 'workspace_monthly_limit_usd'):
+ result[key] = number(result[key], key, 0, 1000000)
+ result['reservation_usd'] = number(result['reservation_usd'], 'Per-run reservation', 0.01, 1000)
+ if result['temperature'] is not None:
+ result['temperature'] = number(result['temperature'], 'Temperature', 0, 2)
+ if result['reasoning_effort'] not in (None, 'low', 'medium', 'high'):
+ raise ValidationError('Invalid reasoning effort.')
+ return result
+
+
+def save(data):
+ result = validate(data)
+ SystemSettings.set('ai_management', result, value_type='json')
+ db.session.commit()
+ return result
+
+
+def references(connection_id):
+ config = settings()
+ return any(v and v['connection_id'] == connection_id for v in
+ [*config['profiles'].values(), *config['pool'], *config['fallbacks']])
+
+
+def model_info(connection, model):
+ """Exact metadata only: never assign upstream pricing to a gateway alias."""
+ from prompture.infra.model_rates import get_model_rates, get_model_info
+ result = {'id': model, 'provider': connection.provider, 'pricing': None,
+ 'context_window': None, 'max_output_tokens': None,
+ 'tools': None, 'vision': None, 'structured': None, 'reasoning': None,
+ 'temperature': None, 'reasoning_effort': False}
+ if connection.provider in ('openai_compatible', 'prompture-hub', 'lmstudio'):
+ return result
+ try:
+ info = get_model_info(connection.provider, model) or {}
+ rates = get_model_rates(connection.provider, model)
+ if rates and all(isinstance(rates.get(k), (int, float)) and math.isfinite(rates[k]) and rates[k] >= 0 for k in ('input', 'output')):
+ result['pricing'] = {k: rates[k] for k in ('input', 'output')}
+ limit = info.get('limit', {})
+ result.update(context_window=limit.get('context'), max_output_tokens=limit.get('output'),
+ tools=info.get('tool_call'), vision=('image' in info.get('modalities', {}).get('input', [])) if info else None,
+ structured=info.get('structured_output'), reasoning=info.get('reasoning'),
+ temperature=info.get('temperature'),
+ reasoning_effort=connection.provider == 'openai' and info.get('reasoning') is True)
+ except Exception:
+ pass # Missing catalog metadata remains unknown, never free/supported.
+ return result
+
+
+def public_profiles():
+ config = settings()
+ return {'profiles': config['profiles'], 'routing_enabled': config['routing_enabled'],
+ 'strategy': config['strategy'], 'workflows': WORKFLOWS}
+
+
+def resolve(*, profile=None, workflow='chat', connection_id=None, model=None, message='', config=None):
+ from prompture.pipeline.resolver import ModelResolver, dict_layer
+ from prompture.pipeline.routing import ModelRouter, RoutingConfig
+ config = config or settings()
+ if not isinstance(workflow, str) or workflow not in WORKFLOWS or (profile is not None and profile not in PROFILES):
+ raise ValidationError('Invalid task or model role.')
+ requested_profile = profile
+ profile = profile or WORKFLOWS[workflow]
+ default = ai_connections.get_connection()
+ base = {'connection_id': default.id, 'model': default.model}
+ choices = {key: value for key, value in config['profiles'].items() if value}
+ choices['default'] = choices.get('standard', base)
+ # The SDK resolves opaque slot values; the matching map retains credentials.
+ key_for = lambda value: value['connection_id'] + ':' + value['model']
+ bindings = {key_for(value): value for value in choices.values()}
+ layers = [dict_layer({key: key_for(value) for key, value in choices.items()})]
+ if connection_id or model:
+ connection = ai_connections.get_connection(connection_id or base['connection_id'])
+ explicit = binding({'connection_id': connection.id, 'model': model or connection.model})
+ bindings[key_for(explicit)] = explicit
+ layers.insert(0, dict_layer({profile: key_for(explicit)}))
+ resolver = ModelResolver(layers=layers, fallback_slots={key: ['default'] for key in PROFILES})
+ selected = dict(bindings[resolver.resolve(profile)])
+ strategy, reason = 'explicit', 'Selected task model' if choices.get(profile) else 'Inherited Standard/default model'
+ if connection_id or model:
+ reason = 'Explicit connection/model override'
+ elif config['routing_enabled'] and requested_profile is None:
+ candidates = []
+ for item in config['pool']:
+ connection = ai_connections.get_connection(item['connection_id'])
+ if workflow == 'diagnose' and model_info(connection, item['model'])['tools'] is False:
+ continue
+ candidates.append((ai_connections.model_name(connection, item['model']), item))
+ if not candidates:
+ raise ValidationError('No approved model supports this task.')
+ # Override discovery: never allow the SDK's ambient provider scan.
+ class ConnectionRouter(ModelRouter):
+ def _get_available_models(self):
+ return list(dict.fromkeys(name for name, _ in candidates))
+
+ def _get_model_tier(self, name):
+ return next(item['tier'] for candidate, item in candidates if candidate == name)
+
+ router = ConnectionRouter(RoutingConfig(strategy=config['strategy']))
+ name, decision = router.select_model(message, {'type': 'object', 'properties': {'answer': {'type': 'string'}}})
+ selected = dict(next(item for candidate, item in candidates if candidate == name))
+ strategy, reason = config['strategy'], decision.reason
+ if strategy == 'fast':
+ reason = 'Fast strategy using configured capability tiers (not measured latency)'
+ selected.pop('tier', None)
+ return {**selected, 'profile': profile, 'workflow': workflow, 'strategy': strategy, 'reason': reason}
+
+
+def workspace_id(user, requested):
+ from app.services.workspace_service import WorkspaceService
+ if requested in (None, '', 'all'):
+ return None
+ resolved = WorkspaceService.resolve_workspace_id(user, requested)
+ if resolved is None:
+ raise PermissionDeniedError('Workspace is unavailable or access was removed.')
+ return resolved
+
+
+def generation_options(config, connection, model):
+ info = model_info(connection, model)
+ output = min(config['max_output_tokens'], info['max_output_tokens'] or config['max_output_tokens'])
+ options = {'max_tokens': output}
+ if config['temperature'] is not None and info['temperature'] is True:
+ options['temperature'] = config['temperature']
+ if config['reasoning_effort'] and info['reasoning_effort']:
+ options['reasoning_effort'] = config['reasoning_effort']
+ return options
diff --git a/backend/app/services/ai_runtime.py b/backend/app/services/ai_runtime.py
new file mode 100644
index 00000000..15c571a9
--- /dev/null
+++ b/backend/app/services/ai_runtime.py
@@ -0,0 +1,181 @@
+"""Adapter around Prompture drivers: isolated fallback and per-call accounting."""
+import time
+
+from app.services import ai_connections, ai_management
+
+
+def managed_driver(driver, connection, model, recorder, max_cost, row):
+ from prompture.drivers.base import Driver
+ from prompture.exceptions import BudgetExceededError
+
+ class ManagedDriver(Driver):
+ def __init__(self):
+ self.delegate = driver
+ self.connection = connection
+ self.model = model
+ self.conversation = None
+ self.tried = {(connection.id, model)}
+ self._sync_flags()
+
+ def _sync_flags(self):
+ for name in ('supports_messages', 'supports_tool_use', 'supports_streaming',
+ 'supports_streaming_tool_use', 'supports_vision', 'supports_json_mode', 'supports_json_schema'):
+ setattr(self, name, getattr(self.delegate, name, False))
+
+ def _switch(self, candidate, reason):
+ target = ai_connections.get_connection(candidate['connection_id'])
+ replacement = ai_connections.build_driver(target, candidate['model'])
+ self.delegate, self.connection, self.model = replacement, target, candidate['model']
+ self.tried.add((target.id, self.model))
+ self._sync_flags()
+ recorder.connection_id = target.id
+ recorder.model = ai_connections.model_name(target, self.model)
+ recorder.run.reason = reason
+ recorder.switched = True
+ row.connection_id, row.model_name = target.id, recorder.model
+ if self.conversation is not None:
+ self.conversation._model_name = recorder.model
+
+ def _fallback(self, error):
+ if (not recorder.config['fallback_enabled'] or recorder.output_started
+ or isinstance(error, (BudgetExceededError, ValueError, PermissionError))):
+ return False
+ for candidate in recorder.config['fallbacks']:
+ key = candidate['connection_id'], candidate['model']
+ if key in self.tried:
+ continue
+ self.tried.add(key)
+ try:
+ self._switch(candidate, 'Configured fallback after provider failure; no output or tools replayed')
+ return True
+ except Exception:
+ # Failed construction is visible without leaking credentials.
+ recorder.attempts.append({**candidate, 'status': 'unavailable', 'cost_source': 'unknown',
+ 'cost': 0, 'total_tokens': 0, 'duration_ms': 0})
+ return False
+
+ def degrade(self):
+ if recorder.config['budget_policy'] != 'degrade' or not max_cost:
+ return
+ if recorder.initial.get('cost', 0) < max_cost * 0.8:
+ return
+ rates = ai_management.model_info(self.connection, self.model)['pricing']
+ if not rates:
+ return
+ for candidate in recorder.config['fallbacks']:
+ target = ai_connections.get_connection(candidate['connection_id'])
+ cheaper = ai_management.model_info(target, candidate['model'])['pricing']
+ if cheaper and sum(cheaper.values()) < sum(rates.values()):
+ self._switch(candidate, 'Lower-cost model selected before the turn at 80% of the conversation allowance')
+ return
+
+ def _options(self, options):
+ # Keep SDK JSON/schema options, while enforcing administrator controls.
+ return {**options, **ai_management.generation_options(recorder.config, self.connection, self.model)}
+
+ def _meta(self, raw):
+ from prompture.infra.tracker import UsageTracker
+ import math
+ raw = raw if isinstance(raw, dict) else {}
+ meta = {}
+ for key in ('prompt_tokens', 'completion_tokens', 'total_tokens', 'cached_prompt_tokens', 'cache_creation_tokens', 'cost'):
+ value = raw.get(key, 0)
+ meta[key] = value if isinstance(value, (int, float)) and math.isfinite(value) and value >= 0 else 0
+ if not meta['total_tokens']:
+ meta['total_tokens'] = meta['prompt_tokens'] + meta['completion_tokens']
+ pricing = ai_management.model_info(self.connection, self.model)['pricing']
+ reported = raw.get('cost')
+ if (raw.get('cost_source') == 'provider_reported' and isinstance(reported, (int, float))
+ and not isinstance(reported, bool) and math.isfinite(reported) and reported >= 0):
+ meta['cost_source'] = 'provider_reported'
+ elif pricing is not None:
+ if not meta['cost']:
+ meta['cost'] = UsageTracker.calculate_cost(ai_connections.model_name(self.connection, self.model), meta['prompt_tokens'], meta['completion_tokens'])
+ meta['cost_source'] = 'estimated'
+ else:
+ meta['cost_source'] = 'estimated' if meta['cost'] > 0 else 'unknown'
+ return meta
+
+ def _call(self, method, *args):
+ while True:
+ recorder.check(max_cost)
+ started = time.monotonic()
+ try:
+ response = getattr(self.delegate, method)(*args[:-1], self._options(args[-1]))
+ except Exception as error:
+ recorder.record({}, (time.monotonic() - started) * 1000, error=True)
+ if self._fallback(error):
+ continue
+ raise
+ response = {**response, 'meta': self._meta(response.get('meta', {}))}
+ recorder.output_started = True
+ recorder.record(response['meta'], (time.monotonic() - started) * 1000)
+ return response
+
+ def _stream(self, method, *args):
+ while True:
+ recorder.check(max_cost)
+ started = time.monotonic()
+ meta = {}
+ try:
+ for event in getattr(self.delegate, method)(*args[:-1], self._options(args[-1])):
+ recorder.output_started = True # conservative: never replay any delivered event
+ if isinstance(event, dict) and 'meta' in event:
+ meta = self._meta(event['meta'])
+ event = {**event, 'meta': meta}
+ elif getattr(event, 'event_type', '') == 'message_stop':
+ meta = self._meta(event.usage)
+ event.usage.update(meta)
+ yield event
+ except Exception as error:
+ recorder.record(meta, (time.monotonic() - started) * 1000, error=True)
+ if self._fallback(error):
+ continue
+ raise
+ except GeneratorExit:
+ recorder.record(meta, (time.monotonic() - started) * 1000, error=True)
+ raise
+ else:
+ recorder.record(meta, (time.monotonic() - started) * 1000)
+ return
+
+ def generate(self, prompt, options):
+ return self._call('generate', prompt, options)
+
+ def generate_messages(self, messages, options):
+ return self._call('generate_messages', messages, options)
+
+ def generate_messages_with_tools(self, messages, tools, options):
+ return self._call('generate_messages_with_tools', messages, tools, options)
+
+ def generate_messages_stream(self, messages, options):
+ yield from self._stream('generate_messages_stream', messages, options)
+
+ def generate_messages_with_tools_stream(self, messages, tools, options):
+ yield from self._stream('generate_messages_with_tools_stream', messages, tools, options)
+
+ wrapped = ManagedDriver()
+ wrapped.degrade()
+ return wrapped
+
+
+EXTRACTION_SCHEMA = {'type': 'object', 'properties': {
+ 'summary': {'type': 'string'}, 'facts': {'type': 'array', 'items': {'type': 'string'}},
+ 'warnings': {'type': 'array', 'items': {'type': 'string'}},
+}, 'required': ['summary', 'facts', 'warnings'], 'additionalProperties': False}
+
+
+def ask(conv, message, workflow='chat'):
+ if workflow == 'extract':
+ result = conv.ask_for_json(message, EXTRACTION_SCHEMA, ai_cleanup=False)
+ return result['json_string']
+ return conv.ask(message)
+
+
+def ask_live(conv, message, workflow='chat'):
+ if workflow == 'extract':
+ from prompture.agents.live_events import TextDelta, TurnComplete
+ yield TextDelta(text=ask(conv, message, workflow))
+ yield TurnComplete(usage=conv.usage)
+ else:
+ yield from conv.ask_live(message)
diff --git a/backend/app/services/ai_service.py b/backend/app/services/ai_service.py
index b4325244..a2638258 100644
--- a/backend/app/services/ai_service.py
+++ b/backend/app/services/ai_service.py
@@ -401,8 +401,10 @@ def build_system_prompt(user, mode: str, page_context: Optional[dict],
# ===========================================================================
# Per-request tool registry (read/write wrappers)
# ===========================================================================
-def build_tool_registry(user, mode: str, gate: Optional["ConfirmationGate"]) -> Optional[ToolRegistry]:
+def build_tool_registry(user, mode: str, gate: Optional["ConfirmationGate"], *, read_only=False) -> Optional[ToolRegistry]:
descriptors = ai_tool_registry.list_for(user, mode)
+ if read_only:
+ descriptors = [d for d in descriptors if not d.is_write]
if not descriptors:
return None
from prompture.agents.tools_schema import ToolDefinition, ToolRegistry
@@ -529,21 +531,36 @@ def summarize_action(descriptor, params: dict) -> str:
# ===========================================================================
def build_conversation(row, user, mode: str, page_context: Optional[dict],
gate: Optional["ConfirmationGate"],
- attachment_context: Optional[list[dict]] = None) -> Conversation:
+ attachment_context: Optional[list[dict]] = None, recorder=None) -> Conversation:
"""Create a fresh Conversation or resume from the stored export (re-supplying tools)."""
from prompture import Conversation
from app.services import ai_connections
-
- registry = build_tool_registry(user, mode, gate)
+ from app.services import ai_management, ai_runtime
+
+ config = ai_management.settings()
+ workflow = (getattr(row, 'management', None) or {}).get('workflow', 'chat')
+ if workflow in ('summarize', 'extract'):
+ registry = None
+ elif workflow == 'diagnose':
+ registry = build_tool_registry(user, 'assistant', gate, read_only=True)
+ else:
+ registry = build_tool_registry(user, mode, gate)
system = build_system_prompt(
user, mode, page_context, attachment_context=attachment_context,
)
+ if workflow == 'summarize':
+ system += '\nSummarize the supplied data concisely. Separate observations from uncertainty. Do not execute actions.'
+ elif workflow == 'diagnose':
+ system += '\nThis is read-only diagnosis. Gather evidence with available read tools, explain likely causes and suggest next steps. Never change server state.'
connection_id = getattr(row, 'connection_id', None)
if connection_id:
connection = ai_connections.get_connection(connection_id)
model = (row.model_name or '').split('/', 1)[-1] or connection.model
driver = ai_connections.build_driver(connection, model)
resolved_model = ai_connections.model_name(connection, model)
+ if recorder is not None:
+ driver = ai_runtime.managed_driver(driver, connection, model, recorder, _max_cost(), row)
+ resolved_model = row.model_name
else:
# Compatibility for pre-migration/test conversations. Never attach a
# different provider's current credentials to an old conversation.
@@ -563,27 +580,27 @@ def build_conversation(row, user, mode: str, page_context: Optional[dict],
def __init__(self, **kwargs):
max_cost = _max_cost()
kwargs.update(driver=driver, model_name=resolved_model,
- max_tool_rounds=(6 if mode == 'assistant' else 0),
- max_tool_result_length=4000, max_cost=max_cost,
- budget_policy='hard_stop' if max_cost is not None else None,
- fallback_models=None, options={})
+ max_tool_rounds=(config['max_tool_rounds'] if registry else 0),
+ max_tool_result_length=config['max_tool_result_length'],
+ max_history_messages=config['max_history_messages'],
+ max_cost=max_cost, max_tokens=config['max_tokens'] or None,
+ budget_policy=('warn_and_continue' if config['budget_policy'] == 'warn_and_continue' else 'hard_stop')
+ if max_cost is not None or config['max_tokens'] else None,
+ fallback_models=None, options={'max_tokens': config['max_output_tokens']})
super().__init__(**kwargs)
export = row.export
if export:
conv = ConfiguredConversation.from_export(_maybe_redact_result(export), tools=registry)
conv.system_prompt = system # refresh page context each turn
- return conv
- return ConfiguredConversation(
- model_name=resolved_model,
- system_prompt=system,
- tools=registry,
- max_tool_rounds=(6 if mode == "assistant" else 0),
- max_tool_result_length=4000,
- max_cost=_max_cost(),
- simulated_tools="auto",
- conversation_id=row.id,
- )
+ else:
+ conv = ConfiguredConversation(model_name=resolved_model, system_prompt=system,
+ tools=registry, simulated_tools='auto', conversation_id=row.id)
+ if recorder is not None:
+ recorder.initial = dict(conv.usage)
+ if connection_id:
+ driver.conversation = conv
+ return conv
def _max_cost() -> Optional[float]:
@@ -594,11 +611,6 @@ def _max_cost() -> Optional[float]:
return None
-def _fallback_models() -> list[str]:
- fb = _setting("ai_fallback_models", []) or []
- return [m for m in fb if m] if isinstance(fb, list) else []
-
-
def persist_conversation(row, conv: Conversation, *, page_context: Optional[dict] = None) -> None:
"""Persist the Prompture export + last page after a completed turn."""
try:
@@ -619,38 +631,64 @@ def derive_title(message: str) -> str:
# ===========================================================================
# In-process invocation (used by plugins via app.plugins_sdk.ai)
# ===========================================================================
-def _oneshot_conversation(user, mode: str, page_context: Optional[dict]) -> Conversation:
- from prompture import Conversation
- from app.services import ai_connections
-
+def _oneshot_run(user, prompt, *, mode, profile, workflow):
+ from types import SimpleNamespace
+ from flask import has_request_context, request
+ from app.exceptions import PermissionDeniedError, ValidationError
+ from app.services import ai_connections, ai_management, ai_usage
+
+ if user is None or _caller_validator(user)() is None:
+ raise PermissionDeniedError('An active user is required for AI calls.')
+ if not is_enabled() or not is_configured():
+ raise PermissionDeniedError('The AI assistant is disabled or unconfigured.')
+ if not isinstance(prompt, str) or not prompt.strip() or len(prompt) > 16000 or mode not in ('simple', 'assistant'):
+ raise ValidationError('Provide a valid AI prompt and mode.')
ensure_initialized()
- registry = build_tool_registry(user, mode, gate=None)
- system = build_system_prompt(user, mode, page_context)
- return Conversation(
- model_name=current_model_name(),
- driver=(ai_connections.build_driver(ai_connections.get_connection())
- if ai_connections.default_id() else None),
- system_prompt=system,
- tools=registry,
- env=build_provider_env(),
- max_tool_rounds=(6 if mode == "assistant" else 0),
- max_tool_result_length=4000,
- simulated_tools="auto",
- )
+ if injection_flagged(prompt):
+ raise AIProtectionError('Your message was flagged by the prompt-injection guardrail.')
+ selection = ai_management.resolve(profile=profile, workflow=workflow, message=prompt)
+ selection['workspace_id'] = ai_management.workspace_id(user, request.headers.get('X-Workspace-Id') if has_request_context() else None)
+ connection = ai_connections.get_connection(selection['connection_id'])
+ # Plugins retain their one-shot API without creating a chat transcript.
+ row = SimpleNamespace(id=None, connection_id=connection.id,
+ model_name=ai_connections.model_name(connection, selection['model']),
+ management=selection, export=None)
+ return row, ai_usage.start(row, user)
def oneshot_ask(user, prompt: str, *, mode: str = "simple",
- page_context: Optional[dict] = None) -> str:
- """Single-shot assistant call (no persistence) for in-process callers."""
- conv = _oneshot_conversation(user, mode, page_context)
- return conv.ask(redact_input(prompt))
+ page_context: Optional[dict] = None, profile=None, workflow='chat') -> str:
+ """Single-shot plugin call with shared policy and durable run accounting."""
+ from app.services import ai_runtime
+ row, recorder = _oneshot_run(user, prompt, mode=mode, profile=profile, workflow=workflow)
+ conv = None
+ try:
+ conv = build_conversation(row, user, mode, page_context, None, recorder=recorder)
+ result = ai_runtime.ask(conv, redact_input(prompt), workflow)
+ except Exception as exc:
+ recorder.finish('budget_exceeded' if budget_error_message(exc) else 'error', conv)
+ raise
+ recorder.finish('success', conv)
+ return result
def oneshot_stream(user, prompt: str, *, mode: str = "simple",
- page_context: Optional[dict] = None) -> Iterator[str]:
- """Single-shot streaming assistant call yielding text chunks."""
- conv = _oneshot_conversation(user, mode, page_context)
- yield from conv.ask_stream(redact_input(prompt))
+ page_context: Optional[dict] = None, profile=None, workflow='chat') -> Iterator[str]:
+ """Single-shot plugin stream using the same driver and spending policy."""
+ from app.services import ai_runtime
+ row, recorder = _oneshot_run(user, prompt, mode=mode, profile=profile, workflow=workflow)
+ conv, outcome = None, 'interrupted'
+ try:
+ conv = build_conversation(row, user, mode, page_context, None, recorder=recorder)
+ for event in ai_runtime.ask_live(conv, redact_input(prompt), workflow):
+ if getattr(event, 'event_type', '') == 'text_delta':
+ yield event.text
+ outcome = 'success'
+ except Exception as exc:
+ outcome = 'budget_exceeded' if budget_error_message(exc) else 'error'
+ raise
+ finally:
+ recorder.finish(outcome, conv)
# ===========================================================================
diff --git a/backend/app/services/ai_usage.py b/backend/app/services/ai_usage.py
new file mode 100644
index 00000000..5d4697e2
--- /dev/null
+++ b/backend/app/services/ai_usage.py
@@ -0,0 +1,201 @@
+"""Per-run accounting and scoped spending admission for the single panel worker."""
+from datetime import datetime
+import threading
+import time
+
+from sqlalchemy import func
+from flask import current_app
+
+from app import db
+from app.exceptions import ConflictError
+from app.models.ai import AiRun
+from app.services import ai_management
+
+_admission_lock = threading.RLock()
+
+
+def recover_interrupted():
+ """Reconcile runs left by the previous single worker, once per app boot."""
+ with _admission_lock:
+ if current_app.extensions.get('ai_usage_recovered'):
+ return
+ for run in AiRun.query.filter_by(status='running').all():
+ run.status = 'interrupted'
+ run.completed_at = datetime.utcnow()
+ run.budget_charge = max(run.cost, run.reserved_cost)
+ run.reserved_cost = 0
+ run.usage = {**run.usage, 'cost_source': 'unknown'}
+ db.session.commit()
+ current_app.extensions['ai_usage_recovered'] = True
+
+
+def _month():
+ return datetime.utcnow().replace(day=1, hour=0, minute=0, second=0, microsecond=0)
+
+
+def quota_status(config, user_id, workspace_id):
+ results = []
+ for scope, limit, predicate in (
+ ('Panel', config['monthly_limit_usd'], None),
+ ('User', config['user_monthly_limit_usd'], AiRun.user_id == user_id),
+ ('Workspace', config['workspace_monthly_limit_usd'], AiRun.workspace_id == workspace_id),
+ ):
+ if not limit or (scope == 'Workspace' and workspace_id is None) or (scope == 'User' and user_id is None):
+ continue
+ query = db.session.query(func.coalesce(func.sum(AiRun.budget_charge + AiRun.reserved_cost), 0)).filter(AiRun.created_at >= _month())
+ if predicate is not None:
+ query = query.filter(predicate)
+ used = float(query.scalar())
+ results.append({'scope': scope, 'limit': limit, 'used': used, 'remaining': max(0, limit - used)})
+ return results
+
+
+def start(row, user):
+ recover_interrupted()
+ config = ai_management.settings()
+ selection = row.management
+ workspace = ai_management.workspace_id(user, selection.get('workspace_id'))
+ with _admission_lock:
+ quotas = quota_status(config, user.id, workspace)
+ reservation = config['reservation_usd'] if quotas else 0
+ if any(item['remaining'] < reservation for item in quotas):
+ raise ConflictError('Monthly AI allowance has insufficient room for another run.', code='ai_quota_exceeded')
+ run = AiRun(conversation_id=row.id, user_id=user.id, workspace_id=workspace,
+ connection_id=row.connection_id, model=row.model_name,
+ profile=selection.get('profile', 'standard'), workflow=selection.get('workflow', 'chat'),
+ strategy=selection.get('strategy', 'explicit'), reason=selection.get('reason', 'Saved conversation'),
+ reserved_cost=reservation)
+ db.session.add(run)
+ db.session.commit()
+ return Recorder(run, config, row.export.get('usage', {}) if row.export else {})
+
+
+class Recorder:
+ def __init__(self, run, config, initial):
+ from prompture import UsageSession
+ self.run = run
+ self.config = config
+ self.initial = initial
+ self.session = UsageSession()
+ self.attempts = []
+ self.started = time.monotonic()
+ self.connection_id = run.connection_id
+ self.model = run.model
+ self.switched = False
+ self.output_started = False
+ self.call_started = 0
+
+ def check(self, max_cost):
+ from prompture.infra.budget import BudgetState, enforce_budget
+ # Conversation's own guard remains active; this also covers SDK JSON
+ # repair calls and spending from partial provider responses.
+ state = BudgetState(cost_used=self.initial.get('cost', 0) + self.session.cost,
+ tokens_used=int(self.initial.get('total_tokens', 0)) + self.session.total_tokens,
+ max_cost=max_cost, max_tokens=self.config['max_tokens'] or None)
+ policy = self.config['budget_policy']
+ enforce_budget(state, 'hard_stop' if policy == 'degrade' else policy)
+ if self.run.reserved_cost:
+ enforce_budget(BudgetState(cost_used=self.session.cost, max_cost=self.run.reserved_cost), 'hard_stop')
+
+ def record(self, meta, elapsed_ms, *, error=False):
+ info = {'driver': (self.connection_id or 'legacy') + ':' + (self.model or 'unknown'), 'meta': meta, 'elapsed_ms': elapsed_ms}
+ if error:
+ self.session.record_error({})
+ if meta:
+ self.session.record(info)
+ source = meta.get('cost_source', 'unknown')
+ if source not in ('provider_reported', 'estimated'):
+ source = 'estimated' if meta.get('cost', 0) > 0 else 'unknown'
+ self.attempts.append({'connection_id': self.connection_id, 'model': self.model,
+ 'status': 'error' if error else 'success', 'cost_source': source,
+ 'cost': meta.get('cost', 0), 'total_tokens': meta.get('total_tokens', 0),
+ 'duration_ms': elapsed_ms})
+ self._persist()
+
+ def summary(self):
+ from prompture.infra.budget import BudgetState
+ from app.services.ai_service import _max_cost
+ result = self.session.summary()
+ result.pop('formatted', None)
+ sources = {a['cost_source'] for a in self.attempts}
+ budget = BudgetState(cost_used=self.initial.get('cost', 0) + self.session.cost,
+ tokens_used=int(self.initial.get('total_tokens', 0)) + self.session.total_tokens,
+ max_cost=_max_cost(), max_tokens=self.config['max_tokens'] or None)
+ return {**result, 'run_id': self.run.id, 'profile': self.run.profile,
+ 'workflow': self.run.workflow, 'model': self.model, 'connection_id': self.connection_id,
+ 'strategy': self.run.strategy, 'reason': self.run.reason, 'attempts': self.attempts,
+ 'cost_source': 'unknown' if not sources or 'unknown' in sources else ('provider_reported' if sources == {'provider_reported'} else 'estimated'),
+ 'duration_ms': (time.monotonic() - self.started) * 1000,
+ 'budget': {'cost_remaining': budget.cost_remaining, 'tokens_remaining': budget.tokens_remaining,
+ 'exceeded': budget.exceeded, 'policy': self.config['budget_policy']}}
+
+ def _persist(self):
+ self.run.usage = self.summary()
+ self.run.cost = self.session.cost
+ self.run.total_tokens = self.session.total_tokens
+ self.run.connection_id, self.run.model = self.connection_id, self.model
+ db.session.commit()
+
+ def finish(self, status, conv=None):
+ with _admission_lock:
+ if self.run.status != 'running':
+ return self.run.usage
+ # Compatibility with unwrapped legacy/test conversations. Difference
+ # cumulative snapshots once, never sum historical turn snapshots.
+ if not self.attempts and conv is not None and getattr(conv, 'usage', None):
+ usage = conv.usage
+ delta = {k: max(0, usage.get(k, 0) - self.initial.get(k, 0)) for k in
+ ('prompt_tokens', 'completion_tokens', 'total_tokens', 'cost')}
+ if any(delta.values()):
+ self.record(delta, (time.monotonic() - self.started) * 1000)
+ self.run.status = status
+ if conv is not None and hasattr(conv, '_usage'):
+ # Prompture may not fold a partial/failed stream or JSON repair
+ # into Conversation usage. Preserve the observed lifetime floor
+ # for the next turn, without counting successful calls twice.
+ for key in ('prompt_tokens', 'completion_tokens', 'total_tokens', 'cost'):
+ conv._usage[key] = max(conv._usage.get(key, 0), self.initial.get(key, 0) + getattr(self.session, key))
+ self.run.completed_at = datetime.utcnow()
+ self.run.duration_ms = (time.monotonic() - self.started) * 1000
+ unknown = any(a['cost_source'] == 'unknown' for a in self.attempts)
+ # Unknown billed usage cannot silently reopen a scoped allowance.
+ self.run.budget_charge = max(self.session.cost, self.run.reserved_cost if unknown else 0)
+ self.run.reserved_cost = 0
+ self._persist()
+ return self.run.usage
+
+
+def report(args):
+ """Admin-only caller. Aggregate the full filtered set, paginate only detail."""
+ recover_interrupted()
+ days = ai_management.number(args.get('days', 30), 'Days', 1, 366, integer=True)
+ from datetime import timedelta
+ query = AiRun.query.filter(AiRun.created_at >= datetime.utcnow() - timedelta(days=days))
+ for key in ('profile', 'connection_id', 'user_id', 'workspace_id'):
+ value = args.get(key)
+ if value:
+ if key.endswith('_id') and key != 'connection_id':
+ value = ai_management.number(value, key, 1, 2147483647, integer=True)
+ query = query.filter(getattr(AiRun, key) == value)
+ rows = query.order_by(AiRun.created_at.desc()).all()
+ groups = {key: {} for key in ('daily', 'profile', 'model', 'connection_id', 'user_id', 'workspace_id')}
+ totals = {'runs': len(rows), 'cost': 0, 'tokens': 0, 'errors': 0, 'unknown_cost_runs': 0, 'duration_ms': 0}
+ for row in rows:
+ totals['cost'] += row.cost
+ totals['tokens'] += row.total_tokens
+ totals['errors'] += row.status in ('error', 'budget_exceeded', 'interrupted')
+ totals['unknown_cost_runs'] += row.usage.get('cost_source') == 'unknown'
+ totals['duration_ms'] += row.duration_ms
+ for key, buckets in groups.items():
+ label = row.created_at.date().isoformat() if key == 'daily' else str(getattr(row, key) or 'Unassigned')
+ item = buckets.setdefault(label, {'label': label, 'cost': 0, 'tokens': 0, 'runs': 0, 'errors': 0})
+ item['cost'] += row.cost
+ item['tokens'] += row.total_tokens
+ item['runs'] += 1
+ item['errors'] += row.status in ('error', 'budget_exceeded', 'interrupted')
+ totals['average_duration_ms'] = totals['duration_ms'] / len(rows) if rows else 0
+ offset = ai_management.number(args.get('offset', 0), 'Offset', 0, 10000000, integer=True)
+ return {'totals': totals, 'groups': {key: list(value.values()) for key, value in groups.items()},
+ 'allowances': quota_status(ai_management.settings(), args.get('user_id'), args.get('workspace_id')),
+ 'runs': [row.to_dict() for row in rows[offset:offset + 50]],
+ 'has_more': offset + 50 < len(rows), 'next_offset': offset + 50}
diff --git a/backend/migrations/versions/099_ai_management.py b/backend/migrations/versions/099_ai_management.py
new file mode 100644
index 00000000..908d7186
--- /dev/null
+++ b/backend/migrations/versions/099_ai_management.py
@@ -0,0 +1,45 @@
+"""Task selection snapshots and durable AI run accounting."""
+from alembic import op
+import sqlalchemy as sa
+
+revision = '099_ai_management'
+down_revision = '098_ai_provider_connections'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ inspector = sa.inspect(op.get_bind())
+ if 'model_catalog_json' not in {c['name'] for c in inspector.get_columns('ai_provider_connections')}:
+ op.add_column('ai_provider_connections', sa.Column('model_catalog_json', sa.Text))
+ if 'management_json' not in {c['name'] for c in inspector.get_columns('ai_conversations')}:
+ op.add_column('ai_conversations', sa.Column('management_json', sa.Text))
+ if 'ai_runs' in inspector.get_table_names():
+ return
+ op.create_table('ai_runs',
+ sa.Column('id', sa.String(64), primary_key=True),
+ sa.Column('conversation_id', sa.String(64), sa.ForeignKey('ai_conversations.id', ondelete='SET NULL')),
+ sa.Column('user_id', sa.Integer, sa.ForeignKey('users.id', ondelete='SET NULL')),
+ sa.Column('workspace_id', sa.Integer, sa.ForeignKey('workspaces.id', ondelete='SET NULL')),
+ sa.Column('connection_id', sa.String(64)), sa.Column('model', sa.String(384)),
+ sa.Column('profile', sa.String(16), nullable=False),
+ sa.Column('workflow', sa.String(16), nullable=False),
+ sa.Column('strategy', sa.String(32), nullable=False), sa.Column('reason', sa.String(512)),
+ sa.Column('status', sa.String(24), nullable=False),
+ sa.Column('cost', sa.Float, nullable=False, server_default='0'),
+ sa.Column('budget_charge', sa.Float, nullable=False, server_default='0'),
+ sa.Column('reserved_cost', sa.Float, nullable=False, server_default='0'),
+ sa.Column('total_tokens', sa.Integer, nullable=False, server_default='0'),
+ sa.Column('duration_ms', sa.Float, nullable=False, server_default='0'),
+ sa.Column('usage_json', sa.Text), sa.Column('created_at', sa.DateTime, nullable=False),
+ sa.Column('completed_at', sa.DateTime))
+ for column in ('conversation_id', 'user_id', 'workspace_id', 'connection_id', 'status', 'created_at'):
+ op.create_index('ix_ai_runs_' + column, 'ai_runs', [column])
+
+
+def downgrade():
+ op.drop_table('ai_runs')
+ with op.batch_alter_table('ai_conversations') as batch:
+ batch.drop_column('management_json')
+ with op.batch_alter_table('ai_provider_connections') as batch:
+ batch.drop_column('model_catalog_json')
diff --git a/backend/tests/test_ai_connections.py b/backend/tests/test_ai_connections.py
index 6c816c48..7c3ceb04 100644
--- a/backend/tests/test_ai_connections.py
+++ b/backend/tests/test_ai_connections.py
@@ -51,7 +51,7 @@ def test_connection_catalog_supports_api_key_clients_without_secrets(app, client
headers = {'X-API-Key': key}
response = client.get('/api/v1/ai/connections', headers=headers)
assert response.status_code == 200
- assert set(response.get_json()['connections'][0]) == {'id', 'name', 'provider', 'model'}
+ assert set(response.get_json()['connections'][0]) == {'id', 'name', 'provider', 'model', 'model_catalog'}
assert 'private-test-key' not in response.get_data(as_text=True)
assert client.post('/api/v1/ai/connections', json=draft(), headers=headers).status_code == 403
@@ -95,7 +95,7 @@ def test_profile_storage_and_api_never_return_secrets(app, client, auth_headers)
body = client.get(path, headers=auth_headers).get_data(as_text=True)
assert 'private-test-key' not in body
selector = client.get('/api/v1/ai/connections', headers=auth_headers).get_json()['connections'][0]
- assert set(selector) == {'id', 'name', 'provider', 'model'}
+ assert set(selector) == {'id', 'name', 'provider', 'model', 'model_catalog'}
def test_secret_retention_clear_and_endpoint_change(app):
diff --git a/backend/tests/test_ai_management.py b/backend/tests/test_ai_management.py
new file mode 100644
index 00000000..d2253494
--- /dev/null
+++ b/backend/tests/test_ai_management.py
@@ -0,0 +1,413 @@
+"""Real SDK integration with offline drivers: no paid calls or ambient credentials."""
+import json
+from types import SimpleNamespace
+
+import pytest
+
+from app import db
+from app.models import User
+from app.models.ai import AiConversation, AiRun
+from app.services import ai_connections, ai_management, ai_service, ai_usage
+from factories import make_user, headers_for
+
+
+def setup_ai(app, auth_headers, monkeypatch):
+ from prompture.drivers.base import Driver
+ calls = []
+
+ class FakeDriver(Driver):
+ supports_messages = True
+ supports_streaming = True
+
+ def __init__(self, connection, model):
+ self.connection_id, self.model = connection.id, model
+ self.key = connection.config['api_key']
+
+ def generate(self, prompt, options):
+ calls.append((self.connection_id, self.model, self.key, options))
+ if self.model == 'fail':
+ raise RuntimeError('provider-secret-must-not-leak')
+ text = json.dumps({'summary': 'Healthy', 'facts': ['CPU normal'], 'warnings': []}) if self.model == 'json' else 'Healthy'
+ return {'text': text, 'meta': {'prompt_tokens': 7, 'completion_tokens': 3, 'total_tokens': 10, 'cost': 0.01}}
+
+ def generate_messages(self, messages, options):
+ return self.generate(messages, options)
+
+ def generate_messages_stream(self, messages, options):
+ result = self.generate(messages, options)
+ yield {'type': 'delta', 'text': result['text']}
+ yield {'type': 'done', **result}
+
+ monkeypatch.setattr(ai_service, 'ensure_initialized', lambda: None)
+ monkeypatch.setattr(ai_service, 'is_enabled', lambda: True)
+ monkeypatch.setattr(ai_service, 'is_configured', lambda: True)
+ monkeypatch.setattr(ai_service, 'injection_flagged', lambda text: False)
+ monkeypatch.setattr(ai_service, 'redact_input', lambda text: text)
+ monkeypatch.setattr(ai_service, '_maybe_redact_result', lambda data: data)
+ monkeypatch.setattr(ai_service, 'build_tool_registry', lambda *a, **kw: None)
+ monkeypatch.setattr(ai_service, 'build_system_prompt', lambda *a, **kw: 'Use provided data')
+ monkeypatch.setattr(ai_connections, 'build_driver', lambda connection, model=None: FakeDriver(connection, model or connection.model))
+ monkeypatch.setattr(ai_management, 'model_info', lambda connection, model: {
+ 'pricing': {'input': 1, 'output': 1}, 'max_output_tokens': 8192,
+ 'temperature': True, 'reasoning_effort': True, 'tools': True,
+ })
+ with app.app_context():
+ def connection(name, key):
+ return ai_connections.save({'name': name, 'provider': 'openai_compatible', 'model': 'main',
+ 'config': {'endpoint': 'http://offline.test/v1', 'api_key': key}}).id
+ first, second = connection('Primary', 'first-secret'), connection('Fallback', 'second-secret')
+ return SimpleNamespace(first=first, second=second, calls=calls, driver=FakeDriver)
+
+
+def configure(client, headers, **changes):
+ response = client.put('/api/v1/ai/management', headers=headers, json=changes)
+ assert response.status_code == 200, response.get_json()
+ return response.get_json()
+
+
+def turn(client, headers, stream=False, **body):
+ response = client.post('/api/v1/ai/chat' + ('/stream' if stream else ''), headers=headers,
+ json={'message': 'Check the server', 'mode': 'simple', **body})
+ if not stream:
+ return response, response.get_json()
+ events = []
+ for block in response.get_data(as_text=True).split('\n\n'):
+ if block.startswith('event:'):
+ lines = block.splitlines()
+ events.append((lines[0][7:], json.loads(lines[1][6:])))
+ errors = [data for name, data in events if name == 'error']
+ done = next(data for name, data in events if name == 'done')
+ return response, {**done, 'errors': errors, 'events': events}
+
+
[email protected]('stream', [False, True])
+def test_profile_runs_resume_and_account_only_new_usage(app, client, auth_headers, setup_ai, stream):
+ config = ai_management.DEFAULTS.copy()
+ config['profiles'] = {'utility': {'connection_id': setup_ai.second, 'model': 'team/cheap'}, 'standard': None, 'advanced': None}
+ configure(client, auth_headers, **config)
+ response, first = turn(client, auth_headers, stream, workflow='summarize')
+ assert response.status_code == 200, first
+ assert first.get('errors', []) == []
+ assert first['usage']['profile'] == 'utility'
+ assert first['usage']['total_tokens'] == 10
+ assert setup_ai.calls[-1][0:3] == (setup_ai.second, 'team/cheap', 'second-secret')
+ configure(client, auth_headers, profiles={'utility': None, 'standard': None, 'advanced': None})
+ _, second = turn(client, auth_headers, stream, conversation_id=first['conversation_id'])
+ assert second['usage']['total_tokens'] == 10
+ assert second['usage']['cost'] == pytest.approx(0.01)
+ assert setup_ai.calls[-1][0] == setup_ai.second
+ report = client.get('/api/v1/ai/usage', headers=auth_headers).get_json()
+ assert report['totals']['tokens'] == 20
+ assert report['totals']['cost'] == pytest.approx(0.02)
+ with app.app_context():
+ row = db.session.get(AiConversation, first['conversation_id'])
+ assert row.export['usage']['total_tokens'] == 20
+ assert row.messages.filter_by(role='assistant').count() == 2
+ assert all(message.usage['total_tokens'] == 10 for message in row.messages.filter_by(role='assistant'))
+
+
[email protected]('stream', [False, True])
+def test_fallback_uses_own_credentials_and_records_failure(client, auth_headers, setup_ai, stream):
+ configure(client, auth_headers, fallback_enabled=True,
+ fallbacks=[{'connection_id': setup_ai.second, 'model': 'recovery'}])
+ response, body = turn(client, auth_headers, stream, connection_id=setup_ai.first, model='fail')
+ assert response.status_code == 200, body
+ assert body.get('errors', []) == []
+ assert [call[2] for call in setup_ai.calls] == ['first-secret', 'second-secret']
+ assert body['usage']['connection_id'] == setup_ai.second
+ assert [a['status'] for a in body['usage']['attempts']] == ['error', 'success']
+ assert 'secret' not in json.dumps(body)
+
+
+def test_no_fallback_after_streamed_output(client, auth_headers, setup_ai, monkeypatch):
+ def partial(self, messages, options):
+ yield {'type': 'delta', 'text': 'Partial'}
+ raise RuntimeError('private-error')
+ monkeypatch.setattr(setup_ai.driver, 'generate_messages_stream', partial)
+ configure(client, auth_headers, fallback_enabled=True,
+ fallbacks=[{'connection_id': setup_ai.second, 'model': 'recovery'}])
+ _, body = turn(client, auth_headers, True)
+ assert body['errors']
+ assert len(body['usage']['attempts']) == 1
+ assert body['usage']['connection_id'] == setup_ai.first
+ assert 'private-error' not in json.dumps(body)
+
+
+def test_preview_and_execution_use_only_approved_candidates(client, auth_headers, setup_ai, monkeypatch):
+ from prompture.pipeline.routing import ModelRouter
+ monkeypatch.setattr(ModelRouter, '_get_available_models', lambda self: pytest.fail('Ambient model discovery'))
+ configure(client, auth_headers, routing_enabled=True, strategy='quality_first', pool=[
+ {'connection_id': setup_ai.first, 'model': 'small', 'tier': 'budget'},
+ {'connection_id': setup_ai.second, 'model': 'deep', 'tier': 'premium'},
+ ])
+ preview = client.post('/api/v1/ai/management/preview', headers=auth_headers, json={'message': 'Check the server'}).get_json()
+ _, result = turn(client, auth_headers)
+ assert preview['connection_id'] == result['usage']['connection_id'] == setup_ai.second
+ assert setup_ai.calls[-1][1] == preview['model'] == 'deep'
+ assert client.delete('/api/v1/ai/connections/' + setup_ai.second, headers=auth_headers).status_code == 409
+
+
+def test_generation_options_are_applied_and_extraction_is_structured(client, auth_headers, setup_ai):
+ configure(client, auth_headers, max_output_tokens=1024, temperature=0.3, reasoning_effort='high',
+ profiles={'utility': {'connection_id': setup_ai.first, 'model': 'json'}, 'standard': None, 'advanced': None})
+ response, body = turn(client, auth_headers, workflow='extract')
+ assert response.status_code == 200, body
+ assert json.loads(body['reply']) == {'summary': 'Healthy', 'facts': ['CPU normal'], 'warnings': []}
+ assert setup_ai.calls[-1][-1]['max_tokens'] == 1024
+ assert setup_ai.calls[-1][-1]['temperature'] == 0.3
+ assert setup_ai.calls[-1][-1]['reasoning_effort'] == 'high'
+
+
+def test_run_finish_is_idempotent_and_quota_reservations_block_concurrent_runs(app, client, auth_headers, setup_ai):
+ configure(client, auth_headers, monthly_limit_usd=0.5, reservation_usd=0.5)
+ with app.app_context():
+ from app.exceptions import ConflictError
+ user = User.query.filter_by(username='testadmin').one()
+ row = AiConversation(user_id=user.id, connection_id=setup_ai.first, model_name='openai_compatible/main', management_json='{}')
+ db.session.add(row); db.session.commit()
+ recorder = ai_usage.start(row, user)
+ with pytest.raises(ConflictError):
+ ai_usage.start(row, user)
+ recorder.record({'cost': 0.1, 'total_tokens': 10}, 10)
+ recorder.finish('success')
+ recorder.finish('success')
+ assert AiRun.query.count() == 1
+ assert AiRun.query.one().cost == pytest.approx(0.1)
+ assert ai_usage.quota_status(ai_management.settings(), user.id, None)[0]['remaining'] == pytest.approx(0.4)
+
+
+def test_management_and_reports_are_admin_only(app, client, setup_ai):
+ with app.app_context():
+ headers = headers_for(make_user(db, role='viewer'))
+ assert client.get('/api/v1/ai/management', headers=headers).status_code == 403
+ assert client.get('/api/v1/ai/usage', headers=headers).status_code == 403
+ assert client.put('/api/v1/ai/management', headers=headers, json={'fallback_enabled': True}).status_code == 403
+ assert client.post('/api/v1/ai/management/preview', headers=headers, json={'message': 'x'}).status_code == 403
+
+
[email protected]('payload', [
+ {'max_tokens': -1}, {'max_output_tokens': 'NaN'}, {'routing_enabled': 'yes'},
+ {'routing_enabled': True, 'pool': []}, {'fallback_enabled': True, 'fallbacks': []},
+ {'profiles': {'utility': None}}, {'reservation_usd': 0}, {'reasoning_effort': 'unknown'},
+])
+def test_invalid_configuration_is_rejected(client, auth_headers, setup_ai, payload):
+ assert client.put('/api/v1/ai/management', headers=auth_headers, json=payload).status_code == 400
+
+
+def test_explicit_role_and_override_win_over_automatic_routing(client, auth_headers, setup_ai):
+ configure(client, auth_headers, routing_enabled=True, pool=[
+ {'connection_id': setup_ai.second, 'model': 'routed', 'tier': 'premium'}],
+ profiles={'utility': {'connection_id': setup_ai.first, 'model': 'cheap'},
+ 'standard': None, 'advanced': None})
+ _, body = turn(client, auth_headers, profile='utility')
+ assert body['usage']['model'] == 'openai_compatible/cheap'
+ _, body = turn(client, auth_headers, profile='utility', connection_id=setup_ai.second, model='manual/path')
+ assert body['usage']['model'] == 'openai_compatible/manual/path'
+
+
[email protected]('stream', [False, True])
+def test_token_budget_is_retained_on_resume(client, auth_headers, setup_ai, stream):
+ configure(client, auth_headers, max_tokens=10)
+ _, first = turn(client, auth_headers, stream)
+ response, second = turn(client, auth_headers, stream, conversation_id=first['conversation_id'])
+ assert len(setup_ai.calls) == 1
+ assert response.status_code == (200 if stream else 409)
+ assert second['usage']['total_tokens'] == 0
+ assert second['usage']['budget']['tokens_remaining'] == 0
+ report = client.get('/api/v1/ai/usage', headers=auth_headers).get_json()
+ assert report['totals']['tokens'] == 10
+ assert report['totals']['errors'] == 1
+
+
[email protected]('known', [False, True])
+def test_degradation_switches_only_to_known_cheaper_connection(app, client, auth_headers, setup_ai, monkeypatch, known):
+ from app.models.system_settings import SystemSettings
+ _, first = turn(client, auth_headers)
+ with app.app_context():
+ SystemSettings.set('ai_max_cost_usd', '0.012')
+ db.session.commit()
+ configure(client, auth_headers, budget_policy='degrade',
+ fallbacks=[{'connection_id': setup_ai.second, 'model': 'cheaper'}])
+ def info(connection, model):
+ return {'pricing': ({'input': 1, 'output': 1} if connection.id == setup_ai.first else
+ {'input': 0.1, 'output': 0.1} if known else None),
+ 'max_output_tokens': 8192, 'temperature': False, 'reasoning_effort': False}
+ monkeypatch.setattr(ai_management, 'model_info', info)
+ _, second = turn(client, auth_headers, conversation_id=first['conversation_id'])
+ assert second['usage']['connection_id'] == (setup_ai.second if known else setup_ai.first)
+ assert second['usage']['budget']['exceeded'] is True
+
+
+def test_scoped_allowances_and_unknown_cost_are_conservative(app, client, auth_headers, setup_ai):
+ from factories import make_workspace
+ configure(client, auth_headers, user_monthly_limit_usd=1, workspace_monthly_limit_usd=1)
+ with app.app_context():
+ user = User.query.filter_by(username='testadmin').one()
+ other = make_user(db)
+ workspace = make_workspace(db, created_by=user.id)
+ row = AiConversation(user_id=user.id, connection_id=setup_ai.first, model_name='openai_compatible/main')
+ row.management = {'workspace_id': workspace.id}
+ db.session.add(row); db.session.commit()
+ recorder = ai_usage.start(row, user)
+ recorder.record({'total_tokens': 10, 'cost': 0, 'cost_source': 'unknown'}, 10)
+ recorder.finish('success')
+ mine = ai_usage.quota_status(ai_management.settings(), user.id, workspace.id)
+ theirs = ai_usage.quota_status(ai_management.settings(), other.id, None)
+ assert all(item['used'] == 0.5 for item in mine)
+ assert theirs[0]['used'] == 0
+ assert AiRun.query.one().cost == 0
+ assert AiRun.query.one().budget_charge == 0.5
+ assert ai_usage.report({'user_id': other.id})['totals']['runs'] == 0
+ assert ai_usage.report({'workspace_id': workspace.id})['totals']['tokens'] == 10
+
+
+def test_resume_rechecks_workspace_membership(app, client, auth_headers, setup_ai):
+ from factories import make_workspace
+ from app.exceptions import PermissionDeniedError
+ with app.app_context():
+ owner, outsider = make_user(db), make_user(db)
+ workspace = make_workspace(db, created_by=owner.id)
+ row = AiConversation(user_id=outsider.id, connection_id=setup_ai.first, model_name='openai_compatible/main')
+ row.management = {'workspace_id': workspace.id}
+ db.session.add(row); db.session.commit()
+ with pytest.raises(PermissionDeniedError):
+ ai_usage.start(row, outsider)
+ assert AiRun.query.count() == 0
+
+
[email protected]('payload', [{'workflow': {}}, {'profile': []}, {'workflow': 10}])
+def test_invalid_task_payload_is_a_client_error(client, auth_headers, setup_ai, payload):
+ assert turn(client, auth_headers, **payload)[0].status_code == 400
+ assert client.post('/api/v1/ai/management/preview', headers=auth_headers,
+ json={'message': 'test', **payload}).status_code == 400
+
+
+def test_usage_survives_transcript_deletion(app, client, auth_headers, setup_ai):
+ _, first = turn(client, auth_headers)
+ assert client.delete('/api/v1/ai/conversations/' + first['conversation_id'], headers=auth_headers).status_code == 200
+ with app.app_context():
+ assert AiRun.query.one().conversation_id is None
+ assert AiRun.query.one().total_tokens == 10
+
+
+def test_read_only_registry_never_exposes_write_tools(app, monkeypatch):
+ from app.services.ai_tool_registry import ToolDescriptor, ai_tool_registry
+ with app.app_context():
+ user = make_user(db, role='admin')
+ descriptors = [ToolDescriptor(name, 'test__' + name, lambda: 'ok', name, {},
+ plugin_slug='test', is_write=write)
+ for name, write in [('read', False), ('write', True)]]
+ monkeypatch.setattr(ai_tool_registry, 'list_for', lambda *a: descriptors)
+ registry = ai_service.build_tool_registry(user, 'assistant', None, read_only=True)
+ assert registry.get('test__read') is not None
+ assert registry.get('test__write') is None
+
+
+def test_management_migration_preserves_existing_data_and_ledger_foreign_keys():
+ import importlib.util
+ from pathlib import Path
+ import sqlalchemy as sa
+ from alembic.migration import MigrationContext
+ from alembic.operations import Operations
+ path = Path(__file__).parents[1] / 'migrations/versions/099_ai_management.py'
+ spec = importlib.util.spec_from_file_location('migration099', path)
+ migration = importlib.util.module_from_spec(spec); spec.loader.exec_module(migration)
+ engine = sa.create_engine('sqlite://')
+ with engine.begin() as conn:
+ conn.exec_driver_sql('PRAGMA foreign_keys=ON')
+ conn.exec_driver_sql('CREATE TABLE users (id INTEGER PRIMARY KEY)')
+ conn.exec_driver_sql('CREATE TABLE workspaces (id INTEGER PRIMARY KEY)')
+ conn.exec_driver_sql('CREATE TABLE ai_provider_connections (id TEXT PRIMARY KEY, config_json TEXT)')
+ conn.exec_driver_sql('CREATE TABLE ai_conversations (id TEXT PRIMARY KEY, export_json TEXT)')
+ conn.exec_driver_sql("INSERT INTO ai_conversations VALUES ('chat', '{\"usage\":{\"cost\":0.5}}')")
+ migration.op = Operations(MigrationContext.configure(conn))
+ migration.upgrade(); migration.upgrade()
+ assert json.loads(conn.exec_driver_sql('SELECT export_json FROM ai_conversations').scalar())['usage']['cost'] == 0.5
+ conn.exec_driver_sql("INSERT INTO ai_runs (id, conversation_id, profile, workflow, strategy, status, created_at) VALUES ('run', 'chat', 'standard', 'chat', 'explicit', 'success', CURRENT_TIMESTAMP)")
+ conn.exec_driver_sql("DELETE FROM ai_conversations WHERE id='chat'")
+ assert conn.exec_driver_sql('SELECT conversation_id FROM ai_runs').scalar() is None
+ assert len(sa.inspect(conn).get_indexes('ai_runs')) == 6
+ migration.downgrade()
+ assert 'ai_runs' not in sa.inspect(conn).get_table_names()
+
+
[email protected]('stream', [False, True])
+def test_plugin_calls_use_task_models_and_shared_usage(app, client, auth_headers, setup_ai, monkeypatch, stream):
+ from app.plugins_sdk import ai as sdk_ai
+ from app import plugins_sdk
+ from app.exceptions import PermissionDeniedError
+ configure(client, auth_headers, profiles={
+ 'utility': {'connection_id': setup_ai.second, 'model': 'plugin/cheap'}, 'standard': None, 'advanced': None})
+ with app.app_context():
+ user = User.query.filter_by(username='testadmin').one()
+ monkeypatch.setattr(plugins_sdk, 'current_user', lambda: user)
+ result = ''.join(sdk_ai.ask_stream('Summarize', workflow='summarize')) if stream else sdk_ai.ask('Summarize', workflow='summarize')
+ assert result == 'Healthy'
+ run = AiRun.query.one()
+ assert run.conversation_id is None
+ assert run.model == 'openai_compatible/plugin/cheap'
+ assert run.status == 'success' and run.total_tokens == 10
+ monkeypatch.setattr(ai_service, 'is_enabled', lambda: False)
+ with pytest.raises(PermissionDeniedError):
+ sdk_ai.ask('Disabled')
+ assert AiRun.query.count() == 1
+
+
+def test_restart_reconciles_abandoned_reservation_once(app, client, auth_headers, setup_ai):
+ with app.app_context():
+ app.extensions.pop('ai_usage_recovered', None) # simulate a new worker
+ run = AiRun(reserved_cost=0.5, cost=0.1)
+ db.session.add(run); db.session.commit()
+ ai_usage.recover_interrupted()
+ assert run.status == 'interrupted'
+ assert run.budget_charge == 0.5 and run.reserved_cost == 0
+ assert run.usage['cost_source'] == 'unknown'
+ active = AiRun(reserved_cost=0.5)
+ db.session.add(active); db.session.commit()
+ ai_usage.recover_interrupted()
+ assert active.status == 'running'
+
+
+def test_fallback_does_not_replay_completed_tools(app, client, auth_headers, setup_ai, monkeypatch):
+ from prompture.agents.tools_schema import ToolDefinition, ToolRegistry
+ invoked, attempts = [], []
+ registry = ToolRegistry()
+ registry.add(ToolDefinition(name='peek', description='Read status',
+ parameters={'type': 'object', 'properties': {}},
+ function=lambda: invoked.append('peek') or 'healthy'))
+ monkeypatch.setattr(ai_service, 'build_tool_registry', lambda *a, **kw: registry)
+ monkeypatch.setattr(setup_ai.driver, 'supports_tool_use', True)
+ def native(self, messages, tools, options):
+ attempts.append(self.connection_id)
+ if len(attempts) > 1:
+ raise RuntimeError('Provider failed after a tool response')
+ return {'text': '', 'tool_calls': [{'id': 'call1', 'name': 'peek', 'arguments': {}}],
+ 'meta': {'total_tokens': 10, 'cost': 0.01}}
+ monkeypatch.setattr(setup_ai.driver, 'generate_messages_with_tools', native)
+ configure(client, auth_headers, fallback_enabled=True,
+ fallbacks=[{'connection_id': setup_ai.second, 'model': 'recovery'}])
+ response, body = turn(client, auth_headers, mode='assistant')
+ assert response.status_code == 500
+ assert invoked == ['peek']
+ assert attempts == [setup_ai.first, setup_ai.first]
+ with app.app_context():
+ run = AiRun.query.one()
+ assert run.cost == pytest.approx(0.01) and run.total_tokens == 10
+ assert run.status == 'error'
+ assert db.session.get(AiConversation, run.conversation_id).export['usage']['cost'] == pytest.approx(0.01)
+
+
+def test_warn_policy_continues_and_reports_exceeded_allowance(client, auth_headers, setup_ai):
+ configure(client, auth_headers, max_tokens=10, budget_policy='warn_and_continue')
+ _, first = turn(client, auth_headers)
+ response, second = turn(client, auth_headers, conversation_id=first['conversation_id'])
+ assert response.status_code == 200
+ assert len(setup_ai.calls) == 2
+ assert second['usage']['budget']['exceeded'] is True
+ assert second['usage']['budget']['policy'] == 'warn_and_continue'
+
+
[email protected]('payload', [{'enabled': 'false'}, {'fallback_models': ['ambient/model']}])
+def test_settings_reject_ambiguous_enable_and_unbound_fallbacks(client, auth_headers, setup_ai, payload):
+ assert client.put('/api/v1/ai/settings', headers=auth_headers, json=payload).status_code == 400