Skip to content

Personal AI, On Personal Devices

Python

feat(operators): wire metrics field and add collect_metrics() to OperatorManager

Abhinav Cherukurue963321114cd
diff --git a/src/openjarvis/operators/manager.py b/src/openjarvis/operators/manager.py
index 55d6abfc..3337e848 100644
--- a/src/openjarvis/operators/manager.py
+++ b/src/openjarvis/operators/manager.py
@@ -1,5 +1,4 @@
"""Operator manager — lifecycle management for autonomous operators."""
-
from __future__ import annotations
import logging
@@ -61,7 +60,6 @@ class OperatorManager:
"""Activate an operator by creating a scheduler task.
Returns the scheduler task ID (deterministic: ``operator:{id}``).
-
Raises ``KeyError`` if the operator is not registered, or
``RuntimeError`` if the scheduler is not available.
"""
@@ -88,12 +86,12 @@ class OperatorManager:
pass
tools_str = ",".join(manifest.tools) if manifest.tools else ""
-
metadata: Dict[str, Any] = {
"operator_id": operator_id,
"system_prompt": manifest.system_prompt,
"temperature": manifest.temperature,
"max_turns": manifest.max_turns,
+ "metrics": manifest.metrics,
}
# Use the scheduler's create_task but with a deterministic ID
@@ -105,11 +103,11 @@ class OperatorManager:
tools=tools_str,
metadata=metadata,
)
+
# Override the random ID with our deterministic one
task_dict = task.to_dict()
task_dict["id"] = task_id
scheduler._store.save_task(task_dict)
-
logger.info("Activated operator %s (task_id=%s)", operator_id, task_id)
return task_id
@@ -148,7 +146,6 @@ class OperatorManager:
"""
results: List[Dict[str, Any]] = []
scheduler = self._system.scheduler
-
for op_id, manifest in self._manifests.items():
info: Dict[str, Any] = {
"id": op_id,
@@ -157,11 +154,11 @@ class OperatorManager:
"schedule_type": manifest.schedule_type,
"schedule_value": manifest.schedule_value,
"tools": manifest.tools,
+ "metrics": manifest.metrics,
"status": "registered",
"next_run": None,
"last_run": None,
}
-
if scheduler is not None:
task_id = f"operator:{op_id}"
try:
@@ -174,7 +171,6 @@ class OperatorManager:
break
except Exception:
pass
-
results.append(info)
return results
@@ -200,6 +196,111 @@ class OperatorManager:
return result.get("content", str(result))
return str(result)
+ # -- Metrics -------------------------------------------------------------
+
+ def collect_metrics(
+ self,
+ operator_id: str,
+ *,
+ since: Optional[float] = None,
+ until: Optional[float] = None,
+ ) -> Dict[str, Any]:
+ """Collect telemetry metrics declared in the operator's manifest.
+
+ Reads the ``metrics`` list from :class:`OperatorManifest` and queries
+ the system's ``TelemetryAggregator`` (when available) for matching
+ statistics. Only fields explicitly listed in ``manifest.metrics`` are
+ returned, giving operators fine-grained control over what data they
+ expose.
+
+ Parameters
+ ----------
+ operator_id:
+ ID of the registered operator.
+ since:
+ Optional Unix timestamp; restrict stats to records after this time.
+ until:
+ Optional Unix timestamp; restrict stats to records before this time.
+
+ Returns
+ -------
+ Dict[str, Any]
+ Mapping of ``{metric_name: value}`` for every metric declared in
+ the manifest that could be resolved. Unknown metric names are
+ silently skipped and logged at DEBUG level.
+
+ Raises
+ ------
+ KeyError
+ If *operator_id* is not registered.
+ """
+ manifest = self._manifests.get(operator_id)
+ if manifest is None:
+ raise KeyError(f"Operator not registered: {operator_id}")
+
+ if not manifest.metrics:
+ logger.debug(
+ "Operator %s declares no metrics; returning empty dict.",
+ operator_id,
+ )
+ return {}
+
+ aggregator = getattr(self._system, "telemetry", None)
+ if aggregator is None:
+ logger.warning(
+ "TelemetryAggregator not available on system; "
+ "cannot collect metrics for operator %s.",
+ operator_id,
+ )
+ return {}
+
+ try:
+ summary = aggregator.summary(since=since, until=until)
+ except Exception as exc:
+ logger.error(
+ "Failed to query telemetry for operator %s: %s",
+ operator_id,
+ exc,
+ )
+ return {}
+
+ # Build a flat lookup of all available summary-level stats
+ available: Dict[str, Any] = {
+ "total_calls": summary.total_calls,
+ "total_tokens": summary.total_tokens,
+ "total_cost": summary.total_cost,
+ "total_latency": summary.total_latency,
+ "total_energy_joules": summary.total_energy_joules,
+ "avg_throughput_tok_per_sec": summary.avg_throughput_tok_per_sec,
+ "avg_gpu_utilization_pct": summary.avg_gpu_utilization_pct,
+ "avg_energy_per_output_token_joules": summary.avg_energy_per_output_token_joules,
+ "avg_throughput_per_watt": summary.avg_throughput_per_watt,
+ "total_prefill_energy_joules": summary.total_prefill_energy_joules,
+ "total_decode_energy_joules": summary.total_decode_energy_joules,
+ "avg_mean_itl_ms": summary.avg_mean_itl_ms,
+ "avg_median_itl_ms": summary.avg_median_itl_ms,
+ "avg_p95_itl_ms": summary.avg_p95_itl_ms,
+ }
+
+ collected: Dict[str, Any] = {}
+ for metric in manifest.metrics:
+ if metric in available:
+ collected[metric] = available[metric]
+ else:
+ logger.debug(
+ "Operator %s requested unknown metric '%s'; skipping.",
+ operator_id,
+ metric,
+ )
+
+ logger.info(
+ "Collected %d/%d metrics for operator %s.",
+ len(collected),
+ len(manifest.metrics),
+ operator_id,
+ )
+ return collected
+
def get_manifest(self, operator_id: str) -> Optional[OperatorManifest]:
"""Return the manifest for an operator, or None."""
return self._manifests.get(operator_id)