Personal AI, On Personal Devices
Python
feat(operators): wire metrics field and add collect_metrics() to OperatorManager
Abhinav Cherukurue963321114cd
diff --git a/src/openjarvis/operators/manager.py b/src/openjarvis/operators/manager.pyindex 55d6abfc..3337e848 100644--- a/src/openjarvis/operators/manager.py+++ b/src/openjarvis/operators/manager.py@@ -1,5 +1,4 @@"""Operator manager — lifecycle management for autonomous operators."""-from __future__ import annotationsimport logging@@ -61,7 +60,6 @@ class OperatorManager:"""Activate an operator by creating a scheduler task.Returns the scheduler task ID (deterministic: ``operator:{id}``).-Raises ``KeyError`` if the operator is not registered, or``RuntimeError`` if the scheduler is not available."""@@ -88,12 +86,12 @@ class OperatorManager:passtools_str = ",".join(manifest.tools) if manifest.tools else ""-metadata: Dict[str, Any] = {"operator_id": operator_id,"system_prompt": manifest.system_prompt,"temperature": manifest.temperature,"max_turns": manifest.max_turns,+ "metrics": manifest.metrics,}# Use the scheduler's create_task but with a deterministic ID@@ -105,11 +103,11 @@ class OperatorManager:tools=tools_str,metadata=metadata,)+# Override the random ID with our deterministic onetask_dict = task.to_dict()task_dict["id"] = task_idscheduler._store.save_task(task_dict)-logger.info("Activated operator %s (task_id=%s)", operator_id, task_id)return task_id@@ -148,7 +146,6 @@ class OperatorManager:"""results: List[Dict[str, Any]] = []scheduler = self._system.scheduler-for op_id, manifest in self._manifests.items():info: Dict[str, Any] = {"id": op_id,@@ -157,11 +154,11 @@ class OperatorManager:"schedule_type": manifest.schedule_type,"schedule_value": manifest.schedule_value,"tools": manifest.tools,+ "metrics": manifest.metrics,"status": "registered","next_run": None,"last_run": None,}-if scheduler is not None:task_id = f"operator:{op_id}"try:@@ -174,7 +171,6 @@ class OperatorManager:breakexcept Exception:pass-results.append(info)return results@@ -200,6 +196,111 @@ class OperatorManager:return result.get("content", str(result))return str(result)+ # -- Metrics -------------------------------------------------------------++ def collect_metrics(+ self,+ operator_id: str,+ *,+ since: Optional[float] = None,+ until: Optional[float] = None,+ ) -> Dict[str, Any]:+ """Collect telemetry metrics declared in the operator's manifest.++ Reads the ``metrics`` list from :class:`OperatorManifest` and queries+ the system's ``TelemetryAggregator`` (when available) for matching+ statistics. Only fields explicitly listed in ``manifest.metrics`` are+ returned, giving operators fine-grained control over what data they+ expose.++ Parameters+ ----------+ operator_id:+ ID of the registered operator.+ since:+ Optional Unix timestamp; restrict stats to records after this time.+ until:+ Optional Unix timestamp; restrict stats to records before this time.++ Returns+ -------+ Dict[str, Any]+ Mapping of ``{metric_name: value}`` for every metric declared in+ the manifest that could be resolved. Unknown metric names are+ silently skipped and logged at DEBUG level.++ Raises+ ------+ KeyError+ If *operator_id* is not registered.+ """+ manifest = self._manifests.get(operator_id)+ if manifest is None:+ raise KeyError(f"Operator not registered: {operator_id}")++ if not manifest.metrics:+ logger.debug(+ "Operator %s declares no metrics; returning empty dict.",+ operator_id,+ )+ return {}++ aggregator = getattr(self._system, "telemetry", None)+ if aggregator is None:+ logger.warning(+ "TelemetryAggregator not available on system; "+ "cannot collect metrics for operator %s.",+ operator_id,+ )+ return {}++ try:+ summary = aggregator.summary(since=since, until=until)+ except Exception as exc:+ logger.error(+ "Failed to query telemetry for operator %s: %s",+ operator_id,+ exc,+ )+ return {}++ # Build a flat lookup of all available summary-level stats+ available: Dict[str, Any] = {+ "total_calls": summary.total_calls,+ "total_tokens": summary.total_tokens,+ "total_cost": summary.total_cost,+ "total_latency": summary.total_latency,+ "total_energy_joules": summary.total_energy_joules,+ "avg_throughput_tok_per_sec": summary.avg_throughput_tok_per_sec,+ "avg_gpu_utilization_pct": summary.avg_gpu_utilization_pct,+ "avg_energy_per_output_token_joules": summary.avg_energy_per_output_token_joules,+ "avg_throughput_per_watt": summary.avg_throughput_per_watt,+ "total_prefill_energy_joules": summary.total_prefill_energy_joules,+ "total_decode_energy_joules": summary.total_decode_energy_joules,+ "avg_mean_itl_ms": summary.avg_mean_itl_ms,+ "avg_median_itl_ms": summary.avg_median_itl_ms,+ "avg_p95_itl_ms": summary.avg_p95_itl_ms,+ }++ collected: Dict[str, Any] = {}+ for metric in manifest.metrics:+ if metric in available:+ collected[metric] = available[metric]+ else:+ logger.debug(+ "Operator %s requested unknown metric '%s'; skipping.",+ operator_id,+ metric,+ )++ logger.info(+ "Collected %d/%d metrics for operator %s.",+ len(collected),+ len(manifest.metrics),+ operator_id,+ )+ return collected+def get_manifest(self, operator_id: str) -> Optional[OperatorManifest]:"""Return the manifest for an operator, or None."""return self._manifests.get(operator_id)