Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI
Python agentagentopsagents-sdkaianthropicautogencost-estimationcrewaievalsevaluation-metricsgroqlangchainllmmistralollamaopenaiopenai-agents
input/output guardrail decorator (#1039)
koreandinnereb34e26216f2
diff --git a/agentops/__init__.py b/agentops/__init__.pyindex 0ed1171a..2f10c142 100755--- a/agentops/__init__.py+++ b/agentops/__init__.py@@ -15,7 +15,7 @@ from agentops.legacy import (from typing import List, Optional, Union, Dict, Anyfrom agentops.client import Clientfrom agentops.sdk.core import TraceContext, tracer-from agentops.sdk.decorators import trace, session, agent, task, workflow, operation, tool+from agentops.sdk.decorators import trace, session, agent, task, workflow, operation, tool, guardrailfrom agentops.enums import TraceState, SUCCESS, ERROR, UNSETfrom opentelemetry.trace.status import StatusCode@@ -265,6 +265,7 @@ __all__ = ["task","workflow","operation",+ "guardrail","tracer","tool",# Trace state enumsdiff --git a/agentops/sdk/decorators/__init__.py b/agentops/sdk/decorators/__init__.pyindex f7524fbb..6881bc19 100644--- a/agentops/sdk/decorators/__init__.py+++ b/agentops/sdk/decorators/__init__.py@@ -18,6 +18,7 @@ workflow = create_entity_decorator(SpanKind.WORKFLOW)trace = create_entity_decorator(SpanKind.SESSION)tool = create_entity_decorator(SpanKind.TOOL)operation = task+guardrail = create_entity_decorator(SpanKind.GUARDRAIL)# For backward compatibility: @session decorator calls @trace decorator@@ -37,4 +38,13 @@ def session(*args, **kwargs): # noqa: F811# For now, keeping the alias as it was, assuming it was intentional for `operation` to be `task`.operation = task-__all__ = ["agent", "task", "workflow", "trace", "session", "operation", "tool"]+__all__ = [+ "agent",+ "task",+ "workflow",+ "trace",+ "session",+ "operation",+ "tool",+ "guardrail",+]diff --git a/agentops/sdk/decorators/factory.py b/agentops/sdk/decorators/factory.pyindex 6951767e..7be5c26b 100644--- a/agentops/sdk/decorators/factory.py+++ b/agentops/sdk/decorators/factory.py@@ -33,9 +33,10 @@ def create_entity_decorator(entity_kind: str) -> Callable[..., Any]:version: Optional[Any] = None,tags: Optional[Union[list, dict]] = None,cost=None,+ spec=None,) -> Callable[..., Any]:if wrapped is None:- return functools.partial(decorator, name=name, version=version, tags=tags, cost=cost)+ return functools.partial(decorator, name=name, version=version, tags=tags, cost=cost, spec=spec)if inspect.isclass(wrapped):# Class decoration wraps __init__ and aenter/aexit for context management.@@ -168,10 +169,13 @@ def create_entity_decorator(entity_kind: str) -> Callable[..., Any]:attributes={CoreAttributes.TAGS: tags} if tags else None,)try:- _record_entity_input(span, args, kwargs)+ _record_entity_input(span, args, kwargs, entity_kind=entity_kind)# Set cost attribute if toolif entity_kind == "tool" and cost is not None:span.set_attribute(SpanAttributes.LLM_USAGE_TOOL_COST, cost)+ # Set spec attribute if guardrail+ if entity_kind == "guardrail" and (spec == "input" or spec == "output"):+ span.set_attribute(SpanAttributes.AGENTOPS_DECORATOR_SPEC.format(entity_kind=entity_kind), spec)except Exception as e:logger.warning(f"Input recording failed for '{operation_name}': {e}")result = wrapped_func(*args, **kwargs)@@ -184,10 +188,13 @@ def create_entity_decorator(entity_kind: str) -> Callable[..., Any]:attributes={CoreAttributes.TAGS: tags} if tags else None,)try:- _record_entity_input(span, args, kwargs)+ _record_entity_input(span, args, kwargs, entity_kind=entity_kind)# Set cost attribute if toolif entity_kind == "tool" and cost is not None:span.set_attribute(SpanAttributes.LLM_USAGE_TOOL_COST, cost)+ # Set spec attribute if guardrail+ if entity_kind == "guardrail" and (spec == "input" or spec == "output"):+ span.set_attribute(SpanAttributes.AGENTOPS_DECORATOR_SPEC.format(entity_kind=entity_kind), spec)except Exception as e:logger.warning(f"Input recording failed for '{operation_name}': {e}")result = wrapped_func(*args, **kwargs)@@ -202,16 +209,21 @@ def create_entity_decorator(entity_kind: str) -> Callable[..., Any]:attributes={CoreAttributes.TAGS: tags} if tags else None,) as span:try:- _record_entity_input(span, args, kwargs)+ _record_entity_input(span, args, kwargs, entity_kind=entity_kind)# Set cost attribute if toolif entity_kind == "tool" and cost is not None:span.set_attribute(SpanAttributes.LLM_USAGE_TOOL_COST, cost)+ # Set spec attribute if guardrail+ if entity_kind == "guardrail" and (spec == "input" or spec == "output"):+ span.set_attribute(+ SpanAttributes.AGENTOPS_DECORATOR_SPEC.format(entity_kind=entity_kind), spec+ )except Exception as e:logger.warning(f"Input recording failed for '{operation_name}': {e}")try:result = await wrapped_func(*args, **kwargs)try:- _record_entity_output(span, result)+ _record_entity_output(span, result, entity_kind=entity_kind)except Exception as e:logger.warning(f"Output recording failed for '{operation_name}': {e}")return result@@ -229,16 +241,21 @@ def create_entity_decorator(entity_kind: str) -> Callable[..., Any]:attributes={CoreAttributes.TAGS: tags} if tags else None,) as span:try:- _record_entity_input(span, args, kwargs)+ _record_entity_input(span, args, kwargs, entity_kind=entity_kind)# Set cost attribute if toolif entity_kind == "tool" and cost is not None:span.set_attribute(SpanAttributes.LLM_USAGE_TOOL_COST, cost)+ # Set spec attribute if guardrail+ if entity_kind == "guardrail" and (spec == "input" or spec == "output"):+ span.set_attribute(+ SpanAttributes.AGENTOPS_DECORATOR_SPEC.format(entity_kind=entity_kind), spec+ )except Exception as e:logger.warning(f"Input recording failed for '{operation_name}': {e}")try:result = wrapped_func(*args, **kwargs)try:- _record_entity_output(span, result)+ _record_entity_output(span, result, entity_kind=entity_kind)except Exception as e:logger.warning(f"Output recording failed for '{operation_name}': {e}")return resultdiff --git a/agentops/sdk/decorators/utility.py b/agentops/sdk/decorators/utility.pyindex 58830149..8abc8820 100644--- a/agentops/sdk/decorators/utility.py+++ b/agentops/sdk/decorators/utility.py@@ -135,27 +135,27 @@ def _create_as_current_span(logger.debug(f"[DEBUG] AFTER {operation_name}.{span_kind} - Returned to context: {after_span}")-def _record_entity_input(span: trace.Span, args: tuple, kwargs: Dict[str, Any]) -> None:+def _record_entity_input(span: trace.Span, args: tuple, kwargs: Dict[str, Any], entity_kind: str = "entity") -> None:"""Record operation input parameters to span if content tracing is enabled"""try:input_data = {"args": args, "kwargs": kwargs}json_data = safe_serialize(input_data)if _check_content_size(json_data):- span.set_attribute(SpanAttributes.AGENTOPS_ENTITY_INPUT, json_data)+ span.set_attribute(SpanAttributes.AGENTOPS_DECORATOR_INPUT.format(entity_kind=entity_kind), json_data)else:logger.debug("Operation input exceeds size limit, not recording")except Exception as err:logger.warning(f"Failed to serialize operation input: {err}")-def _record_entity_output(span: trace.Span, result: Any) -> None:+def _record_entity_output(span: trace.Span, result: Any, entity_kind: str = "entity") -> None:"""Record operation output value to span if content tracing is enabled"""try:json_data = safe_serialize(result)if _check_content_size(json_data):- span.set_attribute(SpanAttributes.AGENTOPS_ENTITY_OUTPUT, json_data)+ span.set_attribute(SpanAttributes.AGENTOPS_DECORATOR_OUTPUT.format(entity_kind=entity_kind), json_data)else:logger.debug("Operation output exceeds size limit, not recording")except Exception as err:diff --git a/agentops/semconv/span_attributes.py b/agentops/semconv/span_attributes.pyindex 67320bff..0daf0ddf 100644--- a/agentops/semconv/span_attributes.py+++ b/agentops/semconv/span_attributes.py@@ -88,6 +88,9 @@ class SpanAttributes:AGENTOPS_ENTITY_INPUT = "agentops.entity.input"AGENTOPS_SPAN_KIND = "agentops.span.kind"AGENTOPS_ENTITY_NAME = "agentops.entity.name"+ AGENTOPS_DECORATOR_SPEC = "agentops.{entity_kind}.spec"+ AGENTOPS_DECORATOR_INPUT = "agentops.{entity_kind}.input"+ AGENTOPS_DECORATOR_OUTPUT = "agentops.{entity_kind}.output"# Operation attributesOPERATION_NAME = "operation.name"diff --git a/agentops/semconv/span_kinds.py b/agentops/semconv/span_kinds.pyindex 0d90a8cc..5a75f6d7 100644--- a/agentops/semconv/span_kinds.py+++ b/agentops/semconv/span_kinds.py@@ -27,6 +27,7 @@ class SpanKind:UNKNOWN = "unknown"CHAIN = "chain"TEXT = "text"+ GUARDRAIL = "guardrail"class AgentOpsSpanKindValues(Enum):diff --git a/docs/v2/concepts/decorators.mdx b/docs/v2/concepts/decorators.mdxindex 4596e419..7782175c 100644--- a/docs/v2/concepts/decorators.mdx+++ b/docs/v2/concepts/decorators.mdx@@ -14,6 +14,7 @@ AgentOps provides the following decorators:| `@workflow` | Track a sequence of operations | WORKFLOW span || `@task` | Track smaller units of work (similar to operations) | TASK span || `@tool` | Track tool usage and cost in agent operations | TOOL span |+| `@guardrail` | Track guardrail input and output | GUARDRAIL span |## Decorator Hierarchy@@ -235,6 +236,27 @@ The tool decorator provides:- Support for all function types (sync, async, generator, async generator)- Cost accumulation in generator and async generator operations+### @guardrail++The `@guardrail` decorator tracks guardrail input and output. You can specify the guardrail type (`"input"` or `"output"`) with the `spec` parameter.++```python+from agentops.sdk.decorators import guardrail+import agentops+import re++# Initialize AgentOps+agentops.init(api_key="YOUR_API_KEY")++@guardrail(spec="input")+def secret_key_guardrail(input):+ pattern = r'\bsk-[a-zA-Z0-9]{10,}\b'+ result = True if re.search(pattern, input) else False+ return {+ "tripwire_triggered" : result+ }+```+## Decorator AttributesYou can pass additional attributes to decorators:diff --git a/examples/google_adk/human_approval.ipynb b/examples/google_adk/human_approval.ipynbindex 92a87178..982d80ad 100644--- a/examples/google_adk/human_approval.ipynb+++ b/examples/google_adk/human_approval.ipynb@@ -183,7 +183,7 @@" \"\"\"\n"," Prompts for human approval and returns the decision as a JSON string.\n"," \"\"\"\n",- " print(f\"🔔 HUMAN APPROVAL REQUIRED:\")\n",+ " print(\"🔔 HUMAN APPROVAL REQUIRED:\")\n"," print(f\" Amount: ${amount:,.2f}\")\n"," print(f\" Reason: {reason}\")\n"," decision = \"\"\n",diff --git a/examples/openai_agents/agent_guardrails.ipynb b/examples/openai_agents/agent_guardrails.ipynbnew file mode 100644index 00000000..4c1739c3--- /dev/null+++ b/examples/openai_agents/agent_guardrails.ipynb@@ -0,0 +1,163 @@+{+ "cells": [+ {+ "cell_type": "markdown",+ "id": "f68ce4af",+ "metadata": {},+ "source": [+ "# OpenAI Agents Guardrails Demonstration\n",+ "\n",+ "This notebook demonstrates guardrails using the Agents SDK and how one can observe them using the AgentOps platform."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "id": "10bcf29b",+ "metadata": {},+ "outputs": [],+ "source": [+ "# Install required packages\n",+ "%pip install agentops\n",+ "%pip install openai-agents\n",+ "%pip install dotenv pydantic"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "id": "a3be4e68",+ "metadata": {},+ "outputs": [],+ "source": [+ "# Import dependencies\n",+ "from pydantic import BaseModel\n",+ "from agents import (\n",+ " Agent,\n",+ " GuardrailFunctionOutput,\n",+ " InputGuardrailTripwireTriggered,\n",+ " RunContextWrapper,\n",+ " Runner,\n",+ " TResponseInputItem,\n",+ " input_guardrail,\n",+ ")"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "id": "2d0dddb6",+ "metadata": {},+ "outputs": [],+ "source": [+ "# Load API keys\n",+ "import os\n",+ "from dotenv import load_dotenv\n",+ "\n",+ "load_dotenv()\n",+ "\n",+ "os.environ[\"AGENTOPS_API_KEY\"] = os.getenv(\"AGENTOPS_API_KEY\", \"your_api_key_here\")\n",+ "os.environ[\"OPENAI_API_KEY\"] = os.getenv(\"OPENAI_API_KEY\", \"your_openai_api_key_here\")"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "id": "114e216b",+ "metadata": {},+ "outputs": [],+ "source": [+ "# Initialize agentops and import the guardrail decorator\n",+ "import agentops\n",+ "from agentops import guardrail\n",+ "\n",+ "agentops.init(api_key=os.environ[\"AGENTOPS_API_KEY\"], tags=[\"agentops-example\"], auto_start_session=False)\n",+ "tracer = agentops.start_trace(trace_name=\"OpenAI Agents Guardrail Example\")"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "id": "0bf8b54d",+ "metadata": {},+ "outputs": [],+ "source": [+ "# OpenAI Agents SDK guardrail example with agentops guardrails decorator for observability\n",+ "class MathHomeworkOutput(BaseModel):\n",+ " is_math_homework: bool\n",+ " reasoning: str\n",+ "\n",+ "\n",+ "guardrail_agent = Agent(\n",+ " name=\"Guardrail check\",\n",+ " instructions=\"Check if the user is asking you to do their math homework.\",\n",+ " output_type=MathHomeworkOutput,\n",+ ")\n",+ "\n",+ "\n",+ "@input_guardrail\n",+ "@guardrail(spec=\"input\") # Specify guardrail type as input or output\n",+ "async def math_guardrail(\n",+ " ctx: RunContextWrapper[None], agent: Agent, input: str | list[TResponseInputItem]\n",+ ") -> GuardrailFunctionOutput:\n",+ " result = await Runner.run(guardrail_agent, input, context=ctx.context)\n",+ "\n",+ " return GuardrailFunctionOutput(\n",+ " output_info=result.final_output,\n",+ " tripwire_triggered=result.final_output.is_math_homework,\n",+ " )\n",+ "\n",+ "\n",+ "agent = Agent(\n",+ " name=\"Customer support agent\",\n",+ " instructions=\"You are a customer support agent. You help customers with their questions.\",\n",+ " input_guardrails=[math_guardrail],\n",+ ")\n",+ "\n",+ "\n",+ "async def main():\n",+ " # This should trip the guardrail\n",+ " try:\n",+ " await Runner.run(agent, \"Hello, can you help me solve for x: 2x + 3 = 11?\")\n",+ " print(\"Guardrail didn't trip - this is unexpected\")\n",+ "\n",+ " except InputGuardrailTripwireTriggered:\n",+ " print(\"Math homework guardrail tripped\")\n",+ "\n",+ "\n",+ "await main()"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "id": "63bf8e09",+ "metadata": {},+ "outputs": [],+ "source": [+ "agentops.end_trace(tracer, end_state=\"Success\")"+ ]+ }+ ],+ "metadata": {+ "kernelspec": {+ "display_name": "agentops (3.11.11)",+ "language": "python",+ "name": "python3"+ },+ "language_info": {+ "codemirror_mode": {+ "name": "ipython",+ "version": 3+ },+ "file_extension": ".py",+ "mimetype": "text/x-python",+ "name": "python",+ "nbconvert_exporter": "python",+ "pygments_lexer": "ipython3",+ "version": "3.11.11"+ }+ },+ "nbformat": 4,+ "nbformat_minor": 5+}diff --git a/examples/openai_agents/agent_guardrails.py b/examples/openai_agents/agent_guardrails.pynew file mode 100644index 00000000..2429dab4--- /dev/null+++ b/examples/openai_agents/agent_guardrails.py@@ -0,0 +1,83 @@+# # OpenAI Agents Guardrails Demonstration+#+# This notebook demonstrates guardrails using the Agents SDK and how one can observe them using the AgentOps platform.++# Install required packages+# %pip install agentops+# %pip install openai-agents+# %pip install dotenv pydantic++# Import dependencies+from pydantic import BaseModel+from agents import (+ Agent,+ GuardrailFunctionOutput,+ InputGuardrailTripwireTriggered,+ RunContextWrapper,+ Runner,+ TResponseInputItem,+ input_guardrail,+)++# Initialize agentops and import the guardrail decorator+import agentops+from agentops import guardrail++# Load API keys+import os+from dotenv import load_dotenv+import asyncio++load_dotenv()++os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here")+os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here")++agentops.init(api_key=os.environ["AGENTOPS_API_KEY"], tags=["agentops-example"])+++# OpenAI Agents SDK guardrail example with agentops guardrails decorator for observability+class MathHomeworkOutput(BaseModel):+ is_math_homework: bool+ reasoning: str+++guardrail_agent = Agent(+ name="Guardrail check",+ instructions="Check if the user is asking you to do their math homework.",+ output_type=MathHomeworkOutput,+)+++@input_guardrail+@guardrail(spec="input") # Specify guardrail type as input or output+async def math_guardrail(+ ctx: RunContextWrapper[None], agent: Agent, input: str | list[TResponseInputItem]+) -> GuardrailFunctionOutput:+ result = await Runner.run(guardrail_agent, input, context=ctx.context)++ return GuardrailFunctionOutput(+ output_info=result.final_output,+ tripwire_triggered=result.final_output.is_math_homework,+ )+++agent = Agent(+ name="Customer support agent",+ instructions="You are a customer support agent. You help customers with their questions.",+ input_guardrails=[math_guardrail],+)+++async def main():+ # This should trip the guardrail+ try:+ await Runner.run(agent, "Hello, can you help me solve for x: 2x + 3 = 11?")+ print("Guardrail didn't trip - this is unexpected")++ except InputGuardrailTripwireTriggered:+ print("Math homework guardrail tripped")+++if __name__ == "__main__":+ asyncio.run(main())diff --git a/examples/openai_agents/agent_patterns.ipynb b/examples/openai_agents/agent_patterns.ipynbindex d3614c86..1af989db 100644--- a/examples/openai_agents/agent_patterns.ipynb+++ b/examples/openai_agents/agent_patterns.ipynb@@ -100,10 +100,9 @@" OutputGuardrailTripwireTriggered,\n"," input_guardrail,\n"," output_guardrail,\n",- " RawResponsesStreamEvent,\n",")\n","\n",- "from openai.types.responses import ResponseContentPartDoneEvent, ResponseTextDeltaEvent"+ "from openai.types.responses import ResponseTextDeltaEvent"]},{diff --git a/examples/openai_agents/agents_tools.ipynb b/examples/openai_agents/agents_tools.ipynbindex a8bf70b6..b27830b0 100755--- a/examples/openai_agents/agents_tools.ipynb+++ b/examples/openai_agents/agents_tools.ipynb@@ -79,24 +79,17 @@"metadata": {},"outputs": [],"source": [- "import asyncio\n","import base64\n","import os\n","import subprocess\n","import sys\n","import tempfile\n",- "from typing import Literal, Union\n","\n","from agents import (\n"," Agent,\n",- " AsyncComputer,\n",- " Button,\n"," CodeInterpreterTool,\n",- " ComputerTool,\n",- " Environment,\n"," FileSearchTool,\n"," ImageGenerationTool,\n",- " ModelSettings,\n"," Runner,\n"," WebSearchTool,\n"," trace,\n",diff --git a/examples/openai_agents/customer_service_agent.ipynb b/examples/openai_agents/customer_service_agent.ipynbindex a873049a..6135c44d 100644--- a/examples/openai_agents/customer_service_agent.ipynb+++ b/examples/openai_agents/customer_service_agent.ipynb@@ -67,10 +67,8 @@"source": ["from __future__ import annotations as _annotations # noqa: F404\n","\n",- "import asyncio\n","import random\n","import uuid\n",- "import sys\n","\n","from pydantic import BaseModel\n","import agentops\n",