Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
[Feat]: Add OpenTelemetry instrumentation for Monitoring LLM Applications using GPT4All (#257)
patcher912e35d3bdf3a
diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.tomlindex 0bb471d5..1d53ea4f 100644--- a/sdk/python/pyproject.toml+++ b/sdk/python/pyproject.toml@@ -1,6 +1,6 @@[tool.poetry]name = "openlit"-version = "1.10.0"+version = "1.11.0"description = "OpenTelemetry-native Auto instrumentation library for monitoring LLM Applications, facilitating the integration of observability into your GenAI-driven projects"authors = ["OpenLIT"]repository = "https://github.com/openlit/openlit/tree/main/openlit/python"diff --git a/sdk/python/src/openlit/__init__.py b/sdk/python/src/openlit/__init__.pyindex 282082e1..9f1fad13 100644--- a/sdk/python/src/openlit/__init__.py+++ b/sdk/python/src/openlit/__init__.py@@ -21,6 +21,7 @@ from openlit.instrumentation.bedrock import BedrockInstrumentorfrom openlit.instrumentation.vertexai import VertexAIInstrumentorfrom openlit.instrumentation.groq import GroqInstrumentorfrom openlit.instrumentation.ollama import OllamaInstrumentor+from openlit.instrumentation.gpt4all import GPT4AllInstrumentorfrom openlit.instrumentation.langchain import LangChainInstrumentorfrom openlit.instrumentation.llamaindex import LlamaIndexInstrumentorfrom openlit.instrumentation.haystack import HaystackInstrumentor@@ -161,6 +162,7 @@ def init(environment="default", application_name="default", tracer=None, otlp_en"vertexai": "vertexai","groq": "groq","ollama": "ollama",+ "gpt4all": "gpt4all","langchain": "langchain","llama_index": "llama_index","haystack": "haystack",@@ -216,6 +218,7 @@ def init(environment="default", application_name="default", tracer=None, otlp_en"vertexai": VertexAIInstrumentor(),"groq": GroqInstrumentor(),"ollama": OllamaInstrumentor(),+ "gpt4all": GPT4AllInstrumentor(),"langchain": LangChainInstrumentor(),"llama_index": LlamaIndexInstrumentor(),"haystack": HaystackInstrumentor(),diff --git a/sdk/python/src/openlit/instrumentation/gpt4all/__init__.py b/sdk/python/src/openlit/instrumentation/gpt4all/__init__.pynew file mode 100644index 00000000..54625787--- /dev/null+++ b/sdk/python/src/openlit/instrumentation/gpt4all/__init__.py@@ -0,0 +1,52 @@+# pylint: disable=useless-return, bad-staticmethod-argument, disable=duplicate-code+"""Initializer of Auto Instrumentation of GPT4All Functions"""++from typing import Collection+import importlib.metadata+from opentelemetry.instrumentation.instrumentor import BaseInstrumentor+from wrapt import wrap_function_wrapper++from openlit.instrumentation.gpt4all.gpt4all import (+ embed, generate+)++_instruments = ("gpt4all >= 2.6.0",)++class GPT4AllInstrumentor(BaseInstrumentor):+ """+ An instrumentor for GPT4All's client library.+ """++ def instrumentation_dependencies(self) -> Collection[str]:+ return _instruments++ def _instrument(self, **kwargs):+ application_name = kwargs.get("application_name", "default_application")+ environment = kwargs.get("environment", "default_environment")+ tracer = kwargs.get("tracer")+ metrics = kwargs.get("metrics_dict")+ pricing_info = kwargs.get("pricing_info", {})+ trace_content = kwargs.get("trace_content", False)+ disable_metrics = kwargs.get("disable_metrics")+ version = importlib.metadata.version("gpt4all")++ # generate+ wrap_function_wrapper(+ "gpt4all",+ "GPT4All.generate",+ generate("gpt4all.generate", version, environment, application_name,+ tracer, pricing_info, trace_content, metrics, disable_metrics),+ )++ # embed+ wrap_function_wrapper(+ "gpt4all",+ "Embed4All.embed",+ embed("gpt4all.embed", version, environment, application_name,+ tracer, pricing_info, trace_content, metrics, disable_metrics),+ )+++ def _uninstrument(self, **kwargs):+ # Proper uninstrumentation logic to revert patched methods+ passdiff --git a/sdk/python/src/openlit/instrumentation/gpt4all/gpt4all.py b/sdk/python/src/openlit/instrumentation/gpt4all/gpt4all.pynew file mode 100644index 00000000..1dc764b8--- /dev/null+++ b/sdk/python/src/openlit/instrumentation/gpt4all/gpt4all.py@@ -0,0 +1,352 @@+# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, possibly-used-before-assignment+"""+Module for monitoring GPT4All API calls.+"""++import logging+from opentelemetry.trace import SpanKind, Status, StatusCode+from opentelemetry.sdk.resources import TELEMETRY_SDK_NAME+from openlit.__helpers import handle_exception, general_tokens+from openlit.semcov import SemanticConvetion++# Initialize logger for logging potential issues and operations+logger = logging.getLogger(__name__)++def generate(gen_ai_endpoint, version, environment, application_name,+ tracer, pricing_info, trace_content, metrics, disable_metrics):+ """+ Generates a telemetry wrapper for generate to collect metrics.++ Args:+ gen_ai_endpoint: Endpoint identifier for logging and tracing.+ version: Version of the monitoring package.+ environment: Deployment environment (e.g., production, staging).+ application_name: Name of the application using the GPT4All API.+ tracer: OpenTelemetry tracer for creating spans.+ pricing_info: Information used for calculating the cost of GPT4All usage.+ trace_content: Flag indicating whether to trace the actual content.++ Returns:+ A function that wraps the generate method to add telemetry.+ """++ def wrapper(wrapped, instance, args, kwargs):+ """+ Wraps the 'generate' API call to add telemetry.++ This collects metrics such as execution time, cost, and token usage, and handles errors+ gracefully, adding details to the trace for observability.++ Args:+ wrapped: The original 'generate' method to be wrapped.+ instance: The instance of the class where the original method is defined.+ args: Positional arguments for the 'generate' method.+ kwargs: Keyword arguments for the 'generate' method.++ Returns:+ The response from the original 'generate' method.+ """++ # Check if streaming is enabled for the API call+ streaming = kwargs.get("streaming", False)++ # pylint: disable=no-else-return+ if streaming:+ # Special handling for streaming response to accommodate the nature of data flow+ def stream_generator():+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:+ # Placeholder for aggregating streaming response+ llmresponse = ""++ # Loop through streaming events capturing relevant details+ for chunk in wrapped(*args, **kwargs):+ # Collect aggregated response from events+ llmresponse += chunk++ yield chunk++ # Handling exception ensure observability without disrupting operation+ try:+ # Calculate cost of the operation+ cost = 0++ # pylint: disable=line-too-long+ model = str(instance.model.model_path).rsplit('/', maxsplit=1)[-1] or "orca-mini-3b-gguf2-q4_0.gguf"+ prompt = kwargs.get("prompt") or args[0] or ""++ # Calculate cost of the operation+ cost = 0+ prompt_tokens = general_tokens(prompt)+ completion_tokens = general_tokens(llmresponse)+ total_tokens = prompt_tokens + completion_tokens++ # Set base span attribues+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_GPT4ALL)+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,+ SemanticConvetion.GEN_AI_TYPE_CHAT)+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,+ gen_ai_endpoint)+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,+ environment)+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,+ application_name)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,+ model)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_K,+ kwargs.get("top_k", 40))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,+ kwargs.get("top_p", 0.4))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,+ kwargs.get("max_tokens", 200))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,+ kwargs.get("temperature", 0.7))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,+ kwargs.get("frequency_penalty", 1.18))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,+ True)+ if trace_content:+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,+ prompt)+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,+ llmresponse)++ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,+ prompt_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,+ completion_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,+ total_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,+ cost)++ span.set_status(Status(StatusCode.OK))++ if disable_metrics is False:+ attributes = {+ TELEMETRY_SDK_NAME:+ "openlit",+ SemanticConvetion.GEN_AI_APPLICATION_NAME:+ application_name,+ SemanticConvetion.GEN_AI_SYSTEM:+ SemanticConvetion.GEN_AI_SYSTEM_GPT4ALL,+ SemanticConvetion.GEN_AI_ENVIRONMENT:+ environment,+ SemanticConvetion.GEN_AI_TYPE:+ SemanticConvetion.GEN_AI_TYPE_CHAT,+ SemanticConvetion.GEN_AI_REQUEST_MODEL:+ model+ }++ metrics["genai_requests"].add(1, attributes)+ metrics["genai_total_tokens"].add(total_tokens, attributes)+ metrics["genai_completion_tokens"].add(completion_tokens, attributes)+ metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)+ metrics["genai_cost"].record(cost, attributes)++ except Exception as e:+ handle_exception(span, e)+ logger.error("Error in trace creation: %s", e)++ return stream_generator()++ # Handling for non-streaming responses+ else:+ # pylint: disable=line-too-long+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:+ response = wrapped(*args, **kwargs)++ # pylint: disable=line-too-long+ model = str(instance.model.model_path).rsplit('/', maxsplit=1)[-1] or "orca-mini-3b-gguf2-q4_0.gguf"+ prompt = kwargs.get("prompt") or args[0] or ""++ # Calculate cost of the operation+ cost = 0+ prompt_tokens = general_tokens(prompt)+ completion_tokens = general_tokens(response)+ total_tokens = prompt_tokens + completion_tokens++ try:+ # Set base span attribues+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_GPT4ALL)+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,+ SemanticConvetion.GEN_AI_TYPE_CHAT)+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,+ gen_ai_endpoint)+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,+ environment)+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,+ application_name)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,+ model)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_K,+ kwargs.get("top_k", 40))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,+ kwargs.get("top_p", 0.4))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,+ kwargs.get("max_tokens", 200))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,+ kwargs.get("temperature", 0.7))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,+ kwargs.get("frequency_penalty", 1.18))+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,+ False)+ if trace_content:+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,+ prompt)+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,+ response)++ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,+ prompt_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,+ completion_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,+ total_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,+ cost)++ span.set_status(Status(StatusCode.OK))++ if disable_metrics is False:+ attributes = {+ TELEMETRY_SDK_NAME:+ "openlit",+ SemanticConvetion.GEN_AI_APPLICATION_NAME:+ application_name,+ SemanticConvetion.GEN_AI_SYSTEM:+ SemanticConvetion.GEN_AI_SYSTEM_GPT4ALL,+ SemanticConvetion.GEN_AI_ENVIRONMENT:+ environment,+ SemanticConvetion.GEN_AI_TYPE:+ SemanticConvetion.GEN_AI_TYPE_CHAT,+ SemanticConvetion.GEN_AI_REQUEST_MODEL:+ model+ }++ metrics["genai_requests"].add(1, attributes)+ metrics["genai_total_tokens"].add(total_tokens, attributes)+ metrics["genai_completion_tokens"].add(completion_tokens, attributes)+ metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)+ metrics["genai_cost"].record(cost, attributes)++ # Return original response+ return response++ except Exception as e:+ handle_exception(span, e)+ logger.error("Error in trace creation: %s", e)++ # Return original response+ return response++ return wrapper++def embed(gen_ai_endpoint, version, environment, application_name,+ tracer, pricing_info, trace_content, metrics, disable_metrics):+ """+ Generates a telemetry wrapper for embeddings to collect metrics.++ Args:+ gen_ai_endpoint: Endpoint identifier for logging and tracing.+ version: Version of the monitoring package.+ environment: Deployment environment (e.g., production, staging).+ application_name: Name of the application using the GPT4All API.+ tracer: OpenTelemetry tracer for creating spans.+ pricing_info: Information used for calculating the cost of GPT4All usage.+ trace_content: Flag indicating whether to trace the actual content.++ Returns:+ A function that wraps the embeddings method to add telemetry.+ """++ def wrapper(wrapped, instance, args, kwargs):+ """+ Wraps the 'embeddings' API call to add telemetry.++ This collects metrics such as execution time, cost, and token usage, and handles errors+ gracefully, adding details to the trace for observability.++ Args:+ wrapped: The original 'embeddings' method to be wrapped.+ instance: The instance of the class where the original method is defined.+ args: Positional arguments for the 'embeddings' method.+ kwargs: Keyword arguments for the 'embeddings' method.++ Returns:+ The response from the original 'embeddings' method.+ """++ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:+ response = wrapped(*args, **kwargs)++ try:+ # pylint: disable=line-too-long+ model = str(instance.gpt4all.model.model_path).rsplit('/', maxsplit=1)[-1] or "all-MiniLM-L6-v2.gguf2.f16.gguf"+ prompt = kwargs.get("prompt") or args[0] or ""++ # Calculate cost of the operation+ cost = 0+ prompt_tokens = general_tokens(prompt)++ # Set Span attributes+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,+ SemanticConvetion.GEN_AI_SYSTEM_GPT4ALL)+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,+ SemanticConvetion.GEN_AI_TYPE_EMBEDDING)+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,+ gen_ai_endpoint)+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,+ environment)+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,+ application_name)+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,+ model)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,+ prompt_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,+ prompt_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,+ cost)+ if trace_content:+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,+ prompt)++ span.set_status(Status(StatusCode.OK))++ if disable_metrics is False:+ attributes = {+ TELEMETRY_SDK_NAME:+ "openlit",+ SemanticConvetion.GEN_AI_APPLICATION_NAME:+ application_name,+ SemanticConvetion.GEN_AI_SYSTEM:+ SemanticConvetion.GEN_AI_SYSTEM_GPT4ALL,+ SemanticConvetion.GEN_AI_ENVIRONMENT:+ environment,+ SemanticConvetion.GEN_AI_TYPE:+ SemanticConvetion.GEN_AI_TYPE_EMBEDDING,+ SemanticConvetion.GEN_AI_REQUEST_MODEL:+ model+ }++ metrics["genai_requests"].add(1, attributes)+ metrics["genai_total_tokens"].add(prompt_tokens, attributes)+ metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)+ metrics["genai_cost"].record(cost, attributes)++ # Return original response+ return response++ except Exception as e:+ handle_exception(span, e)+ logger.error("Error in trace creation: %s", e)++ # Return original response+ return response++ return wrapperdiff --git a/sdk/python/src/openlit/semcov/__init__.py b/sdk/python/src/openlit/semcov/__init__.pyindex 5db3ea10..e61eb71a 100644--- a/sdk/python/src/openlit/semcov/__init__.py+++ b/sdk/python/src/openlit/semcov/__init__.py@@ -95,6 +95,7 @@ class SemanticConvetion:GEN_AI_SYSTEM_VERTEXAI = "vertexai"GEN_AI_SYSTEM_GROQ = "groq"GEN_AI_SYSTEM_OLLAMA = "ollama"+ GEN_AI_SYSTEM_GPT4ALL = "gpt4all"GEN_AI_SYSTEM_LANGCHAIN = "langchain"GEN_AI_SYSTEM_LLAMAINDEX = "llama_index"GEN_AI_SYSTEM_HAYSTACK = "haystack"