Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

[Feat]: Add OpenTelemetry instrumentation for Monitoring LLM Applications using GPT4All (#257)

patcher912e35d3bdf3a
diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml
index 0bb471d5..1d53ea4f 100644
--- a/sdk/python/pyproject.toml
+++ b/sdk/python/pyproject.toml
@@ -1,6 +1,6 @@
[tool.poetry]
name = "openlit"
-version = "1.10.0"
+version = "1.11.0"
description = "OpenTelemetry-native Auto instrumentation library for monitoring LLM Applications, facilitating the integration of observability into your GenAI-driven projects"
authors = ["OpenLIT"]
repository = "https://github.com/openlit/openlit/tree/main/openlit/python"
diff --git a/sdk/python/src/openlit/__init__.py b/sdk/python/src/openlit/__init__.py
index 282082e1..9f1fad13 100644
--- a/sdk/python/src/openlit/__init__.py
+++ b/sdk/python/src/openlit/__init__.py
@@ -21,6 +21,7 @@ from openlit.instrumentation.bedrock import BedrockInstrumentor
from openlit.instrumentation.vertexai import VertexAIInstrumentor
from openlit.instrumentation.groq import GroqInstrumentor
from openlit.instrumentation.ollama import OllamaInstrumentor
+from openlit.instrumentation.gpt4all import GPT4AllInstrumentor
from openlit.instrumentation.langchain import LangChainInstrumentor
from openlit.instrumentation.llamaindex import LlamaIndexInstrumentor
from openlit.instrumentation.haystack import HaystackInstrumentor
@@ -161,6 +162,7 @@ def init(environment="default", application_name="default", tracer=None, otlp_en
"vertexai": "vertexai",
"groq": "groq",
"ollama": "ollama",
+ "gpt4all": "gpt4all",
"langchain": "langchain",
"llama_index": "llama_index",
"haystack": "haystack",
@@ -216,6 +218,7 @@ def init(environment="default", application_name="default", tracer=None, otlp_en
"vertexai": VertexAIInstrumentor(),
"groq": GroqInstrumentor(),
"ollama": OllamaInstrumentor(),
+ "gpt4all": GPT4AllInstrumentor(),
"langchain": LangChainInstrumentor(),
"llama_index": LlamaIndexInstrumentor(),
"haystack": HaystackInstrumentor(),
diff --git a/sdk/python/src/openlit/instrumentation/gpt4all/__init__.py b/sdk/python/src/openlit/instrumentation/gpt4all/__init__.py
new file mode 100644
index 00000000..54625787
--- /dev/null
+++ b/sdk/python/src/openlit/instrumentation/gpt4all/__init__.py
@@ -0,0 +1,52 @@
+# pylint: disable=useless-return, bad-staticmethod-argument, disable=duplicate-code
+"""Initializer of Auto Instrumentation of GPT4All Functions"""
+
+from typing import Collection
+import importlib.metadata
+from opentelemetry.instrumentation.instrumentor import BaseInstrumentor
+from wrapt import wrap_function_wrapper
+
+from openlit.instrumentation.gpt4all.gpt4all import (
+ embed, generate
+)
+
+_instruments = ("gpt4all >= 2.6.0",)
+
+class GPT4AllInstrumentor(BaseInstrumentor):
+ """
+ An instrumentor for GPT4All's client library.
+ """
+
+ def instrumentation_dependencies(self) -> Collection[str]:
+ return _instruments
+
+ def _instrument(self, **kwargs):
+ application_name = kwargs.get("application_name", "default_application")
+ environment = kwargs.get("environment", "default_environment")
+ tracer = kwargs.get("tracer")
+ metrics = kwargs.get("metrics_dict")
+ pricing_info = kwargs.get("pricing_info", {})
+ trace_content = kwargs.get("trace_content", False)
+ disable_metrics = kwargs.get("disable_metrics")
+ version = importlib.metadata.version("gpt4all")
+
+ # generate
+ wrap_function_wrapper(
+ "gpt4all",
+ "GPT4All.generate",
+ generate("gpt4all.generate", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+
+ # embed
+ wrap_function_wrapper(
+ "gpt4all",
+ "Embed4All.embed",
+ embed("gpt4all.embed", version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics),
+ )
+
+
+ def _uninstrument(self, **kwargs):
+ # Proper uninstrumentation logic to revert patched methods
+ pass
diff --git a/sdk/python/src/openlit/instrumentation/gpt4all/gpt4all.py b/sdk/python/src/openlit/instrumentation/gpt4all/gpt4all.py
new file mode 100644
index 00000000..1dc764b8
--- /dev/null
+++ b/sdk/python/src/openlit/instrumentation/gpt4all/gpt4all.py
@@ -0,0 +1,352 @@
+# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument, possibly-used-before-assignment
+"""
+Module for monitoring GPT4All API calls.
+"""
+
+import logging
+from opentelemetry.trace import SpanKind, Status, StatusCode
+from opentelemetry.sdk.resources import TELEMETRY_SDK_NAME
+from openlit.__helpers import handle_exception, general_tokens
+from openlit.semcov import SemanticConvetion
+
+# Initialize logger for logging potential issues and operations
+logger = logging.getLogger(__name__)
+
+def generate(gen_ai_endpoint, version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for generate to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the GPT4All API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of GPT4All usage.
+ trace_content: Flag indicating whether to trace the actual content.
+
+ Returns:
+ A function that wraps the generate method to add telemetry.
+ """
+
+ def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'generate' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and token usage, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'generate' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'generate' method.
+ kwargs: Keyword arguments for the 'generate' method.
+
+ Returns:
+ The response from the original 'generate' method.
+ """
+
+ # Check if streaming is enabled for the API call
+ streaming = kwargs.get("streaming", False)
+
+ # pylint: disable=no-else-return
+ if streaming:
+ # Special handling for streaming response to accommodate the nature of data flow
+ def stream_generator():
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ # Placeholder for aggregating streaming response
+ llmresponse = ""
+
+ # Loop through streaming events capturing relevant details
+ for chunk in wrapped(*args, **kwargs):
+ # Collect aggregated response from events
+ llmresponse += chunk
+
+ yield chunk
+
+ # Handling exception ensure observability without disrupting operation
+ try:
+ # Calculate cost of the operation
+ cost = 0
+
+ # pylint: disable=line-too-long
+ model = str(instance.model.model_path).rsplit('/', maxsplit=1)[-1] or "orca-mini-3b-gguf2-q4_0.gguf"
+ prompt = kwargs.get("prompt") or args[0] or ""
+
+ # Calculate cost of the operation
+ cost = 0
+ prompt_tokens = general_tokens(prompt)
+ completion_tokens = general_tokens(llmresponse)
+ total_tokens = prompt_tokens + completion_tokens
+
+ # Set base span attribues
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_GPT4ALL)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ model)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_K,
+ kwargs.get("top_k", 40))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,
+ kwargs.get("top_p", 0.4))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,
+ kwargs.get("max_tokens", 200))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,
+ kwargs.get("temperature", 0.7))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,
+ kwargs.get("frequency_penalty", 1.18))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ True)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ llmresponse)
+
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ total_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_GPT4ALL,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ model
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(total_tokens, attributes)
+ metrics["genai_completion_tokens"].add(completion_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ return stream_generator()
+
+ # Handling for non-streaming responses
+ else:
+ # pylint: disable=line-too-long
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ response = wrapped(*args, **kwargs)
+
+ # pylint: disable=line-too-long
+ model = str(instance.model.model_path).rsplit('/', maxsplit=1)[-1] or "orca-mini-3b-gguf2-q4_0.gguf"
+ prompt = kwargs.get("prompt") or args[0] or ""
+
+ # Calculate cost of the operation
+ cost = 0
+ prompt_tokens = general_tokens(prompt)
+ completion_tokens = general_tokens(response)
+ total_tokens = prompt_tokens + completion_tokens
+
+ try:
+ # Set base span attribues
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_GPT4ALL)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ model)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_K,
+ kwargs.get("top_k", 40))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,
+ kwargs.get("top_p", 0.4))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,
+ kwargs.get("max_tokens", 200))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,
+ kwargs.get("temperature", 0.7))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_FREQUENCY_PENALTY,
+ kwargs.get("frequency_penalty", 1.18))
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
+ False)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
+ response)
+
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ completion_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ total_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_GPT4ALL,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ model
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(total_tokens, attributes)
+ metrics["genai_completion_tokens"].add(completion_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
+
+def embed(gen_ai_endpoint, version, environment, application_name,
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
+ """
+ Generates a telemetry wrapper for embeddings to collect metrics.
+
+ Args:
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
+ version: Version of the monitoring package.
+ environment: Deployment environment (e.g., production, staging).
+ application_name: Name of the application using the GPT4All API.
+ tracer: OpenTelemetry tracer for creating spans.
+ pricing_info: Information used for calculating the cost of GPT4All usage.
+ trace_content: Flag indicating whether to trace the actual content.
+
+ Returns:
+ A function that wraps the embeddings method to add telemetry.
+ """
+
+ def wrapper(wrapped, instance, args, kwargs):
+ """
+ Wraps the 'embeddings' API call to add telemetry.
+
+ This collects metrics such as execution time, cost, and token usage, and handles errors
+ gracefully, adding details to the trace for observability.
+
+ Args:
+ wrapped: The original 'embeddings' method to be wrapped.
+ instance: The instance of the class where the original method is defined.
+ args: Positional arguments for the 'embeddings' method.
+ kwargs: Keyword arguments for the 'embeddings' method.
+
+ Returns:
+ The response from the original 'embeddings' method.
+ """
+
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
+ response = wrapped(*args, **kwargs)
+
+ try:
+ # pylint: disable=line-too-long
+ model = str(instance.gpt4all.model.model_path).rsplit('/', maxsplit=1)[-1] or "all-MiniLM-L6-v2.gguf2.f16.gguf"
+ prompt = kwargs.get("prompt") or args[0] or ""
+
+ # Calculate cost of the operation
+ cost = 0
+ prompt_tokens = general_tokens(prompt)
+
+ # Set Span attributes
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
+ SemanticConvetion.GEN_AI_SYSTEM_GPT4ALL)
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
+ SemanticConvetion.GEN_AI_TYPE_EMBEDDING)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
+ gen_ai_endpoint)
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
+ environment)
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
+ application_name)
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
+ model)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ prompt_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
+ if trace_content:
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
+ prompt)
+
+ span.set_status(Status(StatusCode.OK))
+
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_GPT4ALL,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_EMBEDDING,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ model
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(prompt_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
+
+ # Return original response
+ return response
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error in trace creation: %s", e)
+
+ # Return original response
+ return response
+
+ return wrapper
diff --git a/sdk/python/src/openlit/semcov/__init__.py b/sdk/python/src/openlit/semcov/__init__.py
index 5db3ea10..e61eb71a 100644
--- a/sdk/python/src/openlit/semcov/__init__.py
+++ b/sdk/python/src/openlit/semcov/__init__.py
@@ -95,6 +95,7 @@ class SemanticConvetion:
GEN_AI_SYSTEM_VERTEXAI = "vertexai"
GEN_AI_SYSTEM_GROQ = "groq"
GEN_AI_SYSTEM_OLLAMA = "ollama"
+ GEN_AI_SYSTEM_GPT4ALL = "gpt4all"
GEN_AI_SYSTEM_LANGCHAIN = "langchain"
GEN_AI_SYSTEM_LLAMAINDEX = "llama_index"
GEN_AI_SYSTEM_HAYSTACK = "haystack"