Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

[Fix]: Model, Prompt and Completion extraction in Transformers and Langchain Instrumentation (#514)

Patcher8ba356a06062
diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml
index 8b383fdd..0aea613e 100644
--- a/sdk/python/pyproject.toml
+++ b/sdk/python/pyproject.toml
@@ -1,6 +1,6 @@
[tool.poetry]
name = "openlit"
-version = "1.30.4"
+version = "1.30.5"
description = "OpenTelemetry-native Auto instrumentation library for monitoring LLM Applications and GPUs, facilitating the integration of observability into your GenAI-driven projects"
authors = ["OpenLIT"]
repository = "https://github.com/openlit/openlit/tree/main/openlit/python"
diff --git a/sdk/python/src/openlit/instrumentation/langchain/langchain.py b/sdk/python/src/openlit/instrumentation/langchain/langchain.py
index fcc86439..9958fc24 100644
--- a/sdk/python/src/openlit/instrumentation/langchain/langchain.py
+++ b/sdk/python/src/openlit/instrumentation/langchain/langchain.py
@@ -12,6 +12,26 @@ from openlit.semcov import SemanticConvetion
# Initialize logger for logging potential issues and operations
logger = logging.getLogger(__name__)
+def get_attribute_from_instance_or_kwargs(instance, attribute_name, default=-1):
+ """Return attribute from instance or kwargs"""
+ # Attempt to retrieve model_kwargs from the instance
+ model_kwargs = getattr(instance, 'model_kwargs', None)
+
+ # Check for attribute in model_kwargs if it exists
+ if model_kwargs and attribute_name in model_kwargs:
+ return model_kwargs[attribute_name]
+
+ # Attempt to get the attribute directly from the instance
+ try:
+ return getattr(instance, attribute_name)
+ except AttributeError:
+ # Special handling for 'model' attribute to consider 'model_id'
+ if attribute_name == 'model':
+ return getattr(instance, 'model_id', 'default_model_id')
+
+ # Default if the attribute isn't found in model_kwargs or the instance
+ return default
+
def general_wrap(gen_ai_endpoint, version, environment, application_name,
tracer, pricing_info, trace_content, metrics, disable_metrics):
"""
@@ -207,15 +227,18 @@ def allm(gen_ai_endpoint, version, environment, application_name,
response = await wrapped(*args, **kwargs)
try:
- prompt = args[0] or ""
- # input_tokens = general_tokens(prompt)
- # output_tokens = general_tokens(response)
+ if args:
+ prompt = str(args[0]) if args[0] is not None else ""
+ else:
+ prompt = ""
+ input_tokens = general_tokens(prompt)
+ output_tokens = general_tokens(response)
- # # Calculate cost of the operation
- # cost = get_chat_model_cost(
- # str(getattr(instance, 'model')),
- # pricing_info, input_tokens, output_tokens
- # )
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(
+ str(get_attribute_from_instance_or_kwargs(instance, 'model')),
+ pricing_info, input_tokens, output_tokens
+ )
span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
@@ -229,23 +252,23 @@ def allm(gen_ai_endpoint, version, environment, application_name,
span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
application_name)
span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
- str(getattr(instance, 'model')))
+ str(get_attribute_from_instance_or_kwargs(instance, 'model')))
span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,
- str(getattr(instance, 'temperature')))
+ str(get_attribute_from_instance_or_kwargs(instance, 'temperature')))
span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_K,
- str(getattr(instance, 'top_k')))
+ str(get_attribute_from_instance_or_kwargs(instance, 'top_k')))
span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,
- str(getattr(instance, 'top_p')))
+ str(get_attribute_from_instance_or_kwargs(instance, 'top_p')))
span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
False)
- # span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
- # input_tokens)
- # span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
- # output_tokens)
- # span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
- # input_tokens + output_tokens)
- # span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
- # cost)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ input_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ output_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ input_tokens + output_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
if trace_content:
span.add_event(
name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,
@@ -262,29 +285,29 @@ def allm(gen_ai_endpoint, version, environment, application_name,
span.set_status(Status(StatusCode.OK))
- # if disable_metrics is False:
- # attributes = {
- # TELEMETRY_SDK_NAME:
- # "openlit",
- # SemanticConvetion.GEN_AI_APPLICATION_NAME:
- # application_name,
- # SemanticConvetion.GEN_AI_SYSTEM:
- # SemanticConvetion.GEN_AI_SYSTEM_LANGCHAIN,
- # SemanticConvetion.GEN_AI_ENVIRONMENT:
- # environment,
- # SemanticConvetion.GEN_AI_TYPE:
- # SemanticConvetion.GEN_AI_TYPE_CHAT,
- # SemanticConvetion.GEN_AI_REQUEST_MODEL:
- # str(getattr(instance, 'model'))
- # }
-
- # metrics["genai_requests"].add(1, attributes)
- # metrics["genai_total_tokens"].add(
- # input_tokens + output_tokens, attributes
- # )
- # metrics["genai_completion_tokens"].add(output_tokens, attributes)
- # metrics["genai_prompt_tokens"].add(input_tokens, attributes)
- # metrics["genai_cost"].record(cost, attributes)
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_LANGCHAIN,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ str(get_attribute_from_instance_or_kwargs(instance, 'model'))
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(
+ input_tokens + output_tokens, attributes
+ )
+ metrics["genai_completion_tokens"].add(output_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(input_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
# Return original response
return response
@@ -344,15 +367,18 @@ def llm(gen_ai_endpoint, version, environment, application_name,
response = wrapped(*args, **kwargs)
try:
- prompt = args[0] or ""
- # input_tokens = general_tokens(prompt)
- # output_tokens = general_tokens(response)
+ if args:
+ prompt = str(args[0]) if args[0] is not None else ""
+ else:
+ prompt = ""
+ input_tokens = general_tokens(prompt)
+ output_tokens = general_tokens(response)
- # # Calculate cost of the operation
- # cost = get_chat_model_cost(
- # str(getattr(instance, 'model')),
- # pricing_info, input_tokens, output_tokens
- # )
+ # Calculate cost of the operation
+ cost = get_chat_model_cost(
+ str(get_attribute_from_instance_or_kwargs(instance, 'model')),
+ pricing_info, input_tokens, output_tokens
+ )
span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
@@ -366,23 +392,23 @@ def llm(gen_ai_endpoint, version, environment, application_name,
span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
application_name)
span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
- str(getattr(instance, 'model')))
+ str(get_attribute_from_instance_or_kwargs(instance, 'model')))
span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,
- str(getattr(instance, 'temperature')))
+ str(get_attribute_from_instance_or_kwargs(instance, 'temperature')))
span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_K,
- str(getattr(instance, 'top_k')))
+ str(get_attribute_from_instance_or_kwargs(instance, 'top_k')))
span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,
- str(getattr(instance, 'top_p')))
+ str(get_attribute_from_instance_or_kwargs(instance, 'top_p')))
span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
False)
- # span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
- # input_tokens)
- # span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
- # output_tokens)
- # span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
- # input_tokens + output_tokens)
- # span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
- # cost)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
+ input_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
+ output_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
+ input_tokens + output_tokens)
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
+ cost)
if trace_content:
span.add_event(
name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,
@@ -399,29 +425,29 @@ def llm(gen_ai_endpoint, version, environment, application_name,
span.set_status(Status(StatusCode.OK))
- # if disable_metrics is False:
- # attributes = {
- # TELEMETRY_SDK_NAME:
- # "openlit",
- # SemanticConvetion.GEN_AI_APPLICATION_NAME:
- # application_name,
- # SemanticConvetion.GEN_AI_SYSTEM:
- # SemanticConvetion.GEN_AI_SYSTEM_LANGCHAIN,
- # SemanticConvetion.GEN_AI_ENVIRONMENT:
- # environment,
- # SemanticConvetion.GEN_AI_TYPE:
- # SemanticConvetion.GEN_AI_TYPE_CHAT,
- # SemanticConvetion.GEN_AI_REQUEST_MODEL:
- # str(getattr(instance, 'model'))
- # }
-
- # metrics["genai_requests"].add(1, attributes)
- # metrics["genai_total_tokens"].add(
- # input_tokens + output_tokens, attributes
- # )
- # metrics["genai_completion_tokens"].add(output_tokens, attributes)
- # metrics["genai_prompt_tokens"].add(input_tokens, attributes)
- # metrics["genai_cost"].record(cost, attributes)
+ if disable_metrics is False:
+ attributes = {
+ TELEMETRY_SDK_NAME:
+ "openlit",
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
+ application_name,
+ SemanticConvetion.GEN_AI_SYSTEM:
+ SemanticConvetion.GEN_AI_SYSTEM_LANGCHAIN,
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
+ environment,
+ SemanticConvetion.GEN_AI_TYPE:
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
+ str(get_attribute_from_instance_or_kwargs(instance, 'model'))
+ }
+
+ metrics["genai_requests"].add(1, attributes)
+ metrics["genai_total_tokens"].add(
+ input_tokens + output_tokens, attributes
+ )
+ metrics["genai_completion_tokens"].add(output_tokens, attributes)
+ metrics["genai_prompt_tokens"].add(input_tokens, attributes)
+ metrics["genai_cost"].record(cost, attributes)
# Return original response
return response
diff --git a/sdk/python/src/openlit/instrumentation/transformers/transformers.py b/sdk/python/src/openlit/instrumentation/transformers/transformers.py
index 16a0c8f0..fd372cad 100644
--- a/sdk/python/src/openlit/instrumentation/transformers/transformers.py
+++ b/sdk/python/src/openlit/instrumentation/transformers/transformers.py
@@ -67,7 +67,7 @@ def text_wrap(gen_ai_endpoint, version, environment, application_name,
else:
prompt = kwargs.get("args", "")
- prompt_tokens = general_tokens(prompt)
+ prompt_tokens = general_tokens(prompt[0])
span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
@@ -106,14 +106,20 @@ def text_wrap(gen_ai_endpoint, version, environment, application_name,
else:
attribute_name = SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENT
if trace_content:
+ # pylint: disable=bare-except
+ try:
+ llm_response = completion.get('generated_text', '')
+ except:
+ llm_response = completion[i].get('generated_text', '')
+
span.add_event(
name=attribute_name,
attributes={
# pylint: disable=line-too-long
- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: completion["generated_text"],
+ SemanticConvetion.GEN_AI_CONTENT_COMPLETION: llm_response,
},
)
- completion_tokens += general_tokens(completion["generated_text"])
+ completion_tokens += general_tokens(llm_response)
i=i+1
span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,