Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
[Fix]: Model, Prompt and Completion extraction in Transformers and Langchain Instrumentation (#514)
Patcher8ba356a06062
diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.tomlindex 8b383fdd..0aea613e 100644--- a/sdk/python/pyproject.toml+++ b/sdk/python/pyproject.toml@@ -1,6 +1,6 @@[tool.poetry]name = "openlit"-version = "1.30.4"+version = "1.30.5"description = "OpenTelemetry-native Auto instrumentation library for monitoring LLM Applications and GPUs, facilitating the integration of observability into your GenAI-driven projects"authors = ["OpenLIT"]repository = "https://github.com/openlit/openlit/tree/main/openlit/python"diff --git a/sdk/python/src/openlit/instrumentation/langchain/langchain.py b/sdk/python/src/openlit/instrumentation/langchain/langchain.pyindex fcc86439..9958fc24 100644--- a/sdk/python/src/openlit/instrumentation/langchain/langchain.py+++ b/sdk/python/src/openlit/instrumentation/langchain/langchain.py@@ -12,6 +12,26 @@ from openlit.semcov import SemanticConvetion# Initialize logger for logging potential issues and operationslogger = logging.getLogger(__name__)+def get_attribute_from_instance_or_kwargs(instance, attribute_name, default=-1):+ """Return attribute from instance or kwargs"""+ # Attempt to retrieve model_kwargs from the instance+ model_kwargs = getattr(instance, 'model_kwargs', None)++ # Check for attribute in model_kwargs if it exists+ if model_kwargs and attribute_name in model_kwargs:+ return model_kwargs[attribute_name]++ # Attempt to get the attribute directly from the instance+ try:+ return getattr(instance, attribute_name)+ except AttributeError:+ # Special handling for 'model' attribute to consider 'model_id'+ if attribute_name == 'model':+ return getattr(instance, 'model_id', 'default_model_id')++ # Default if the attribute isn't found in model_kwargs or the instance+ return default+def general_wrap(gen_ai_endpoint, version, environment, application_name,tracer, pricing_info, trace_content, metrics, disable_metrics):"""@@ -207,15 +227,18 @@ def allm(gen_ai_endpoint, version, environment, application_name,response = await wrapped(*args, **kwargs)try:- prompt = args[0] or ""- # input_tokens = general_tokens(prompt)- # output_tokens = general_tokens(response)+ if args:+ prompt = str(args[0]) if args[0] is not None else ""+ else:+ prompt = ""+ input_tokens = general_tokens(prompt)+ output_tokens = general_tokens(response)- # # Calculate cost of the operation- # cost = get_chat_model_cost(- # str(getattr(instance, 'model')),- # pricing_info, input_tokens, output_tokens- # )+ # Calculate cost of the operation+ cost = get_chat_model_cost(+ str(get_attribute_from_instance_or_kwargs(instance, 'model')),+ pricing_info, input_tokens, output_tokens+ )span.set_attribute(TELEMETRY_SDK_NAME, "openlit")span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,@@ -229,23 +252,23 @@ def allm(gen_ai_endpoint, version, environment, application_name,span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,application_name)span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- str(getattr(instance, 'model')))+ str(get_attribute_from_instance_or_kwargs(instance, 'model')))span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,- str(getattr(instance, 'temperature')))+ str(get_attribute_from_instance_or_kwargs(instance, 'temperature')))span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_K,- str(getattr(instance, 'top_k')))+ str(get_attribute_from_instance_or_kwargs(instance, 'top_k')))span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,- str(getattr(instance, 'top_p')))+ str(get_attribute_from_instance_or_kwargs(instance, 'top_p')))span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,False)- # span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,- # input_tokens)- # span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,- # output_tokens)- # span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- # input_tokens + output_tokens)- # span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- # cost)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,+ input_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,+ output_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,+ input_tokens + output_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,+ cost)if trace_content:span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,@@ -262,29 +285,29 @@ def allm(gen_ai_endpoint, version, environment, application_name,span.set_status(Status(StatusCode.OK))- # if disable_metrics is False:- # attributes = {- # TELEMETRY_SDK_NAME:- # "openlit",- # SemanticConvetion.GEN_AI_APPLICATION_NAME:- # application_name,- # SemanticConvetion.GEN_AI_SYSTEM:- # SemanticConvetion.GEN_AI_SYSTEM_LANGCHAIN,- # SemanticConvetion.GEN_AI_ENVIRONMENT:- # environment,- # SemanticConvetion.GEN_AI_TYPE:- # SemanticConvetion.GEN_AI_TYPE_CHAT,- # SemanticConvetion.GEN_AI_REQUEST_MODEL:- # str(getattr(instance, 'model'))- # }-- # metrics["genai_requests"].add(1, attributes)- # metrics["genai_total_tokens"].add(- # input_tokens + output_tokens, attributes- # )- # metrics["genai_completion_tokens"].add(output_tokens, attributes)- # metrics["genai_prompt_tokens"].add(input_tokens, attributes)- # metrics["genai_cost"].record(cost, attributes)+ if disable_metrics is False:+ attributes = {+ TELEMETRY_SDK_NAME:+ "openlit",+ SemanticConvetion.GEN_AI_APPLICATION_NAME:+ application_name,+ SemanticConvetion.GEN_AI_SYSTEM:+ SemanticConvetion.GEN_AI_SYSTEM_LANGCHAIN,+ SemanticConvetion.GEN_AI_ENVIRONMENT:+ environment,+ SemanticConvetion.GEN_AI_TYPE:+ SemanticConvetion.GEN_AI_TYPE_CHAT,+ SemanticConvetion.GEN_AI_REQUEST_MODEL:+ str(get_attribute_from_instance_or_kwargs(instance, 'model'))+ }++ metrics["genai_requests"].add(1, attributes)+ metrics["genai_total_tokens"].add(+ input_tokens + output_tokens, attributes+ )+ metrics["genai_completion_tokens"].add(output_tokens, attributes)+ metrics["genai_prompt_tokens"].add(input_tokens, attributes)+ metrics["genai_cost"].record(cost, attributes)# Return original responsereturn response@@ -344,15 +367,18 @@ def llm(gen_ai_endpoint, version, environment, application_name,response = wrapped(*args, **kwargs)try:- prompt = args[0] or ""- # input_tokens = general_tokens(prompt)- # output_tokens = general_tokens(response)+ if args:+ prompt = str(args[0]) if args[0] is not None else ""+ else:+ prompt = ""+ input_tokens = general_tokens(prompt)+ output_tokens = general_tokens(response)- # # Calculate cost of the operation- # cost = get_chat_model_cost(- # str(getattr(instance, 'model')),- # pricing_info, input_tokens, output_tokens- # )+ # Calculate cost of the operation+ cost = get_chat_model_cost(+ str(get_attribute_from_instance_or_kwargs(instance, 'model')),+ pricing_info, input_tokens, output_tokens+ )span.set_attribute(TELEMETRY_SDK_NAME, "openlit")span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,@@ -366,23 +392,23 @@ def llm(gen_ai_endpoint, version, environment, application_name,span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,application_name)span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,- str(getattr(instance, 'model')))+ str(get_attribute_from_instance_or_kwargs(instance, 'model')))span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,- str(getattr(instance, 'temperature')))+ str(get_attribute_from_instance_or_kwargs(instance, 'temperature')))span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_K,- str(getattr(instance, 'top_k')))+ str(get_attribute_from_instance_or_kwargs(instance, 'top_k')))span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,- str(getattr(instance, 'top_p')))+ str(get_attribute_from_instance_or_kwargs(instance, 'top_p')))span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,False)- # span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,- # input_tokens)- # span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,- # output_tokens)- # span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,- # input_tokens + output_tokens)- # span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,- # cost)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,+ input_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,+ output_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,+ input_tokens + output_tokens)+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,+ cost)if trace_content:span.add_event(name=SemanticConvetion.GEN_AI_CONTENT_PROMPT_EVENT,@@ -399,29 +425,29 @@ def llm(gen_ai_endpoint, version, environment, application_name,span.set_status(Status(StatusCode.OK))- # if disable_metrics is False:- # attributes = {- # TELEMETRY_SDK_NAME:- # "openlit",- # SemanticConvetion.GEN_AI_APPLICATION_NAME:- # application_name,- # SemanticConvetion.GEN_AI_SYSTEM:- # SemanticConvetion.GEN_AI_SYSTEM_LANGCHAIN,- # SemanticConvetion.GEN_AI_ENVIRONMENT:- # environment,- # SemanticConvetion.GEN_AI_TYPE:- # SemanticConvetion.GEN_AI_TYPE_CHAT,- # SemanticConvetion.GEN_AI_REQUEST_MODEL:- # str(getattr(instance, 'model'))- # }-- # metrics["genai_requests"].add(1, attributes)- # metrics["genai_total_tokens"].add(- # input_tokens + output_tokens, attributes- # )- # metrics["genai_completion_tokens"].add(output_tokens, attributes)- # metrics["genai_prompt_tokens"].add(input_tokens, attributes)- # metrics["genai_cost"].record(cost, attributes)+ if disable_metrics is False:+ attributes = {+ TELEMETRY_SDK_NAME:+ "openlit",+ SemanticConvetion.GEN_AI_APPLICATION_NAME:+ application_name,+ SemanticConvetion.GEN_AI_SYSTEM:+ SemanticConvetion.GEN_AI_SYSTEM_LANGCHAIN,+ SemanticConvetion.GEN_AI_ENVIRONMENT:+ environment,+ SemanticConvetion.GEN_AI_TYPE:+ SemanticConvetion.GEN_AI_TYPE_CHAT,+ SemanticConvetion.GEN_AI_REQUEST_MODEL:+ str(get_attribute_from_instance_or_kwargs(instance, 'model'))+ }++ metrics["genai_requests"].add(1, attributes)+ metrics["genai_total_tokens"].add(+ input_tokens + output_tokens, attributes+ )+ metrics["genai_completion_tokens"].add(output_tokens, attributes)+ metrics["genai_prompt_tokens"].add(input_tokens, attributes)+ metrics["genai_cost"].record(cost, attributes)# Return original responsereturn responsediff --git a/sdk/python/src/openlit/instrumentation/transformers/transformers.py b/sdk/python/src/openlit/instrumentation/transformers/transformers.pyindex 16a0c8f0..fd372cad 100644--- a/sdk/python/src/openlit/instrumentation/transformers/transformers.py+++ b/sdk/python/src/openlit/instrumentation/transformers/transformers.py@@ -67,7 +67,7 @@ def text_wrap(gen_ai_endpoint, version, environment, application_name,else:prompt = kwargs.get("args", "")- prompt_tokens = general_tokens(prompt)+ prompt_tokens = general_tokens(prompt[0])span.set_attribute(TELEMETRY_SDK_NAME, "openlit")span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,@@ -106,14 +106,20 @@ def text_wrap(gen_ai_endpoint, version, environment, application_name,else:attribute_name = SemanticConvetion.GEN_AI_CONTENT_COMPLETION_EVENTif trace_content:+ # pylint: disable=bare-except+ try:+ llm_response = completion.get('generated_text', '')+ except:+ llm_response = completion[i].get('generated_text', '')+span.add_event(name=attribute_name,attributes={# pylint: disable=line-too-long- SemanticConvetion.GEN_AI_CONTENT_COMPLETION: completion["generated_text"],+ SemanticConvetion.GEN_AI_CONTENT_COMPLETION: llm_response,},)- completion_tokens += general_tokens(completion["generated_text"])+ completion_tokens += general_tokens(llm_response)i=i+1span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,