Skip to content

The agent engineering platform.

fix(typesafe): trace usage metadata (#40570)

ccurme6520b7f1ab21
diff --git a/libs/partners/typesafe/langchain_typesafe/classifier.py b/libs/partners/typesafe/langchain_typesafe/classifier.py
index b4880b041d..6621afbfd0 100644
--- a/libs/partners/typesafe/langchain_typesafe/classifier.py
+++ b/libs/partners/typesafe/langchain_typesafe/classifier.py
@@ -2,12 +2,15 @@
from __future__ import annotations
+import logging
from typing import Any
import httpx2
from langchain_core._api import beta
from langchain_core.runnables import RunnableConfig, RunnableSerializable
+from langchain_core.runnables.config import ensure_config
from langchain_core.utils import from_env, secret_from_env
+from langsmith.run_helpers import get_current_run_tree
from pydantic import (
ConfigDict,
Field,
@@ -30,6 +33,9 @@ from langchain_typesafe.types import ClassificationResponse, Question, State
_DEFAULT_BASE_URL = "https://api.typesafe.ai"
_DEFAULT_MODEL = "jev-latest"
_DEFAULT_TIMEOUT = 30.0
+_LS_PROVIDER = "typesafe"
+
+logger = logging.getLogger(__name__)
@beta()
@@ -322,8 +328,8 @@ class TypeSafeClassifier(RunnableSerializable[State, ClassificationResponse]):
return self._call_with_config(
self._classify,
input,
- config,
- run_type="chain",
+ self._traced_config(config),
+ run_type="llm",
)
@override
@@ -355,8 +361,8 @@ class TypeSafeClassifier(RunnableSerializable[State, ClassificationResponse]):
return await self._acall_with_config(
self._aclassify,
input,
- config,
- run_type="chain",
+ self._traced_config(config),
+ run_type="llm",
)
def _classify(self, state: State) -> ClassificationResponse:
@@ -375,7 +381,7 @@ class TypeSafeClassifier(RunnableSerializable[State, ClassificationResponse]):
except httpx2.HTTPError as error:
message = "Unable to connect to the TypeSafe API."
raise TypeSafeAPIConnectionError(message) from error
- return parse_response(response)
+ return self._record_usage(parse_response(response))
async def _aclassify(self, state: State) -> ClassificationResponse:
payload = self._payload(state)
@@ -393,7 +399,38 @@ class TypeSafeClassifier(RunnableSerializable[State, ClassificationResponse]):
except httpx2.HTTPError as error:
message = "Unable to connect to the TypeSafe API."
raise TypeSafeAPIConnectionError(message) from error
- return parse_response(response)
+ return self._record_usage(parse_response(response))
+
+ def _traced_config(self, config: RunnableConfig | None) -> RunnableConfig:
+ """Set `ls_provider` and `ls_model_name` when run is created."""
+ config = ensure_config(config)
+ config["metadata"] = {
+ **(config.get("metadata") or {}),
+ "ls_provider": _LS_PROVIDER,
+ "ls_model_name": self.model,
+ "ls_model_type": "chat",
+ }
+ return config
+
+ def _record_usage(self, response: ClassificationResponse) -> ClassificationResponse:
+ """Attach TypeSafe token usage to the active run, if there is one.
+
+ Nothing is written when tracing is disabled, and a tracing failure never fails
+ an otherwise successful classification.
+ """
+ input_tokens = response.usage.input_tokens or 0
+ output_tokens = response.usage.output_tokens or 0
+ try:
+ run_tree = get_current_run_tree()
+ if run_tree is not None:
+ run_tree.extra.setdefault("metadata", {})["usage_metadata"] = {
+ "input_tokens": input_tokens,
+ "output_tokens": output_tokens,
+ "total_tokens": input_tokens + output_tokens,
+ }
+ except Exception: # noqa: BLE001 - tracing must not break classification
+ logger.debug("Could not attach TypeSafe usage.", exc_info=True)
+ return response
@property
def _endpoint(self) -> str:
diff --git a/libs/partners/typesafe/tests/unit_tests/test_classifier.py b/libs/partners/typesafe/tests/unit_tests/test_classifier.py
index d266396aa7..f63355a7ab 100644
--- a/libs/partners/typesafe/tests/unit_tests/test_classifier.py
+++ b/libs/partners/typesafe/tests/unit_tests/test_classifier.py
@@ -22,6 +22,7 @@ from langchain_typesafe import (
TypeSafeClassifier,
__version__,
)
+from langchain_typesafe import classifier as classifier_module
from langchain_typesafe.client import (
TypeSafeAPIConnectionError,
TypeSafeAPIError,
@@ -29,6 +30,26 @@ from langchain_typesafe.client import (
TypeSafeAPITimeoutError,
)
+
+class _RunTreeStub:
+ """Stand-in for the LangSmith run tree that `_record_usage` writes to."""
+
+ def __init__(self) -> None:
+ self.extra: dict[str, Any] = {}
+
+
+class _RunRecorder(BaseCallbackHandler):
+ """Record the run type and metadata the classifier starts its run with."""
+
+ def __init__(self) -> None:
+ self.metadata: dict[str, Any] = {}
+ self.run_type: str | None = None
+
+ def on_chain_start(self, *_: Any, **kwargs: Any) -> None:
+ self.metadata = kwargs.get("metadata") or {}
+ self.run_type = kwargs.get("run_type")
+
+
API_KEY = "test-api-key"
REQUEST_ID = "req_test"
@@ -537,3 +558,104 @@ def test_callbacks_receive_classifier_run() -> None:
assert callback.starts == 1
assert callback.ends == 1
client.close()
+
+
+def test_usage_is_recorded_on_the_active_run(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Token usage is written where LangSmith totals it.
+
+ LangSmith only sums tokens for `llm` runs whose metadata carries
+ `usage_metadata`, so both the payload and the run type are pinned.
+ """
+ stub = _RunTreeStub()
+ monkeypatch.setattr(classifier_module, "get_current_run_tree", lambda: stub)
+
+ def handler(_: httpx2.Request) -> httpx2.Response:
+ return httpx2.Response(200, json=_response_payload())
+
+ client = httpx2.Client(transport=httpx2.MockTransport(handler))
+ recorder = _RunRecorder()
+ classifier = TypeSafeClassifier(
+ api_key=API_KEY,
+ questions=_questions(),
+ client=client,
+ )
+
+ classifier.invoke("hello", config={"callbacks": [recorder]})
+ client.close()
+
+ assert recorder.run_type == "llm"
+ assert stub.extra["metadata"]["usage_metadata"] == {
+ "input_tokens": 42,
+ "output_tokens": 12,
+ "total_tokens": 54,
+ }
+
+
+async def test_async_usage_is_recorded_on_the_active_run(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """`ainvoke` records the same usage as `invoke`."""
+ stub = _RunTreeStub()
+ monkeypatch.setattr(classifier_module, "get_current_run_tree", lambda: stub)
+
+ async def handler(_: httpx2.Request) -> httpx2.Response:
+ return httpx2.Response(200, json=_response_payload())
+
+ client = httpx2.AsyncClient(transport=httpx2.MockTransport(handler))
+ classifier = TypeSafeClassifier(
+ api_key=API_KEY,
+ questions=_questions(),
+ async_client=client,
+ )
+
+ await classifier.ainvoke("hello")
+ await client.aclose()
+
+ assert stub.extra["metadata"]["usage_metadata"]["total_tokens"] == 54
+
+
+def test_run_carries_model_identity_without_losing_caller_metadata() -> None:
+ """Identity tags LangSmith prices by are added alongside caller metadata."""
+
+ def handler(_: httpx2.Request) -> httpx2.Response:
+ return httpx2.Response(200, json=_response_payload())
+
+ client = httpx2.Client(transport=httpx2.MockTransport(handler))
+ recorder = _RunRecorder()
+ classifier = TypeSafeClassifier(
+ api_key=API_KEY,
+ questions=_questions(),
+ client=client,
+ )
+
+ classifier.invoke(
+ "hello",
+ config={"callbacks": [recorder], "metadata": {"tenant": "acme"}},
+ )
+ client.close()
+
+ assert recorder.metadata["tenant"] == "acme"
+ assert recorder.metadata["ls_provider"] == "typesafe"
+ assert recorder.metadata["ls_model_name"] == "jev-latest"
+ assert API_KEY not in json.dumps(recorder.metadata, default=str)
+
+
+def test_untraced_invocation_is_unaffected() -> None:
+ """With no tracer active, recording usage is a no-op rather than an error."""
+
+ def handler(_: httpx2.Request) -> httpx2.Response:
+ return httpx2.Response(200, json=_response_payload())
+
+ client = httpx2.Client(transport=httpx2.MockTransport(handler))
+ classifier = TypeSafeClassifier(
+ api_key=API_KEY,
+ questions=_questions(),
+ client=client,
+ )
+
+ result = classifier.invoke("hello")
+ client.close()
+
+ assert result.usage.input_tokens == 42