The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
fix(typesafe): trace usage metadata (#40570)
ccurme6520b7f1ab21
diff --git a/libs/partners/typesafe/langchain_typesafe/classifier.py b/libs/partners/typesafe/langchain_typesafe/classifier.pyindex b4880b041d..6621afbfd0 100644--- a/libs/partners/typesafe/langchain_typesafe/classifier.py+++ b/libs/partners/typesafe/langchain_typesafe/classifier.py@@ -2,12 +2,15 @@from __future__ import annotations+import loggingfrom typing import Anyimport httpx2from langchain_core._api import betafrom langchain_core.runnables import RunnableConfig, RunnableSerializable+from langchain_core.runnables.config import ensure_configfrom langchain_core.utils import from_env, secret_from_env+from langsmith.run_helpers import get_current_run_treefrom pydantic import (ConfigDict,Field,@@ -30,6 +33,9 @@ from langchain_typesafe.types import ClassificationResponse, Question, State_DEFAULT_BASE_URL = "https://api.typesafe.ai"_DEFAULT_MODEL = "jev-latest"_DEFAULT_TIMEOUT = 30.0+_LS_PROVIDER = "typesafe"++logger = logging.getLogger(__name__)@beta()@@ -322,8 +328,8 @@ class TypeSafeClassifier(RunnableSerializable[State, ClassificationResponse]):return self._call_with_config(self._classify,input,- config,- run_type="chain",+ self._traced_config(config),+ run_type="llm",)@override@@ -355,8 +361,8 @@ class TypeSafeClassifier(RunnableSerializable[State, ClassificationResponse]):return await self._acall_with_config(self._aclassify,input,- config,- run_type="chain",+ self._traced_config(config),+ run_type="llm",)def _classify(self, state: State) -> ClassificationResponse:@@ -375,7 +381,7 @@ class TypeSafeClassifier(RunnableSerializable[State, ClassificationResponse]):except httpx2.HTTPError as error:message = "Unable to connect to the TypeSafe API."raise TypeSafeAPIConnectionError(message) from error- return parse_response(response)+ return self._record_usage(parse_response(response))async def _aclassify(self, state: State) -> ClassificationResponse:payload = self._payload(state)@@ -393,7 +399,38 @@ class TypeSafeClassifier(RunnableSerializable[State, ClassificationResponse]):except httpx2.HTTPError as error:message = "Unable to connect to the TypeSafe API."raise TypeSafeAPIConnectionError(message) from error- return parse_response(response)+ return self._record_usage(parse_response(response))++ def _traced_config(self, config: RunnableConfig | None) -> RunnableConfig:+ """Set `ls_provider` and `ls_model_name` when run is created."""+ config = ensure_config(config)+ config["metadata"] = {+ **(config.get("metadata") or {}),+ "ls_provider": _LS_PROVIDER,+ "ls_model_name": self.model,+ "ls_model_type": "chat",+ }+ return config++ def _record_usage(self, response: ClassificationResponse) -> ClassificationResponse:+ """Attach TypeSafe token usage to the active run, if there is one.++ Nothing is written when tracing is disabled, and a tracing failure never fails+ an otherwise successful classification.+ """+ input_tokens = response.usage.input_tokens or 0+ output_tokens = response.usage.output_tokens or 0+ try:+ run_tree = get_current_run_tree()+ if run_tree is not None:+ run_tree.extra.setdefault("metadata", {})["usage_metadata"] = {+ "input_tokens": input_tokens,+ "output_tokens": output_tokens,+ "total_tokens": input_tokens + output_tokens,+ }+ except Exception: # noqa: BLE001 - tracing must not break classification+ logger.debug("Could not attach TypeSafe usage.", exc_info=True)+ return response@propertydef _endpoint(self) -> str:diff --git a/libs/partners/typesafe/tests/unit_tests/test_classifier.py b/libs/partners/typesafe/tests/unit_tests/test_classifier.pyindex d266396aa7..f63355a7ab 100644--- a/libs/partners/typesafe/tests/unit_tests/test_classifier.py+++ b/libs/partners/typesafe/tests/unit_tests/test_classifier.py@@ -22,6 +22,7 @@ from langchain_typesafe import (TypeSafeClassifier,__version__,)+from langchain_typesafe import classifier as classifier_modulefrom langchain_typesafe.client import (TypeSafeAPIConnectionError,TypeSafeAPIError,@@ -29,6 +30,26 @@ from langchain_typesafe.client import (TypeSafeAPITimeoutError,)++class _RunTreeStub:+ """Stand-in for the LangSmith run tree that `_record_usage` writes to."""++ def __init__(self) -> None:+ self.extra: dict[str, Any] = {}+++class _RunRecorder(BaseCallbackHandler):+ """Record the run type and metadata the classifier starts its run with."""++ def __init__(self) -> None:+ self.metadata: dict[str, Any] = {}+ self.run_type: str | None = None++ def on_chain_start(self, *_: Any, **kwargs: Any) -> None:+ self.metadata = kwargs.get("metadata") or {}+ self.run_type = kwargs.get("run_type")++API_KEY = "test-api-key"REQUEST_ID = "req_test"@@ -537,3 +558,104 @@ def test_callbacks_receive_classifier_run() -> None:assert callback.starts == 1assert callback.ends == 1client.close()+++def test_usage_is_recorded_on_the_active_run(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Token usage is written where LangSmith totals it.++ LangSmith only sums tokens for `llm` runs whose metadata carries+ `usage_metadata`, so both the payload and the run type are pinned.+ """+ stub = _RunTreeStub()+ monkeypatch.setattr(classifier_module, "get_current_run_tree", lambda: stub)++ def handler(_: httpx2.Request) -> httpx2.Response:+ return httpx2.Response(200, json=_response_payload())++ client = httpx2.Client(transport=httpx2.MockTransport(handler))+ recorder = _RunRecorder()+ classifier = TypeSafeClassifier(+ api_key=API_KEY,+ questions=_questions(),+ client=client,+ )++ classifier.invoke("hello", config={"callbacks": [recorder]})+ client.close()++ assert recorder.run_type == "llm"+ assert stub.extra["metadata"]["usage_metadata"] == {+ "input_tokens": 42,+ "output_tokens": 12,+ "total_tokens": 54,+ }+++async def test_async_usage_is_recorded_on_the_active_run(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """`ainvoke` records the same usage as `invoke`."""+ stub = _RunTreeStub()+ monkeypatch.setattr(classifier_module, "get_current_run_tree", lambda: stub)++ async def handler(_: httpx2.Request) -> httpx2.Response:+ return httpx2.Response(200, json=_response_payload())++ client = httpx2.AsyncClient(transport=httpx2.MockTransport(handler))+ classifier = TypeSafeClassifier(+ api_key=API_KEY,+ questions=_questions(),+ async_client=client,+ )++ await classifier.ainvoke("hello")+ await client.aclose()++ assert stub.extra["metadata"]["usage_metadata"]["total_tokens"] == 54+++def test_run_carries_model_identity_without_losing_caller_metadata() -> None:+ """Identity tags LangSmith prices by are added alongside caller metadata."""++ def handler(_: httpx2.Request) -> httpx2.Response:+ return httpx2.Response(200, json=_response_payload())++ client = httpx2.Client(transport=httpx2.MockTransport(handler))+ recorder = _RunRecorder()+ classifier = TypeSafeClassifier(+ api_key=API_KEY,+ questions=_questions(),+ client=client,+ )++ classifier.invoke(+ "hello",+ config={"callbacks": [recorder], "metadata": {"tenant": "acme"}},+ )+ client.close()++ assert recorder.metadata["tenant"] == "acme"+ assert recorder.metadata["ls_provider"] == "typesafe"+ assert recorder.metadata["ls_model_name"] == "jev-latest"+ assert API_KEY not in json.dumps(recorder.metadata, default=str)+++def test_untraced_invocation_is_unaffected() -> None:+ """With no tracer active, recording usage is a no-op rather than an error."""++ def handler(_: httpx2.Request) -> httpx2.Response:+ return httpx2.Response(200, json=_response_payload())++ client = httpx2.Client(transport=httpx2.MockTransport(handler))+ classifier = TypeSafeClassifier(+ api_key=API_KEY,+ questions=_questions(),+ client=client,+ )++ result = classifier.invoke("hello")+ client.close()++ assert result.usage.input_tokens == 42