The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
feat(standard-tests): forward LangSmith CI env vars to traces (#37645)
Mason Daugherty2638f75e1331
diff --git a/libs/standard-tests/langchain_tests/_langsmith_plugin.py b/libs/standard-tests/langchain_tests/_langsmith_plugin.pynew file mode 100644index 0000000000..f1c4147723--- /dev/null+++ b/libs/standard-tests/langchain_tests/_langsmith_plugin.py@@ -0,0 +1,91 @@+"""Pytest plugin that forwards CI-set LangSmith env vars into tracing context.++The LangSmith SDK does not natively read `LANGSMITH_TAGS` / `LANGSMITH_METADATA`+from the environment, so tags/metadata written to `$GITHUB_ENV` by CI workflows+(e.g. `integration_tests.yml`) would otherwise be silently dropped. This plugin+bridges that gap for the duration of the pytest session by entering+`langsmith.run_helpers.tracing_context`.++To avoid surprising developers who happen to export these vars locally, the+plugin only activates when running under GitHub Actions (`GITHUB_ACTIONS=true`).++Auto-discovered by pytest in any package that depends on `langchain-tests`+(declared via the `pytest11` entry point in+`libs/standard-tests/pyproject.toml`).+"""++from __future__ import annotations++import contextlib+import json+import os+import sys+import warnings+from typing import TYPE_CHECKING, Any++import pytest+from langsmith.run_helpers import tracing_context++if TYPE_CHECKING:+ from collections.abc import Iterator+++def _is_github_actions() -> bool:+ return os.environ.get("GITHUB_ACTIONS") == "true"+++def _parse_tags(raw: str) -> list[str]:+ return [tag.strip() for tag in raw.split(",") if tag.strip()]+++def _parse_metadata(raw: str) -> dict[str, Any] | None:+ raw = raw.strip()+ if not raw:+ return None+ try:+ parsed = json.loads(raw)+ except json.JSONDecodeError as exc:+ _warn_loud(f"Ignoring LANGSMITH_METADATA: invalid JSON ({exc}).")+ return None+ if not isinstance(parsed, dict):+ _warn_loud("Ignoring LANGSMITH_METADATA: expected a JSON object.")+ return None+ return parsed+++def _warn_loud(message: str) -> None:+ """Emit a `UserWarning` and mirror it to stderr so CI logs surface it."""+ warnings.warn(message, UserWarning, stacklevel=3)+ sys.stderr.write(f"[langchain-tests] {message}\n")+++def _langsmith_ci_cm() -> Iterator[None]:+ """Yield with tracing context applied from current env vars.++ No-op when neither env var is set or when not running on GitHub Actions.+ """+ if not _is_github_actions():+ yield+ return++ tags = _parse_tags(os.environ.get("LANGSMITH_TAGS", ""))+ metadata = _parse_metadata(os.environ.get("LANGSMITH_METADATA", ""))++ if not tags and not metadata:+ yield+ return++ keys = sorted(metadata or {})+ sys.stderr.write(+ f"[langchain-tests] langsmith CI context: tags={tags}, metadata_keys={keys}\n",+ )+ with tracing_context(tags=tags or None, metadata=metadata):+ yield++[email protected](scope="session", autouse=True)+def _langsmith_ci_context() -> Iterator[None]:+ """Apply `LANGSMITH_TAGS`/`LANGSMITH_METADATA` to traces in this session."""+ with _langsmith_ci_cm():+ yielddiff --git a/libs/standard-tests/pyproject.toml b/libs/standard-tests/pyproject.tomlindex d7c0cfa490..ba594d0c06 100644--- a/libs/standard-tests/pyproject.toml+++ b/libs/standard-tests/pyproject.toml@@ -48,6 +48,9 @@ Twitter = "https://x.com/langchain_oss"Slack = "https://www.langchain.com/join-community"Reddit = "https://www.reddit.com/r/LangChain/"+[project.entry-points.pytest11]+langsmith_ci = "langchain_tests._langsmith_plugin"+[dependency-groups]test = ["langchain-core>=1.4.0,<2.0.0"]test_integration = []diff --git a/libs/standard-tests/tests/unit_tests/test_langsmith_plugin.py b/libs/standard-tests/tests/unit_tests/test_langsmith_plugin.pynew file mode 100644index 0000000000..b17c508a1f--- /dev/null+++ b/libs/standard-tests/tests/unit_tests/test_langsmith_plugin.py@@ -0,0 +1,150 @@+"""Tests for the CI LangSmith env-var pytest plugin."""++from __future__ import annotations++from textwrap import dedent++import pytest+from langsmith.run_helpers import get_tracing_context++from langchain_tests._langsmith_plugin import (+ _langsmith_ci_cm,+ _parse_metadata,+ _parse_tags,+)++pytest_plugins = ["pytester"]++[email protected](autouse=True)+def _force_ci_env(monkeypatch: pytest.MonkeyPatch) -> None:+ """Activate the plugin's CI-gated code paths for every test."""+ monkeypatch.setenv("GITHUB_ACTIONS", "true")+++class TestParseTags:+ def test_splits_and_strips(self) -> None:+ assert _parse_tags("a, b ,c") == ["a", "b", "c"]++ def test_drops_empty_segments(self) -> None:+ assert _parse_tags(",a,,b,") == ["a", "b"]++ def test_empty_string(self) -> None:+ assert _parse_tags("") == []++ def test_whitespace_only(self) -> None:+ assert _parse_tags(" , , ") == []+++class TestParseMetadata:+ def test_valid_object(self) -> None:+ assert _parse_metadata('{"sha": "abc", "n": 1}') == {"sha": "abc", "n": 1}++ def test_empty_string_returns_none(self) -> None:+ assert _parse_metadata("") is None+ assert _parse_metadata(" ") is None++ def test_invalid_json_warns_and_returns_none(self) -> None:+ with pytest.warns(UserWarning, match="invalid JSON"):+ result = _parse_metadata("{not json")+ assert result is None++ def test_non_object_warns_and_returns_none(self) -> None:+ with pytest.warns(UserWarning, match="JSON object"):+ result = _parse_metadata('["a", "b"]')+ assert result is None+++class TestContextManager:+ def test_applies_tags_and_metadata(self, monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setenv("LANGSMITH_TAGS", "github-actions,sha-deadbeef")+ monkeypatch.setenv("LANGSMITH_METADATA", '{"github_run_id": "42"}')+ with _langsmith_ci_cm():+ ctx = get_tracing_context()+ assert ctx["tags"] == ["github-actions", "sha-deadbeef"]+ assert ctx["metadata"] == {"github_run_id": "42"}++ def test_restores_context_on_exit(self, monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setenv("LANGSMITH_TAGS", "x")+ monkeypatch.delenv("LANGSMITH_METADATA", raising=False)+ before = get_tracing_context()+ with _langsmith_ci_cm():+ pass+ after = get_tracing_context()+ assert before["tags"] == after["tags"]+ assert before["metadata"] == after["metadata"]++ def test_no_env_is_noop(self, monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.delenv("LANGSMITH_TAGS", raising=False)+ monkeypatch.delenv("LANGSMITH_METADATA", raising=False)+ before = get_tracing_context()+ with _langsmith_ci_cm():+ inside = get_tracing_context()+ assert inside["tags"] == before["tags"]+ assert inside["metadata"] == before["metadata"]++ def test_whitespace_only_tags_is_noop(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ monkeypatch.setenv("LANGSMITH_TAGS", " , , ")+ monkeypatch.delenv("LANGSMITH_METADATA", raising=False)+ before = get_tracing_context()+ with _langsmith_ci_cm():+ inside = get_tracing_context()+ assert inside["tags"] == before["tags"]++ def test_only_metadata(self, monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.delenv("LANGSMITH_TAGS", raising=False)+ monkeypatch.setenv("LANGSMITH_METADATA", '{"k": "v"}')+ with _langsmith_ci_cm():+ ctx = get_tracing_context()+ assert ctx["metadata"] == {"k": "v"}++ def test_only_tags(self, monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setenv("LANGSMITH_TAGS", "solo")+ monkeypatch.delenv("LANGSMITH_METADATA", raising=False)+ with _langsmith_ci_cm():+ ctx = get_tracing_context()+ assert ctx["tags"] == ["solo"]++ def test_bad_metadata_does_not_block_tags(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ monkeypatch.setenv("LANGSMITH_TAGS", "kept")+ monkeypatch.setenv("LANGSMITH_METADATA", "not-json")+ with pytest.warns(UserWarning, match="invalid JSON"), _langsmith_ci_cm(): # noqa: PT031+ ctx = get_tracing_context()+ assert ctx["tags"] == ["kept"]+ assert ctx["metadata"] is None++ def test_inactive_outside_github_actions(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ monkeypatch.delenv("GITHUB_ACTIONS", raising=False)+ monkeypatch.setenv("LANGSMITH_TAGS", "should-not-apply")+ before = get_tracing_context()+ with _langsmith_ci_cm():+ inside = get_tracing_context()+ assert inside["tags"] == before["tags"]+++class TestPluginDiscovery:+ """End-to-end: the `pytest11` entry point wires up the autouse fixture."""++ def test_autouse_fixture_applies_env_in_subprocess(+ self, pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ monkeypatch.setenv("GITHUB_ACTIONS", "true")+ monkeypatch.setenv("LANGSMITH_TAGS", "discovered,from-entrypoint")+ monkeypatch.delenv("LANGSMITH_METADATA", raising=False)+ pytester.makepyfile(+ dedent("""+ from langsmith.run_helpers import get_tracing_context++ def test_tags_visible_via_autouse_fixture():+ ctx = get_tracing_context()+ assert ctx["tags"] == ["discovered", "from-entrypoint"]+ """),+ )+ result = pytester.runpytest_subprocess("-q")+ result.assert_outcomes(passed=1)