Skip to content

The agent engineering platform.

feat(standard-tests): forward LangSmith CI env vars to traces (#37645)

Mason Daugherty2638f75e1331
diff --git a/libs/standard-tests/langchain_tests/_langsmith_plugin.py b/libs/standard-tests/langchain_tests/_langsmith_plugin.py
new file mode 100644
index 0000000000..f1c4147723
--- /dev/null
+++ b/libs/standard-tests/langchain_tests/_langsmith_plugin.py
@@ -0,0 +1,91 @@
+"""Pytest plugin that forwards CI-set LangSmith env vars into tracing context.
+
+The LangSmith SDK does not natively read `LANGSMITH_TAGS` / `LANGSMITH_METADATA`
+from the environment, so tags/metadata written to `$GITHUB_ENV` by CI workflows
+(e.g. `integration_tests.yml`) would otherwise be silently dropped. This plugin
+bridges that gap for the duration of the pytest session by entering
+`langsmith.run_helpers.tracing_context`.
+
+To avoid surprising developers who happen to export these vars locally, the
+plugin only activates when running under GitHub Actions (`GITHUB_ACTIONS=true`).
+
+Auto-discovered by pytest in any package that depends on `langchain-tests`
+(declared via the `pytest11` entry point in
+`libs/standard-tests/pyproject.toml`).
+"""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import os
+import sys
+import warnings
+from typing import TYPE_CHECKING, Any
+
+import pytest
+from langsmith.run_helpers import tracing_context
+
+if TYPE_CHECKING:
+ from collections.abc import Iterator
+
+
+def _is_github_actions() -> bool:
+ return os.environ.get("GITHUB_ACTIONS") == "true"
+
+
+def _parse_tags(raw: str) -> list[str]:
+ return [tag.strip() for tag in raw.split(",") if tag.strip()]
+
+
+def _parse_metadata(raw: str) -> dict[str, Any] | None:
+ raw = raw.strip()
+ if not raw:
+ return None
+ try:
+ parsed = json.loads(raw)
+ except json.JSONDecodeError as exc:
+ _warn_loud(f"Ignoring LANGSMITH_METADATA: invalid JSON ({exc}).")
+ return None
+ if not isinstance(parsed, dict):
+ _warn_loud("Ignoring LANGSMITH_METADATA: expected a JSON object.")
+ return None
+ return parsed
+
+
+def _warn_loud(message: str) -> None:
+ """Emit a `UserWarning` and mirror it to stderr so CI logs surface it."""
+ warnings.warn(message, UserWarning, stacklevel=3)
+ sys.stderr.write(f"[langchain-tests] {message}\n")
+
+
+def _langsmith_ci_cm() -> Iterator[None]:
+ """Yield with tracing context applied from current env vars.
+
+ No-op when neither env var is set or when not running on GitHub Actions.
+ """
+ if not _is_github_actions():
+ yield
+ return
+
+ tags = _parse_tags(os.environ.get("LANGSMITH_TAGS", ""))
+ metadata = _parse_metadata(os.environ.get("LANGSMITH_METADATA", ""))
+
+ if not tags and not metadata:
+ yield
+ return
+
+ keys = sorted(metadata or {})
+ sys.stderr.write(
+ f"[langchain-tests] langsmith CI context: tags={tags}, metadata_keys={keys}\n",
+ )
+ with tracing_context(tags=tags or None, metadata=metadata):
+ yield
+
+
[email protected](scope="session", autouse=True)
+def _langsmith_ci_context() -> Iterator[None]:
+ """Apply `LANGSMITH_TAGS`/`LANGSMITH_METADATA` to traces in this session."""
+ with _langsmith_ci_cm():
+ yield
diff --git a/libs/standard-tests/pyproject.toml b/libs/standard-tests/pyproject.toml
index d7c0cfa490..ba594d0c06 100644
--- a/libs/standard-tests/pyproject.toml
+++ b/libs/standard-tests/pyproject.toml
@@ -48,6 +48,9 @@ Twitter = "https://x.com/langchain_oss"
Slack = "https://www.langchain.com/join-community"
Reddit = "https://www.reddit.com/r/LangChain/"
+[project.entry-points.pytest11]
+langsmith_ci = "langchain_tests._langsmith_plugin"
+
[dependency-groups]
test = ["langchain-core>=1.4.0,<2.0.0"]
test_integration = []
diff --git a/libs/standard-tests/tests/unit_tests/test_langsmith_plugin.py b/libs/standard-tests/tests/unit_tests/test_langsmith_plugin.py
new file mode 100644
index 0000000000..b17c508a1f
--- /dev/null
+++ b/libs/standard-tests/tests/unit_tests/test_langsmith_plugin.py
@@ -0,0 +1,150 @@
+"""Tests for the CI LangSmith env-var pytest plugin."""
+
+from __future__ import annotations
+
+from textwrap import dedent
+
+import pytest
+from langsmith.run_helpers import get_tracing_context
+
+from langchain_tests._langsmith_plugin import (
+ _langsmith_ci_cm,
+ _parse_metadata,
+ _parse_tags,
+)
+
+pytest_plugins = ["pytester"]
+
+
[email protected](autouse=True)
+def _force_ci_env(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Activate the plugin's CI-gated code paths for every test."""
+ monkeypatch.setenv("GITHUB_ACTIONS", "true")
+
+
+class TestParseTags:
+ def test_splits_and_strips(self) -> None:
+ assert _parse_tags("a, b ,c") == ["a", "b", "c"]
+
+ def test_drops_empty_segments(self) -> None:
+ assert _parse_tags(",a,,b,") == ["a", "b"]
+
+ def test_empty_string(self) -> None:
+ assert _parse_tags("") == []
+
+ def test_whitespace_only(self) -> None:
+ assert _parse_tags(" , , ") == []
+
+
+class TestParseMetadata:
+ def test_valid_object(self) -> None:
+ assert _parse_metadata('{"sha": "abc", "n": 1}') == {"sha": "abc", "n": 1}
+
+ def test_empty_string_returns_none(self) -> None:
+ assert _parse_metadata("") is None
+ assert _parse_metadata(" ") is None
+
+ def test_invalid_json_warns_and_returns_none(self) -> None:
+ with pytest.warns(UserWarning, match="invalid JSON"):
+ result = _parse_metadata("{not json")
+ assert result is None
+
+ def test_non_object_warns_and_returns_none(self) -> None:
+ with pytest.warns(UserWarning, match="JSON object"):
+ result = _parse_metadata('["a", "b"]')
+ assert result is None
+
+
+class TestContextManager:
+ def test_applies_tags_and_metadata(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv("LANGSMITH_TAGS", "github-actions,sha-deadbeef")
+ monkeypatch.setenv("LANGSMITH_METADATA", '{"github_run_id": "42"}')
+ with _langsmith_ci_cm():
+ ctx = get_tracing_context()
+ assert ctx["tags"] == ["github-actions", "sha-deadbeef"]
+ assert ctx["metadata"] == {"github_run_id": "42"}
+
+ def test_restores_context_on_exit(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv("LANGSMITH_TAGS", "x")
+ monkeypatch.delenv("LANGSMITH_METADATA", raising=False)
+ before = get_tracing_context()
+ with _langsmith_ci_cm():
+ pass
+ after = get_tracing_context()
+ assert before["tags"] == after["tags"]
+ assert before["metadata"] == after["metadata"]
+
+ def test_no_env_is_noop(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.delenv("LANGSMITH_TAGS", raising=False)
+ monkeypatch.delenv("LANGSMITH_METADATA", raising=False)
+ before = get_tracing_context()
+ with _langsmith_ci_cm():
+ inside = get_tracing_context()
+ assert inside["tags"] == before["tags"]
+ assert inside["metadata"] == before["metadata"]
+
+ def test_whitespace_only_tags_is_noop(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ monkeypatch.setenv("LANGSMITH_TAGS", " , , ")
+ monkeypatch.delenv("LANGSMITH_METADATA", raising=False)
+ before = get_tracing_context()
+ with _langsmith_ci_cm():
+ inside = get_tracing_context()
+ assert inside["tags"] == before["tags"]
+
+ def test_only_metadata(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.delenv("LANGSMITH_TAGS", raising=False)
+ monkeypatch.setenv("LANGSMITH_METADATA", '{"k": "v"}')
+ with _langsmith_ci_cm():
+ ctx = get_tracing_context()
+ assert ctx["metadata"] == {"k": "v"}
+
+ def test_only_tags(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv("LANGSMITH_TAGS", "solo")
+ monkeypatch.delenv("LANGSMITH_METADATA", raising=False)
+ with _langsmith_ci_cm():
+ ctx = get_tracing_context()
+ assert ctx["tags"] == ["solo"]
+
+ def test_bad_metadata_does_not_block_tags(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ monkeypatch.setenv("LANGSMITH_TAGS", "kept")
+ monkeypatch.setenv("LANGSMITH_METADATA", "not-json")
+ with pytest.warns(UserWarning, match="invalid JSON"), _langsmith_ci_cm(): # noqa: PT031
+ ctx = get_tracing_context()
+ assert ctx["tags"] == ["kept"]
+ assert ctx["metadata"] is None
+
+ def test_inactive_outside_github_actions(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ monkeypatch.delenv("GITHUB_ACTIONS", raising=False)
+ monkeypatch.setenv("LANGSMITH_TAGS", "should-not-apply")
+ before = get_tracing_context()
+ with _langsmith_ci_cm():
+ inside = get_tracing_context()
+ assert inside["tags"] == before["tags"]
+
+
+class TestPluginDiscovery:
+ """End-to-end: the `pytest11` entry point wires up the autouse fixture."""
+
+ def test_autouse_fixture_applies_env_in_subprocess(
+ self, pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ monkeypatch.setenv("GITHUB_ACTIONS", "true")
+ monkeypatch.setenv("LANGSMITH_TAGS", "discovered,from-entrypoint")
+ monkeypatch.delenv("LANGSMITH_METADATA", raising=False)
+ pytester.makepyfile(
+ dedent("""
+ from langsmith.run_helpers import get_tracing_context
+
+ def test_tags_visible_via_autouse_fixture():
+ ctx = get_tracing_context()
+ assert ctx["tags"] == ["discovered", "from-entrypoint"]
+ """),
+ )
+ result = pytester.runpytest_subprocess("-q")
+ result.assert_outcomes(passed=1)