Skip to content

The agent engineering platform.

fix(langchain): sanitize evaluation Git remote tags (#39254)

John Kennedy3e871a148d7b
diff --git a/libs/langchain/langchain_classic/smith/evaluation/runner_utils.py b/libs/langchain/langchain_classic/smith/evaluation/runner_utils.py
index c830a3d749..e58e31fc9a 100644
--- a/libs/langchain/langchain_classic/smith/evaluation/runner_utils.py
+++ b/libs/langchain/langchain_classic/smith/evaluation/runner_utils.py
@@ -7,6 +7,8 @@ import dataclasses
import functools
import inspect
import logging
+import re
+import unicodedata
import uuid
from collections.abc import Callable
from datetime import datetime, timezone
@@ -15,6 +17,7 @@ from typing import (
Any,
cast,
)
+from urllib.parse import urlsplit, urlunsplit
from langchain_core._api import warn_deprecated
from langchain_core.callbacks import Callbacks
@@ -69,6 +72,70 @@ MODEL_OR_CHAIN_FACTORY = (
)
MCF = Callable[[], Chain | Runnable] | BaseLanguageModel
+_GIT_REMOTE_URL_SCHEMES = {"git", "http", "https", "ssh"}
+_SCP_STYLE_GIT_REMOTE_RE = re.compile(
+ r"^(?:[^@/:\s]+@)?(?P<host>[^@/:\s]+):(?P<path>[^\s:][^\s]*)$"
+)
+
+
+def _sanitize_git_remote_url(remote_url: object) -> str | None:
+ if not isinstance(remote_url, str) or not remote_url:
+ return None
+ if any(
+ character.isspace() or unicodedata.category(character) == "Cc"
+ for character in remote_url
+ ):
+ return None
+
+ if "://" not in remote_url:
+ match = _SCP_STYLE_GIT_REMOTE_RE.fullmatch(remote_url)
+ if match is None:
+ return None
+ host = match.group("host")
+ path = match.group("path")
+ if host.lower() in _GIT_REMOTE_URL_SCHEMES or (
+ len(host) == 1
+ and host.isascii()
+ and host.isalpha()
+ and path.startswith(("/", "\\"))
+ ):
+ return None
+ return f"{host}:{path}"
+
+ try:
+ parsed = urlsplit(remote_url)
+ host = parsed.hostname
+ if (
+ parsed.scheme not in _GIT_REMOTE_URL_SCHEMES
+ or host is None
+ or "%" in host
+ or not parsed.path
+ or parsed.query
+ or parsed.fragment
+ ):
+ return None
+ port = parsed.port
+ except ValueError:
+ return None
+
+ if ":" in host:
+ host = f"[{host}]"
+ netloc = f"{host}:{port}" if port is not None else host
+ return urlunsplit((parsed.scheme, netloc, parsed.path, "", ""))
+
+
+def _format_git_tags(git_info: dict[str, Any]) -> list[str]:
+ tags = []
+ for key, value in git_info.items():
+ if key == "remote_url":
+ sanitized_remote_url = _sanitize_git_remote_url(value)
+ if sanitized_remote_url is None:
+ continue
+ tags.append(f"git:{key}={sanitized_remote_url}")
+ else:
+ tags.append(f"git:{key}={value}")
+ return tags
+
class InputFormatError(Exception):
"""Raised when the input format is invalid."""
@@ -1247,8 +1314,7 @@ class _DatasetRunContainer:
dataset_version=dataset_version,
)
tags = tags or []
- for k, v in (project.metadata.get("git") or {}).items():
- tags.append(f"git:{k}={v}")
+ tags.extend(_format_git_tags(project.metadata.get("git") or {}))
run_metadata = {"dataset_version": project.metadata["dataset_version"]}
if revision_id:
run_metadata["revision_id"] = revision_id
diff --git a/libs/langchain/tests/unit_tests/smith/evaluation/test_runner_utils.py b/libs/langchain/tests/unit_tests/smith/evaluation/test_runner_utils.py
index 6f1cf90ffc..8744bfbdd8 100644
--- a/libs/langchain/tests/unit_tests/smith/evaluation/test_runner_utils.py
+++ b/libs/langchain/tests/unit_tests/smith/evaluation/test_runner_utils.py
@@ -14,13 +14,16 @@ from langsmith.schemas import Dataset, Example
from langchain_classic.chains.transform import TransformChain
from langchain_classic.smith.evaluation.runner_utils import (
InputFormatError,
+ _format_git_tags,
_get_messages,
_get_prompt,
_run_llm,
_run_llm_or_chain,
+ _sanitize_git_remote_url,
_validate_example_inputs_for_chain,
_validate_example_inputs_for_language_model,
arun_on_dataset,
+ run_on_dataset,
)
from tests.unit_tests.llms.fake_chat_model import FakeChatModel
from tests.unit_tests.llms.fake_llm import FakeLLM
@@ -58,6 +61,74 @@ _INVALID_PROMPTS = (
)
+ ("remote_url", "expected"),
+ [
+ (
+ "https://user:[email protected]/langchain-ai/langchain.git",
+ "https://github.com/langchain-ai/langchain.git",
+ ),
+ (
+ "https://user%40example.com:token%[email protected]/org/repo.git",
+ "https://github.com/org/repo.git",
+ ),
+ (
+ "ssh://[email protected]:2222/langchain-ai/langchain.git",
+ "ssh://github.com:2222/langchain-ai/langchain.git",
+ ),
+ (
+ "[email protected]:langchain-ai/langchain.git",
+ "github.com:langchain-ai/langchain.git",
+ ),
+ (
+ "github.com:langchain-ai/langchain.git",
+ "github.com:langchain-ai/langchain.git",
+ ),
+ (
+ "https://github.com/langchain-ai/langchain.git",
+ "https://github.com/langchain-ai/langchain.git",
+ ),
+ ("https://alice%3As3cret%40github.com/org/repo.git", None),
+ ("not a remote", None),
+ ("https:user:[email protected]/org/repo.git", None),
+ ("https:/user:[email protected]/org/repo.git", None),
+ ("C:/Users/alice/private.git", None),
+ (r"C:\Users\alice\private.git", None),
+ ("https://user:token@", None),
+ ("file:///tmp/repo", None),
+ ("https://github.com/org/repo.git?token=secret", None),
+ ("https://github.com/org/repo.git\nsecret", None),
+ ("https://github.com/org/repo.git\x7fsecret", None),
+ ("https://github.com/org/repo.git\x9bsecret", None),
+ (None, None),
+ ],
+)
+def test_sanitize_git_remote_url(remote_url: object, expected: str | None) -> None:
+ assert _sanitize_git_remote_url(remote_url) == expected
+
+
+def test_format_git_tags_sanitizes_only_remote_url() -> None:
+ git_info = {
+ "remote_url": "https://user:[email protected]/langchain-ai/langchain.git",
+ "commit": "abc123",
+ "branch": None,
+ "dirty": False,
+ }
+
+ assert _format_git_tags(git_info) == [
+ "git:remote_url=https://github.com/langchain-ai/langchain.git",
+ "git:commit=abc123",
+ "git:branch=None",
+ "git:dirty=False",
+ ]
+
+
+def test_format_git_tags_omits_malformed_remote_url() -> None:
+ assert _format_git_tags({"remote_url": "not a remote", "commit": "abc123"}) == [
+ "git:commit=abc123"
+ ]
+
+
@pytest.mark.parametrize(
"inputs",
_VALID_MESSAGES,
@@ -240,6 +311,72 @@ def test_run_chat_model_all_formats(inputs: dict[str, Any]) -> None:
_run_llm(llm, inputs, mock.MagicMock())
+@freeze_time("2023-01-01")
+def test_run_on_dataset_sanitizes_git_remote_tag() -> None:
+ dataset = Dataset(
+ id=uuid.uuid4(),
+ name="test",
+ description="Test dataset",
+ owner_id="owner",
+ created_at=_CREATED_AT,
+ tenant_id=_TENANT_ID,
+ _host_url="http://localhost:1984",
+ )
+ example = Example(
+ id=uuid.uuid4(),
+ created_at=_CREATED_AT,
+ inputs={"input": "1"},
+ outputs={"output": "2"},
+ dataset_id=str(dataset.id),
+ )
+ captured_tags = []
+
+ def mock_run_chain(
+ _: Example,
+ config: dict[str, Any],
+ **__: Any,
+ ) -> dict[str, str]:
+ captured_tags.append(config["tags"])
+ return {"result": "done"}
+
+ project = mock.MagicMock()
+ project.id = "123"
+ project.metadata = {
+ "dataset_version": None,
+ "git": {
+ "remote_url": "https://user:[email protected]/langchain-ai/langchain.git",
+ "commit": "abc123",
+ },
+ }
+
+ with (
+ mock.patch.object(Client, "read_dataset", return_value=dataset),
+ mock.patch.object(Client, "list_examples", return_value=iter([example])),
+ mock.patch(
+ "langchain_classic.smith.evaluation.runner_utils._run_llm_or_chain",
+ new=mock_run_chain,
+ ),
+ mock.patch.object(Client, "create_project", return_value=project),
+ ):
+ client = Client(api_url="http://localhost:1984", api_key="123")
+ chain = mock.MagicMock()
+ chain.input_keys = ["input"]
+ run_on_dataset(
+ dataset_name="test",
+ llm_or_chain_factory=lambda: chain,
+ concurrency_level=0,
+ project_name="test_project",
+ client=client,
+ )
+
+ assert captured_tags == [
+ [
+ "git:remote_url=https://github.com/langchain-ai/langchain.git",
+ "git:commit=abc123",
+ ]
+ ]
+
+
@freeze_time("2023-01-01")
async def test_arun_on_dataset() -> None:
dataset = Dataset(
@@ -302,16 +439,27 @@ async def test_arun_on_dataset() -> None:
def mock_list_examples(*_: Any, **__: Any) -> Iterator[Example]:
return iter(examples)
+ captured_tags = []
+
async def mock_arun_chain(
example: Example,
+ config: dict[str, Any],
*_: Any,
**__: Any,
) -> dict[str, Any]:
+ captured_tags.append(config["tags"])
return {"result": f"Result for example {example.id}"}
def mock_create_project(*_: Any, **__: Any) -> Any:
proj = mock.MagicMock()
proj.id = "123"
+ proj.metadata = {
+ "dataset_version": None,
+ "git": {
+ "remote_url": "https://user:[email protected]/langchain-ai/langchain.git",
+ "commit": "abc123",
+ },
+ }
return proj
with (
@@ -352,3 +500,9 @@ async def test_arun_on_dataset() -> None:
for example in examples
}
assert results["results"] == expected
+ assert captured_tags == [
+ [
+ "git:remote_url=https://github.com/langchain-ai/langchain.git",
+ "git:commit=abc123",
+ ]
+ ] * len(examples)