The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
fix(langchain): sanitize evaluation Git remote tags (#39254)
John Kennedy3e871a148d7b
diff --git a/libs/langchain/langchain_classic/smith/evaluation/runner_utils.py b/libs/langchain/langchain_classic/smith/evaluation/runner_utils.pyindex c830a3d749..e58e31fc9a 100644--- a/libs/langchain/langchain_classic/smith/evaluation/runner_utils.py+++ b/libs/langchain/langchain_classic/smith/evaluation/runner_utils.py@@ -7,6 +7,8 @@ import dataclassesimport functoolsimport inspectimport logging+import re+import unicodedataimport uuidfrom collections.abc import Callablefrom datetime import datetime, timezone@@ -15,6 +17,7 @@ from typing import (Any,cast,)+from urllib.parse import urlsplit, urlunsplitfrom langchain_core._api import warn_deprecatedfrom langchain_core.callbacks import Callbacks@@ -69,6 +72,70 @@ MODEL_OR_CHAIN_FACTORY = ()MCF = Callable[[], Chain | Runnable] | BaseLanguageModel+_GIT_REMOTE_URL_SCHEMES = {"git", "http", "https", "ssh"}+_SCP_STYLE_GIT_REMOTE_RE = re.compile(+ r"^(?:[^@/:\s]+@)?(?P<host>[^@/:\s]+):(?P<path>[^\s:][^\s]*)$"+)+++def _sanitize_git_remote_url(remote_url: object) -> str | None:+ if not isinstance(remote_url, str) or not remote_url:+ return None+ if any(+ character.isspace() or unicodedata.category(character) == "Cc"+ for character in remote_url+ ):+ return None++ if "://" not in remote_url:+ match = _SCP_STYLE_GIT_REMOTE_RE.fullmatch(remote_url)+ if match is None:+ return None+ host = match.group("host")+ path = match.group("path")+ if host.lower() in _GIT_REMOTE_URL_SCHEMES or (+ len(host) == 1+ and host.isascii()+ and host.isalpha()+ and path.startswith(("/", "\\"))+ ):+ return None+ return f"{host}:{path}"++ try:+ parsed = urlsplit(remote_url)+ host = parsed.hostname+ if (+ parsed.scheme not in _GIT_REMOTE_URL_SCHEMES+ or host is None+ or "%" in host+ or not parsed.path+ or parsed.query+ or parsed.fragment+ ):+ return None+ port = parsed.port+ except ValueError:+ return None++ if ":" in host:+ host = f"[{host}]"+ netloc = f"{host}:{port}" if port is not None else host+ return urlunsplit((parsed.scheme, netloc, parsed.path, "", ""))+++def _format_git_tags(git_info: dict[str, Any]) -> list[str]:+ tags = []+ for key, value in git_info.items():+ if key == "remote_url":+ sanitized_remote_url = _sanitize_git_remote_url(value)+ if sanitized_remote_url is None:+ continue+ tags.append(f"git:{key}={sanitized_remote_url}")+ else:+ tags.append(f"git:{key}={value}")+ return tags+class InputFormatError(Exception):"""Raised when the input format is invalid."""@@ -1247,8 +1314,7 @@ class _DatasetRunContainer:dataset_version=dataset_version,)tags = tags or []- for k, v in (project.metadata.get("git") or {}).items():- tags.append(f"git:{k}={v}")+ tags.extend(_format_git_tags(project.metadata.get("git") or {}))run_metadata = {"dataset_version": project.metadata["dataset_version"]}if revision_id:run_metadata["revision_id"] = revision_iddiff --git a/libs/langchain/tests/unit_tests/smith/evaluation/test_runner_utils.py b/libs/langchain/tests/unit_tests/smith/evaluation/test_runner_utils.pyindex 6f1cf90ffc..8744bfbdd8 100644--- a/libs/langchain/tests/unit_tests/smith/evaluation/test_runner_utils.py+++ b/libs/langchain/tests/unit_tests/smith/evaluation/test_runner_utils.py@@ -14,13 +14,16 @@ from langsmith.schemas import Dataset, Examplefrom langchain_classic.chains.transform import TransformChainfrom langchain_classic.smith.evaluation.runner_utils import (InputFormatError,+ _format_git_tags,_get_messages,_get_prompt,_run_llm,_run_llm_or_chain,+ _sanitize_git_remote_url,_validate_example_inputs_for_chain,_validate_example_inputs_for_language_model,arun_on_dataset,+ run_on_dataset,)from tests.unit_tests.llms.fake_chat_model import FakeChatModelfrom tests.unit_tests.llms.fake_llm import FakeLLM@@ -58,6 +61,74 @@ _INVALID_PROMPTS = ()+ ("remote_url", "expected"),+ [+ (+ "https://user:[email protected]/langchain-ai/langchain.git",+ "https://github.com/langchain-ai/langchain.git",+ ),+ (+ "https://user%40example.com:token%[email protected]/org/repo.git",+ "https://github.com/org/repo.git",+ ),+ (+ "ssh://[email protected]:2222/langchain-ai/langchain.git",+ "ssh://github.com:2222/langchain-ai/langchain.git",+ ),+ (+ "[email protected]:langchain-ai/langchain.git",+ "github.com:langchain-ai/langchain.git",+ ),+ (+ "github.com:langchain-ai/langchain.git",+ "github.com:langchain-ai/langchain.git",+ ),+ (+ "https://github.com/langchain-ai/langchain.git",+ "https://github.com/langchain-ai/langchain.git",+ ),+ ("https://alice%3As3cret%40github.com/org/repo.git", None),+ ("not a remote", None),+ ("https:user:[email protected]/org/repo.git", None),+ ("https:/user:[email protected]/org/repo.git", None),+ ("C:/Users/alice/private.git", None),+ (r"C:\Users\alice\private.git", None),+ ("https://user:token@", None),+ ("file:///tmp/repo", None),+ ("https://github.com/org/repo.git?token=secret", None),+ ("https://github.com/org/repo.git\nsecret", None),+ ("https://github.com/org/repo.git\x7fsecret", None),+ ("https://github.com/org/repo.git\x9bsecret", None),+ (None, None),+ ],+)+def test_sanitize_git_remote_url(remote_url: object, expected: str | None) -> None:+ assert _sanitize_git_remote_url(remote_url) == expected+++def test_format_git_tags_sanitizes_only_remote_url() -> None:+ git_info = {+ "remote_url": "https://user:[email protected]/langchain-ai/langchain.git",+ "commit": "abc123",+ "branch": None,+ "dirty": False,+ }++ assert _format_git_tags(git_info) == [+ "git:remote_url=https://github.com/langchain-ai/langchain.git",+ "git:commit=abc123",+ "git:branch=None",+ "git:dirty=False",+ ]+++def test_format_git_tags_omits_malformed_remote_url() -> None:+ assert _format_git_tags({"remote_url": "not a remote", "commit": "abc123"}) == [+ "git:commit=abc123"+ ]++@pytest.mark.parametrize("inputs",_VALID_MESSAGES,@@ -240,6 +311,72 @@ def test_run_chat_model_all_formats(inputs: dict[str, Any]) -> None:_run_llm(llm, inputs, mock.MagicMock())+@freeze_time("2023-01-01")+def test_run_on_dataset_sanitizes_git_remote_tag() -> None:+ dataset = Dataset(+ id=uuid.uuid4(),+ name="test",+ description="Test dataset",+ owner_id="owner",+ created_at=_CREATED_AT,+ tenant_id=_TENANT_ID,+ _host_url="http://localhost:1984",+ )+ example = Example(+ id=uuid.uuid4(),+ created_at=_CREATED_AT,+ inputs={"input": "1"},+ outputs={"output": "2"},+ dataset_id=str(dataset.id),+ )+ captured_tags = []++ def mock_run_chain(+ _: Example,+ config: dict[str, Any],+ **__: Any,+ ) -> dict[str, str]:+ captured_tags.append(config["tags"])+ return {"result": "done"}++ project = mock.MagicMock()+ project.id = "123"+ project.metadata = {+ "dataset_version": None,+ "git": {+ "remote_url": "https://user:[email protected]/langchain-ai/langchain.git",+ "commit": "abc123",+ },+ }++ with (+ mock.patch.object(Client, "read_dataset", return_value=dataset),+ mock.patch.object(Client, "list_examples", return_value=iter([example])),+ mock.patch(+ "langchain_classic.smith.evaluation.runner_utils._run_llm_or_chain",+ new=mock_run_chain,+ ),+ mock.patch.object(Client, "create_project", return_value=project),+ ):+ client = Client(api_url="http://localhost:1984", api_key="123")+ chain = mock.MagicMock()+ chain.input_keys = ["input"]+ run_on_dataset(+ dataset_name="test",+ llm_or_chain_factory=lambda: chain,+ concurrency_level=0,+ project_name="test_project",+ client=client,+ )++ assert captured_tags == [+ [+ "git:remote_url=https://github.com/langchain-ai/langchain.git",+ "git:commit=abc123",+ ]+ ]++@freeze_time("2023-01-01")async def test_arun_on_dataset() -> None:dataset = Dataset(@@ -302,16 +439,27 @@ async def test_arun_on_dataset() -> None:def mock_list_examples(*_: Any, **__: Any) -> Iterator[Example]:return iter(examples)+ captured_tags = []+async def mock_arun_chain(example: Example,+ config: dict[str, Any],*_: Any,**__: Any,) -> dict[str, Any]:+ captured_tags.append(config["tags"])return {"result": f"Result for example {example.id}"}def mock_create_project(*_: Any, **__: Any) -> Any:proj = mock.MagicMock()proj.id = "123"+ proj.metadata = {+ "dataset_version": None,+ "git": {+ "remote_url": "https://user:[email protected]/langchain-ai/langchain.git",+ "commit": "abc123",+ },+ }return projwith (@@ -352,3 +500,9 @@ async def test_arun_on_dataset() -> None:for example in examples}assert results["results"] == expected+ assert captured_tags == [+ [+ "git:remote_url=https://github.com/langchain-ai/langchain.git",+ "git:commit=abc123",+ ]+ ] * len(examples)