Skip to content

The agent engineering platform.

feat(model-profiles): plain-English summary for profile refresh PRs (#38218)

Mason Daughertyf7e87f7ab8a4
diff --git a/.github/workflows/_refresh_model_profiles.yml b/.github/workflows/_refresh_model_profiles.yml
index cc5abbe949..33bae64d8a 100644
--- a/.github/workflows/_refresh_model_profiles.yml
+++ b/.github/workflows/_refresh_model_profiles.yml
@@ -167,6 +167,32 @@ jobs:
exit 1
fi
+ - name: "πŸ“ Build PR body with change summary"
+ id: pr-body
+ env:
+ PROVIDERS_JSON: ${{ inputs.providers }}
+ PR_BODY: ${{ inputs.pr-body }}
+ run: |
+ # The refresh step modified the working tree without committing, so
+ # comparing against HEAD yields exactly the refresh's changes.
+ cli_dir="${GITHUB_WORKSPACE}/${{ steps.cli.outputs.dir }}"
+ body_file="${RUNNER_TEMP}/pr_body.md"
+ printf '%s\n\n' "${PR_BODY}" > "${body_file}"
+ # `summarize` builds the whole summary in memory and prints it once,
+ # so a failure exits non-zero before any stdout reaches the append β€”
+ # the body keeps only the static note, never a half-written summary.
+ if ! uv run --frozen --project "${cli_dir}" \
+ langchain-profiles summarize \
+ --providers "${PROVIDERS_JSON}" \
+ --base-ref HEAD \
+ --repo-root "${GITHUB_WORKSPACE}" >> "${body_file}"; then
+ echo "::warning::Could not generate change summary; see job log."
+ # Surface the degradation in the PR body too: the warning above only
+ # lands in the Actions log, which a PR reviewer won't see.
+ printf '\n> [!NOTE]\n> Automated change summary unavailable β€” see the workflow run log.\n' >> "${body_file}"
+ fi
+ echo "path=${body_file}" >> "$GITHUB_OUTPUT"
+
- name: "πŸ”‘ Generate GitHub App token"
id: app-token
uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3
@@ -182,7 +208,7 @@ jobs:
branch: ${{ inputs.pr-branch }}
commit-message: ${{ inputs.pr-title }}
title: ${{ inputs.pr-title }}
- body: ${{ inputs.pr-body }}
+ body-path: ${{ steps.pr-body.outputs.path }}
labels: ${{ inputs.pr-labels }}
add-paths: ${{ inputs.add-paths }}
diff --git a/libs/model-profiles/langchain_model_profiles/_summary.py b/libs/model-profiles/langchain_model_profiles/_summary.py
new file mode 100644
index 0000000000..391a1f384e
--- /dev/null
+++ b/libs/model-profiles/langchain_model_profiles/_summary.py
@@ -0,0 +1,475 @@
+"""Generate a plain-English summary of model profile changes.
+
+The `refresh_model_profiles` workflow opens an automated PR whenever the data
+behind `_profiles.py` files changes. Those diffs are large blocks of generated
+data, so a reviewer otherwise has to open *Files changed* and eyeball raw values
+to learn what actually moved. This module turns the structured before/after data
+into a skimmable Markdown summary (new models, removed models, and per-field
+capability/metadata changes) for the PR body. The summary is generated
+deterministically from the data, so there is no risk of an LLM misdescribing it.
+"""
+
+from __future__ import annotations
+
+import ast
+import subprocess
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, NamedTuple, TypedDict
+
+if TYPE_CHECKING:
+ from collections.abc import Mapping
+
+ from langchain_core.language_models.model_profile import (
+ ModelProfile,
+ ModelProfileRegistry,
+ )
+
+# Maximum number of bullet rows rendered per section before truncating.
+_MAX_ROWS = 25
+
+# Human-readable labels for profile fields.
+_FIELD_LABELS: dict[str, str] = {
+ "name": "display name",
+ "status": "status",
+ "release_date": "release date",
+ "last_updated": "last updated",
+ "open_weights": "open weights",
+ "max_input_tokens": "max input tokens",
+ "max_output_tokens": "max output tokens",
+ "text_inputs": "text input",
+ "image_inputs": "image input",
+ "audio_inputs": "audio input",
+ "pdf_inputs": "PDF input",
+ "video_inputs": "video input",
+ "text_outputs": "text output",
+ "image_outputs": "image output",
+ "audio_outputs": "audio output",
+ "video_outputs": "video output",
+ "reasoning_output": "reasoning",
+ "tool_calling": "tool calling",
+ "tool_choice": "tool choice",
+ "tool_call_streaming": "tool call streaming",
+ "structured_output": "structured output",
+ "attachment": "attachments",
+ "temperature": "temperature control",
+ "image_url_inputs": "image URL input",
+ "image_tool_message": "image tool messages",
+ "pdf_tool_message": "PDF tool messages",
+}
+
+# Token fields rendered with thousands separators.
+_TOKEN_FIELDS = frozenset({"max_input_tokens", "max_output_tokens"})
+
+
+class ProfileParseError(ValueError):
+ """A `_profiles.py` source exists but its `_PROFILES` data is unparseable.
+
+ Distinguished from a genuinely absent file (which yields an empty mapping)
+ so that corrupt working-tree or committed data surfaces as an error rather
+ than being silently diffed as a mass addition or removal of models.
+ """
+
+
+class FieldChange(NamedTuple):
+ """Old and new values for a single changed profile field.
+
+ Named rather than a bare `tuple` so the old β†’ new ordering the renderer
+ relies on is part of the type instead of a positional convention. Values are
+ heterogeneous profile data (bool, int, str, or unset), so `Any` is the
+ honest element type here.
+ """
+
+ old: Any
+ """Value before the refresh, or `None` if absent/unset."""
+
+ new: Any
+ """Value after the refresh, or `None` if absent/unset."""
+
+
+class ProviderEntry(TypedDict):
+ """One provider's identity and the data dir holding its `_profiles.py`."""
+
+ provider: str
+ """Provider identifier (e.g. `'openai'`)."""
+
+ data_dir: str
+ """Path to the provider's data directory, relative to the repo root."""
+
+
+@dataclass
+class ProfileDiff:
+ """Structured difference between two sets of model profiles."""
+
+ added: list[str] = field(default_factory=list)
+ """Model IDs present after the refresh but not before, sorted."""
+
+ removed: list[str] = field(default_factory=list)
+ """Model IDs present before the refresh but not after, sorted."""
+
+ changed: dict[str, dict[str, FieldChange]] = field(default_factory=dict)
+ """Per-model field changes, keyed by model ID then field name."""
+
+ added_profiles: ModelProfileRegistry = field(default_factory=dict)
+ """Full profiles for each added model, keyed by model ID."""
+
+ @property
+ def is_empty(self) -> bool:
+ """Whether there are no model additions, removals, or field changes."""
+ return not (self.added or self.removed or self.changed)
+
+
+def extract_profiles(source: str) -> ModelProfileRegistry:
+ """Extract the `_PROFILES` mapping from `_profiles.py` source.
+
+ Uses `ast.literal_eval` rather than importing/executing the module so the
+ generated data file is never run as code.
+
+ Args:
+ source: Contents of a `_profiles.py` module. An empty string (e.g. a
+ file absent at a git ref) yields an empty mapping.
+
+ Returns:
+ The `_PROFILES` mapping, or an empty dict when the source contains no
+ `_PROFILES` assignment.
+
+ Raises:
+ ProfileParseError: If the source is present but cannot be parsed, or its
+ `_PROFILES` value is not a dict literal. Surfacing this rather than
+ returning `{}` prevents a corrupt file from being misreported as
+ every model added or removed.
+ """
+ try:
+ tree = ast.parse(source)
+ except SyntaxError as e:
+ msg = f"Could not parse profile source as Python: {e}"
+ raise ProfileParseError(msg) from e
+
+ for node in tree.body:
+ if isinstance(node, ast.AnnAssign):
+ targets: list[ast.expr] = [node.target]
+ elif isinstance(node, ast.Assign):
+ targets = list(node.targets)
+ else:
+ continue
+ is_profiles = any(
+ isinstance(t, ast.Name) and t.id == "_PROFILES" for t in targets
+ )
+ if is_profiles and node.value is not None:
+ try:
+ value = ast.literal_eval(node.value)
+ except (ValueError, SyntaxError) as e:
+ msg = f"`_PROFILES` is not a literal expression: {e}"
+ raise ProfileParseError(msg) from e
+ if not isinstance(value, dict):
+ msg = f"`_PROFILES` is not a dict (got {type(value).__name__})"
+ raise ProfileParseError(msg)
+ return value
+ return {}
+
+
+def diff_profiles(old: ModelProfileRegistry, new: ModelProfileRegistry) -> ProfileDiff:
+ """Compute the difference between two `_PROFILES` mappings.
+
+ Args:
+ old: Profiles before the refresh.
+ new: Profiles after the refresh.
+
+ Returns:
+ A `ProfileDiff` describing added, removed, and changed models.
+ """
+ added = sorted(set(new) - set(old))
+ removed = sorted(set(old) - set(new))
+
+ changed: dict[str, dict[str, FieldChange]] = {}
+ for model_id in sorted(set(old) & set(new)):
+ # View profiles as plain mappings so we can iterate dynamic keys (the
+ # `ModelProfile` TypedDict only permits literal-key access).
+ old_profile: Mapping[str, Any] = old[model_id]
+ new_profile: Mapping[str, Any] = new[model_id]
+ fields: dict[str, FieldChange] = {}
+ for key in sorted(set(old_profile) | set(new_profile)):
+ old_val = old_profile.get(key)
+ new_val = new_profile.get(key)
+ if old_val != new_val:
+ fields[key] = FieldChange(old_val, new_val)
+ if fields:
+ changed[model_id] = fields
+
+ added_profiles = {model_id: new[model_id] for model_id in added}
+ return ProfileDiff(
+ added=added,
+ removed=removed,
+ changed=changed,
+ added_profiles=added_profiles,
+ )
+
+
+def _format_value(field_name: str, value: Any) -> str: # noqa: ANN401
+ """Render a single field value for display."""
+ if value is None:
+ return "unset"
+ if isinstance(value, bool):
+ return "yes" if value else "no"
+ if isinstance(value, int) and field_name in _TOKEN_FIELDS:
+ return f"{value:,}"
+ return f"`{value}`" if isinstance(value, str) else str(value)
+
+
+def _describe_field_change(
+ field_name: str,
+ old_val: Any, # noqa: ANN401
+ new_val: Any, # noqa: ANN401
+) -> str:
+ """Produce a plain-English phrase for one field change."""
+ label = _FIELD_LABELS.get(field_name, field_name)
+ if isinstance(old_val, bool) or isinstance(new_val, bool):
+ if new_val and not old_val:
+ return f"added {label}"
+ if old_val and not new_val:
+ return f"removed {label}"
+ old_str = _format_value(field_name, old_val)
+ new_str = _format_value(field_name, new_val)
+ return f"{label} {old_str} β†’ {new_str}"
+
+
+def _describe_new_model(profile: ModelProfile) -> str:
+ """Produce a short descriptor for a newly added model."""
+ parts: list[str] = []
+ context = profile.get("max_input_tokens")
+ if context:
+ parts.append(f"{context:,} ctx")
+ output = profile.get("max_output_tokens")
+ if output:
+ parts.append(f"{output:,} out")
+ modalities = [
+ name
+ for key, name in (
+ ("image_inputs", "image"),
+ ("audio_inputs", "audio"),
+ ("video_inputs", "video"),
+ ("pdf_inputs", "pdf"),
+ )
+ if profile.get(key)
+ ]
+ if modalities:
+ parts.append("text+" + "+".join(modalities) + " in")
+ if profile.get("reasoning_output"):
+ parts.append("reasoning")
+ if profile.get("tool_calling"):
+ parts.append("tools")
+ return ", ".join(parts)
+
+
+def _truncate(rows: list[str]) -> list[str]:
+ """Cap a list of bullet rows, appending an ellipsis row when truncated."""
+ if len(rows) <= _MAX_ROWS:
+ return rows
+ hidden = len(rows) - _MAX_ROWS
+ return [*rows[:_MAX_ROWS], f"- …and {hidden} more"]
+
+
+def render_provider_section(provider: str, diff: ProfileDiff) -> str | None:
+ """Render the Markdown section for a single provider, or None if unchanged.
+
+ Args:
+ provider: Provider identifier (e.g. `'openai'`).
+ diff: The computed `ProfileDiff` for the provider.
+
+ Returns:
+ Markdown for the provider's changes, or `None` when there are none.
+ """
+ if diff.is_empty:
+ return None
+
+ lines = [f"### {provider}"]
+
+ if diff.added:
+ lines.append(f"\n**βž• {len(diff.added)} added**") # noqa: RUF001
+ rows = []
+ for model_id in diff.added:
+ descriptor = _describe_new_model(diff.added_profiles[model_id])
+ suffix = f" β€” {descriptor}" if descriptor else ""
+ rows.append(f"- `{model_id}`{suffix}")
+ lines.extend(_truncate(rows))
+
+ if diff.removed:
+ lines.append(f"\n**βž– {len(diff.removed)} removed**") # noqa: RUF001
+ lines.extend(_truncate([f"- `{m}`" for m in diff.removed]))
+
+ if diff.changed:
+ lines.append(f"\n**✏️ {len(diff.changed)} changed**")
+ rows = []
+ for model_id, fields in diff.changed.items():
+ phrases = [
+ _describe_field_change(name, change.old, change.new)
+ for name, change in fields.items()
+ ]
+ rows.append(f"- `{model_id}`: " + "; ".join(phrases))
+ lines.extend(_truncate(rows))
+
+ return "\n".join(lines)
+
+
+def build_summary(provider_diffs: dict[str, ProfileDiff]) -> str:
+ """Assemble the full Markdown summary across all providers.
+
+ Args:
+ provider_diffs: Mapping of provider name to its `ProfileDiff`.
+
+ Returns:
+ Markdown summary. When nothing changed, a short note is returned.
+ """
+ sections = [
+ section
+ for provider in sorted(provider_diffs)
+ if (section := render_provider_section(provider, provider_diffs[provider]))
+ ]
+ if not sections:
+ return "No model profile data changed."
+
+ total_added = sum(len(d.added) for d in provider_diffs.values())
+ total_removed = sum(len(d.removed) for d in provider_diffs.values())
+ total_changed = sum(len(d.changed) for d in provider_diffs.values())
+ headline = (
+ f"**{total_added} added Β· {total_removed} removed Β· "
+ f"{total_changed} changed** across {len(sections)} provider(s)."
+ )
+
+ return "\n\n".join(["## Summary of changes", headline, *sections])
+
+
+def _verify_ref(repo_root: Path, ref: str) -> None:
+ """Confirm `ref` resolves to a commit in `repo_root`.
+
+ Validating once up front lets `_git_show` treat a non-zero exit
+ unambiguously as "path absent at this ref", rather than conflating a typo'd
+ ref, an unfetched ref, or a non-repository root with a genuinely new file β€”
+ which would otherwise render every existing model as newly added.
+
+ Raises:
+ RuntimeError: If git is unavailable, `repo_root` is not a repository, or
+ `ref` cannot be resolved.
+ """
+ try:
+ result = subprocess.run( # noqa: S603
+ [ # noqa: S607
+ "git",
+ "-C",
+ str(repo_root),
+ "rev-parse",
+ "--verify",
+ "--quiet",
+ f"{ref}^{{commit}}",
+ ],
+ capture_output=True,
+ text=True,
+ check=False,
+ )
+ except OSError as e:
+ msg = f"Could not run git (is it installed and on PATH?): {e}"
+ raise RuntimeError(msg) from e
+ if result.returncode != 0:
+ msg = (
+ f"Could not resolve base ref {ref!r} in {repo_root}; "
+ "is it a valid git ref in this repository?"
+ )
+ raise RuntimeError(msg)
+
+
+def _git_show(repo_root: Path, ref: str, rel_path: str) -> str | None:
+ """Return file contents at `ref`, or None if the file does not exist there.
+
+ Assumes `ref` has already been validated by `_verify_ref`, so a non-zero
+ exit here means the path is absent at `ref` rather than a bad ref.
+ """
+ try:
+ result = subprocess.run( # noqa: S603
+ ["git", "-C", str(repo_root), "show", f"{ref}:{rel_path}"], # noqa: S607
+ capture_output=True,
+ text=True,
+ check=False,
+ )
+ except OSError:
+ return None
+ return result.stdout if result.returncode == 0 else None
+
+
+def summarize(
+ providers: list[ProviderEntry],
+ *,
+ base_ref: str = "HEAD",
+ repo_root: Path | None = None,
+) -> str:
+ """Build a Markdown summary of profile changes vs `base_ref`.
+
+ Args:
+ providers: List of `{'provider': ..., 'data_dir': ...}` entries,
+ matching the workflow input. `data_dir` is relative to the repo
+ root and contains `_profiles.py`.
+ base_ref: Git ref to compare the working tree against.
+ repo_root: Repository root. Defaults to the current directory.
+
+ Returns:
+ Markdown summary suitable for a PR body.
+
+ Raises:
+ RuntimeError: If `base_ref` cannot be resolved, or a profiles file
+ exists but cannot be read or parsed.
+ ValueError: If a `providers` entry is missing a required key.
+ TypeError: If a `providers` entry's `provider`/`data_dir` is not a
+ string.
+ """
+ root = (repo_root or Path.cwd()).resolve()
+ _verify_ref(root, base_ref)
+ provider_diffs: dict[str, ProfileDiff] = {}
+
+ for entry in providers:
+ # View the entry as an untrusted mapping: at the CLI boundary it is
+ # arbitrary parsed JSON, not a guaranteed `ProviderEntry`.
+ entry_map: Mapping[str, Any] = entry
+ try:
+ provider = entry_map["provider"]
+ data_dir = entry_map["data_dir"]
+ except (KeyError, TypeError) as e:
+ msg = (
+ f"Invalid provider entry {entry!r}: expected 'provider' and "
+ f"'data_dir' keys ({e})"
+ )
+ raise ValueError(msg) from e
+ if not isinstance(provider, str) or not isinstance(data_dir, str):
+ msg = (
+ f"Invalid provider entry {entry!r}: 'provider' and 'data_dir' "
+ "must be strings"
+ )
+ raise TypeError(msg)
+ rel_path = f"{data_dir.rstrip('/')}/_profiles.py"
+
+ old_source = _git_show(root, base_ref, rel_path) or ""
+ new_path = root / rel_path
+ if new_path.exists():
+ try:
+ new_source = new_path.read_text(encoding="utf-8")
+ except (OSError, UnicodeDecodeError) as e:
+ msg = f"Could not read {new_path}: {e}"
+ raise RuntimeError(msg) from e
+ else:
+ new_source = ""
+
+ # A corrupt-but-readable file must surface as an error: extracting `{}`
+ # from it would otherwise be diffed as every model added (old side) or
+ # removed (new side), yielding a confident but wrong summary.
+ try:
+ old_profiles = extract_profiles(old_source)
+ except ProfileParseError as e:
+ msg = f"Profile data for {provider!r} at {base_ref!r} is unparseable: {e}"
+ raise RuntimeError(msg) from e
+ try:
+ new_profiles = extract_profiles(new_source)
+ except ProfileParseError as e:
+ msg = f"Profile data for {provider!r} at {new_path} is unparseable: {e}"
+ raise RuntimeError(msg) from e
+
+ provider_diffs[provider] = diff_profiles(old_profiles, new_profiles)
+
+ return build_summary(provider_diffs)
diff --git a/libs/model-profiles/langchain_model_profiles/cli.py b/libs/model-profiles/langchain_model_profiles/cli.py
index b7b2c56ea7..456d70c55e 100644
--- a/libs/model-profiles/langchain_model_profiles/cli.py
+++ b/libs/model-profiles/langchain_model_profiles/cli.py
@@ -399,10 +399,53 @@ def main() -> None:
help="Data directory containing profile_augmentations.toml",
)
+ # summarize command
+ summarize_parser = subparsers.add_parser(
+ "summarize",
+ help="Summarize profile changes vs a git ref as Markdown (for PR bodies)",
+ )
+ summarize_parser.add_argument(
+ "--providers",
+ required=True,
+ help=(
+ "JSON array of objects with 'provider' and 'data_dir' keys "
+ "(data_dir relative to the repo root)."
+ ),
+ )
+ summarize_parser.add_argument(
+ "--base-ref",
+ default="HEAD",
+ help="Git ref to compare the working tree against (default: HEAD).",
+ )
+ summarize_parser.add_argument(
+ "--repo-root",
+ type=Path,
+ default=None,
+ help="Repository root (default: current directory).",
+ )
+
args = parser.parse_args()
if args.command == "refresh":
refresh(args.provider, args.data_dir)
+ elif args.command == "summarize":
+ from langchain_model_profiles._summary import summarize
+
+ try:
+ providers = json.loads(args.providers)
+ except json.JSONDecodeError as e:
+ parser.error(f"--providers is not valid JSON: {e}")
+
+ if not isinstance(providers, list):
+ parser.error("--providers must be a JSON array")
+
+ try:
+ output = summarize(
+ providers, base_ref=args.base_ref, repo_root=args.repo_root
+ )
+ except (RuntimeError, ValueError, TypeError) as e:
+ parser.error(str(e))
+ print(output)
if __name__ == "__main__":
diff --git a/libs/model-profiles/tests/unit_tests/test_summary.py b/libs/model-profiles/tests/unit_tests/test_summary.py
new file mode 100644
index 0000000000..f0615aad28
--- /dev/null
+++ b/libs/model-profiles/tests/unit_tests/test_summary.py
@@ -0,0 +1,470 @@
+"""Tests for the profile change summary generator."""
+
+import json
+import subprocess
+import sys
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import pytest
+
+from langchain_model_profiles import cli
+from langchain_model_profiles._summary import (
+ _MAX_ROWS,
+ FieldChange,
+ ProfileDiff,
+ ProfileParseError,
+ _describe_new_model,
+ _format_value,
+ _truncate,
+ build_summary,
+ diff_profiles,
+ extract_profiles,
+ render_provider_section,
+ summarize,
+)
+
+if TYPE_CHECKING:
+ from langchain_core.language_models.model_profile import (
+ ModelProfile,
+ ModelProfileRegistry,
+ )
+
+_OLD_SOURCE = '''"""Auto-generated."""
+
+from typing import Any
+
+_PROFILES: dict[str, dict[str, Any]] = {
+ "gpt-4": {
+ "name": "GPT-4",
+ "max_input_tokens": 8192,
+ "max_output_tokens": 4096,
+ "image_inputs": False,
+ "tool_calling": True,
+ },
+ "old-model": {
+ "name": "Old",
+ "max_input_tokens": 1000,
+ },
+}
+'''
+
+_NEW_SOURCE = '''"""Auto-generated."""
+
+from typing import Any
+
+_PROFILES: dict[str, dict[str, Any]] = {
+ "gpt-4": {
+ "name": "GPT-4",
+ "max_input_tokens": 8192,
+ "max_output_tokens": 16384,
+ "image_inputs": True,
+ "tool_calling": True,
+ },
+ "gpt-5": {
+ "name": "GPT-5",
+ "max_input_tokens": 400000,
+ "max_output_tokens": 128000,
+ "image_inputs": True,
+ "reasoning_output": True,
+ "tool_calling": True,
+ },
+}
+'''
+
+
+def _git(repo: Path, *args: str) -> None:
+ """Run a git command inside `repo` (test helper)."""
+ subprocess.run( # noqa: S603
+ ["git", "-C", str(repo), *args], # noqa: S607
+ check=True,
+ capture_output=True,
+ text=True,
+ )
+
+
+def _init_repo(repo: Path) -> None:
+ """Initialize a git repo with a deterministic identity."""
+ _git(repo, "init", "-q")
+ _git(repo, "config", "user.email", "[email protected]")
+ _git(repo, "config", "user.name", "Test")
+
+
+def _write_profiles(path: Path, source: str) -> None:
+ """Write a `_profiles.py` file, creating parent directories."""
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(source)
+
+
+def test_extract_profiles() -> None:
+ """`_PROFILES` literal is extracted via ast without executing the module."""
+ profiles = extract_profiles(_OLD_SOURCE)
+ assert set(profiles) == {"gpt-4", "old-model"}
+ assert profiles["gpt-4"]["max_input_tokens"] == 8192
+
+
+def test_extract_profiles_handles_missing_or_invalid() -> None:
+ """Absent `_PROFILES` yields `{}`; present-but-unparseable sources raise."""
+ # No `_PROFILES` assignment, and an empty file, are both legitimately empty.
+ assert extract_profiles("x = 1") == {}
+ assert extract_profiles("") == {}
+ # A syntactically broken file is corrupt, not empty.
+ with pytest.raises(ProfileParseError):
+ extract_profiles("def (:")
+ # A non-literal or non-dict `_PROFILES` is corrupt too.
+ with pytest.raises(ProfileParseError):
+ extract_profiles("_PROFILES = some_function()")
+ with pytest.raises(ProfileParseError):
+ extract_profiles("_PROFILES = [1, 2, 3]")
+
+
+def test_diff_profiles() -> None:
+ """Diff reports added, removed, and per-field changes."""
+ diff = diff_profiles(extract_profiles(_OLD_SOURCE), extract_profiles(_NEW_SOURCE))
+ assert diff.added == ["gpt-5"]
+ assert diff.removed == ["old-model"]
+ assert set(diff.changed) == {"gpt-4"}
+ assert diff.changed["gpt-4"]["max_output_tokens"] == (4096, 16384)
+ assert diff.changed["gpt-4"]["image_inputs"] == (False, True)
+ assert diff.added_profiles["gpt-5"]["max_input_tokens"] == 400000
+
+
+def test_diff_profiles_no_changes() -> None:
+ """Identical inputs produce an empty diff."""
+ profiles = extract_profiles(_OLD_SOURCE)
+ diff = diff_profiles(profiles, profiles)
+ assert diff.is_empty
+
+
+def test_render_provider_section_content() -> None:
+ """Rendered section describes additions, removals, and field changes."""
+ diff = diff_profiles(extract_profiles(_OLD_SOURCE), extract_profiles(_NEW_SOURCE))
+ section = render_provider_section("openai", diff)
+ assert section is not None
+ assert "### openai" in section
+ assert "1 added" in section
+ assert "`gpt-5`" in section
+ assert "400,000 ctx" in section
+ assert "reasoning" in section
+ assert "1 removed" in section
+ assert "`old-model`" in section
+ assert "1 changed" in section
+ assert "max output tokens 4,096 β†’ 16,384" in section
+ assert "added image input" in section
+
+
+def test_render_provider_section_empty() -> None:
+ """An empty diff renders no section."""
+ assert render_provider_section("openai", ProfileDiff()) is None
+
+
+def test_build_summary_headline() -> None:
+ """The summary leads with a header and an aggregate headline."""
+ diff = diff_profiles(extract_profiles(_OLD_SOURCE), extract_profiles(_NEW_SOURCE))
+ summary = build_summary({"openai": diff})
+ assert summary.startswith("## Summary of changes")
+ assert "1 added" in summary
+ assert "1 removed" in summary
+ assert "1 changed" in summary
+
+
+def test_build_summary_no_changes() -> None:
+ """An all-empty diff produces a short no-change note."""
+ assert build_summary({"openai": ProfileDiff()}) == "No model profile data changed."
+
+
+def test_truncation() -> None:
+ """Long lists are truncated with a trailing count of hidden rows."""
+ new: ModelProfileRegistry = {f"model-{i}": {"name": f"m{i}"} for i in range(40)}
+ diff = diff_profiles({}, new)
+ section = render_provider_section("openai", diff)
+ assert section is not None
+ assert "…and 15 more" in section
+
+
+def test_summarize_against_git(tmp_path: Path) -> None:
+ """Summarize compares the working tree against a committed baseline."""
+ repo = tmp_path
+ _init_repo(repo)
+
+ data_dir = "libs/partners/openai/data"
+ profiles_path = repo / data_dir / "_profiles.py"
+ _write_profiles(profiles_path, _OLD_SOURCE)
+ _git(repo, "add", "-A")
+ _git(repo, "commit", "-q", "-m", "init")
+
+ # Simulate a refresh by overwriting the working-tree file.
+ profiles_path.write_text(_NEW_SOURCE)
+
+ summary = summarize([{"provider": "openai", "data_dir": data_dir}], repo_root=repo)
+ assert "## Summary of changes" in summary
+ assert "`gpt-5`" in summary
+ assert "`old-model`" in summary
+ # The changed-field path is exercised end-to-end, not just at the unit layer.
+ assert "max output tokens 4,096 β†’ 16,384" in summary
+
+
+def test_summarize_new_provider_file(tmp_path: Path) -> None:
+ """A brand-new profiles file is treated as all-added."""
+ repo = tmp_path
+ _init_repo(repo)
+ (repo / "README.md").write_text("x")
+ _git(repo, "add", "-A")
+ _git(repo, "commit", "-q", "-m", "init")
+
+ data_dir = "libs/partners/new/data"
+ _write_profiles(repo / data_dir / "_profiles.py", _NEW_SOURCE)
+
+ summary = summarize([{"provider": "new", "data_dir": data_dir}], repo_root=repo)
+ assert "2 added" in summary
+
+
+def test_field_change_is_tuple() -> None:
+ """`FieldChange` unpacks and compares like a plain (old, new) tuple."""
+ change = FieldChange(1, 2)
+ # Access the named fields before the tuple comparison: `== (1, 2)` would
+ # otherwise narrow `change` to a plain `tuple` for the rest of the scope.
+ assert change.old == 1
+ assert change.new == 2
+ assert change == (1, 2)
+
+
+def test_format_value_variants() -> None:
+ """Each `_format_value` branch renders the expected string."""
+ assert _format_value("x", None) == "unset"
+ assert _format_value("tool_calling", True) == "yes" # noqa: FBT003
+ assert _format_value("tool_calling", False) == "no" # noqa: FBT003
+ assert _format_value("max_input_tokens", 200000) == "200,000"
+ # Plain int outside the token fields is rendered without separators.
+ assert _format_value("foo", 42) == "42"
+ # Floats fall through to str().
+ assert _format_value("temperature", 1.5) == "1.5"
+ assert _format_value("name", "GPT") == "`GPT`"
+
+
+def test_render_non_bool_field_change() -> None:
+ """Non-boolean field changes render an `old β†’ new` phrase."""
+ old: ModelProfileRegistry = {"m": {"status": "active", "name": "M"}}
+ new: ModelProfileRegistry = {"m": {"status": "deprecated", "name": "M2"}}
+ section = render_provider_section("openai", diff_profiles(old, new))
+ assert section is not None
+ assert "status `active` β†’ `deprecated`" in section
+ assert "display name `M` β†’ `M2`" in section
+
+
+def test_render_removed_bool_field_change() -> None:
+ """A boolean field flipped off renders a `removed <label>` phrase."""
+ old: ModelProfileRegistry = {"m": {"image_inputs": True}}
+ new: ModelProfileRegistry = {"m": {"image_inputs": False}}
+ section = render_provider_section("openai", diff_profiles(old, new))
+ assert section is not None
+ assert "removed image input" in section
+
+
+def test_describe_new_model_modalities() -> None:
+ """A new model descriptor lists context, output, modalities, and tools."""
+ profile: ModelProfile = {
+ "max_input_tokens": 200000,
+ "max_output_tokens": 64000,
+ "image_inputs": True,
+ "audio_inputs": True,
+ "video_inputs": True,
+ "pdf_inputs": True,
+ "tool_calling": True,
+ }
+ descriptor = _describe_new_model(profile)
+ assert "200,000 ctx" in descriptor
+ assert "64,000 out" in descriptor
+ assert "text+image+audio+video+pdf in" in descriptor
+ assert "tools" in descriptor
+
+
+def test_describe_new_model_empty() -> None:
+ """A profile with no notable fields yields an empty descriptor."""
+ assert _describe_new_model({"name": "x"}) == ""
+
+
+def test_render_added_model_without_descriptor() -> None:
+ """An added model with no descriptor renders no ` β€” ` suffix."""
+ section = render_provider_section("p", diff_profiles({}, {"bare": {"name": "B"}}))
+ assert section is not None
+ assert "- `bare`" in section
+ assert "- `bare` β€”" not in section
+
+
+def test_truncate_boundary() -> None:
+ """`_truncate` keeps exactly `_MAX_ROWS` rows but caps one more."""
+ exactly = [f"- r{i}" for i in range(_MAX_ROWS)]
+ assert _truncate(exactly) == exactly
+
+ over = [f"- r{i}" for i in range(_MAX_ROWS + 1)]
+ result = _truncate(over)
+ assert len(result) == _MAX_ROWS + 1
+ assert result[-1] == "- …and 1 more"
+
+
+def test_build_summary_multi_provider_sorted() -> None:
+ """Providers are rendered in sorted order regardless of input order."""
+ diff_a = diff_profiles({}, {"a": {"name": "A"}})
+ diff_z = diff_profiles({}, {"z": {"name": "Z"}})
+ summary = build_summary({"zzz": diff_z, "aaa": diff_a})
+ assert summary.index("### aaa") < summary.index("### zzz")
+
+
+def test_summarize_removed_when_file_deleted(tmp_path: Path) -> None:
+ """Deleting the working-tree file reports every model as removed."""
+ repo = tmp_path
+ _init_repo(repo)
+ data_dir = "libs/partners/openai/data"
+ profiles_path = repo / data_dir / "_profiles.py"
+ _write_profiles(profiles_path, _OLD_SOURCE)
+ _git(repo, "add", "-A")
+ _git(repo, "commit", "-q", "-m", "init")
+
+ profiles_path.unlink()
+
+ summary = summarize([{"provider": "openai", "data_dir": data_dir}], repo_root=repo)
+ assert "2 removed" in summary
+
+
+def test_summarize_bad_base_ref(tmp_path: Path) -> None:
+ """An unresolvable base ref raises rather than fabricating an all-added diff."""
+ repo = tmp_path
+ _init_repo(repo)
+ (repo / "README.md").write_text("x")
+ _git(repo, "add", "-A")
+ _git(repo, "commit", "-q", "-m", "init")
+
+ data_dir = "libs/partners/openai/data"
+ _write_profiles(repo / data_dir / "_profiles.py", _NEW_SOURCE)
+
+ with pytest.raises(RuntimeError, match="Could not resolve base ref"):
+ summarize(
+ [{"provider": "openai", "data_dir": data_dir}],
+ base_ref="no-such-ref",
+ repo_root=repo,
+ )
+
+
+def test_summarize_malformed_entry(tmp_path: Path) -> None:
+ """A provider entry missing a required key raises a clear error."""
+ repo = tmp_path
+ _init_repo(repo)
+ (repo / "README.md").write_text("x")
+ _git(repo, "add", "-A")
+ _git(repo, "commit", "-q", "-m", "init")
+
+ with pytest.raises(ValueError, match="Invalid provider entry"):
+ summarize([{"provider": "openai"}], repo_root=repo) # type: ignore[typeddict-item]
+
+
+def test_summarize_non_string_entry(tmp_path: Path) -> None:
+ """Non-string `provider`/`data_dir` raises `TypeError`, not `AttributeError`.
+
+ A non-string value would otherwise reach `data_dir.rstrip(...)` and raise an
+ `AttributeError` that escapes the CLI's `except (RuntimeError, ValueError,
+ TypeError)`, surfacing a raw traceback instead of a clean error.
+ """
+ repo = tmp_path
+ _init_repo(repo)
+ (repo / "README.md").write_text("x")
+ _git(repo, "add", "-A")
+ _git(repo, "commit", "-q", "-m", "init")
+
+ with pytest.raises(TypeError, match="must be strings"):
+ summarize(
+ [{"provider": 5, "data_dir": 7}], # type: ignore[typeddict-item]
+ repo_root=repo,
+ )
+
+
+def test_summarize_corrupt_working_tree_file(tmp_path: Path) -> None:
+ """A present-but-unparseable working-tree file raises, not a mass removal.
+
+ Mirrors the `_verify_ref` guard on the base-ref side: a corrupt new file
+ must surface as an error rather than be diffed as every model removed.
+ """
+ repo = tmp_path
+ _init_repo(repo)
+ data_dir = "libs/partners/openai/data"
+ profiles_path = repo / data_dir / "_profiles.py"
+ _write_profiles(profiles_path, _OLD_SOURCE)
+ _git(repo, "add", "-A")
+ _git(repo, "commit", "-q", "-m", "init")
+
+ # Simulate a refresh that left the file truncated / syntactically broken.
+ profiles_path.write_text("_PROFILES = {")
+
+ with pytest.raises(RuntimeError, match="unparseable"):
+ summarize([{"provider": "openai", "data_dir": data_dir}], repo_root=repo)
+
+
+def test_summarize_corrupt_base_ref_file(tmp_path: Path) -> None:
+ """An unparseable file at the base ref raises, not an all-added diff."""
+ repo = tmp_path
+ _init_repo(repo)
+ data_dir = "libs/partners/openai/data"
+ profiles_path = repo / data_dir / "_profiles.py"
+ _write_profiles(profiles_path, "_PROFILES = {") # committed broken
+ _git(repo, "add", "-A")
+ _git(repo, "commit", "-q", "-m", "init")
+
+ profiles_path.write_text(_NEW_SOURCE) # working tree now valid
+
+ with pytest.raises(RuntimeError, match="unparseable"):
+ summarize([{"provider": "openai", "data_dir": data_dir}], repo_root=repo)
+
+
+def test_cli_summarize_success(
+ tmp_path: Path,
+ capsys: pytest.CaptureFixture[str],
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """A valid `--providers` array prints the Markdown summary to stdout."""
+ repo = tmp_path
+ _init_repo(repo)
+ data_dir = "libs/partners/openai/data"
+ profiles_path = repo / data_dir / "_profiles.py"
+ _write_profiles(profiles_path, _OLD_SOURCE)
+ _git(repo, "add", "-A")
+ _git(repo, "commit", "-q", "-m", "init")
+ profiles_path.write_text(_NEW_SOURCE)
+
+ providers = json.dumps([{"provider": "openai", "data_dir": data_dir}])
+ monkeypatch.setattr(
+ sys,
+ "argv",
+ [
+ "langchain-profiles",
+ "summarize",
+ "--providers",
+ providers,
+ "--repo-root",
+ str(repo),
+ ],
+ )
+
+ cli.main()
+
+ out = capsys.readouterr().out
+ assert "## Summary of changes" in out
+ assert "`gpt-5`" in out
+ assert "max output tokens 4,096 β†’ 16,384" in out
+
+
+def test_cli_summarize_invalid_json(monkeypatch: pytest.MonkeyPatch) -> None:
+ """The CLI rejects a `--providers` value that is not valid JSON."""
+ monkeypatch.setattr(
+ sys, "argv", ["langchain-profiles", "summarize", "--providers", "not json"]
+ )
+ with pytest.raises(SystemExit):
+ cli.main()
+
+
+def test_cli_summarize_non_list(monkeypatch: pytest.MonkeyPatch) -> None:
+ """The CLI rejects a `--providers` value that is not a JSON array."""
+ monkeypatch.setattr(
+ sys, "argv", ["langchain-profiles", "summarize", "--providers", '{"a": 1}']
+ )
+ with pytest.raises(SystemExit):
+ cli.main()
diff --git a/libs/model-profiles/uv.lock b/libs/model-profiles/uv.lock
index ce528ad369..3cd50fd708 100644
--- a/libs/model-profiles/uv.lock
+++ b/libs/model-profiles/uv.lock
@@ -504,7 +504,7 @@ wheels = [
[[package]]
name = "langchain"
-version = "1.3.10"
+version = "1.3.11"
source = { editable = "../langchain_v1" }
dependencies = [
{ name = "langchain-core" },
@@ -703,7 +703,7 @@ typing = [
[[package]]
name = "langchain-openai"
-version = "1.3.2"
+version = "1.3.3"
source = { editable = "../partners/openai" }
dependencies = [
{ name = "langchain-core" },