A programming framework for agentic AI
Feature add Add LlamaCppChatCompletionClient and llama-cpp (#5326)
PythicCoder6a3acc454831
diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.ymlindex 99714fe7e..3b204f5f9 100644--- a/.github/workflows/checks.yml+++ b/.github/workflows/checks.yml@@ -197,7 +197,7 @@ jobs:- name: Install Python depsrun: |- uv sync --locked --all-extras+ uv sync --locked --all-extras --no-extra llama-cppshell: pwshworking-directory: ./pythondiff --git a/python/packages/autogen-core/docs/src/reference/index.md b/python/packages/autogen-core/docs/src/reference/index.mdindex e312c288d..d498a60fc 100644--- a/python/packages/autogen-core/docs/src/reference/index.md+++ b/python/packages/autogen-core/docs/src/reference/index.md@@ -54,6 +54,7 @@ python/autogen_ext.models.azurepython/autogen_ext.models.anthropicpython/autogen_ext.models.semantic_kernelpython/autogen_ext.models.ollama+python/autogen_ext.models.llama_cpppython/autogen_ext.tools.code_executionpython/autogen_ext.tools.graphragpython/autogen_ext.tools.httpdiff --git a/python/packages/autogen-core/docs/src/reference/python/autogen_ext.models.llama_cpp.rst b/python/packages/autogen-core/docs/src/reference/python/autogen_ext.models.llama_cpp.rstnew file mode 100644index 000000000..56470d36d--- /dev/null+++ b/python/packages/autogen-core/docs/src/reference/python/autogen_ext.models.llama_cpp.rst@@ -0,0 +1,9 @@+autogen\_ext.models.llama\_cpp+==============================+++.. automodule:: autogen_ext.models.llama_cpp+ :members:+ :undoc-members:+ :show-inheritance:+ :member-order: bysourcediff --git a/python/packages/autogen-ext/pyproject.toml b/python/packages/autogen-ext/pyproject.tomlindex e8a1ae27c..22fed0c7f 100644--- a/python/packages/autogen-ext/pyproject.toml+++ b/python/packages/autogen-ext/pyproject.toml@@ -32,6 +32,11 @@ file-surfer = ["autogen-agentchat==0.4.8","markitdown~=0.0.1",]++llama-cpp = [+ "llama-cpp-python>=0.1.9",+]+graphrag = ["graphrag>=1.0.1"]chromadb = ["chromadb"]web-surfer = [diff --git a/python/packages/autogen-ext/src/autogen_ext/models/llama_cpp/__init__.py b/python/packages/autogen-ext/src/autogen_ext/models/llama_cpp/__init__.pynew file mode 100644index 000000000..0324e4005--- /dev/null+++ b/python/packages/autogen-ext/src/autogen_ext/models/llama_cpp/__init__.py@@ -0,0 +1,10 @@+try:+ from ._llama_cpp_completion_client import LlamaCppChatCompletionClient+except ImportError as e:+ raise ImportError(+ "Dependencies for Llama Cpp not found. "+ "Please install llama-cpp-python: "+ "pip install autogen-ext[llama-cpp]"+ ) from e++__all__ = ["LlamaCppChatCompletionClient"]diff --git a/python/packages/autogen-ext/src/autogen_ext/models/llama_cpp/_llama_cpp_completion_client.py b/python/packages/autogen-ext/src/autogen_ext/models/llama_cpp/_llama_cpp_completion_client.pynew file mode 100644index 000000000..dc7772114--- /dev/null+++ b/python/packages/autogen-ext/src/autogen_ext/models/llama_cpp/_llama_cpp_completion_client.py@@ -0,0 +1,426 @@+import logging # added import+import re+from typing import Any, AsyncGenerator, Dict, List, Literal, Mapping, Optional, Sequence, TypedDict, Union, cast++from autogen_core import EVENT_LOGGER_NAME, CancellationToken, FunctionCall, MessageHandlerContext+from autogen_core.logging import LLMCallEvent+from autogen_core.models import (+ AssistantMessage,+ ChatCompletionClient,+ CreateResult,+ FinishReasons,+ FunctionExecutionResultMessage,+ LLMMessage,+ ModelInfo,+ RequestUsage,+ SystemMessage,+ UserMessage,+ validate_model_info,+)+from autogen_core.tools import Tool, ToolSchema+from llama_cpp import (+ ChatCompletionFunctionParameters,+ ChatCompletionRequestAssistantMessage,+ ChatCompletionRequestFunctionMessage,+ ChatCompletionRequestSystemMessage,+ ChatCompletionRequestToolMessage,+ ChatCompletionRequestUserMessage,+ ChatCompletionTool,+ ChatCompletionToolFunction,+ Llama,+ llama_chat_format,+)+from typing_extensions import Unpack++logger = logging.getLogger(EVENT_LOGGER_NAME) # initialize logger+++def normalize_stop_reason(stop_reason: str | None) -> FinishReasons:+ if stop_reason is None:+ return "unknown"++ # Convert to lower case+ stop_reason = stop_reason.lower()++ KNOWN_STOP_MAPPINGS: Dict[str, FinishReasons] = {+ "stop": "stop",+ "length": "length",+ "content_filter": "content_filter",+ "function_calls": "function_calls",+ "end_turn": "stop",+ "tool_calls": "function_calls",+ }++ return KNOWN_STOP_MAPPINGS.get(stop_reason, "unknown")+++def normalize_name(name: str) -> str:+ """+ LLMs sometimes ask functions while ignoring their own format requirements, this function should be used to replace invalid characters with "_".++ Prefer _assert_valid_name for validating user configuration or input+ """+ return re.sub(r"[^a-zA-Z0-9_-]", "_", name)[:64]+++def assert_valid_name(name: str) -> str:+ """+ Ensure that configured names are valid, raises ValueError if not.++ For munging LLM responses use _normalize_name to ensure LLM specified names don't break the API.+ """+ if not re.match(r"^[a-zA-Z0-9_-]+$", name):+ raise ValueError(f"Invalid name: {name}. Only letters, numbers, '_' and '-' are allowed.")+ if len(name) > 64:+ raise ValueError(f"Invalid name: {name}. Name must be less than 64 characters.")+ return name+++def convert_tools(+ tools: Sequence[Tool | ToolSchema],+) -> List[ChatCompletionTool]:+ result: List[ChatCompletionTool] = []+ for tool in tools:+ if isinstance(tool, Tool):+ tool_schema = tool.schema+ else:+ assert isinstance(tool, dict)+ tool_schema = tool++ result.append(+ ChatCompletionTool(+ type="function",+ function=ChatCompletionToolFunction(+ name=tool_schema["name"],+ description=(tool_schema["description"] if "description" in tool_schema else ""),+ parameters=(+ cast(ChatCompletionFunctionParameters, tool_schema["parameters"])+ if "parameters" in tool_schema+ else {}+ ),+ ),+ )+ )+ # Check if all tools have valid names.+ for tool_param in result:+ assert_valid_name(tool_param["function"]["name"])+ return result+++class LlamaCppParams(TypedDict, total=False):+ # from_pretrained parameters:+ repo_id: Optional[str]+ filename: Optional[str]+ additional_files: Optional[List[Any]]+ local_dir: Optional[str]+ local_dir_use_symlinks: Union[bool, Literal["auto"]]+ cache_dir: Optional[str]+ # __init__ parameters:+ model_path: str+ n_gpu_layers: int+ split_mode: int+ main_gpu: int+ tensor_split: Optional[List[float]]+ rpc_servers: Optional[str]+ vocab_only: bool+ use_mmap: bool+ use_mlock: bool+ kv_overrides: Optional[Dict[str, Union[bool, int, float, str]]]+ seed: int+ n_ctx: int+ n_batch: int+ n_ubatch: int+ n_threads: Optional[int]+ n_threads_batch: Optional[int]+ rope_scaling_type: Optional[int]+ pooling_type: int+ rope_freq_base: float+ rope_freq_scale: float+ yarn_ext_factor: float+ yarn_attn_factor: float+ yarn_beta_fast: float+ yarn_beta_slow: float+ yarn_orig_ctx: int+ logits_all: bool+ embedding: bool+ offload_kqv: bool+ flash_attn: bool+ no_perf: bool+ last_n_tokens_size: int+ lora_base: Optional[str]+ lora_scale: float+ lora_path: Optional[str]+ numa: Union[bool, int]+ chat_format: Optional[str]+ chat_handler: Optional[llama_chat_format.LlamaChatCompletionHandler]+ draft_model: Optional[Any] # LlamaDraftModel not exposed by llama_cpp+ tokenizer: Optional[Any] # BaseLlamaTokenizer not exposed by llama_cpp+ type_k: Optional[int]+ type_v: Optional[int]+ spm_infill: bool+ verbose: bool+++class LlamaCppChatCompletionClient(ChatCompletionClient):+ """Chat completion client for LlamaCpp models.+ To use this client, you must install the `llama-cpp` extra:++ .. code-block:: bash++ pip install "autogen-ext[llama-cpp]"++ This client allows you to interact with LlamaCpp models, either by specifying a local model path or by downloading a model from Hugging Face Hub.++ Args:+ model_path (optional, str): The path to the LlamaCpp model file. Required if repo_id and filename are not provided.+ repo_id (optional, str): The Hugging Face Hub repository ID. Required if model_path is not provided.+ filename (optional, str): The filename of the model within the Hugging Face Hub repository. Required if model_path is not provided.+ n_gpu_layers (optional, int): The number of layers to put on the GPU.+ n_ctx (optional, int): The context size.+ n_batch (optional, int): The batch size.+ verbose (optional, bool): Whether to print verbose output.+ model_info (optional, ModelInfo): The capabilities of the model. Defaults to a ModelInfo instance with function_calling set to True.+ **kwargs: Additional parameters to pass to the Llama class.++ Examples:++ The following code snippet shows how to use the client with a local model file:++ .. code-block:: python++ import asyncio++ from autogen_core.models import UserMessage+ from autogen_ext.models.llama_cpp import LlamaCppChatCompletionClient+++ async def main():+ llama_client = LlamaCppChatCompletionClient(model_path="/path/to/your/model.gguf")+ result = await llama_client.create([UserMessage(content="What is the capital of France?", source="user")])+ print(result)+++ asyncio.run(main())++ The following code snippet shows how to use the client with a model from Hugging Face Hub:++ .. code-block:: python++ import asyncio++ from autogen_core.models import UserMessage+ from autogen_ext.models.llama_cpp import LlamaCppChatCompletionClient+++ async def main():+ llama_client = LlamaCppChatCompletionClient(+ repo_id="unsloth/phi-4-GGUF", filename="phi-4-Q2_K_L.gguf", n_gpu_layers=-1, seed=1337, n_ctx=5000+ )+ result = await llama_client.create([UserMessage(content="What is the capital of France?", source="user")])+ print(result)+++ asyncio.run(main())+ """++ def __init__(+ self,+ model_info: Optional[ModelInfo] = None,+ **kwargs: Unpack[LlamaCppParams],+ ) -> None:+ """+ Initialize the LlamaCpp client.+ """++ if model_info:+ validate_model_info(model_info)++ if "repo_id" in kwargs and "filename" in kwargs and kwargs["repo_id"] and kwargs["filename"]:+ repo_id: str = cast(str, kwargs.pop("repo_id"))+ filename: str = cast(str, kwargs.pop("filename"))+ pretrained = Llama.from_pretrained(repo_id=repo_id, filename=filename, **kwargs) # type: ignore+ assert isinstance(pretrained, Llama)+ self.llm = pretrained++ elif "model_path" in kwargs:+ self.llm = Llama(**kwargs) # pyright: ignore[reportUnknownMemberType]+ else:+ raise ValueError("Please provide model_path if ... or provide repo_id and filename if ....")+ self._total_usage = {"prompt_tokens": 0, "completion_tokens": 0}++ async def create(+ self,+ messages: Sequence[LLMMessage],+ *,+ tools: Sequence[Tool | ToolSchema] = [],+ # None means do not override the default+ # A value means to override the client default - often specified in the constructor+ json_output: Optional[bool] = None,+ extra_create_args: Mapping[str, Any] = {},+ cancellation_token: Optional[CancellationToken] = None,+ ) -> CreateResult:+ # Convert LLMMessage objects to dictionaries with 'role' and 'content'+ # converted_messages: List[Dict[str, str | Image | list[str | Image] | list[FunctionCall]]] = []+ converted_messages: list[+ ChatCompletionRequestSystemMessage+ | ChatCompletionRequestUserMessage+ | ChatCompletionRequestAssistantMessage+ | ChatCompletionRequestUserMessage+ | ChatCompletionRequestToolMessage+ | ChatCompletionRequestFunctionMessage+ ] = []+ for msg in messages:+ if isinstance(msg, SystemMessage):+ converted_messages.append({"role": "system", "content": msg.content})+ elif isinstance(msg, UserMessage) and isinstance(msg.content, str):+ converted_messages.append({"role": "user", "content": msg.content})+ elif isinstance(msg, AssistantMessage) and isinstance(msg.content, str):+ converted_messages.append({"role": "assistant", "content": msg.content})+ elif (+ isinstance(msg, SystemMessage) or isinstance(msg, UserMessage) or isinstance(msg, AssistantMessage)+ ) and isinstance(msg.content, list):+ raise ValueError("Multi-part messages such as those containing images are currently not supported.")+ else:+ raise ValueError(f"Unsupported message type: {type(msg)}")++ if self.model_info["function_calling"]:+ response = self.llm.create_chat_completion(+ messages=converted_messages, tools=convert_tools(tools), stream=False+ )+ else:+ response = self.llm.create_chat_completion(messages=converted_messages, stream=False)++ if not isinstance(response, dict):+ raise ValueError("Unexpected response type from LlamaCpp model.")++ self._total_usage["prompt_tokens"] += response["usage"]["prompt_tokens"]+ self._total_usage["completion_tokens"] += response["usage"]["completion_tokens"]++ # Parse the response+ response_tool_calls: ChatCompletionTool | None = None+ response_text: str | None = None+ if "choices" in response and len(response["choices"]) > 0:+ if "message" in response["choices"][0]:+ response_text = response["choices"][0]["message"]["content"]+ if "tool_calls" in response["choices"][0]:+ response_tool_calls = response["choices"][0]["tool_calls"] # type: ignore++ content: List[FunctionCall] | str = ""+ thought: str | None = None+ if response_tool_calls:+ content = []+ for tool_call in response_tool_calls:+ if not isinstance(tool_call, dict):+ raise ValueError("Unexpected tool call type from LlamaCpp model.")+ content.append(+ FunctionCall(+ id=tool_call["id"],+ arguments=tool_call["function"]["arguments"],+ name=normalize_name(tool_call["function"]["name"]),+ )+ )+ if response_text and len(response_text) > 0:+ thought = response_text+ else:+ if response_text:+ content = response_text++ # Detect tool usage in the response+ if not response_tool_calls and not response_text:+ logger.debug("DEBUG: No response text found. Returning empty response.")+ return CreateResult(+ content="", usage=RequestUsage(prompt_tokens=0, completion_tokens=0), finish_reason="stop", cached=False+ )++ # Create a CreateResult object+ if "finish_reason" in response["choices"][0]:+ finish_reason = response["choices"][0]["finish_reason"]+ else:+ finish_reason = "unknown"+ if finish_reason not in ("stop", "length", "function_calls", "content_filter", "unknown"):+ finish_reason = "unknown"+ create_result = CreateResult(+ content=content,+ thought=thought,+ usage=cast(RequestUsage, response["usage"]),+ finish_reason=normalize_stop_reason(finish_reason), # type: ignore+ cached=False,+ )++ # If we are running in the context of a handler we can get the agent_id+ try:+ agent_id = MessageHandlerContext.agent_id()+ except RuntimeError:+ agent_id = None++ logger.info(+ LLMCallEvent(+ messages=cast(List[Dict[str, Any]], converted_messages),+ response=create_result.model_dump(),+ prompt_tokens=response["usage"]["prompt_tokens"],+ completion_tokens=response["usage"]["completion_tokens"],+ agent_id=agent_id,+ )+ )+ return create_result++ async def create_stream(+ self,+ messages: Sequence[LLMMessage],+ *,+ tools: Sequence[Tool | ToolSchema] = [],+ # None means do not override the default+ # A value means to override the client default - often specified in the constructor+ json_output: Optional[bool] = None,+ extra_create_args: Mapping[str, Any] = {},+ cancellation_token: Optional[CancellationToken] = None,+ ) -> AsyncGenerator[Union[str, CreateResult], None]:+ raise NotImplementedError("Stream not yet implemented for LlamaCppChatCompletionClient")+ yield ""++ # Implement abstract methods+ def actual_usage(self) -> RequestUsage:+ return RequestUsage(+ prompt_tokens=self._total_usage.get("prompt_tokens", 0),+ completion_tokens=self._total_usage.get("completion_tokens", 0),+ )++ @property+ def capabilities(self) -> ModelInfo:+ return self.model_info++ def count_tokens(+ self,+ messages: Sequence[SystemMessage | UserMessage | AssistantMessage | FunctionExecutionResultMessage],+ **kwargs: Any,+ ) -> int:+ total = 0+ for msg in messages:+ # Use the Llama model's tokenizer to encode the content+ tokens = self.llm.tokenize(str(msg.content).encode("utf-8"))+ total += len(tokens)+ return total++ @property+ def model_info(self) -> ModelInfo:+ return ModelInfo(vision=False, json_output=False, family="llama-cpp", function_calling=True)++ def remaining_tokens(+ self,+ messages: Sequence[SystemMessage | UserMessage | AssistantMessage | FunctionExecutionResultMessage],+ **kwargs: Any,+ ) -> int:+ used_tokens = self.count_tokens(messages)+ return max(self.llm.n_ctx() - used_tokens, 0)++ def total_usage(self) -> RequestUsage:+ return RequestUsage(+ prompt_tokens=self._total_usage.get("prompt_tokens", 0),+ completion_tokens=self._total_usage.get("completion_tokens", 0),+ )++ async def close(self) -> None:+ """+ Close the LlamaCpp client.+ """+ self.llm.close()diff --git a/python/packages/autogen-ext/tests/code_executors/test_commandline_code_executor.py b/python/packages/autogen-ext/tests/code_executors/test_commandline_code_executor.pyindex 7ff87909a..4e138a53b 100644--- a/python/packages/autogen-ext/tests/code_executors/test_commandline_code_executor.py+++ b/python/packages/autogen-ext/tests/code_executors/test_commandline_code_executor.py@@ -3,8 +3,8 @@import asyncioimport os-import shutilimport platform+import shutilimport sysimport tempfileimport venv@@ -18,7 +18,6 @@ from autogen_core import CancellationTokenfrom autogen_core.code_executor import CodeBlockfrom autogen_ext.code_executors.local import LocalCommandLineCodeExecutor-HAS_POWERSHELL: bool = platform.system() == "Windows" and (shutil.which("powershell") is not None or shutil.which("pwsh") is not None)diff --git a/python/packages/autogen-ext/tests/models/test_llama_cpp_model_client.py b/python/packages/autogen-ext/tests/models/test_llama_cpp_model_client.pynew file mode 100644index 000000000..1898fe7a7--- /dev/null+++ b/python/packages/autogen-ext/tests/models/test_llama_cpp_model_client.py@@ -0,0 +1,205 @@+import contextlib+import sys+from typing import TYPE_CHECKING, Any, ContextManager, Generator, List, Sequence, Union++import pytest+import torch++# from autogen_agentchat.agents import AssistantAgent+# from autogen_agentchat.messages import TextMessage+# from autogen_core import CancellationToken+from autogen_core.models import RequestUsage, SystemMessage, UserMessage++# from autogen_core.tools import FunctionTool+try:+ from llama_cpp import ChatCompletionMessageToolCalls++ if TYPE_CHECKING:+ from autogen_ext.models.llama_cpp._llama_cpp_completion_client import LlamaCppChatCompletionClient+except ImportError:+ # If llama_cpp is not installed, we can't run the tests.+ pytest.skip("Skipping LlamaCppChatCompletionClient tests: llama-cpp-python not installed", allow_module_level=True)+++# Fake Llama class to simulate responses+class FakeLlama:+ def __init__(+ self,+ model_path: str,+ **_: Any,+ ) -> None:+ self.model_path = model_path+ self.n_ctx = lambda: 1024++ # Added tokenize method for testing purposes.+ def tokenize(self, b: bytes) -> list[int]:+ return list(b)++ def create_chat_completion(+ self, messages: Any, tools: List[ChatCompletionMessageToolCalls] | None, stream: bool = False+ ) -> dict[str, Any]:+ # Return fake non-streaming response.++ return {+ "usage": {"prompt_tokens": 1, "completion_tokens": 2},+ "choices": [{"message": {"content": "Fake response"}}],+ }++ def __call__(self, prompt: str, stream: bool = True) -> Generator[dict[str, Any], None, None]:+ # Yield fake streaming tokens.+ yield {"choices": [{"text": "Hello "}]}+ yield {"choices": [{"text": "World"}]}+++def get_completion_client(+ monkeypatch: pytest.MonkeyPatch,+) -> "Generator[type[LlamaCppChatCompletionClient], None, None]":+ with monkeypatch.context() as m:+ m.setattr("llama_cpp.Llama", FakeLlama)+ from autogen_ext.models.llama_cpp._llama_cpp_completion_client import LlamaCppChatCompletionClient++ yield LlamaCppChatCompletionClient+ sys.modules.pop("autogen_ext.models.llama_cpp._llama_cpp_completion_client", None)+ sys.modules.pop("llama_cpp", None)+++async def test_llama_cpp_create(get_completion_client: "ContextManager[type[LlamaCppChatCompletionClient]]") -> None:+ with get_completion_client as Client:+ client = Client(model_path="dummy")+ messages: Sequence[Union[SystemMessage, UserMessage]] = [+ SystemMessage(content="Test system"),+ UserMessage(content="Test user", source="user"),+ ]+ result = await client.create(messages=messages)+ assert result.content == "Fake response"+ usage: RequestUsage = result.usage+ assert usage.prompt_tokens == 1+ assert usage.completion_tokens == 2+ assert result.finish_reason in ("stop", "unknown")+++# Commmented out due to raising not implemented error will leave in case streaming is supported in the future.+# @pytest.mark.asyncio+# async def test_llama_cpp_create_stream(+# get_completion_client: "ContextManager[type[LlamaCppChatCompletionClient]]",+# ) -> None:+# with get_completion_client as Client:+# client = Client(filename="dummy")+# messages: Sequence[Union[SystemMessage, UserMessage]] = [+# SystemMessage(content="Test system"),+# UserMessage(content="Test user", source="user"),+# ]+# collected = ""+# async for token in client.create_stream(messages=messages):+# collected += token+# assert collected == "Hello World"+++async def test_create_invalid_message(+ get_completion_client: "ContextManager[type[LlamaCppChatCompletionClient]]",+) -> None:+ with get_completion_client as Client:+ client = Client(model_path="dummy")+ # Pass an unsupported message type (integer) to trigger ValueError.+ with pytest.raises(ValueError, match="Unsupported message type"):+ await client.create(messages=[123]) # type: ignore+++async def test_count_and_remaining_tokens(+ get_completion_client: "ContextManager[type[LlamaCppChatCompletionClient]]", monkeypatch: pytest.MonkeyPatch+) -> None:+ with get_completion_client as Client:+ client = Client(model_path="dummy")+ msg = SystemMessage(content="Test")+ # count_tokens should count the bytes+ token_count = client.count_tokens([msg])+ # Since "Test" encoded is 4 bytes, expect 4 tokens.+ assert token_count >= 4+ remaining = client.remaining_tokens([msg])+ # remaining should be (1024 - token_count); ensure non-negative.+ assert remaining == max(1024 - token_count, 0)+++async def test_llama_cpp_integration_non_streaming() -> None:+ if not ((hasattr(torch.backends, "mps") and torch.backends.mps.is_available()) or torch.cuda.is_available()):+ pytest.skip("Skipping LlamaCpp integration tests: GPU not available not set")++ from autogen_ext.models.llama_cpp._llama_cpp_completion_client import LlamaCppChatCompletionClient++ client = LlamaCppChatCompletionClient(+ repo_id="unsloth/phi-4-GGUF", filename="phi-4-Q2_K_L.gguf", n_gpu_layers=-1, seed=1337, n_ctx=5000+ )+ messages: Sequence[Union[SystemMessage, UserMessage]] = [+ SystemMessage(content="You are a helpful assistant."),+ UserMessage(content="Hello, how are you?", source="user"),+ ]+ result = await client.create(messages=messages)+ assert isinstance(result.content, str) and len(result.content.strip()) > 0+++# Commmented out due to raising not implemented error will leave in case streaming is supported in the future.+# @pytest.mark.asyncio+# async def test_llama_cpp_integration_streaming() -> None:+# if not ((hasattr(torch.backends, "mps") and torch.backends.mps.is_available()) or torch.cuda.is_available()):+# pytest.skip("Skipping LlamaCpp integration tests: GPU not available not set")++# from autogen_ext.models.llama_cpp._llama_cpp_completion_client import LlamaCppChatCompletionClient+# client = LlamaCppChatCompletionClient(+# repo_id="unsloth/phi-4-GGUF", filename="phi-4-Q2_K_L.gguf", n_gpu_layers=-1, seed=1337, n_ctx=5000+# )+# messages: Sequence[Union[SystemMessage, UserMessage]] = [+# SystemMessage(content="You are a helpful assistant."),+# UserMessage(content="Please stream your response.", source="user"),+# ]+# collected = ""+# async for token in client.create_stream(messages=messages):+# collected += token+# assert isinstance(collected, str) and len(collected.strip()) > 0++# Commented out tool use as this functionality is not yet implemented for Phi-4.+# Define tools (functions) for the AssistantAgent+# def add(num1: int, num2: int) -> int:+# """Add two numbers together"""+# return num1 + num2+++# @pytest.mark.asyncio+# async def test_llama_cpp_integration_tool_use() -> None:+# if not ((hasattr(torch.backends, "mps") and torch.backends.mps.is_available()) or torch.cuda.is_available()):+# pytest.skip("Skipping LlamaCpp integration tests: GPU not available not set")++# from autogen_ext.models.llama_cpp._llama_cpp_completion_client import LlamaCppChatCompletionClient++# model_client = LlamaCppChatCompletionClient(+# repo_id="unsloth/phi-4-GGUF", filename="phi-4-Q2_K_L.gguf", n_gpu_layers=-1, seed=1337, n_ctx=5000+# )++# # Initialize the AssistantAgent+# assistant = AssistantAgent(+# name="assistant",+# system_message=("You can add two numbers together using the `add` function. "),+# model_client=model_client,+# tools=[+# FunctionTool(+# add,+# description="Add two numbers together. The first argument is num1 and second is num2. The return value is num1 + num2",+# )+# ],+# reflect_on_tool_use=True, # Reflect on tool results+# )++# # Test the tool+# response = await assistant.on_messages(+# [+# TextMessage(content="add 3 and 4", source="user"),+# ],+# CancellationToken(),+# )++# assert "7" in response.chat_message.contentdiff --git a/python/uv.lock b/python/uv.lockindex 12ef4315f..b6c38aa8d 100644--- a/python/uv.lock+++ b/python/uv.lock@@ -628,6 +628,9 @@ jupyter-executor = [langchain = [{ name = "langchain-core" },]+llama-cpp = [+ { name = "llama-cpp-python" },+]magentic-one = [{ name = "autogen-agentchat" },{ name = "markitdown" },@@ -735,6 +738,7 @@ requires-dist = [{ name = "json-schema-to-pydantic", marker = "extra == 'http-tool'", specifier = ">=0.2.0" },{ name = "json-schema-to-pydantic", marker = "extra == 'mcp'", specifier = ">=0.2.2" },{ name = "langchain-core", marker = "extra == 'langchain'", specifier = "~=0.3.3" },+ { name = "llama-cpp-python", marker = "extra == 'llama-cpp'", specifier = ">=0.1.9" },{ name = "markitdown", marker = "extra == 'file-surfer'", specifier = "~=0.0.1" },{ name = "markitdown", marker = "extra == 'magentic-one'", specifier = "~=0.0.1" },{ name = "markitdown", marker = "extra == 'web-surfer'", specifier = "~=0.0.1" },@@ -3501,6 +3505,18 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/43/b1/9355547c3b9043ba2821e7797f322c753dfa4d2a3da7bb05690fce689eaa/llama_cloud-0.1.11-py3-none-any.whl", hash = "sha256:b703765d03783a5a0fc57a52adc9892f8b91b0c19bbecb85a54ad4e813342951", size = 250609 },]+[[package]]+name = "llama-cpp-python"+version = "0.3.7"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "diskcache" },+ { name = "jinja2" },+ { name = "numpy" },+ { name = "typing-extensions" },+]+sdist = { url = "https://files.pythonhosted.org/packages/a6/38/7a47b1fb1d83eaddd86ca8ddaf20f141cbc019faf7b425283d8e5ef710e5/llama_cpp_python-0.3.7.tar.gz", hash = "sha256:0566a0dcc0f38005c4093309a87f67c2452449522e3e17e15cd735a62957894c", size = 66715891 }+[[package]]name = "llama-index"version = "0.12.14"