Skip to content

Personal AI, On Personal Devices

Python

merge: resolve conflicts with main — keep both DeepResearch tools + MCP functions

Jon Saad-Falconbce8a2fe12cb
diff --cc pyproject.toml
index bc51a610,8049b1dd..56f38c6a
--- a/pyproject.toml
+++ b/pyproject.toml
@@@ -82,7 -83,12 +83,13 @@@ channel-rocketchat = ["rocketchat-API>=
channel-zulip = ["zulip>=0.9"]
channel-twitch = ["twitchio>=2.6"]
channel-nostr = ["pynostr>=0.6"]
+channel-twilio = ["twilio>=9.0"]
+ channel-gmail = [
+ "google-api-python-client>=2.0",
+ "google-auth-oauthlib>=1.0",
+ "google-auth-httplib2>=0.2",
+ ]
+ channel-twitter = ["tweepy>=4.14"]
browser = ["playwright>=1.40"]
media = ["openai>=1.30"]
pdf = ["pdfplumber>=0.10"]
diff --cc src/openjarvis/channels/__init__.py
index fb52ce90,e98ce6e8..4dc37062
--- a/src/openjarvis/channels/__init__.py
+++ b/src/openjarvis/channels/__init__.py
@@@ -38,7 -38,8 +38,9 @@@ _CHANNEL_MODULES =
"zulip_channel",
"twitch_channel",
"nostr_channel",
+ "twilio_sms",
+ "twitter",
+ "gmail",
]
for _mod in _CHANNEL_MODULES:
diff --cc src/openjarvis/server/agent_manager_routes.py
index 02d63f12,03c6b945..76f3a585
--- a/src/openjarvis/server/agent_manager_routes.py
+++ b/src/openjarvis/server/agent_manager_routes.py
@@@ -207,41 -297,142 +297,177 @@@ def build_tools_list() -> List[Dict[str
return items
+def _build_deep_research_tools(
+ engine: Any,
+ model: str,
+ knowledge_db_path: str = "",
+) -> list:
+ """Build the 4 DeepResearch tools from a KnowledgeStore.
+
+ Returns an empty list if the knowledge DB does not exist.
+ """
+ from pathlib import Path
+
+ if not knowledge_db_path:
+ from openjarvis.core.config import DEFAULT_CONFIG_DIR
+ knowledge_db_path = str(DEFAULT_CONFIG_DIR / "knowledge.db")
+
+ if not Path(knowledge_db_path).exists():
+ return []
+
+ from openjarvis.connectors.retriever import TwoStageRetriever
+ from openjarvis.connectors.store import KnowledgeStore
+ from openjarvis.tools.knowledge_search import KnowledgeSearchTool
+ from openjarvis.tools.knowledge_sql import KnowledgeSQLTool
+ from openjarvis.tools.scan_chunks import ScanChunksTool
+ from openjarvis.tools.think import ThinkTool
+
+ store = KnowledgeStore(knowledge_db_path)
+ retriever = TwoStageRetriever(store)
+ return [
+ KnowledgeSearchTool(retriever=retriever),
+ KnowledgeSQLTool(store=store),
+ ScanChunksTool(store=store, engine=engine, model=model),
+ ThinkTool(),
+ ]
+
+
+ def _merge_tool_call_fragments(
+ accumulated: Dict[int, Dict[str, Any]],
+ fragments: List[Dict[str, Any]],
+ ) -> None:
+ """Merge incremental tool_call delta fragments into accumulated state.
+
+ OpenAI-compatible APIs send tool_calls as incremental fragments keyed
+ by ``index``. Each fragment may contain partial ``function.name`` and/or
+ ``function.arguments`` strings that must be concatenated.
+ """
+ for frag in fragments:
+ idx = frag.get("index", 0)
+ if idx not in accumulated:
+ accumulated[idx] = {
+ "id": frag.get("id", ""),
+ "type": "function",
+ "function": {"name": "", "arguments": ""},
+ }
+ entry = accumulated[idx]
+ if frag.get("id"):
+ entry["id"] = frag["id"]
+ fn = frag.get("function", {})
+ if fn.get("name"):
+ entry["function"]["name"] += fn["name"]
+ if fn.get("arguments"):
+ entry["function"]["arguments"] += fn["arguments"]
+
+
+ def _get_mcp_tools(app_state: Any) -> Tuple[List[Dict[str, Any]], Dict[str, Any]]:
+ """Return (openai_tools_list, mcp_adapters_by_name).
+
+ Lazily discovers MCP tools from config and caches them on ``app_state``
+ so that subsequent requests reuse the same connections.
+ """
+ cached = getattr(app_state, "_mcp_tools_cache", None)
+ if cached is not None:
+ return cached
+
+ import json as _json
+
+ from openjarvis.core.config import load_config
+
+ openai_tools: List[Dict[str, Any]] = []
+ adapters_by_name: Dict[str, Any] = {}
+
+ try:
+ app_config = load_config()
+ except Exception as exc:
+ logger.warning("Failed to load config for MCP discovery: %s", exc)
+ return openai_tools, adapters_by_name
+
+ if not app_config.tools.mcp.enabled or not app_config.tools.mcp.servers:
+ return openai_tools, adapters_by_name
+
+ from openjarvis.mcp.client import MCPClient
+ from openjarvis.mcp.transport import StdioTransport, StreamableHTTPTransport
+ from openjarvis.tools.mcp_adapter import MCPToolProvider
+
+ # Keep clients alive so transports persist for tool calls at runtime
+ mcp_clients: list = getattr(app_state, "_mcp_clients", [])
+
+ try:
+ server_list = _json.loads(app_config.tools.mcp.servers)
+ except (_json.JSONDecodeError, TypeError) as exc:
+ logger.warning("Failed to parse MCP server config: %s", exc)
+ return openai_tools, adapters_by_name
+
+ if not isinstance(server_list, list):
+ return openai_tools, adapters_by_name
+
+ for server_cfg in server_list:
+ cfg = _json.loads(server_cfg) if isinstance(server_cfg, str) else server_cfg
+ name = cfg.get("name", "<unnamed>")
+ url = cfg.get("url")
+ command = cfg.get("command", "")
+ args = cfg.get("args", [])
+
+ try:
+ if url:
+ transport = StreamableHTTPTransport(url=url)
+ elif command:
+ transport = StdioTransport(command=[command] + args)
+ else:
+ logger.warning(
+ "MCP server '%s' has neither 'url' nor 'command' — skipping",
+ name,
+ )
+ continue
+
+ client = MCPClient(transport)
+ client.initialize()
+ mcp_clients.append(client)
+
+ provider = MCPToolProvider(client)
+ discovered = provider.discover()
+
+ # Per-server tool filtering
+ include_tools = set(cfg.get("include_tools", []))
+ exclude_tools = set(cfg.get("exclude_tools", []))
+ if include_tools:
+ discovered = [t for t in discovered if t.spec.name in include_tools]
+ if exclude_tools:
+ discovered = [t for t in discovered if t.spec.name not in exclude_tools]
+
+ for adapter in discovered:
+ spec = adapter.spec
+ openai_tools.append(
+ {
+ "type": "function",
+ "function": {
+ "name": spec.name,
+ "description": spec.description,
+ "parameters": spec.parameters,
+ },
+ }
+ )
+ adapters_by_name[spec.name] = adapter
+
+ logger.info(
+ "Discovered %d MCP tools from server '%s'",
+ len(discovered),
+ name,
+ )
+ except Exception as exc:
+ logger.warning(
+ "Failed to discover MCP tools from '%s': %s",
+ name,
+ exc,
+ )
+
+ app_state._mcp_clients = mcp_clients
+ if openai_tools:
+ app_state._mcp_tools_cache = (openai_tools, adapters_by_name)
+ return openai_tools, adapters_by_name
+
+
async def _stream_managed_agent(
*,
manager: AgentManager,
@@@ -268,56 -456,20 +491,53 @@@
agent_id = agent_record["id"]
config = agent_record.get("config", {})
- agent_type = agent_record.get("agent_type", "orchestrator")
model = config.get("model", getattr(engine, "_model", ""))
+ system_prompt = config.get("system_prompt")
+ temperature = config.get("temperature", 0.7)
+ max_tokens = config.get("max_tokens", 1024)
+ max_turns = config.get("max_turns", 10)
- # Resolve the agent class from registry
- agent_cls = AgentRegistry.get(agent_type)
- if agent_cls is None:
- # Fallback to orchestrator if the type is not registered
- agent_cls = AgentRegistry.get("orchestrator")
- if agent_cls is None:
- raise HTTPException(
- status_code=500, detail=f"Agent type '{agent_type}' not found in registry",
- )
+ # Build conversation messages from history + current input
+ llm_messages: List[Message] = []
+ if system_prompt:
+ llm_messages.append(Message(role=Role.SYSTEM, content=system_prompt))
- # Load prior conversation context (DESC order, reverse for chronological)
- history = manager.list_messages(agent_id, limit=50)
- for m in reversed(history):
+ # Build agent constructor kwargs from config
+ agent_kwargs: Dict[str, Any] = {
+ "engine": engine,
+ "model": model,
+ }
+ if bus is not None:
+ agent_kwargs["bus"] = bus
+ if config.get("system_prompt"):
+ agent_kwargs["system_prompt"] = config["system_prompt"]
+ if config.get("temperature") is not None:
+ agent_kwargs["temperature"] = config["temperature"]
+ if config.get("max_tokens") is not None:
+ agent_kwargs["max_tokens"] = config["max_tokens"]
+ if config.get("max_turns") is not None:
+ agent_kwargs["max_turns"] = config["max_turns"]
+
+ # Build DeepResearch tools when applicable
+ if agent_type == "deep_research":
+ dr_tools = _build_deep_research_tools(engine=engine, model=model)
+ if dr_tools:
+ agent_kwargs["tools"] = dr_tools
+
+ try:
+ agent = agent_cls(**agent_kwargs)
+ except TypeError as exc:
+ logger.warning(
+ "Agent instantiation failed with all kwargs, retrying minimal: %s",
+ exc,
+ )
+ agent = agent_cls(engine=engine, model=model)
+
+ # Build conversation context from existing messages
+ ctx = AgentContext()
+ messages = manager.list_messages(agent_id, limit=50)
+ # Messages come in DESC order, reverse for chronological
+ for m in reversed(messages):
- # Skip the message we just stored (it will be the input)
if m["id"] == message_id:
continue
if m["direction"] == "user_to_agent":
diff --cc uv.lock
index ae25b8b3,5d81ec3e..d50a2c74
--- a/uv.lock
+++ b/uv.lock
@@@ -184,6 -184,6 +184,18 @@@ wheels =
{ url = "https://files.pythonhosted.org/packages/b4/63/278a98c715ae467624eafe375542d8ba9b4383a016df8fdefe0ae28382a7/aiohttp-3.13.3-cp314-cp314t-win_amd64.whl", hash = "sha256:44531a36aa2264a1860089ffd4dce7baf875ee5a6079d5fb42e261c704ef7344", size = 499694, upload-time = "2026-01-03T17:32:24.546Z" },
]
++[[package]]
++name = "aiohttp-retry"
++version = "2.9.1"
++source = { registry = "https://pypi.org/simple" }
++dependencies = [
++ { name = "aiohttp" },
++]
++sdist = { url = "https://files.pythonhosted.org/packages/9d/61/ebda4d8e3d8cfa1fd3db0fb428db2dd7461d5742cea35178277ad180b033/aiohttp_retry-2.9.1.tar.gz", hash = "sha256:8eb75e904ed4ee5c2ec242fefe85bf04240f685391c4879d8f541d6028ff01f1", size = 13608, upload-time = "2024-11-06T10:44:54.574Z" }
++wheels = [
++ { url = "https://files.pythonhosted.org/packages/1a/99/84ba7273339d0f3dfa57901b846489d2e5c2cd731470167757f1935fffbd/aiohttp_retry-2.9.1-py3-none-any.whl", hash = "sha256:66d2759d1921838256a05a3f80ad7e724936f083e35be5abb5e16eed6be6dc54", size = 9981, upload-time = "2024-11-06T10:44:52.917Z" },
++]
++
[[package]]
name = "aiosignal"
version = "1.4.0"
@@@ -4752,6 -4815,6 +4827,9 @@@ channel-slack =
channel-telegram = [
{ name = "python-telegram-bot" },
]
++channel-twilio = [
++ { name = "twilio" },
++]
channel-twitch = [
{ name = "twitchio", version = "2.10.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
{ name = "twitchio", version = "3.2.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
@@@ -4963,6 -5033,7 +5048,8 @@@ requires-dist =
{ name = "torch", marker = "extra == 'memory-colbert'", specifier = ">=2.0" },
{ name = "torch", marker = "extra == 'orchestrator-training'", specifier = ">=2.0" },
{ name = "transformers", marker = "extra == 'orchestrator-training'", specifier = ">=4.40" },
+ { name = "tweepy", marker = "extra == 'channel-twitter'", specifier = ">=4.14" },
++ { name = "twilio", marker = "extra == 'channel-twilio'", specifier = ">=9.0" },
{ name = "twitchio", marker = "extra == 'channel-twitch'", specifier = ">=2.6" },
{ name = "uvicorn", marker = "extra == 'server'", specifier = ">=0.30" },
{ name = "viberbot", marker = "extra == 'channel-viber'", specifier = ">=1.0" },
@@@ -4973,7 -5044,7 +5060,7 @@@
{ name = "zeus-ml", extras = ["apple"], marker = "extra == 'energy-apple'" },
{ name = "zulip", marker = "extra == 'channel-zulip'", specifier = ">=0.9" },
]
- provides-extras = ["dev", "inference-mlx", "inference-vllm", "inference-cloud", "inference-google", "inference-litellm", "inference-gemma", "tools-search", "memory-faiss", "memory-colbert", "memory-pdf", "memory-bm25", "server", "openhands", "gpu-metrics", "energy-amd", "energy-apple", "energy-all", "orchestrator-training", "learning-dspy", "learning-gepa", "channel-telegram", "channel-discord", "channel-slack", "channel-line", "channel-viber", "channel-messenger", "channel-reddit", "channel-mastodon", "channel-xmpp", "channel-rocketchat", "channel-zulip", "channel-twitch", "channel-nostr", "browser", "media", "pdf", "scheduler", "security-signing", "sandbox-wasm", "sandbox-docker", "dashboard", "speech", "speech-deepgram", "eval-wandb", "eval-sheets", "docs"]
-provides-extras = ["dev", "inference-mlx", "inference-vllm", "inference-cloud", "inference-google", "inference-litellm", "inference-gemma", "tools-search", "memory-faiss", "memory-colbert", "memory-pdf", "memory-bm25", "server", "openhands", "gpu-metrics", "energy-amd", "energy-apple", "energy-all", "orchestrator-training", "learning-dspy", "learning-gepa", "channel-telegram", "channel-discord", "channel-slack", "channel-line", "channel-viber", "channel-messenger", "channel-reddit", "channel-mastodon", "channel-xmpp", "channel-rocketchat", "channel-zulip", "channel-twitch", "channel-nostr", "channel-gmail", "channel-twitter", "browser", "media", "pdf", "scheduler", "security-signing", "sandbox-wasm", "sandbox-docker", "dashboard", "speech", "speech-deepgram", "eval-wandb", "eval-sheets", "docs"]
++provides-extras = ["dev", "inference-mlx", "inference-vllm", "inference-cloud", "inference-google", "inference-litellm", "inference-gemma", "tools-search", "memory-faiss", "memory-colbert", "memory-pdf", "memory-bm25", "server", "openhands", "gpu-metrics", "energy-amd", "energy-apple", "energy-all", "orchestrator-training", "learning-dspy", "learning-gepa", "channel-telegram", "channel-discord", "channel-slack", "channel-line", "channel-viber", "channel-messenger", "channel-reddit", "channel-mastodon", "channel-xmpp", "channel-rocketchat", "channel-zulip", "channel-twitch", "channel-nostr", "channel-twilio", "channel-gmail", "channel-twitter", "browser", "media", "pdf", "scheduler", "security-signing", "sandbox-wasm", "sandbox-docker", "dashboard", "speech", "speech-deepgram", "eval-wandb", "eval-sheets", "docs"]
[package.metadata.requires-dev]
dev = [{ name = "maturin", specifier = ">=1.12.6" }]
@@@ -8703,6 -8804,20 +8820,35 @@@ wheels =
{ url = "https://files.pythonhosted.org/packages/f6/56/6113c23ff46c00aae423333eb58b3e60bdfe9179d542781955a5e1514cb3/triton-3.6.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46bd1c1af4b6704e554cad2eeb3b0a6513a980d470ccfa63189737340c7746a7", size = 188397994, upload-time = "2026-01-20T16:01:14.236Z" },
]
+ [[package]]
+ name = "tweepy"
+ version = "4.16.0"
+ source = { registry = "https://pypi.org/simple" }
+ dependencies = [
+ { name = "oauthlib" },
+ { name = "requests" },
+ { name = "requests-oauthlib" },
+ ]
+ sdist = { url = "https://files.pythonhosted.org/packages/6e/45/a73bb812b1817247d3f79b3b9a4784ab93a081853b697e87428caa8c287b/tweepy-4.16.0.tar.gz", hash = "sha256:1d95cbdc50bf6353a387f881f2584eaf60d14e00dbbdd8872a73de79c66878e3", size = 87646, upload-time = "2025-06-22T01:17:51.34Z" }
+ wheels = [
+ { url = "https://files.pythonhosted.org/packages/ee/7c/3375cd1fbefcb8ead580fe324b1b6dcdc21aabf51562ee6def7266fcf363/tweepy-4.16.0-py3-none-any.whl", hash = "sha256:48d1a1eb311d2c4b8990abcfa6f9fa2b2ad61be05c723b1a9b4f242656badae2", size = 98843, upload-time = "2025-06-22T01:17:49.823Z" },
+ ]
+
++[[package]]
++name = "twilio"
++version = "9.10.4"
++source = { registry = "https://pypi.org/simple" }
++dependencies = [
++ { name = "aiohttp" },
++ { name = "aiohttp-retry" },
++ { name = "pyjwt" },
++ { name = "requests" },
++]
++sdist = { url = "https://files.pythonhosted.org/packages/d7/28/9ffb8f01fc014bf634f0731bd562fed6209bc54d4249c69acfe68cf70302/twilio-9.10.4.tar.gz", hash = "sha256:2125e0027288903e7e8954ae6d5b1b593a205c28b3d72fa3fdd0b5f25825de04", size = 1637481, upload-time = "2026-03-24T13:04:11.822Z" }
++wheels = [
++ { url = "https://files.pythonhosted.org/packages/bc/3c/3296ee9f141b83623ff2c6c4d22093f7d2a8ea7455f98f705ee5f969bcd7/twilio-9.10.4-py2.py3-none-any.whl", hash = "sha256:5e8414a88c009f40b2ab2a1f00232f5f459f724c7b4bdf25a935c7d7b297a8a3", size = 2279546, upload-time = "2026-03-24T13:04:10.383Z" },
++]
++
[[package]]
name = "twitchio"
version = "2.10.0"