๐ซ CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
feat(toolkits): Add Querit content fetch API integration to SearchToolkit (#4259)
wzm-nhf56cda207888
diff --git a/camel/toolkits/search_toolkit.py b/camel/toolkits/search_toolkit.pyindex 77ff977c6..8937f94f9 100644--- a/camel/toolkits/search_toolkit.py+++ b/camel/toolkits/search_toolkit.py@@ -1666,6 +1666,173 @@ class SearchToolkit(BaseToolkit):except Exception as e:return {"error": f"Unexpected error during Querit search: {e!s}"}+ @api_keys_required([(None, 'QUERIT_API_KEY')])+ def fetch_querit_content(+ self,+ urls: List[str],+ content_format: Literal["text", "markdown", "html"] = "markdown",+ crawl_timeout: int = 10,+ extras_meta: bool = False,+ ) -> Dict[str, Any]:+ r"""Use Querit contents API to crawl web pages and return their full+ content.++ Querit (https://www.querit.ai) crawls the given URLs and returns the+ page body as text, markdown or HTML. This is typically used after+ `search_querit` to read the full text behind the result URLs, since+ search results only contain short snippets.++ Args:+ urls (List[str]): URLs of the pages to crawl. At least 1 and at+ most 10 URLs per call.+ content_format (Literal["text", "markdown", "html"]): The format+ of the returned page content. Prefer "markdown" or "text";+ "html" returns the raw page and can be tens of times larger.+ (default: :obj:`"markdown"`)+ crawl_timeout (int): The per-page crawl timeout in seconds, from+ 1 to 60. (default: :obj:`10`)+ extras_meta (bool): Whether to also return page metadata such as+ title, publish time and site name. (default: :obj:`False`)++ Returns:+ Dict[str, Any]: A dictionary containing either:++ - 'results': A list of dictionaries, each with:++ - 'result_id': The index of the result (starting from 1).+ - 'id': The fetch id, matching the id in 'statuses'.+ - 'url': The URL of the crawled page.+ - 'content': The page content in the requested format.+ A URL that could not be crawled is still returned here+ with an empty 'content', so check 'status' before+ treating the page as empty.+ - 'status': "success" or "failed" for this URL.+ - 'title', 'publish_time', 'site_name', 'site_icon': Page+ metadata, only present when `extras_meta` is True.++ - 'statuses': The raw per-URL status list from the API.+ - 'search_id': A unique request reference ID.+ - 'search_time': The server-side crawl time in seconds.+ - or 'error': An error message if something went wrong.+ """+ import json++ if not 1 <= len(urls) <= 10:+ return {+ "error": (+ f"urls must contain between 1 and 10 URLs, "+ f"got {len(urls)}."+ )+ }++ if not 1 <= crawl_timeout <= 60:+ return {+ "error": (+ f"crawl_timeout must be between 1 and 60 seconds, "+ f"got {crawl_timeout}."+ )+ }++ QUERIT_API_KEY = os.getenv("QUERIT_API_KEY")++ url = "https://api.querit.ai/v1/contents"+ headers = {+ "Authorization": f"Bearer {QUERIT_API_KEY}",+ "Content-Type": "application/json",+ }++ payload: Dict[str, Any] = {+ "urls": urls,+ "format": content_format,+ "crawlTimeout": crawl_timeout,+ "extrasMeta": extras_meta,+ }++ try:+ response = requests.post(+ url,+ headers=headers,+ data=json.dumps(payload, ensure_ascii=False),+ timeout=self.timeout,+ )++ # The API reports parameter errors as HTTP 4xx with a JSON body+ # carrying a human readable `error_msg`; surface that instead of+ # the raw body when it is available.+ if response.status_code != 200:+ detail = response.text+ try:+ body = response.json()+ except ValueError:+ body = None+ if isinstance(body, dict) and body.get("error_msg"):+ detail = str(body["error_msg"])+ return {+ "error": (+ f"Querit API failed with status "+ f"{response.status_code}: {detail}"+ )+ }++ data = response.json()++ # Check for API-level errors. `error_code` is an int on success+ # but a string on failure, so normalize before comparing.+ error_code = data.get("error_code", 200)+ try:+ error_code = int(error_code)+ except (TypeError, ValueError):+ pass+ if error_code != 200:+ return {+ "error": (+ f"Querit API error {error_code}: "+ f"{data.get('error_msg', 'Unknown error')}"+ )+ }++ # Parse results into CAMEL-consistent format+ statuses = data.get("statuses", [])+ status_by_id = {+ entry.get("id"): entry.get("status", "")+ for entry in statuses+ if isinstance(entry, dict)+ }+ results = []+ for idx, item in enumerate(data.get("results", []), 1):+ result: Dict[str, Any] = {+ "result_id": idx,+ "id": item.get("id", ""),+ "url": item.get("url", ""),+ "content": item.get("content", ""),+ "status": status_by_id.get(item.get("id"), ""),+ }+ meta = item.get("extrasMeta") or {}+ if meta:+ result.update(+ {+ "title": meta.get("title", ""),+ "publish_time": meta.get("publishTime", ""),+ "site_name": meta.get("siteName", ""),+ "site_icon": meta.get("siteIcon", ""),+ }+ )+ results.append(result)++ return {+ "search_id": data.get("search_id", 0),+ "search_time": data.get("searchTime", 0),+ "results": results,+ "statuses": statuses,+ }++ except requests.exceptions.RequestException as e:+ return {"error": f"Querit content request failed: {e!s}"}+ except Exception as e:+ return {+ "error": f"Unexpected error during Querit content fetch: {e!s}"+ }+@api_keys_required([(None, 'PERPLEXITY_API_KEY')])def search_perplexity(self,@@ -1789,6 +1956,7 @@ class SearchToolkit(BaseToolkit):FunctionTool(self.search_metaso),FunctionTool(self.search_serpapi),FunctionTool(self.search_querit),+ FunctionTool(self.fetch_querit_content),FunctionTool(self.search_perplexity),]diff --git a/docs/reference/camel.toolkits.search_toolkit.md b/docs/reference/camel.toolkits.search_toolkit.mdindex ead85c93a..1bb078279 100644--- a/docs/reference/camel.toolkits.search_toolkit.md+++ b/docs/reference/camel.toolkits.search_toolkit.md@@ -601,6 +601,53 @@ such as site names, favicons, and page ages.- 'took': The server-side response time.- or 'error': An error message if something went wrong.+<a id="camel.toolkits.search_toolkit.SearchToolkit.fetch_querit_content"></a>++### fetch_querit_content++```python+def fetch_querit_content(+ self,+ urls: List[str],+ content_format: Literal['text', 'markdown', 'html'] = 'markdown',+ crawl_timeout: int = 10,+ extras_meta: bool = False,+) -> Dict[str, Any]:+```++Use Querit contents API to crawl web pages and return their full+content.++Querit (https://www.querit.ai) crawls the given URLs and returns the+page body as text, markdown or HTML. This is typically used after+`search_querit` to read the full text behind the result URLs, since+search results only contain short snippets.++**Parameters:**++- **urls** (List[str]): URLs of the pages to crawl. At least 1 and at most 10 URLs per call.+- **content_format** (Literal['text', 'markdown', 'html']): The format of the returned page content. Prefer "markdown" or "text"; "html" returns the raw page and can be tens of times larger. (default: :obj:`"markdown"`)+- **crawl_timeout** (int): The per-page crawl timeout in seconds, from 1 to 60. (default: :obj:`10`)+- **extras_meta** (bool): Whether to also return page metadata such as title, publish time and site name. (default: :obj:`False`)++**Returns:**++ Dict[str, Any]: A dictionary containing either:++- 'results': A list of dictionaries, each with:++- 'result_id': The index of the result (starting from 1).+- 'id': The fetch id, matching the id in 'statuses'.+- 'url': The URL of the crawled page.+- 'content': The page content in the requested format. A URL that could not be crawled is still returned here with an empty 'content', so check 'status' before treating the page as empty.+- 'status': "success" or "failed" for this URL.+- 'title', 'publish_time', 'site_name', 'site_icon': Page metadata, only present when `extras_meta` is True.++- 'statuses': The raw per-URL status list from the API.+- 'search_id': A unique request reference ID.+- 'search_time': The server-side crawl time in seconds.+- or 'error': An error message if something went wrong.+<a id="camel.toolkits.search_toolkit.SearchToolkit.search_perplexity"></a>### search_perplexitydiff --git a/examples/toolkits/querit_search_example.py b/examples/toolkits/querit_search_example.pyindex 77530421f..e7c1e16fa 100644--- a/examples/toolkits/querit_search_example.py+++ b/examples/toolkits/querit_search_example.py@@ -13,7 +13,7 @@# ========= Copyright 2023-2026 @ CAMEL-AI.org. All Rights Reserved. ========="""-Querit Search - Real API Call Examples+Querit Search & Content - Real API Call ExamplesBefore running, set your API key:export QUERIT_API_KEY="your_api_key_here"@@ -90,9 +90,46 @@ for item in result_geo.get("results", []):print(f"\n[{item['result_id']}] {item['title']}")print(f" URL : {item['url']}")-# โโ 4. ChatAgent with Querit tool โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ+# โโ 4. Fetch full page content โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโprint("\n" + "=" * 60)-print("Example 4: ChatAgent using Querit as a tool")+print("Example 4: Fetch full page content with metadata")+print("=" * 60)++content_result = toolkit.fetch_querit_content(+ urls=["https://www.camel-ai.org/"],+ content_format="markdown",+ extras_meta=True,+)+print(f"Search ID : {content_result.get('search_id')}")+print(f"Crawl time : {content_result.get('search_time')}s")+print(f"Statuses : {content_result.get('statuses')}")+for item in content_result.get("results", []):+ print(f"\n[{item['result_id']}] {item.get('title', item['url'])}")+ print(f" Site : {item.get('site_name')}")+ print(f" Content : {item['content'][:200]}...")++# โโ 5. Search, then read the full text behind the results โโโโโโโโโโโโโโโโโโโโ+print("\n" + "=" * 60)+print("Example 5: Search then fetch content of the top results")+print("=" * 60)++search_result = toolkit.search_querit(+ query="CAMEL-AI multi-agent framework",+ number_of_result_pages=3,+)+top_urls = [item["url"] for item in search_result.get("results", [])[:3]]+print(f"Fetching content for: {top_urls}")++if top_urls:+ pages = toolkit.fetch_querit_content(urls=top_urls, content_format="text")+ for item in pages.get("results", []):+ print(f"\n[{item['result_id']}] {item['url']}")+ print(f" Chars : {len(item['content'])}")+ print(f" Preview : {item['content'][:150]}...")++# โโ 6. ChatAgent with Querit search + content tools โโโโโโโโโโโโโโโโโโโโโโโโโโ+print("\n" + "=" * 60)+print("Example 6: ChatAgent using Querit search and content as tools")print("=" * 60)model = ModelFactory.create(@@ -103,11 +140,15 @@ model = ModelFactory.create(agent = ChatAgent(system_message=("You are a helpful research assistant. "- "Use the Querit search tool to find up-to-date information "- "and summarize the results concisely."+ "Use the Querit search tool to find up-to-date information, then "+ "use the Querit content tool to read the full text of the most "+ "relevant pages before answering."),model=model,- tools=[FunctionTool(toolkit.search_querit)],+ tools=[+ FunctionTool(toolkit.search_querit),+ FunctionTool(toolkit.fetch_querit_content),+ ],)response = agent.step(diff --git a/examples/toolkits/search_toolkit.py b/examples/toolkits/search_toolkit.pyindex f40f8f205..2fe0ee995 100644--- a/examples/toolkits/search_toolkit.py+++ b/examples/toolkits/search_toolkit.py@@ -556,6 +556,45 @@ usr_msg = "What are the latest developments in multi-agent AI systems?"response = querit_agent.step(input_message=usr_msg, response_format=None)print(response.msgs[0].content)+# Example using Querit content to read the full text behind a URL+querit_content_response = SearchToolkit().fetch_querit_content(+ urls=["https://www.camel-ai.org/"],+ content_format="markdown",+ extras_meta=True,+)+print(querit_content_response)+"""+===============================================================================+{'search_id': 4317875391774580556, 'search_time': 0.217695743, 'results': [+ {'result_id': 1, 'id': '128b90d9-9bad-497b-8e8b-d75c2e502385',+ 'url': 'https://www.camel-ai.org/',+ 'content': 'HuggingFace-like Community for Multi-agent systems\n\npip+ install camel-ai...', 'status': 'success',+ 'title': 'Build Multi-Agent System for Task Automation',+ 'publish_time': '2026-07-29 08:06:25', 'site_name': 'CAMEL-AI',+ 'site_icon': 'https://www.camel-ai.org/favicon.ico'},+], 'statuses': [{'id': '128b90d9-9bad-497b-8e8b-d75c2e502385',+ 'status': 'success'}]}+===============================================================================+"""++# Example with ChatAgent using Querit search plus content+querit_research_agent = ChatAgent(+ system_message="""You are a helpful assistant that can use Querit search+ to find pages and Querit content to read their full text before+ answering.""",+ tools=[+ FunctionTool(SearchToolkit().search_querit),+ FunctionTool(SearchToolkit().fetch_querit_content),+ ],+)++usr_msg = "Summarize what CAMEL-AI is, based on its official website."+response = querit_research_agent.step(+ input_message=usr_msg, response_format=None+)+print(response.msgs[0].content)+# Example with ChatAgent using Perplexity searchperplexity_agent = ChatAgent(system_message="""You are a helpful assistant that can use Perplexitydiff --git a/test/toolkits/test_search_functions.py b/test/toolkits/test_search_functions.pyindex 9d944eee0..195bd378a 100644--- a/test/toolkits/test_search_functions.py+++ b/test/toolkits/test_search_functions.py@@ -1214,6 +1214,290 @@ def test_search_querit_empty_results(mock_post, search_toolkit):assert result["search_id"] == 12348+# ==================== Querit Content Tests ====================+++@patch('requests.post')+def test_fetch_querit_content_success(mock_post, search_toolkit):+ """Test successful Querit content fetch with metadata."""+ mock_response = MagicMock()+ mock_response.status_code = 200+ mock_response.json.return_value = {+ "error_code": 200,+ "error_msg": "",+ "search_id": 22345,+ "results": [+ {+ "id": "fetch-1",+ "url": "https://www.camel-ai.org/",+ "content": "# CAMEL-AI\n\nMulti-agent framework...",+ "extrasMeta": {+ "title": "CAMEL-AI",+ "url": "https://www.camel-ai.org/",+ "publishTime": "2025-01-15T10:30:00Z",+ "siteName": "camel-ai.org",+ "siteIcon": "https://www.camel-ai.org/favicon.ico",+ },+ }+ ],+ "statuses": [{"id": "fetch-1", "status": "success"}],+ "searchTime": 0.217695743,+ }+ mock_post.return_value = mock_response++ with patch.dict(os.environ, {'QUERIT_API_KEY': 'fake_api_key'}):+ result = search_toolkit.fetch_querit_content(+ urls=["https://www.camel-ai.org/"],+ content_format="markdown",+ crawl_timeout=20,+ extras_meta=True,+ )++ assert result["search_id"] == 22345+ assert result["search_time"] == 0.217695743+ assert result["statuses"] == [{"id": "fetch-1", "status": "success"}]+ assert result["results"][0] == {+ "result_id": 1,+ "id": "fetch-1",+ "url": "https://www.camel-ai.org/",+ "content": "# CAMEL-AI\n\nMulti-agent framework...",+ "status": "success",+ "title": "CAMEL-AI",+ "publish_time": "2025-01-15T10:30:00Z",+ "site_name": "camel-ai.org",+ "site_icon": "https://www.camel-ai.org/favicon.ico",+ }++ args, kwargs = mock_post.call_args+ assert args[0] == "https://api.querit.ai/v1/contents"+ assert kwargs['headers']['Authorization'] == 'Bearer fake_api_key'++ import json++ payload = json.loads(kwargs['data'])+ assert payload == {+ "urls": ["https://www.camel-ai.org/"],+ "format": "markdown",+ "crawlTimeout": 20,+ "extrasMeta": True,+ }+++@patch('requests.post')+def test_fetch_querit_content_defaults(mock_post, search_toolkit):+ """Test Querit content defaults and result without metadata."""+ mock_response = MagicMock()+ mock_response.status_code = 200+ mock_response.json.return_value = {+ "error_code": 200,+ "error_msg": "",+ "search_id": 22346,+ "results": [+ {+ "id": "fetch-1",+ "url": "https://example.com",+ "content": "plain content",+ }+ ],+ "statuses": [{"id": "fetch-1", "status": "success"}],+ "searchTime": 1,+ }+ mock_post.return_value = mock_response++ with patch.dict(os.environ, {'QUERIT_API_KEY': 'fake_api_key'}):+ result = search_toolkit.fetch_querit_content(+ urls=["https://example.com"]+ )++ assert result["results"][0] == {+ "result_id": 1,+ "id": "fetch-1",+ "url": "https://example.com",+ "content": "plain content",+ "status": "success",+ }++ import json++ payload = json.loads(mock_post.call_args[1]['data'])+ assert payload["format"] == "markdown"+ assert payload["crawlTimeout"] == 10+ assert payload["extrasMeta"] is False+++@patch('requests.post')+def test_fetch_querit_content_failed_url(mock_post, search_toolkit):+ """Test that a URL which failed to crawl is marked in the result."""+ mock_response = MagicMock()+ mock_response.status_code = 200+ mock_response.json.return_value = {+ "error_code": 200,+ "search_id": 22347,+ "results": [+ {+ "id": "fetch-1",+ "url": "https://this-domain-does-not-exist.com/",+ "content": "",+ }+ ],+ "statuses": [{"id": "fetch-1", "status": "failed"}],+ "searchTime": 3.6271142,+ }+ mock_post.return_value = mock_response++ with patch.dict(os.environ, {'QUERIT_API_KEY': 'fake_api_key'}):+ result = search_toolkit.fetch_querit_content(+ urls=["https://this-domain-does-not-exist.com/"]+ )++ assert result["results"][0]["status"] == "failed"+ assert result["results"][0]["content"] == ""+++ "urls", [[], [f"https://example.com/{i}" for i in range(11)]]+)+def test_fetch_querit_content_invalid_urls(search_toolkit, urls):+ """Test URL count validation in Querit content fetch."""+ with patch.dict(os.environ, {'QUERIT_API_KEY': 'fake_key'}):+ result = search_toolkit.fetch_querit_content(urls=urls)++ assert "error" in result+ assert "between 1 and 10 URLs" in result["error"]+++def test_fetch_querit_content_invalid_timeout(search_toolkit):+ """Test crawl_timeout validation in Querit content fetch."""+ with patch.dict(os.environ, {'QUERIT_API_KEY': 'fake_key'}):+ result = search_toolkit.fetch_querit_content(+ urls=["https://example.com"], crawl_timeout=61+ )++ assert "error" in result+ assert "between 1 and 60 seconds" in result["error"]+++@patch('requests.post')+def test_fetch_querit_content_http_error(mock_post, search_toolkit):+ """Test HTTP error handling in Querit content fetch."""+ mock_response = MagicMock()+ mock_response.status_code = 401+ mock_response.text = "Unauthorized"+ mock_response.json.side_effect = ValueError("not json")+ mock_post.return_value = mock_response++ with patch.dict(os.environ, {'QUERIT_API_KEY': 'invalid_key'}):+ result = search_toolkit.fetch_querit_content(+ urls=["https://example.com"]+ )++ assert "error" in result+ assert "Querit API failed with status 401" in result["error"]+++@patch('requests.post')+def test_fetch_querit_content_http_error_message(mock_post, search_toolkit):+ """Test that error_msg is surfaced for HTTP 4xx responses."""+ mock_response = MagicMock()+ mock_response.status_code = 400+ mock_response.text = '{"error_code":"400","error_msg":"..."}'+ mock_response.json.return_value = {+ "error_code": "400",+ "error_msg": "crawlTimeout must be in [1,60]",+ "search_id": 1,+ }+ mock_post.return_value = mock_response++ with patch.dict(os.environ, {'QUERIT_API_KEY': 'fake_key'}):+ result = search_toolkit.fetch_querit_content(+ urls=["https://example.com"]+ )++ assert "error" in result+ assert "Querit API failed with status 400" in result["error"]+ assert "crawlTimeout must be in [1,60]" in result["error"]+++@patch('requests.post')+def test_fetch_querit_content_string_error_code(mock_post, search_toolkit):+ """Test that a string error_code in a HTTP 200 body is detected."""+ mock_response = MagicMock()+ mock_response.status_code = 200+ mock_response.json.return_value = {+ "error_code": "400",+ "error_msg": "Invalid url",+ "search_id": 0,+ }+ mock_post.return_value = mock_response++ with patch.dict(os.environ, {'QUERIT_API_KEY': 'fake_key'}):+ result = search_toolkit.fetch_querit_content(urls=["not-a-url"])++ assert "error" in result+ assert "Querit API error 400" in result["error"]+ assert "Invalid url" in result["error"]+++@patch('requests.post')+def test_fetch_querit_content_string_success_code(mock_post, search_toolkit):+ """Test that a string error_code of "200" is treated as success."""+ mock_response = MagicMock()+ mock_response.status_code = 200+ mock_response.json.return_value = {+ "error_code": "200",+ "error_msg": "",+ "search_id": 22348,+ "results": [+ {"id": "fetch-1", "url": "https://example.com", "content": "x"}+ ],+ "statuses": [{"id": "fetch-1", "status": "success"}],+ "searchTime": 0.1,+ }+ mock_post.return_value = mock_response++ with patch.dict(os.environ, {'QUERIT_API_KEY': 'fake_key'}):+ result = search_toolkit.fetch_querit_content(+ urls=["https://example.com"]+ )++ assert "error" not in result+ assert result["results"][0]["content"] == "x"+++@patch('requests.post')+def test_fetch_querit_content_api_error(mock_post, search_toolkit):+ """Test API-level error handling in Querit content fetch."""+ mock_response = MagicMock()+ mock_response.status_code = 200+ mock_response.json.return_value = {+ "error_code": 400,+ "error_msg": "Invalid url",+ "search_id": 0,+ }+ mock_post.return_value = mock_response++ with patch.dict(os.environ, {'QUERIT_API_KEY': 'fake_key'}):+ result = search_toolkit.fetch_querit_content(urls=["not-a-url"])++ assert "error" in result+ assert "Querit API error 400" in result["error"]+ assert "Invalid url" in result["error"]+++@patch('requests.post')+def test_fetch_querit_content_request_exception(mock_post, search_toolkit):+ """Test request exception handling in Querit content fetch."""+ mock_post.side_effect = requests.exceptions.Timeout("Connection timed out")++ with patch.dict(os.environ, {'QUERIT_API_KEY': 'fake_key'}):+ result = search_toolkit.fetch_querit_content(+ urls=["https://example.com"]+ )++ assert "error" in result+ assert "Querit content request failed" in result["error"]++# ==================== Perplexity Search Tests ====================