Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

chore: update data common toolkit to v2 api (#3750)

JINO ROHITf7b5cf005e98
diff --git a/camel/toolkits/data_commons_toolkit.py b/camel/toolkits/data_commons_toolkit.py
index caadccc64..6ef9bd188 100644
--- a/camel/toolkits/data_commons_toolkit.py
+++ b/camel/toolkits/data_commons_toolkit.py
@@ -12,11 +12,12 @@
# limitations under the License.
# ========= Copyright 2023-2026 @ CAMEL-AI.org. All Rights Reserved. =========
import logging
+import os
from typing import Any, Dict, List, Optional, Union
from camel.toolkits import FunctionTool
from camel.toolkits.base import BaseToolkit
-from camel.utils import MCPServer
+from camel.utils import MCPServer, api_keys_required
logger = logging.getLogger(__name__)
@@ -38,6 +39,11 @@ class DataCommonsToolkit(BaseToolkit):
Refer to https://datacommons.org/browser/ for more details.
"""
+ @api_keys_required(
+ [
+ (None, "DATACOMMONS_API_KEY"),
+ ]
+ )
def __init__(self, timeout: Optional[float] = None):
r"""Initialize the DataCommonsToolkit.
@@ -47,131 +53,86 @@ class DataCommonsToolkit(BaseToolkit):
(default: :obj:`None`)
"""
super().__init__(timeout=timeout)
+ from datacommons_client import DataCommonsClient
- def query_data_commons(
- self,
- query_string: str,
- ) -> Optional[List[Dict[str, Any]]]:
- r"""Query the Data Commons knowledge graph using SPARQL.
+ datacommons_api_key = os.environ.get("DATACOMMONS_API_KEY")
+
+ self.client = DataCommonsClient(datacommons_api_key)
+
+ def get_triples_outgoing(
+ self, dcids: Union[str, List[str]]
+ ) -> Optional[Dict[str, List[tuple]]]:
+ r"""Retrieve triples associated with nodes for outgoing relationships.
Args:
- query_string (str): A SPARQL query string.
+ dcids (Union[str, List[str]]): A single DCID or a list of DCIDs
+ to query.
Returns:
- Optional[List[Dict[str, Any]]]: A list of dictionaries, each
- representing a node matching the query conditions if success,
+ Optional[Dict[str, List[tuple]]]: A dictionary where keys are
+ DCIDs and values are lists of associated triples if success,
(default: :obj:`None`) otherwise.
-
- Note:
- - Only supports a limited subset of SPARQL functionality (ORDER BY,
- DISTINCT, LIMIT).
- - Each variable in the query should have a 'typeOf' condition.
- - The Python SPARQL library currently only supports the V1 version
- of the API.
-
- Reference:
- https://docs.datacommons.org/api/python/query.html
"""
- import datacommons
try:
- results = datacommons.query(query_string)
-
- processed_results = [
- {key: value for key, value in row.items()} for row in results
- ]
-
- return processed_results
+ result = self.client.node.fetch(dcids, expression="->*")
+ return result.data
except Exception as e:
- logger.error(
- f"An error occurred while querying Data Commons: {e!s}"
- )
+ logger.error(f"An error occurred: {e!s}")
return None
- def get_triples(
- self, dcids: Union[str, List[str]], limit: int = 500
+ def get_triples_incoming(
+ self, dcids: Union[str, List[str]]
) -> Optional[Dict[str, List[tuple]]]:
- r"""Retrieve triples associated with nodes.
+ r"""Retrieve triples associated with nodes for incoming relationships.
Args:
dcids (Union[str, List[str]]): A single DCID or a list of DCIDs
to query.
- limit (int): The maximum number of triples per
- combination of property and type. (default: :obj:`500`)
Returns:
Optional[Dict[str, List[tuple]]]: A dictionary where keys are
DCIDs and values are lists of associated triples if success,
(default: :obj:`None`) otherwise.
-
- Note:
- - The function will raise a ValueError if any of the required
- arguments are missing.
- - The function will raise a TypeError if the dcids are not a string
- or a list of strings.
- - The function will raise a ValueError if the limit is not between
- 1 and 500.
- - The function will raise a KeyError if one or more of the provided
- DCIDs do not exist in the Data Commons knowledge graph.
- - The function will raise an Exception if an unexpected error occurs.
-
- Reference:
- https://docs.datacommons.org/api/python/triple.html
"""
- import datacommons
try:
- result = datacommons.get_triples(dcids, limit)
- return result
+ result = self.client.node.fetch(dcids, expression="<-*")
+ return result.data
except Exception as e:
logger.error(f"An error occurred: {e!s}")
return None
- def get_stat_time_series(
+ def get_stat(
self,
- place: str,
- stat_var: str,
- measurement_method: Optional[str] = None,
- observation_period: Optional[str] = None,
- unit: Optional[str] = None,
- scaling_factor: Optional[str] = None,
+ date: str,
+ entity_dcids: Union[str, List[str]],
+ variable_dcids: Union[str, List[str]],
) -> Optional[Dict[str, Any]]:
r"""Retrieve statistical time series for a place.
Args:
- place (str): The dcid of the Place to query for.
- stat_var (str): The dcid of the StatisticalVariable.
- measurement_method (str, optional): The technique used for
- measuring a statistical variable. (default: :obj:`None`)
- observation_period (str, optional): The time period over which an
- observation is made. (default: :obj:`None`)
- scaling_factor (str, optional): Property of statistical variables
- indicating factor by which a measurement is multiplied to fit
- a certain format. (default: :obj:`None`)
- unit (str, optional): The unit of measurement. (default:
- :obj:`None`)
+ date (str): The date option for the observations.
+ Use 'all' for all dates, 'latest' for the most recent data,
+ or provide a date as a string (e.g., "2026").
+ entity_dcids (Union[str, List[str]]): Entity IDs to filter the
+ data.
+ variable_dcids (Union[str, List[str]]): The variable(s) to fetch
+ observations for. This can be a single variable ID or a list
+ of IDs.
Returns:
Optional[Dict[str, Any]]: A dictionary containing the statistical
time series data if success, (default: :obj:`None`) otherwise.
-
- Reference:
- https://docs.datacommons.org/api/python/stat_series.html
"""
- import datacommons_pandas
try:
- result = datacommons_pandas.get_stat_series(
- place,
- stat_var,
- measurement_method,
- observation_period,
- unit,
- scaling_factor,
+ result = self.client.observation.fetch_observations_by_entity_dcid(
+ date, entity_dcids, variable_dcids
)
- return result
+ return result.byVariable
except Exception as e:
logger.error(
f"An error occurred while querying Data Commons: {e!s}"
@@ -184,22 +145,19 @@ class DataCommonsToolkit(BaseToolkit):
r"""Retrieves and analyzes property labels for given DCIDs.
Args:
- dcids (list): A list of Data Commons IDs (DCIDs) to analyze.
- out (bool): Direction of properties to retrieve. (default:
+ dcids (Union[str, List[str]]): A single DCID or a list of DCIDs
+ to query.
+ out (bool): Whether to fetch outgoing properties (default:
:obj:`True`)
Returns:
Optional[Dict[str, List[str]]]: Analysis results for each DCID if
success, (default: :obj:`None`) otherwise.
-
- Reference:
- https://docs.datacommons.org/api/python/property_label.html
"""
- import datacommons
try:
- result = datacommons.get_property_labels(dcids, out=out)
- return result
+ result = self.client.node.fetch_property_labels(dcids, out)
+ return result.data
except Exception as e:
logger.error(
f"An error occurred while analyzing property labels: {e!s}"
@@ -209,40 +167,32 @@ class DataCommonsToolkit(BaseToolkit):
def get_property_values(
self,
dcids: Union[str, List[str]],
- prop: str,
+ properties: Union[str, List[str]],
+ constraints: Optional[str] = None,
out: Optional[bool] = True,
- value_type: Optional[str] = None,
- limit: Optional[int] = None,
) -> Optional[Dict[str, Any]]:
r"""Retrieves and analyzes property values for given DCIDs.
Args:
- dcids (list): A list of Data Commons IDs (DCIDs) to analyze.
- prop (str): The property to analyze.
- value_type (str, optional): The type of the property value to
- filter by. Defaults to NONE. Only applicable if the value
- refers to a node.
- out (bool, optional): The label's direction. (default: :obj:`True`)
- (only returning response nodes directed towards the requested
- node). If set to False, will only return response nodes
- directed away from the request node. (default: :obj:`None`)
- limit (int, optional): (≤ 500) Maximum number of values returned
- per node. (default: :obj:`datacommons.utils._MAX_LIMIT`)
+ dcids (Union[str, List[str]]): A single DCID or a list of DCIDs
+ to query.
+ properties (Union[str, List[str]]): The property or properties to
+ analyze.
+ constraints (Optional[str]): Additional constraints for the query.
+ (default: :obj:`None`)
+ out (bool, optional): Whether to fetch outgoing properties.
+ (default: :obj:`True`)
Returns:
Optional[Dict[str, Any]]: Analysis results for each DCID if
success, (default: :obj:`None`) otherwise.
-
- Reference:
- https://docs.datacommons.org/api/python/property_value.html
"""
- import datacommons
try:
- result = datacommons.get_property_values(
- dcids, prop, out, value_type, limit
+ result = self.client.node.fetch_property_values(
+ dcids, properties, constraints, out
)
- return result
+ return result.data
except Exception as e:
logger.error(
@@ -251,25 +201,29 @@ class DataCommonsToolkit(BaseToolkit):
return None
def get_places_in(
- self, dcids: list, place_type: str
+ self,
+ place_dcids: Union[str, List[str]],
+ children_type: Optional[str] = None,
) -> Optional[Dict[str, Any]]:
r"""Retrieves places within a given place type.
Args:
- dcids (list): A list of Data Commons IDs (DCIDs) to analyze.
- place_type (str): The type of the place to filter by.
+ place_dcids (Union[str, List[str]]): A single DCID or a list of
+ DCIDs to query.
+ children_type (Optional[str]): The type of the child entities to
+ fetch. If None, fetches all child types.
+ (default: :obj:`None`)
Returns:
Optional[Dict[str, Any]]: Analysis results for each DCID if
success, (default: :obj:`None`) otherwise.
- Reference:
- https://docs.datacommons.org/api/python/place_in.html
"""
- import datacommons
try:
- result = datacommons.get_places_in(dcids, place_type)
+ result = self.client.node.fetch_place_children(
+ place_dcids, children_type
+ )
return result
except Exception as e:
@@ -279,95 +233,6 @@ class DataCommonsToolkit(BaseToolkit):
)
return None
- def get_stat_value(
- self,
- place: str,
- stat_var: str,
- date: Optional[str] = None,
- measurement_method: Optional[str] = None,
- observation_period: Optional[str] = None,
- unit: Optional[str] = None,
- scaling_factor: Optional[str] = None,
- ) -> Optional[float]:
- r"""Retrieves the value of a statistical variable for a given place
- and date.
-
- Args:
- place (str): The DCID of the Place to query for.
- stat_var (str): The DCID of the StatisticalVariable.
- date (str, optional): The preferred date of observation in ISO
- 8601 format. If not specified, returns the latest observation.
- (default: :obj:`None`)
- measurement_method (str, optional): The DCID of the preferred
- measurementMethod value. (default: :obj:`None`)
- observation_period (str, optional): The preferred observationPeriod
- value. (default: :obj:`None`)
- unit (str, optional): The DCID of the preferred unit value.
- (default: :obj:`None`)
- scaling_factor (str, optional): The preferred scalingFactor value.
- (default: :obj:`None`)
-
- Returns:
- Optional[float]: The value of the statistical variable for the
- given place and date if success, (default: :obj:`None`)
- otherwise.
-
- Reference:
- https://docs.datacommons.org/api/python/stat_value.html
- """
- import datacommons
-
- try:
- result = datacommons.get_stat_value(
- place,
- stat_var,
- date,
- measurement_method,
- observation_period,
- unit,
- scaling_factor,
- )
- return result
-
- except Exception as e:
- logger.error(
- "An error occurred while retrieving the value of a "
- f"statistical variable: {e!s}"
- )
- return None
-
- def get_stat_all(self, places: str, stat_vars: str) -> Optional[dict]:
- r"""Retrieves the value of a statistical variable for a given place
- and date.
-
- Args:
- places (str): The DCID IDs of the Place objects to query for.
- (Here DCID stands for Data Commons ID, the unique identifier
- assigned to all entities in Data Commons.)
- stat_vars (str): The dcids of the StatisticalVariables at
- https://datacommons.org/browser/StatisticalVariable
-
- Returns:
- Optional[dict]: A dictionary with the DCID of the place as the key
- and a list of tuples as the value if success, (default:
- :obj:`None`) otherwise.
-
- Reference:
- https://docs.datacommons.org/api/python/stat_all.html
- """
- import datacommons
-
- try:
- result = datacommons.get_stat_all(places, stat_vars)
- return result
-
- except Exception as e:
- logger.error(
- "An error occurred while retrieving the value of a "
- f"statistical variable: {e!s}"
- )
- return None
-
def get_tools(self) -> List[FunctionTool]:
r"""Returns a list of FunctionTool objects representing the functions
in the toolkit.
@@ -377,12 +242,10 @@ class DataCommonsToolkit(BaseToolkit):
the functions in the toolkit.
"""
return [
- FunctionTool(self.query_data_commons),
- FunctionTool(self.get_triples),
- FunctionTool(self.get_stat_time_series),
+ FunctionTool(self.get_triples_incoming),
+ FunctionTool(self.get_triples_outgoing),
+ FunctionTool(self.get_stat),
FunctionTool(self.get_property_labels),
FunctionTool(self.get_property_values),
FunctionTool(self.get_places_in),
- FunctionTool(self.get_stat_value),
- FunctionTool(self.get_stat_all),
]
diff --git a/examples/toolkits/data_commons_toolkit.py b/examples/toolkits/data_commons_toolkit.py
index 712a0d03d..4fba2f7aa 100644
--- a/examples/toolkits/data_commons_toolkit.py
+++ b/examples/toolkits/data_commons_toolkit.py
@@ -16,106 +16,14 @@ from camel.toolkits.data_commons_toolkit import DataCommonsToolkit
# Initialize the DataCommonsToolkit
dc_toolkit = DataCommonsToolkit()
-# Example 1: Query Data Commons
-geoId06_name_query = '''
-SELECT ?name ?dcid
-WHERE {
- ?a typeOf Place .
- ?a name ?name .
- ?a dcid ("geoId/06" "geoId/21" "geoId/24") .
- ?a dcid ?dcid
-}
-'''
-result = dc_toolkit.query_data_commons(geoId06_name_query)
-print("Query Result:")
-print(result)
-
-'''
-===============================================================================
-Query Result:
-[{'?name': 'Kentucky', '?dcid': 'geoId/21'},
- {'?name': 'California', '?dcid': 'geoId/06'},
- {'?name': 'Maryland', '?dcid': 'geoId/24'}]
-===============================================================================
-'''
-
-# Example 2: Get Triples
+# Example 1: Get Outgoing Triples
dcids = ["geoId/06", "geoId/21", "geoId/24"]
-triples = dc_toolkit.get_triples(dcids)
-print("\nTriples for California, Kentucky, and Maryland:")
+triples = dc_toolkit.get_triples_outgoing(dcids)
+print("\nOutgoing Triples for California, Kentucky, and Maryland:")
print(triples)
-'''
-===============================================================================
-Triples for California, Kentucky, and Maryland:
-{
- "geoId/06": [
- ("name", "California"),
- ("containedInPlace", "country/USA"),
- ...
- ],
- "geoId/21": [
- ("name", "Kentucky"),
- ("containedInPlace", "country/USA"),
- ...
- ],
- "geoId/24": [
- ("name", "Maryland"),
- ("containedInPlace", "country/USA"),
- ...
- ]
-}
-===============================================================================
-'''
-
-# Example 3: Get Statistical Time Series
-place = "geoId/06"
-stat_var = "Count_Person"
-series = dc_toolkit.get_stat_time_series(place, stat_var)
-print("\nPopulation Time Series for California:")
-print(series)
-
-'''
-===============================================================================
-Population Time Series for California:
-{
- "2010": 37253956,
- "2011": 37594778,
- "2012": 37971427,
- ...
-}
-===============================================================================
-'''
-
-# Example 4: Get Property Values
+# Example 2: Get Incoming Triples
dcids = ["geoId/06", "geoId/21", "geoId/24"]
-prop = "containedInPlace"
-values = dc_toolkit.get_property_values(dcids, prop)
-print("\nContained In Place for California, Kentucky, and Maryland:")
-print(values)
-
-'''
-===============================================================================
-Contained In Place for California, Kentucky, and Maryland:
-{
- "geoId/06": ["country/USA"],
- "geoId/21": ["country/USA"],
- "geoId/24": ["country/USA"]
-}
-===============================================================================
-'''
-
-# Example 5: Get Statistical Value
-place = "geoId/06"
-stat_var = "Count_Person"
-date = "2021"
-value = dc_toolkit.get_stat_value(place, stat_var, date)
-print("\nPopulation of California in 2021:")
-print(value)
-
-'''
-===============================================================================
-Population of California in 2021:
-39237836
-===============================================================================
-'''
+triples = dc_toolkit.get_triples_incoming(dcids)
+print("\nIncoming Triples for California, Kentucky, and Maryland:")
+print(triples)
diff --git a/pyproject.toml b/pyproject.toml
index d23fcba7b..61083eb74 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -159,11 +159,10 @@ communication_tools = [
"msal>=1.34.0,<2"
]
data_tools = [
+ "datacommons-client[pandas]>=2.1.5",
"numpy>=1.2,<=2.2",
"pandas>=2",
"textblob>=0.17.1,<0.18",
- "datacommons>=1.4.3,<2",
- "datacommons_pandas>=0.0.4",
"rouge>=1.0.1,<2",
"aiosqlite>=0.20.0,<0.21",
"stripe>=11.3.0,<12",
@@ -370,8 +369,7 @@ all = [
"notion-client>=2.2.1,<3",
"yt-dlp>=2024.11.4,<2025",
"ffmpeg-python>=0.2.0,<0.3",
- "datacommons>=1.4.3,<2",
- "datacommons_pandas>=0.0.4",
+ "datacommons-client[pandas]>=2.1.5",
"tavily-python>=0.5.0,<0.6",
"apify_client>=1.8.1,<2",
"stripe>=11.3.0,<12",
@@ -638,14 +636,13 @@ module = [
"praw",
"resend",
"textblob",
- "datacommons",
- "datacommons_pandas",
"tavily-python",
"scholarly",
"notion-client",
"ffmpeg",
"yt_dlp",
"datasets",
+ "datacommons_client",
"pandas",
"tree-sitter-python",
"tree-sitter",
diff --git a/test/toolkits/test_data_commons_toolkit.py b/test/toolkits/test_data_commons_toolkit.py
index 3cc4107a1..0fe6a00fb 100644
--- a/test/toolkits/test_data_commons_toolkit.py
+++ b/test/toolkits/test_data_commons_toolkit.py
@@ -11,237 +11,190 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2023-2026 @ CAMEL-AI.org. All Rights Reserved. =========
-from unittest.mock import patch
+import os
+import sys
+from unittest.mock import MagicMock, patch
-from camel.toolkits.data_commons_toolkit import DataCommonsToolkit
+# Create mock for datacommons_client module
+mock_datacommons_client = MagicMock()
+sys.modules['datacommons_client'] = mock_datacommons_client
-def test_query_data_commons():
[email protected](os.environ, {"DATACOMMONS_API_KEY": "test_api_key"})
+def test_get_triples_outgoing():
+ # Reset mock for fresh test
+ mock_client = MagicMock()
+ mock_datacommons_client.DataCommonsClient.return_value = mock_client
+
+ from camel.toolkits.data_commons_toolkit import DataCommonsToolkit
+
+ dcids = ['geoId/06', 'geoId/21']
+ expected_data = {
+ 'geoId/06': [('name', 'California'), ('typeOf', 'State')],
+ 'geoId/21': [('name', 'Kentucky'), ('typeOf', 'State')],
+ }
+
+ mock_response = MagicMock()
+ mock_response.data = expected_data
+ mock_client.node.fetch.return_value = mock_response
+
dc_toolkit = DataCommonsToolkit()
- query = '''
- SELECT ?name ?dcid
- WHERE {
- ?a typeOf Place .
- ?a name ?name .
- ?a dcid ("geoId/06" "geoId/21" "geoId/24") .
- ?a dcid ?dcid
+ result = dc_toolkit.get_triples_outgoing(dcids)
+
+ assert result == expected_data
+ mock_client.node.fetch.assert_called_once_with(dcids, expression="->*")
+ print("test_get_triples_outgoing passed successfully")
+
+
[email protected](os.environ, {"DATACOMMONS_API_KEY": "test_api_key"})
+def test_get_triples_incoming():
+ mock_client = MagicMock()
+ mock_datacommons_client.DataCommonsClient.return_value = mock_client
+
+ from camel.toolkits.data_commons_toolkit import DataCommonsToolkit
+
+ dcids = ['geoId/06', 'geoId/21']
+ expected_data = {
+ 'geoId/06': [('containedInPlace', 'country/USA')],
+ 'geoId/21': [('containedInPlace', 'country/USA')],
}
- '''
- expected_result = [
- {'?name': 'Kentucky', '?dcid': 'geoId/21'},
- {'?name': 'California', '?dcid': 'geoId/06'},
- {'?name': 'Maryland', '?dcid': 'geoId/24'},
- ]
-
- with patch('datacommons.query') as mock_query:
- mock_query.return_value = expected_result
- try:
- result = dc_toolkit.query_data_commons(query)
- assert result == expected_result
- print("test_query_data_commons passed successfully")
- except Exception as e:
- print(f"An error occurred: {e!s}")
- print(f"Result: {result}")
- raise
-
-
-def test_get_triples():
+
+ mock_response = MagicMock()
+ mock_response.data = expected_data
+ mock_client.node.fetch.return_value = mock_response
+
dc_toolkit = DataCommonsToolkit()
- dcids = ['dc/c3j78rpyssdmf', 'dc/7hfhd2ek8ppd2']
- expected_result = {
- 'dc/c3j78rpyssdmf': [
- ('dc/c3j78rpyssdmf', 'provenance', 'dc/h2lkz1'),
- ('dc/zn6l0flenf3m6', 'biosampleOntology', 'dc/c3j78rpyssdmf'),
- ('dc/tkcknpfwxfrhf', 'biosampleOntology', 'dc/c3j78rpyssdmf'),
- ('dc/jdzbbfhgzghv1', 'biosampleOntology', 'dc/c3j78rpyssdmf'),
- ('dc/4f9w8lhcwggxc', 'biosampleOntology', 'dc/c3j78rpyssdmf'),
- ],
- 'dc/7hfhd2ek8ppd2': [
- ('dc/4mjs95b1meh1h', 'biosampleOntology', 'dc/7hfhd2ek8ppd2'),
- ('dc/13xcyzcr819cb', 'biosampleOntology', 'dc/7hfhd2ek8ppd2'),
- ('dc/7hfhd2ek8ppd2', 'provenance', 'dc/h2lkz1'),
- ],
+ result = dc_toolkit.get_triples_incoming(dcids)
+
+ assert result == expected_data
+ mock_client.node.fetch.assert_called_once_with(dcids, expression="<-*")
+ print("test_get_triples_incoming passed successfully")
+
+
[email protected](os.environ, {"DATACOMMONS_API_KEY": "test_api_key"})
+def test_get_stat():
+ mock_client = MagicMock()
+ mock_datacommons_client.DataCommonsClient.return_value = mock_client
+
+ from camel.toolkits.data_commons_toolkit import DataCommonsToolkit
+
+ date = "2023"
+ entity_dcids = "geoId/06"
+ variable_dcids = "Count_Person"
+ expected_by_variable = {
+ 'Count_Person': {'geoId/06': {'date': '2023', 'value': 39538223}}
}
- with patch('datacommons.get_triples') as mock_get_triples:
- mock_get_triples.return_value = expected_result
- result = dc_toolkit.get_triples(dcids)
- assert result == expected_result
- mock_get_triples.assert_called_once_with(dcids, 500)
- print("test_get_triples passed successfully")
+ mock_response = MagicMock()
+ mock_response.byVariable = expected_by_variable
+ mock_client.observation.fetch_observations_by_entity_dcid.return_value = (
+ mock_response
+ )
+
+ dc_toolkit = DataCommonsToolkit()
+ result = dc_toolkit.get_stat(date, entity_dcids, variable_dcids)
+ assert result == expected_by_variable
+ mock_client.observation.fetch_observations_by_entity_dcid.assert_called_once_with(
+ date, entity_dcids, variable_dcids
+ )
+ print("test_get_stat passed successfully")
+
[email protected](os.environ, {"DATACOMMONS_API_KEY": "test_api_key"})
def test_get_property_labels():
- dc_toolkit = DataCommonsToolkit()
+ mock_client = MagicMock()
+ mock_datacommons_client.DataCommonsClient.return_value = mock_client
+
+ from camel.toolkits.data_commons_toolkit import DataCommonsToolkit
+
dcids = ['geoId/5508']
- expected_result = {
+ expected_data = {
'geoId/5508': [
'containedInPlace',
'geoId',
- 'geoJsonCoordinates',
- 'geoOverlaps',
- 'kmlCoordinates',
- 'landArea',
- 'latitude',
- 'longitude',
'name',
- 'provenance',
'typeOf',
- 'usCensusGeoId',
- 'waterArea',
]
}
- with patch('datacommons.get_property_labels') as mock_get_property_labels:
- mock_get_property_labels.return_value = expected_result
- result = dc_toolkit.get_property_labels(dcids, out=True)
- assert result == expected_result
- mock_get_property_labels.assert_called_once_with(dcids, out=True)
+ mock_response = MagicMock()
+ mock_response.data = expected_data
+ mock_client.node.fetch_property_labels.return_value = mock_response
+
+ dc_toolkit = DataCommonsToolkit()
+ result = dc_toolkit.get_property_labels(dcids, out=True)
+
+ assert result == expected_data
+ mock_client.node.fetch_property_labels.assert_called_once_with(dcids, True)
print("test_get_property_labels passed successfully")
[email protected](os.environ, {"DATACOMMONS_API_KEY": "test_api_key"})
def test_get_property_values():
- dc_toolkit = DataCommonsToolkit()
+ mock_client = MagicMock()
+ mock_datacommons_client.DataCommonsClient.return_value = mock_client
+
+ from camel.toolkits.data_commons_toolkit import DataCommonsToolkit
+
dcids = ["country/MDG"]
- prop, out, value_type = 'affectedPlace', False, 'EarthquakeEvent'
- expected_result = {
+ properties = 'affectedPlace'
+ constraints = None
+ out = False
+ expected_data = {
'country/MDG': [
'earthquake/us10007snb',
'earthquake/us200040me',
- 'earthquake/us60003r15',
- 'earthquake/us70009fim',
- 'earthquake/us7000h164',
- 'earthquake/usc000evr6',
- 'earthquake/usp00005zf',
- 'earthquake/usp00006yt',
- 'earthquake/usp0000afz',
- 'earthquake/usp0001fcd',
- 'earthquake/usp0001ss5',
- 'earthquake/usp00020ud',
- 'earthquake/usp0002kfd',
- 'earthquake/usp0004qn4',
- 'earthquake/usp0005gu9',
- 'earthquake/usp0007k9j',
- 'earthquake/usp0008vc6',
- 'earthquake/usp00091ab',
- 'earthquake/usp000dckw',
- 'earthquake/usp000fu24',
- 'earthquake/usp000gd93',
- 'earthquake/usp000gmuf',
- 'earthquake/usp000h6zw',
- 'earthquake/usp000jgbb',
]
}
- with patch('datacommons.get_property_values') as mock_get_property_values:
- mock_get_property_values.return_value = expected_result
- result = dc_toolkit.get_property_values(
- dcids, prop, out, value_type, 10
- )
- assert result == expected_result
- mock_get_property_values.assert_called_once_with(
- dcids, prop, out, value_type, 10
- )
+ mock_response = MagicMock()
+ mock_response.data = expected_data
+ mock_client.node.fetch_property_values.return_value = mock_response
+
+ dc_toolkit = DataCommonsToolkit()
+ result = dc_toolkit.get_property_values(
+ dcids, properties, constraints, out
+ )
+
+ assert result == expected_data
+ mock_client.node.fetch_property_values.assert_called_once_with(
+ dcids, properties, constraints, out
+ )
print("test_get_property_values passed successfully")
[email protected](os.environ, {"DATACOMMONS_API_KEY": "test_api_key"})
def test_get_places_in():
- dc_toolkit = DataCommonsToolkit()
- dcids = ["geoId/15", "geoId/02"]
- place_type = "CongressionalDistrict"
+ mock_client = MagicMock()
+ mock_datacommons_client.DataCommonsClient.return_value = mock_client
+
+ from camel.toolkits.data_commons_toolkit import DataCommonsToolkit
+
+ place_dcids = ["geoId/15", "geoId/02"]
+ children_type = "CongressionalDistrict"
expected_result = {
'geoId/15': ['geoId/1501', 'geoId/1502'],
'geoId/02': ['geoId/0200'],
}
- with patch('datacommons.get_places_in') as mock_get_places_in:
- mock_get_places_in.return_value = expected_result
- result = dc_toolkit.get_places_in(dcids, place_type)
- assert result == expected_result
- mock_get_places_in.assert_called_once_with(dcids, place_type)
- print("test_get_places_in passed successfully")
-
+ mock_client.node.fetch_place_children.return_value = expected_result
-def test_get_stat_value():
- dc_toolkit = DataCommonsToolkit()
- place = "geoId/12086" # Miami-Dade County
- stat_var = "RetailDrugDistribution_DrugDistribution_Naloxone"
- unit = "Grams"
- expected_result = 118.79
-
- with patch('datacommons.get_stat_value') as mock_get_stat_value:
- mock_get_stat_value.return_value = expected_result
- result = dc_toolkit.get_stat_value(place, stat_var, unit=unit)
- assert result == expected_result
- mock_get_stat_value.assert_called_once_with(
- place, stat_var, None, None, None, unit, None
- )
- print("test_get_stat_value passed successfully")
-
-
-def test_get_stat_all():
dc_toolkit = DataCommonsToolkit()
- places = ["geoId/27", "geoId/55"]
- stat_vars = ["Count_Person_EducationalAttainmentDoctorateDegree"]
- expected_result = {
- 'geoId/27': {
- 'Count_Person_EducationalAttainmentDoctorateDegree': {
- 'sourceSeries': [
- {
- 'val': {
- '2016': 50039,
- '2017': 52737,
- '2018': 54303,
- '2012': 40961,
- '2013': 42511,
- '2014': 44713,
- '2015': 47323,
- },
- 'measurementMethod': 'CensusACS5yrSurvey',
- 'importName': 'CensusACS5YearSurvey',
- 'provenanceDomain': 'census.gov',
- 'provenanceUrl': 'https://www.census.gov/',
- }
- ]
- }
- },
- 'geoId/55': {
- 'Count_Person_EducationalAttainmentDoctorateDegree': {
- 'sourceSeries': [
- {
- 'val': {
- '2017': 43737,
- '2018': 46071,
- '2012': 38052,
- '2013': 38711,
- '2014': 40133,
- '2015': 41387,
- '2016': 42590,
- },
- 'measurementMethod': 'CensusACS5yrSurvey',
- 'importName': 'CensusACS5YearSurvey',
- 'provenanceDomain': 'census.gov',
- 'provenanceUrl': 'https://www.census.gov/',
- }
- ]
- }
- },
- }
+ result = dc_toolkit.get_places_in(place_dcids, children_type)
- with patch('datacommons.get_stat_all') as mock_get_stat_all:
- mock_get_stat_all.return_value = expected_result
- result = dc_toolkit.get_stat_all(places, stat_vars)
- assert result == expected_result
- mock_get_stat_all.assert_called_once_with(places, stat_vars)
- print("test_get_stat_all passed successfully")
+ assert result == expected_result
+ mock_client.node.fetch_place_children.assert_called_once_with(
+ place_dcids, children_type
+ )
+ print("test_get_places_in passed successfully")
if __name__ == '__main__':
- test_query_data_commons()
- test_get_triples()
+ test_get_triples_outgoing()
+ test_get_triples_incoming()
+ test_get_stat()
test_get_property_labels()
test_get_property_values()
test_get_places_in()
- test_get_stat_all()
- test_get_stat_value()
print("All tests completed successfully!")
diff --git a/uv.lock b/uv.lock
index b728f46b5..4c95f7467 100644
--- a/uv.lock
+++ b/uv.lock
@@ -1,5 +1,5 @@
version = 1
-revision = 3
+revision = 2
requires-python = ">=3.10, <3.15"
resolution-markers = [
"python_full_version >= '3.14' and sys_platform == 'win32'",
@@ -923,8 +923,7 @@ all = [
{ name = "cohere" },
{ name = "crawl4ai" },
{ name = "dappier" },
- { name = "datacommons" },
- { name = "datacommons-pandas" },
+ { name = "datacommons-client", extra = ["pandas"] },
{ name = "datasets" },
{ name = "daytona-sdk" },
{ name = "ddgs" },
@@ -1071,8 +1070,7 @@ communication-tools = [
]
data-tools = [
{ name = "aiosqlite" },
- { name = "datacommons" },
- { name = "datacommons-pandas" },
+ { name = "datacommons-client", extra = ["pandas"] },
{ name = "math-verify" },
{ name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
{ name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
@@ -1404,10 +1402,8 @@ requires-dist = [
{ name = "crawl4ai", marker = "extra == 'rag'", specifier = ">=0.3.745" },
{ name = "dappier", marker = "extra == 'all'", specifier = ">=0.3.3,<0.4" },
{ name = "dappier", marker = "extra == 'web-tools'", specifier = ">=0.3.3,<0.4" },
- { name = "datacommons", marker = "extra == 'all'", specifier = ">=1.4.3,<2" },
- { name = "datacommons", marker = "extra == 'data-tools'", specifier = ">=1.4.3,<2" },
- { name = "datacommons-pandas", marker = "extra == 'all'", specifier = ">=0.0.4" },
- { name = "datacommons-pandas", marker = "extra == 'data-tools'", specifier = ">=0.0.4" },
+ { name = "datacommons-client", extras = ["pandas"], marker = "extra == 'all'", specifier = ">=2.1.5" },
+ { name = "datacommons-client", extras = ["pandas"], marker = "extra == 'data-tools'", specifier = ">=2.1.5" },
{ name = "datasets", marker = "extra == 'all'", specifier = ">=3,<4" },
{ name = "datasets", marker = "extra == 'eigent'", specifier = ">=3,<4" },
{ name = "datasets", marker = "extra == 'huggingface'", specifier = ">=3,<4" },
@@ -2563,30 +2559,22 @@ wheels = [
]
[[package]]
-name = "datacommons"
-version = "1.4.4"
+name = "datacommons-client"
+version = "2.1.5"
source = { registry = "https://pypi.org/simple" }
dependencies = [
+ { name = "pydantic" },
{ name = "requests" },
- { name = "six" },
+ { name = "typing-extensions" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/25/36/12e64ec5931a48dd86dd930979bf323881652ddfdebf71e34810ec6ee798/datacommons-1.4.4.tar.gz", hash = "sha256:7f7f84181f6e8933cfd31f29078a06110c4ce9bcfc20340733edf59c5864dace", size = 17065, upload-time = "2026-01-12T22:39:32.428Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/3f/be/378f623f2e7dd3d06108a940b2916bb6afdc3a132598e23cee5239012725/datacommons_client-2.1.5.tar.gz", hash = "sha256:1cc691778a6d0d251b6565dc20669c9529746e4cae71743e5c4a1030c331b94a", size = 60037, upload-time = "2026-01-12T20:04:47.683Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/fa/17/6313fcd1a8d93ee6311f3a83dc396753e32548f06dfbe479376ba99d66d7/datacommons-1.4.4-py3-none-any.whl", hash = "sha256:efd5acad708e56ac1216794e5efcf49134c1a7fb0de2034445a9a27ca7ce7ba4", size = 43688, upload-time = "2026-01-12T22:39:30.305Z" },
+ { url = "https://files.pythonhosted.org/packages/e7/99/fc2814651c64ce2a6e5c2da1dce10d7092852ac5ab23a58092981721f2ae/datacommons_client-2.1.5-py3-none-any.whl", hash = "sha256:d8ebf3cd568ed2a7cdcbdb42fd48fc09c124c2fd3cdbb51ac7f9115721374cc2", size = 77052, upload-time = "2026-01-12T20:04:46.659Z" },
]
-[[package]]
-name = "datacommons-pandas"
-version = "0.0.4"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
+[package.optional-dependencies]
+pandas = [
{ name = "pandas" },
- { name = "requests" },
- { name = "six" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/f9/fb/36c9245449713c622cfd26b9fc9647b6804c81075d45cf8b4a9b86359afc/datacommons_pandas-0.0.4.tar.gz", hash = "sha256:99423f23f49e6e3f95ca787ddc991561525acf9eb3d48dafada41951a6193a1f", size = 19644, upload-time = "2026-01-12T22:39:31.603Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/92/3c/50063fcb93a5057a75809acca2651a92035b320cc6cc4c19a423f131bd6f/datacommons_pandas-0.0.4-py3-none-any.whl", hash = "sha256:b086fdfe87bcb86079ed155917899b6fa37e418871b835d3f0d43e840a4d30b3", size = 43785, upload-time = "2026-01-12T22:39:28.999Z" },
]
[[package]]
@@ -7299,7 +7287,7 @@ name = "pexpect"
version = "4.9.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
- { name = "ptyprocess" },
+ { name = "ptyprocess", marker = "sys_platform != 'win32'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/42/92/cc564bf6381ff43ce1f4d06852fc19a2f11d180f23dc32d9588bee2f149d/pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f", size = 166450, upload-time = "2023-11-25T09:07:26.339Z" }
wheels = [
@@ -8270,7 +8258,7 @@ name = "pyobjc-framework-cocoa"
version = "12.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
- { name = "pyobjc-core", marker = "python_full_version >= '3.13' or sys_platform == 'darwin'" },
+ { name = "pyobjc-core", marker = "(python_full_version >= '3.13' and sys_platform != 'win32') or sys_platform == 'darwin'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/02/a3/16ca9a15e77c061a9250afbae2eae26f2e1579eb8ca9462ae2d2c71e1169/pyobjc_framework_cocoa-12.1.tar.gz", hash = "sha256:5556c87db95711b985d5efdaaf01c917ddd41d148b1e52a0c66b1a2e2c5c1640", size = 2772191, upload-time = "2025-11-14T10:13:02.069Z" }
wheels = [
@@ -8288,8 +8276,8 @@ name = "pyobjc-framework-quartz"
version = "12.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
- { name = "pyobjc-core", marker = "python_full_version >= '3.13' or sys_platform == 'darwin'" },
- { name = "pyobjc-framework-cocoa", marker = "python_full_version >= '3.13' or sys_platform == 'darwin'" },
+ { name = "pyobjc-core", marker = "(python_full_version >= '3.13' and sys_platform != 'win32') or sys_platform == 'darwin'" },
+ { name = "pyobjc-framework-cocoa", marker = "(python_full_version >= '3.13' and sys_platform != 'win32') or sys_platform == 'darwin'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/94/18/cc59f3d4355c9456fc945eae7fe8797003c4da99212dd531ad1b0de8a0c6/pyobjc_framework_quartz-12.1.tar.gz", hash = "sha256:27f782f3513ac88ec9b6c82d9767eef95a5cf4175ce88a1e5a65875fee799608", size = 3159099, upload-time = "2025-11-14T10:21:24.31Z" }
wheels = [