🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
refactor: Pandasai not using llm by default (#2023)
Wendong-Fan9846f952164e
diff --git a/camel/loaders/__init__.py b/camel/loaders/__init__.pyindex 08e70e98d..7dc4e0d53 100644--- a/camel/loaders/__init__.py+++ b/camel/loaders/__init__.py@@ -18,7 +18,7 @@ from .chunkr_reader import ChunkrReaderfrom .firecrawl_reader import Firecrawlfrom .jina_url_reader import JinaURLReaderfrom .mineru_extractor import MinerU-from .panda_reader import PandaReader+from .pandas_reader import PandasReaderfrom .unstructured_io import UnstructuredIO__all__ = [@@ -30,6 +30,6 @@ __all__ = ['Firecrawl','Apify','ChunkrReader',- 'PandaReader',+ 'PandasReader','MinerU',]diff --git a/camel/loaders/panda_reader.py b/camel/loaders/pandas_reader.pysimilarity index 83%rename from camel/loaders/panda_reader.pyrename to camel/loaders/pandas_reader.pyindex d86edde48..36b60ba76 100644--- a/camel/loaders/panda_reader.py+++ b/camel/loaders/pandas_reader.py@@ -11,13 +11,10 @@# See the License for the specific language governing permissions and# limitations under the License.# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========-import osfrom functools import wrapsfrom pathlib import Pathfrom typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union-import pandas as pd-if TYPE_CHECKING:from pandas import DataFramefrom pandasai import SmartDataframe@@ -50,25 +47,18 @@ def check_suffix(valid_suffixs: List[str]) -> Callable:return decorator-class PandaReader:+class PandasReader:def __init__(self, config: Optional[Dict[str, Any]] = None) -> None:- r"""Initializes the PandaReader class.+ r"""Initializes the PandasReader class.Args:config (Optional[Dict[str, Any]], optional): The configurationdictionary that can include LLM API settings for LLM-based- processing. If not provided, it will use OpenAI with the API- key from the OPENAI_API_KEY environment variable. You can- customize the LLM configuration by providing a 'llm' key in- the config dictionary. (default: :obj:`None`)+ processing. If not provided, no LLM will be configured by+ default. You can customize the LLM configuration by providing+ a 'llm' key in the config dictionary. (default: :obj:`None`)"""- from pandasai.llm import OpenAI # type: ignore[import-untyped]-self.config = config or {}- if "llm" not in self.config:- self.config["llm"] = OpenAI(- api_token=os.getenv("OPENAI_API_KEY"),- )self.__LOADER = {".csv": self.read_csv,@@ -91,8 +81,13 @@ class PandaReader:data: Union["DataFrame", str],*args: Any,**kwargs: Dict[str, Any],- ) -> "SmartDataframe":- r"""Loads a file or DataFrame and returns a SmartDataframe object.+ ) -> Union["DataFrame", "SmartDataframe"]:+ r"""Loads a file or DataFrame and returns a DataFrame or+ SmartDataframe object.++ If an LLM is configured in the config dictionary, a SmartDataframe+ will be returned, otherwise a regular pandas DataFrame will be+ returned.args:data (Union[DataFrame, str]): The data to load.@@ -100,24 +95,32 @@ class PandaReader:**kwargs (Dict[str, Any]): Additional keyword arguments.Returns:- SmartDataframe: The SmartDataframe object.+ Union[DataFrame, SmartDataframe]: The DataFrame or SmartDataframe+ object."""from pandas import DataFrame- from pandasai import SmartDataframe+ # Load the data into a pandas DataFrameif isinstance(data, DataFrame):- return SmartDataframe(data, config=self.config)- file_path = str(data)- path = Path(file_path)- if not file_path.startswith("http") and not path.exists():- raise FileNotFoundError(f"File {file_path} not found")- if path.suffix in self.__LOADER:- return SmartDataframe(- self.__LOADER[path.suffix](file_path, *args, **kwargs), # type: ignore[operator]- config=self.config,- )+ df = dataelse:- raise ValueError(f"Unsupported file format: {path.suffix}")+ file_path = str(data)+ path = Path(file_path)+ if not file_path.startswith("http") and not path.exists():+ raise FileNotFoundError(f"File {file_path} not found")+ if path.suffix in self.__LOADER:+ df = self.__LOADER[path.suffix](file_path, *args, **kwargs) # type: ignore[operator]+ else:+ raise ValueError(f"Unsupported file format: {path.suffix}")++ # If an LLM is configured, return a SmartDataframe, otherwise return a+ # regular DataFrame+ if "llm" in self.config:+ from pandasai import SmartDataframe++ return SmartDataframe(df, config=self.config)+ else:+ return df@check_suffix([".csv"])def read_csv(@@ -133,6 +136,8 @@ class PandaReader:Returns:DataFrame: The DataFrame object."""+ import pandas as pd+return pd.read_csv(file_path, *args, **kwargs)@check_suffix([".xlsx", ".xls"])@@ -149,6 +154,8 @@ class PandaReader:Returns:DataFrame: The DataFrame object."""+ import pandas as pd+return pd.read_excel(file_path, *args, **kwargs)@check_suffix([".json"])@@ -165,6 +172,8 @@ class PandaReader:Returns:DataFrame: The DataFrame object."""+ import pandas as pd+return pd.read_json(file_path, *args, **kwargs)@check_suffix([".parquet"])@@ -181,6 +190,8 @@ class PandaReader:Returns:DataFrame: The DataFrame object."""+ import pandas as pd+return pd.read_parquet(file_path, *args, **kwargs)def read_sql(self, *args: Any, **kwargs: Dict[str, Any]) -> "DataFrame":@@ -193,6 +204,8 @@ class PandaReader:Returns:DataFrame: The DataFrame object."""+ import pandas as pd+return pd.read_sql(*args, **kwargs)def read_table(@@ -208,6 +221,8 @@ class PandaReader:Returns:DataFrame: The DataFrame object."""+ import pandas as pd+return pd.read_table(file_path, *args, **kwargs)def read_clipboard(@@ -222,6 +237,8 @@ class PandaReader:Returns:DataFrame: The DataFrame object."""+ import pandas as pd+return pd.read_clipboard(*args, **kwargs)@check_suffix([".html"])@@ -238,6 +255,8 @@ class PandaReader:Returns:DataFrame: The DataFrame object."""+ import pandas as pd+return pd.read_html(file_path, *args, **kwargs)@check_suffix([".feather"])@@ -254,6 +273,8 @@ class PandaReader:Returns:DataFrame: The DataFrame object."""+ import pandas as pd+return pd.read_feather(file_path, *args, **kwargs)@check_suffix([".dta"])@@ -270,6 +291,8 @@ class PandaReader:Returns:DataFrame: The DataFrame object."""+ import pandas as pd+return pd.read_stata(file_path, *args, **kwargs)@check_suffix([".sas"])@@ -286,6 +309,8 @@ class PandaReader:Returns:DataFrame: The DataFrame object."""+ import pandas as pd+return pd.read_sas(file_path, *args, **kwargs)@check_suffix([".pkl"])@@ -302,6 +327,8 @@ class PandaReader:Returns:DataFrame: The DataFrame object."""+ import pandas as pd+return pd.read_pickle(file_path, *args, **kwargs)@check_suffix([".h5"])@@ -318,6 +345,8 @@ class PandaReader:Returns:DataFrame: The DataFrame object."""+ import pandas as pd+return pd.read_hdf(file_path, *args, **kwargs)@check_suffix([".orc"])@@ -334,4 +363,6 @@ class PandaReader:Returns:DataFrame: The DataFrame object."""+ import pandas as pd+return pd.read_orc(file_path, *args, **kwargs)diff --git a/examples/loaders/panda_example.py b/examples/loaders/pandas_example.pysimilarity index 50%rename from examples/loaders/panda_example.pyrename to examples/loaders/pandas_example.pyindex 9848af045..3b8fbdf7f 100644--- a/examples/loaders/panda_example.py+++ b/examples/loaders/pandas_example.py@@ -13,11 +13,14 @@# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+import os+import pandas as pd+from pandasai.llm import OpenAI # type: ignore[import-untyped]-from camel.loaders import PandaReader+from camel.loaders import PandasReader-reader = PandaReader()+# Create sample datasales_by_country = pd.DataFrame({"country": [@@ -46,10 +49,49 @@ sales_by_country = pd.DataFrame(],})-doc = reader.load(sales_by_country)-print(doc.chat("Which are the top 5 countries by sales?"))++# Example 1: Using PandasReader without an LLM (default behavior)+print("Example 1: PandasReader without LLM")+reader_no_llm = PandasReader()+# Without an LLM, load() returns a regular pandas DataFrame+df_no_llm = reader_no_llm.load(sales_by_country)+print(f"Loaded DataFrame shape: {df_no_llm.shape}")+print("Top 5 countries by sales:")+print(df_no_llm.sort_values(by="sales", ascending=False).head(5))+print()++# Example 2: Using PandasReader with an LLM configuration+print("Example 2: PandasReader with LLM")+# Only run this example if OPENAI_API_KEY is set+if os.getenv("OPENAI_API_KEY"):+ llm_config = {+ "llm": OpenAI(+ api_token=os.getenv("OPENAI_API_KEY"),+ )+ }+ reader_with_llm = PandasReader(config=llm_config)+ # With an LLM, load() returns a SmartDataframe+ df_with_llm = reader_with_llm.load(sales_by_country)+ print("Querying data with LLM:")+ print(df_with_llm.chat("Which are the top 5 countries by sales?"))+else:+ print("Skipping LLM example: OPENAI_API_KEY environment variable not set")'''+Example output:++Example 1: PandasReader without LLM+Loaded DataFrame shape: (10, 2)+Top 5 countries by sales:+ country sales+9 China 7000+0 United States 5000+8 Japan 4500+3 Germany 4100+1 United Kingdom 3200++Example 2: PandasReader with LLM+Querying data with LLM:===============================================================================country sales9 China 7000diff --git a/test/loaders/test_pandaai.py b/test/loaders/test_pandaai.pydeleted file mode 100644index fd19e94e3..000000000--- a/test/loaders/test_pandaai.py+++ /dev/null@@ -1,114 +0,0 @@-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========--import pandas as pd # type: ignore[import-untyped]--from camel.loaders import PandaReader---def test_topk():- reader = PandaReader()- sales_by_country = {- "country": [- "United States",- "United Kingdom",- "France",- "Germany",- "Italy",- "Spain",- "Canada",- "Australia",- "Japan",- "China",- ],- "sales": [- 5000,- 3200,- 2900,- 4100,- 2300,- 2100,- 2500,- 2600,- 4500,- 7000,- ],- }- doc = reader.load(pd.DataFrame(sales_by_country))- resp: pd.DataFrame = doc.chat("Which are the top 5 countries by sales?")- assert isinstance(- resp, pd.DataFrame- ), f"Expected a DataFrame but got: {resp}"- assert resp["country"].tolist() == [- "China",- "United States",- "Japan",- "Germany",- "United Kingdom",- ]---def test_multi_rows():- reader = PandaReader()- sales_by_country = {- "country": [- "United States",- "United Kingdom",- "France",- "Germany",- "Italy",- "Spain",- "Canada",- "Australia",- "Japan",- "China",- ],- "sales": [- 5000,- 3200,- 2900,- 4100,- 2300,- 2100,- 2500,- 2600,- 4500,- 7000,- ],- "profit": [- 1000,- 500,- 400,- 600,- 300,- 200,- 250,- 300,- 800,- 1200,- ],- }- doc = reader.load(pd.DataFrame(sales_by_country))- resp: pd.DataFrame = doc.chat(- "Calculate the profit margin for each country."- )- assert "profit_margin" in resp.columns- assert resp["profit_margin"].tolist()[0] == 20.0---def test_load_from_url():- reader = PandaReader()- url = "https://raw.githubusercontent.com/cs109/2014_data/master/countries.csv"- doc = reader.load(url)- assert doc is not Nonediff --git a/test/loaders/test_pandasai.py b/test/loaders/test_pandasai.pynew file mode 100644index 000000000..9db97a88d--- /dev/null+++ b/test/loaders/test_pandasai.py@@ -0,0 +1,162 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++import os+import tempfile++import pandas as pd # type: ignore[import-untyped]+import pytest+from pandasai.llm import OpenAI # type: ignore[import-untyped]++from camel.loaders import PandasReader+++def test_load_dataframe():+ reader = PandasReader()+ sales_by_country = {+ "country": [+ "United States",+ "United Kingdom",+ "France",+ "Germany",+ "Italy",+ "Spain",+ "Canada",+ "Australia",+ "Japan",+ "China",+ ],+ "sales": [+ 5000,+ 3200,+ 2900,+ 4100,+ 2300,+ 2100,+ 2500,+ 2600,+ 4500,+ 7000,+ ],+ }+ df = reader.load(pd.DataFrame(sales_by_country))+ # Test that the dataframe was loaded correctly+ assert isinstance(df, pd.DataFrame)+ assert df.shape == (10, 2)+ assert "country" in df.columns+ assert "sales" in df.columns+ assert df["sales"].sum() == 36200+++def test_multi_column_dataframe():+ reader = PandasReader()+ sales_by_country = {+ "country": [+ "United States",+ "United Kingdom",+ "France",+ "Germany",+ "Italy",+ "Spain",+ "Canada",+ "Australia",+ "Japan",+ "China",+ ],+ "sales": [+ 5000,+ 3200,+ 2900,+ 4100,+ 2300,+ 2100,+ 2500,+ 2600,+ 4500,+ 7000,+ ],+ "profit": [+ 1000,+ 500,+ 400,+ 600,+ 300,+ 200,+ 250,+ 300,+ 800,+ 1200,+ ],+ }+ df = reader.load(pd.DataFrame(sales_by_country))+ # Test that the dataframe was loaded correctly with multiple columns+ assert isinstance(df, pd.DataFrame)+ assert df.shape == (10, 3)+ assert "country" in df.columns+ assert "sales" in df.columns+ assert "profit" in df.columns+ assert df["profit"].sum() == 5550+++def test_load_from_url():+ """Test loading from a file path instead of URL to avoid hanging."""+ reader = PandasReader()++ # Create a temporary CSV file+ with tempfile.NamedTemporaryFile(suffix='.csv', delete=False) as tmp:+ # Write sample data to the file+ tmp.write(+ b"Country,Population,Area\nUSA,331000000,9834000\nChina,1444000000,9597000\nIndia,1393000000,3287000\n"+ )+ tmp_path = tmp.name++ try:+ # Test loading from the file+ df = reader.load(tmp_path)+ assert isinstance(df, pd.DataFrame)+ assert "Country" in df.columns+ assert df.shape == (3, 3)+ finally:+ # Clean up the temporary file+ if os.path.exists(tmp_path):+ os.remove(tmp_path)+++ os.getenv("OPENAI_API_KEY") is None,+ reason="OPENAI_API_KEY environment variable not set",+)+def test_with_llm():+ """Test with LLM explicitly configured."""+ llm_config = {+ "llm": OpenAI(+ api_token=os.getenv("OPENAI_API_KEY"),+ )+ }+ reader = PandasReader(config=llm_config)+ sales_by_country = {+ "country": ["United States", "United Kingdom", "China"],+ "sales": [5000, 3200, 7000],+ }+ df = reader.load(pd.DataFrame(sales_by_country))+ # With LLM config, should return a SmartDataframe+ from pandasai import SmartDataframe++ assert isinstance(df, SmartDataframe)++ # Test chat functionality if needed+ if os.getenv("OPENAI_API_KEY"):+ resp = df.chat("Which country has the highest sales?")+ # The response format can vary, so we just check that we got something+ assert resp is not None