🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
feat: add EarthScienceToolkit param types and supported formats in MarkItDownLoader (#3764)
Florianc45d501e73e1
diff --git a/camel/loaders/markitdown.py b/camel/loaders/markitdown.pyindex bea9da0b3..7c4c2288b 100644--- a/camel/loaders/markitdown.py+++ b/camel/loaders/markitdown.py@@ -62,6 +62,8 @@ class MarkItDownLoader:".xml",".zip",".txt",+ # the file_paths may be markdown files when using FileToolkit.read_file+ ".md",]def __init__(diff --git a/camel/toolkits/earth_science_toolkit.py b/camel/toolkits/earth_science_toolkit.pyindex 95fe8253a..142895fea 100644--- a/camel/toolkits/earth_science_toolkit.py+++ b/camel/toolkits/earth_science_toolkit.py@@ -1583,7 +1583,12 @@ class EarthScienceToolkit(BaseToolkit):return float(np.nanmean(combined_lst_values))def calculate_max_lst_by_ndvi(- self, red_path, nir_path, lst_path, ndvi_threshold, mode='above'+ self,+ red_path: str,+ nir_path: str,+ lst_path: str,+ ndvi_threshold: float,+ mode: str = 'above'):"""Calculate the maximum Land Surface Temperature (LST) in areas where@@ -2275,7 +2280,9 @@ class EarthScienceToolkit(BaseToolkit):dst.write(ntu.astype(rasterio.float32), 1)return f'Result saved at {TEMP_DIR / output_path}'- def threshold_segmentation(self, input_image_path, threshold, output_path):+ def threshold_segmentation(+ self, input_image_path: str, threshold: float | int, output_path: str+ ):"""Perform threshold-based segmentation on a single-band raster image.Reads a raster image, converts it to a binary mask by applying a fixed@@ -2359,7 +2366,7 @@ class EarthScienceToolkit(BaseToolkit):count = np.sum(x > threshold)return int(count)- def count_skeleton_contours(self, image_path):+ def count_skeleton_contours(self, image_path: str):"""Count external contours in a skeletonized binary image.Reads a binary image, applies erosion and skeletonization, then@@ -2394,7 +2401,7 @@ class EarthScienceToolkit(BaseToolkit):)return len(contours)- def bboxes2centroids(self, bboxes):+ def bboxes2centroids(self, bboxes: list[list[float]]):"""Convert bounding boxes to centroid coordinates.Converts from [x_min, y_min, x_max, y_max] format to (x, y) centroids.@@ -2413,7 +2420,7 @@ class EarthScienceToolkit(BaseToolkit):"""return [((x1 + x2) / 2, (y1 + y2) / 2) for x1, y1, x2, y2 in bboxes]- def centroid_distance_extremes(self, centroids):+ def centroid_distance_extremes(self, centroids: list[tuple[float, float]]):"""Find closest and farthest centroid pairs.Computes pairwise distances between centroids and returns both the@@ -2452,7 +2459,9 @@ class EarthScienceToolkit(BaseToolkit):'max': (int(max_idx[0]), int(max_idx[1]), float(max_dist)),}- def calculate_bbox_area(self, bboxes, gsd=None):+ def calculate_bbox_area(+ self, bboxes: list[list[float]], gsd: float | None = None+ ):"""Calculate the total area of bounding boxes in [x, y, w, h] format.Args:@@ -2650,7 +2659,9 @@ class EarthScienceToolkit(BaseToolkit):'resid': [float(x) for x in result.resid],}- def detect_change_points(self, signal, model='l2', penalty=10):+ def detect_change_points(+ self, signal: list[float], model: str = 'l2', penalty: float = 10+ ):"""Description:Detect change points in a one-dimensional time series using the@@ -2870,7 +2881,10 @@ class EarthScienceToolkit(BaseToolkit):return max_directiondef count_spikes_from_values(- self, values, spike_threshold=0.1, verbose=True+ self,+ values: list[float],+ spike_threshold: float = 0.1,+ verbose: bool = True):"""Count the number of upward spikes in a sequence of numerical values.@@ -3662,7 +3676,7 @@ class EarthScienceToolkit(BaseToolkit):min_value = x_arr[min_index]return float(min_value), int(min_index)- def multiply(self, a, b):+ def multiply(self, a: float | int, b: float | int):"""Description:Multiply two numbers and return their product.@@ -3706,7 +3720,9 @@ class EarthScienceToolkit(BaseToolkit):"""return math.ceil(n)- def get_list_object_via_indexes(self, input_list, indexes):+ def get_list_object_via_indexes(+ self, input_list: list, indexes: list[int]+ ):"""Description:Retrieve elements from a list using a list or tuple of indices.@@ -3714,7 +3730,7 @@ class EarthScienceToolkit(BaseToolkit):Args:input_list (list):The source list from which elements will be extracted.- indexes (list[int] or tuple[int]):+ indexes (list[int]):A sequence of indices specifying the positions of elementsto retrieve.@@ -3967,7 +3983,9 @@ class EarthScienceToolkit(BaseToolkit):threshold_value)- def get_percentile_value_from_image(self, image_path, percentile):+ def get_percentile_value_from_image(+ self, image_path: str, percentile: int | float+ ):"""Description:Calculate the N-th percentile value of pixel values in a raster@@ -4011,7 +4029,11 @@ class EarthScienceToolkit(BaseToolkit):raise TypeError(f'Unsupported raster data type: {dtype}')def image_division_mean(- self, image_path1, image_path2=None, band1=1, band2=2+ self,+ image_path1: str,+ image_path2: str | None = None,+ band1: int | None = 1,+ band2: int | None = 2):"""Description:@@ -4063,7 +4085,7 @@ class EarthScienceToolkit(BaseToolkit):return float(np.nanmean(ratio))def calculate_intersection_percentage(- self, path1, threshold1, path2, threshold2+ self, path1: str, threshold1: float, path2: str, threshold2: float):"""Description:@@ -5037,7 +5059,7 @@ class EarthScienceToolkit(BaseToolkit):dst.write(result, 1)return f'Result save at {output_path_full}'- def calculate_area(self, input_image_path, gsd):+ def calculate_area(self, input_image_path: str, gsd: float | None):"""Description:This function calculates the area of non-zero pixels in the input@@ -5134,7 +5156,7 @@ class EarthScienceToolkit(BaseToolkit):[_ for _ in os.listdir(dir_path) if not _.startswith('.')])- def radiometric_correction_sr(self, input_band_path, output_path):+ def radiometric_correction_sr(self, input_band_path: str, output_path: str):"""Apply Landsat 8 surface reflectance (SR_B*) radiometric correction.@@ -5163,7 +5185,9 @@ class EarthScienceToolkit(BaseToolkit):dst.write(corrected_band.astype(rasterio.float32), 1)return f'Result saved at {TEMP_DIR / output_path}'- def apply_cloud_mask(self, sr_band_path, qa_pixel_path, output_path):+ def apply_cloud_mask(+ self, sr_band_path: str, qa_pixel_path: str, output_path: str+ ):"""Apply cloud/shadow mask to a single Landsat 8 surface reflectance bandusing QA_PIXEL band.diff --git a/camel/toolkits/retrieval_toolkit.py b/camel/toolkits/retrieval_toolkit.pyindex 10f4ec45f..5ff8d6c2d 100644--- a/camel/toolkits/retrieval_toolkit.py+++ b/camel/toolkits/retrieval_toolkit.py@@ -58,11 +58,9 @@ class RetrievalToolkit(BaseToolkit):contents (Union[str, List[str]]): Local file paths, remote URLs orstring contents.top_k (int, optional): The number of top results to return during- retrieve. Must be a positive integer. Defaults to- `DEFAULT_TOP_K_RESULTS`.+ retrieve. Must be a positive integer. Defaults to 1.similarity_threshold (float, optional): The similarity threshold- for filtering results. Defaults to- `DEFAULT_SIMILARITY_THRESHOLD`.+ for filtering results. Defaults to 0.7.Returns:str: The information retrieved in response to the query, aggregateddiff --git a/test/loaders/test_markitdown.py b/test/loaders/test_markitdown.pyindex eef8d5805..9f89779eb 100644--- a/test/loaders/test_markitdown.py+++ b/test/loaders/test_markitdown.py@@ -41,6 +41,10 @@ def mock_files():"demo.html","<html><body><h1>Mock HTML </h1></body></html>",),+ "demo_md": (+ "demo.md",+ "## Mock Markdown content",+ ),"report_pdf": ("report.pdf", "Mock PDF content"),"presentation_pptx": ("presentation.pptx", "Mock PPTX content"),"data_xlsx": ("data.xlsx", "Mock XLSX content"),@@ -97,12 +101,13 @@ def test_convert_file_conversion_error(mock_files):def test_convert_files_success(mock_files):converter = MarkItDownLoader()- file_paths = [mock_files["demo_html"], mock_files["report_pdf"]]+ file_paths = [mock_files["demo_html"], mock_files["demo_md"], mock_files["report_pdf"]]results = converter.convert_files(file_paths)assert mock_files["demo_html"] in results+ assert mock_files["demo_md"] in resultsassert mock_files["report_pdf"] in resultsassert all(isinstance(result, str) for result in results.values())- assert len(results) == 2+ assert len(results) == 3def test_convert_files_with_errors(mock_files):