Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

feat: add EarthScienceToolkit param types and supported formats in MarkItDownLoader (#3764)

Florianc45d501e73e1
diff --git a/camel/loaders/markitdown.py b/camel/loaders/markitdown.py
index bea9da0b3..7c4c2288b 100644
--- a/camel/loaders/markitdown.py
+++ b/camel/loaders/markitdown.py
@@ -62,6 +62,8 @@ class MarkItDownLoader:
".xml",
".zip",
".txt",
+ # the file_paths may be markdown files when using FileToolkit.read_file
+ ".md",
]
def __init__(
diff --git a/camel/toolkits/earth_science_toolkit.py b/camel/toolkits/earth_science_toolkit.py
index 95fe8253a..142895fea 100644
--- a/camel/toolkits/earth_science_toolkit.py
+++ b/camel/toolkits/earth_science_toolkit.py
@@ -1583,7 +1583,12 @@ class EarthScienceToolkit(BaseToolkit):
return float(np.nanmean(combined_lst_values))
def calculate_max_lst_by_ndvi(
- self, red_path, nir_path, lst_path, ndvi_threshold, mode='above'
+ self,
+ red_path: str,
+ nir_path: str,
+ lst_path: str,
+ ndvi_threshold: float,
+ mode: str = 'above'
):
"""
Calculate the maximum Land Surface Temperature (LST) in areas where
@@ -2275,7 +2280,9 @@ class EarthScienceToolkit(BaseToolkit):
dst.write(ntu.astype(rasterio.float32), 1)
return f'Result saved at {TEMP_DIR / output_path}'
- def threshold_segmentation(self, input_image_path, threshold, output_path):
+ def threshold_segmentation(
+ self, input_image_path: str, threshold: float | int, output_path: str
+ ):
"""Perform threshold-based segmentation on a single-band raster image.
Reads a raster image, converts it to a binary mask by applying a fixed
@@ -2359,7 +2366,7 @@ class EarthScienceToolkit(BaseToolkit):
count = np.sum(x > threshold)
return int(count)
- def count_skeleton_contours(self, image_path):
+ def count_skeleton_contours(self, image_path: str):
"""Count external contours in a skeletonized binary image.
Reads a binary image, applies erosion and skeletonization, then
@@ -2394,7 +2401,7 @@ class EarthScienceToolkit(BaseToolkit):
)
return len(contours)
- def bboxes2centroids(self, bboxes):
+ def bboxes2centroids(self, bboxes: list[list[float]]):
"""Convert bounding boxes to centroid coordinates.
Converts from [x_min, y_min, x_max, y_max] format to (x, y) centroids.
@@ -2413,7 +2420,7 @@ class EarthScienceToolkit(BaseToolkit):
"""
return [((x1 + x2) / 2, (y1 + y2) / 2) for x1, y1, x2, y2 in bboxes]
- def centroid_distance_extremes(self, centroids):
+ def centroid_distance_extremes(self, centroids: list[tuple[float, float]]):
"""Find closest and farthest centroid pairs.
Computes pairwise distances between centroids and returns both the
@@ -2452,7 +2459,9 @@ class EarthScienceToolkit(BaseToolkit):
'max': (int(max_idx[0]), int(max_idx[1]), float(max_dist)),
}
- def calculate_bbox_area(self, bboxes, gsd=None):
+ def calculate_bbox_area(
+ self, bboxes: list[list[float]], gsd: float | None = None
+ ):
"""Calculate the total area of bounding boxes in [x, y, w, h] format.
Args:
@@ -2650,7 +2659,9 @@ class EarthScienceToolkit(BaseToolkit):
'resid': [float(x) for x in result.resid],
}
- def detect_change_points(self, signal, model='l2', penalty=10):
+ def detect_change_points(
+ self, signal: list[float], model: str = 'l2', penalty: float = 10
+ ):
"""
Description:
Detect change points in a one-dimensional time series using the
@@ -2870,7 +2881,10 @@ class EarthScienceToolkit(BaseToolkit):
return max_direction
def count_spikes_from_values(
- self, values, spike_threshold=0.1, verbose=True
+ self,
+ values: list[float],
+ spike_threshold: float = 0.1,
+ verbose: bool = True
):
"""
Count the number of upward spikes in a sequence of numerical values.
@@ -3662,7 +3676,7 @@ class EarthScienceToolkit(BaseToolkit):
min_value = x_arr[min_index]
return float(min_value), int(min_index)
- def multiply(self, a, b):
+ def multiply(self, a: float | int, b: float | int):
"""
Description:
Multiply two numbers and return their product.
@@ -3706,7 +3720,9 @@ class EarthScienceToolkit(BaseToolkit):
"""
return math.ceil(n)
- def get_list_object_via_indexes(self, input_list, indexes):
+ def get_list_object_via_indexes(
+ self, input_list: list, indexes: list[int]
+ ):
"""
Description:
Retrieve elements from a list using a list or tuple of indices.
@@ -3714,7 +3730,7 @@ class EarthScienceToolkit(BaseToolkit):
Args:
input_list (list):
The source list from which elements will be extracted.
- indexes (list[int] or tuple[int]):
+ indexes (list[int]):
A sequence of indices specifying the positions of elements
to retrieve.
@@ -3967,7 +3983,9 @@ class EarthScienceToolkit(BaseToolkit):
threshold_value
)
- def get_percentile_value_from_image(self, image_path, percentile):
+ def get_percentile_value_from_image(
+ self, image_path: str, percentile: int | float
+ ):
"""
Description:
Calculate the N-th percentile value of pixel values in a raster
@@ -4011,7 +4029,11 @@ class EarthScienceToolkit(BaseToolkit):
raise TypeError(f'Unsupported raster data type: {dtype}')
def image_division_mean(
- self, image_path1, image_path2=None, band1=1, band2=2
+ self,
+ image_path1: str,
+ image_path2: str | None = None,
+ band1: int | None = 1,
+ band2: int | None = 2
):
"""
Description:
@@ -4063,7 +4085,7 @@ class EarthScienceToolkit(BaseToolkit):
return float(np.nanmean(ratio))
def calculate_intersection_percentage(
- self, path1, threshold1, path2, threshold2
+ self, path1: str, threshold1: float, path2: str, threshold2: float
):
"""
Description:
@@ -5037,7 +5059,7 @@ class EarthScienceToolkit(BaseToolkit):
dst.write(result, 1)
return f'Result save at {output_path_full}'
- def calculate_area(self, input_image_path, gsd):
+ def calculate_area(self, input_image_path: str, gsd: float | None):
"""
Description:
This function calculates the area of non-zero pixels in the input
@@ -5134,7 +5156,7 @@ class EarthScienceToolkit(BaseToolkit):
[_ for _ in os.listdir(dir_path) if not _.startswith('.')]
)
- def radiometric_correction_sr(self, input_band_path, output_path):
+ def radiometric_correction_sr(self, input_band_path: str, output_path: str):
"""
Apply Landsat 8 surface reflectance (SR_B*) radiometric correction.
@@ -5163,7 +5185,9 @@ class EarthScienceToolkit(BaseToolkit):
dst.write(corrected_band.astype(rasterio.float32), 1)
return f'Result saved at {TEMP_DIR / output_path}'
- def apply_cloud_mask(self, sr_band_path, qa_pixel_path, output_path):
+ def apply_cloud_mask(
+ self, sr_band_path: str, qa_pixel_path: str, output_path: str
+ ):
"""
Apply cloud/shadow mask to a single Landsat 8 surface reflectance band
using QA_PIXEL band.
diff --git a/camel/toolkits/retrieval_toolkit.py b/camel/toolkits/retrieval_toolkit.py
index 10f4ec45f..5ff8d6c2d 100644
--- a/camel/toolkits/retrieval_toolkit.py
+++ b/camel/toolkits/retrieval_toolkit.py
@@ -58,11 +58,9 @@ class RetrievalToolkit(BaseToolkit):
contents (Union[str, List[str]]): Local file paths, remote URLs or
string contents.
top_k (int, optional): The number of top results to return during
- retrieve. Must be a positive integer. Defaults to
- `DEFAULT_TOP_K_RESULTS`.
+ retrieve. Must be a positive integer. Defaults to 1.
similarity_threshold (float, optional): The similarity threshold
- for filtering results. Defaults to
- `DEFAULT_SIMILARITY_THRESHOLD`.
+ for filtering results. Defaults to 0.7.
Returns:
str: The information retrieved in response to the query, aggregated
diff --git a/test/loaders/test_markitdown.py b/test/loaders/test_markitdown.py
index eef8d5805..9f89779eb 100644
--- a/test/loaders/test_markitdown.py
+++ b/test/loaders/test_markitdown.py
@@ -41,6 +41,10 @@ def mock_files():
"demo.html",
"<html><body><h1>Mock HTML </h1></body></html>",
),
+ "demo_md": (
+ "demo.md",
+ "## Mock Markdown content",
+ ),
"report_pdf": ("report.pdf", "Mock PDF content"),
"presentation_pptx": ("presentation.pptx", "Mock PPTX content"),
"data_xlsx": ("data.xlsx", "Mock XLSX content"),
@@ -97,12 +101,13 @@ def test_convert_file_conversion_error(mock_files):
def test_convert_files_success(mock_files):
converter = MarkItDownLoader()
- file_paths = [mock_files["demo_html"], mock_files["report_pdf"]]
+ file_paths = [mock_files["demo_html"], mock_files["demo_md"], mock_files["report_pdf"]]
results = converter.convert_files(file_paths)
assert mock_files["demo_html"] in results
+ assert mock_files["demo_md"] in results
assert mock_files["report_pdf"] in results
assert all(isinstance(result, str) for result in results.values())
- assert len(results) == 2
+ assert len(results) == 3
def test_convert_files_with_errors(mock_files):