Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

feat: markitdown loader (#2248)

JINO ROHITd310f90ea837
diff --git a/camel/loaders/__init__.py b/camel/loaders/__init__.py
index 23cad7f98..880a2e092 100644
--- a/camel/loaders/__init__.py
+++ b/camel/loaders/__init__.py
@@ -18,6 +18,7 @@ from .chunkr_reader import ChunkrReader
from .crawl4ai_reader import Crawl4AI
from .firecrawl_reader import Firecrawl
from .jina_url_reader import JinaURLReader
+from .markitdown import MarkItDownLoader
from .mineru_extractor import MinerU
from .pandas_reader import PandasReader
from .scrapegraph_reader import ScrapeGraphAI
@@ -35,5 +36,6 @@ __all__ = [
'PandasReader',
'MinerU',
'Crawl4AI',
+ 'MarkItDownLoader',
'ScrapeGraphAI',
]
diff --git a/camel/loaders/markitdown.py b/camel/loaders/markitdown.py
new file mode 100644
index 000000000..a604f74b0
--- /dev/null
+++ b/camel/loaders/markitdown.py
@@ -0,0 +1,204 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import os
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import ClassVar, Dict, List, Optional
+
+from camel.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class MarkItDownLoader:
+ r"""MarkitDown convert various file types into Markdown format.
+
+ Supported Input Formats:
+ - PDF
+ - Microsoft Office documents:
+ - Word (.doc, .docx)
+ - Excel (.xls, .xlsx)
+ - PowerPoint (.ppt, .pptx)
+ - EPUB
+ - HTML
+ - Images (with EXIF metadata and OCR support)
+ - Audio files (with EXIF metadata and speech transcription)
+ - Text-based formats:
+ - CSV
+ - JSON
+ - XML
+ - ZIP archives (iterates over contents)
+ - YouTube URLs (via transcript extraction)
+ """
+
+ SUPPORTED_FORMATS: ClassVar[List[str]] = [
+ ".pdf",
+ ".doc",
+ ".docx",
+ ".xls",
+ ".xlsx",
+ ".ppt",
+ ".pptx",
+ ".epub",
+ ".html",
+ ".htm",
+ ".jpg",
+ ".jpeg",
+ ".png",
+ ".mp3",
+ ".wav",
+ ".csv",
+ ".json",
+ ".xml",
+ ".zip",
+ ".txt",
+ ]
+
+ def __init__(
+ self,
+ llm_client: Optional[object] = None,
+ llm_model: Optional[str] = None,
+ ):
+ r"""Initializes the Converter.
+
+ Args:
+ llm_client (Optional[object]): Optional client for LLM integration.
+ (default: :obj:`None`)
+ llm_model (Optional[str]): Optional model name for the LLM.
+ (default: :obj:`None`)
+ """
+ from markitdown import MarkItDown
+
+ try:
+ self.converter = MarkItDown(
+ llm_client=llm_client, llm_model=llm_model
+ )
+ logger.info("MarkItDownLoader initialized successfully.")
+ except Exception as e:
+ logger.error(f"Failed to initialize MarkItDown Converter: {e}")
+ raise Exception(f"Failed to initialize MarkItDown Converter: {e}")
+
+ def _validate_format(self, file_path: str) -> bool:
+ r"""Validates if the file format is supported.
+
+ Args:
+ file_path (str): Path to the input file.
+
+ Returns:
+ bool: True if the format is supported, False otherwise.
+ """
+ _, ext = os.path.splitext(file_path)
+ return ext.lower() in self.SUPPORTED_FORMATS
+
+ def convert_file(self, file_path: str) -> str:
+ r"""Converts the given file to Markdown format.
+
+ Args:
+ file_path (str): Path to the input file.
+
+ Returns:
+ str: Converted Markdown text.
+
+ Raises:
+ FileNotFoundError: If the specified file does not exist.
+ ValueError: If the file format is not supported.
+ Exception: For other errors during conversion.
+ """
+ if not os.path.isfile(file_path):
+ logger.error(f"File not found: {file_path}")
+ raise FileNotFoundError(f"File not found: {file_path}")
+
+ if not self._validate_format(file_path):
+ logger.error(
+ f"Unsupported file format: {file_path}."
+ f"Supported formats are "
+ f"{MarkItDownLoader.SUPPORTED_FORMATS}"
+ )
+ raise ValueError(f"Unsupported file format: {file_path}")
+
+ try:
+ logger.info(f"Converting file: {file_path}")
+ result = self.converter.convert(file_path)
+ logger.info(f"File converted successfully: {file_path}")
+ return result.text_content
+ except Exception as e:
+ logger.error(f"Error converting file '{file_path}': {e}")
+ raise Exception(f"Error converting file '{file_path}': {e}")
+
+ def convert_files(
+ self,
+ file_paths: List[str],
+ parallel: bool = False,
+ skip_failed: bool = False,
+ ) -> Dict[str, str]:
+ r"""Converts multiple files to Markdown format.
+
+ Args:
+ file_paths (List[str]): List of file paths to convert.
+ parallel (bool): Whether to process files in parallel.
+ (default: :obj:`False`)
+ skip_failed (bool): Whether to skip failed files instead
+ of including error messages.
+ (default: :obj:`False`)
+
+ Returns:
+ Dict[str, str]: Dictionary mapping file paths to their
+ converted Markdown text.
+
+ Raises:
+ Exception: For errors during conversion of any file if
+ skip_failed is False.
+ """
+ from tqdm.auto import tqdm
+
+ converted_files = {}
+
+ if parallel:
+ with ThreadPoolExecutor() as executor:
+ future_to_path = {
+ executor.submit(self.convert_file, path): path
+ for path in file_paths
+ }
+ for future in tqdm(
+ as_completed(future_to_path),
+ total=len(file_paths),
+ desc="Converting files (parallel)",
+ ):
+ path = future_to_path[future]
+ try:
+ converted_files[path] = future.result()
+ except Exception as e:
+ if skip_failed:
+ logger.warning(
+ f"Skipping file '{path}' due to error: {e}"
+ )
+ else:
+ logger.error(
+ f"Error processing file '{path}': {e}"
+ )
+ converted_files[path] = f"Error: {e}"
+ else:
+ for path in tqdm(file_paths, desc="Converting files (sequential)"):
+ try:
+ logger.info(f"Processing file: {path}")
+ converted_files[path] = self.convert_file(path)
+ except Exception as e:
+ if skip_failed:
+ logger.warning(
+ f"Skipping file '{path}' due to error: {e}"
+ )
+ else:
+ logger.error(f"Error processing file '{path}': {e}")
+ converted_files[path] = f"Error: {e}"
+
+ return converted_files
diff --git a/docs/key_modules/loaders.md b/docs/key_modules/loaders.md
index dcdd1c0a0..5a62a3a94 100644
--- a/docs/key_modules/loaders.md
+++ b/docs/key_modules/loaders.md
@@ -25,6 +25,9 @@ Firecrawl Reader provides a Python interface to interact with the Firecrawl API,
### 2.6. Jina_url Reader
JinaURL Reader is a Python client for Jina AI's URL reading service, optimized to provide cleaner, LLM-friendly content from URLs.
+### 2.7 MarkitDown Reader
+MarkItDown is a lightweight Python utility for converting various files to Markdown for use with LLMs and related text analysis pipelines.
+
## 3. Get Started
### 3.1. Using `Base IO`
@@ -339,6 +342,20 @@ response = jina_reader.read_content("https://docs.camel-ai.org/")
print(response)
```
+
+### 3.6. Using `MarkitDown Reader`
+
+Initialize the loader and pass in the path of the file to retrieve information, then print the response.
+
+```python
+from camel.loaders import MarkItDownLoader
+
+loader = MarkItDownLoader()
+response = converter.convert_file("demo.html")
+
+print(response)
+```
+
```markdown
>>>Welcome to CAMEL’s documentation! — CAMEL 0.2.57 documentation
===============
diff --git a/examples/loaders/markitdown_example.py b/examples/loaders/markitdown_example.py
new file mode 100644
index 000000000..93ee49c9b
--- /dev/null
+++ b/examples/loaders/markitdown_example.py
@@ -0,0 +1,70 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+import os
+import tempfile
+
+from camel.loaders import MarkItDownLoader
+
+mock_files = {
+ "demo.html": "<html><body><h1>Demo HTML</h1></body></html>",
+ "report.pdf": "%PDF-1.4\n% Mock PDF content",
+ "presentation.pptx": "Mock PPTX content",
+ "data.xlsx": "Mock XLSX content",
+}
+
+with tempfile.TemporaryDirectory() as temp_dir:
+ file_paths = {}
+
+ for filename, content in mock_files.items():
+ file_path = os.path.join(temp_dir, filename)
+ with open(file_path, "w") as f:
+ f.write(content)
+ file_paths[filename] = file_path
+
+ converter = MarkItDownLoader()
+
+ # Convert a single file
+ try:
+ markdown_text = converter.convert_file(file_paths["demo.html"])
+ print(markdown_text)
+ except Exception as e:
+ print(f"An error occurred: {e}")
+
+ # Convert multiple files
+ files = [
+ file_paths["report.pdf"],
+ file_paths["presentation.pptx"],
+ file_paths["data.xlsx"],
+ ]
+ converted = converter.convert_files(files, parallel=True, skip_failed=True)
+ for path, md in converted.items():
+ print(f"Markdown for {path}:\n{md}\n")
+
+"""
+===============================================================================
+# Demo HTML
+Markdown for /var/folders/93/f_71_t957cq9cmq2gsybs4_40000gn/T/tmp86hqcsxc/
+report.pdf:
+%PDF-1.4
+% Mock PDF content
+
+Markdown for /var/folders/93/f_71_t957cq9cmq2gsybs4_40000gn/T/tmp86hqcsxc/data.
+xlsx:
+Mock XLSX content
+
+Markdown for /var/folders/93/f_71_t957cq9cmq2gsybs4_40000gn/T/tmp86hqcsxc/
+presentation.pptx:
+===============================================================================
+"""
diff --git a/examples/loaders/mineru_example.py b/examples/loaders/mineru_example.py
index d4adc7327..1f854bc59 100644
--- a/examples/loaders/mineru_example.py
+++ b/examples/loaders/mineru_example.py
@@ -74,6 +74,7 @@ if __name__ == "__main__":
main()
"""
+===============================================================================
Example output:
Example 1: Single URL extraction
@@ -94,4 +95,5 @@ Download URL: https://cdn-mineru.openxlab.org.cn/pdf/690a7956-eaaa-4fb2-ad7d-605
Document: doc2
Filename: 2310.07298.pdf
Download URL: https://cdn-mineru.openxlab.org.cn/pdf/250a3762-406e-4279-aa80-47e5ea934509.zip
+===============================================================================
"""
diff --git a/examples/loaders/pandas_example.py b/examples/loaders/pandas_example.py
index 3b8fbdf7f..7eca1bf5d 100644
--- a/examples/loaders/pandas_example.py
+++ b/examples/loaders/pandas_example.py
@@ -77,7 +77,8 @@ if os.getenv("OPENAI_API_KEY"):
else:
print("Skipping LLM example: OPENAI_API_KEY environment variable not set")
-'''
+"""
+===============================================================================
Example output:
Example 1: PandasReader without LLM
@@ -100,4 +101,4 @@ Querying data with LLM:
3 Germany 4100
1 United Kingdom 3200
===============================================================================
-'''
+"""
diff --git a/pyproject.toml b/pyproject.toml
index fe4aa1a18..fd00805a5 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -120,6 +120,7 @@ document_tools = [
"docx>=0.2.4",
"fpdf>=1.7.2",
"crawl4ai>=0.3.745",
+ "markitdown==0.1.1",
]
media_tools = [
"imageio[pyav]>=2.34.2,<3",
@@ -319,6 +320,7 @@ all = [
"cohere>=5.11.0,<6",
"sentence-transformers>=3.0.1,<4",
"neo4j>=5.18.0,<6",
+ "markitdown==0.1.1",
"nebula3-python==3.8.2",
"rank-bm25>=0.2.2,<0.3",
"litellm>=1.38.1,<2",
@@ -365,6 +367,7 @@ all = [
"ibm-watsonx-ai>=1.3.11",
"google-genai>=1.13.0",
"aci-sdk>=1.0.0b1",
+ "markitdown==0.1.1",
]
[project.urls]
diff --git a/test/loaders/test_markitdown.py b/test/loaders/test_markitdown.py
new file mode 100644
index 000000000..fe95dec39
--- /dev/null
+++ b/test/loaders/test_markitdown.py
@@ -0,0 +1,131 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import os
+import tempfile
+from unittest.mock import patch
+
+import pytest
+
+from camel.loaders import MarkItDownLoader
+
+
+def mock_files():
+ files = {
+ "demo_html": (
+ "demo.html",
+ "<html><body><h1>Mock HTML </h1></body></html>",
+ ),
+ "report_pdf": ("report.pdf", "Mock PDF content"),
+ "presentation_pptx": ("presentation.pptx", "Mock PPTX content"),
+ "data_xlsx": ("data.xlsx", "Mock XLSX content"),
+ "unsupported_xxx": ("unsupported.xxx", "unsupported extension"),
+ }
+ created_paths = {}
+
+ for key, (filename, content) in files.items():
+ temp_file = tempfile.NamedTemporaryFile(
+ delete=False, suffix=os.path.splitext(filename)[1], mode="w"
+ )
+ temp_file.write(content)
+ temp_file.close()
+ created_paths[key] = temp_file.name
+
+ yield created_paths
+
+ for path in created_paths.values():
+ if os.path.exists(path):
+ os.remove(path)
+
+
+def test_convert_file_success(mock_files):
+ converter = MarkItDownLoader()
+ markdown_text = converter.convert_file(mock_files["demo_html"])
+ assert markdown_text is not None
+ assert isinstance(markdown_text, str)
+
+
+def test_convert_file_not_found():
+ converter = MarkItDownLoader()
+ with pytest.raises(
+ FileNotFoundError, match="File not found: nonexistent.txt"
+ ):
+ converter.convert_file("nonexistent.txt")
+
+
+def test_convert_file_unsupported_format(mock_files):
+ converter = MarkItDownLoader()
+ with pytest.raises(ValueError, match="Unsupported file format"):
+ converter.convert_file(mock_files["unsupported_xxx"])
+
+
+def test_convert_file_conversion_error(mock_files):
+ converter = MarkItDownLoader()
+ with patch.object(
+ converter.converter,
+ "convert",
+ side_effect=Exception("Mock conversion error"),
+ ):
+ with pytest.raises(Exception, match="Mock conversion error"):
+ converter.convert_file(mock_files["demo_html"])
+
+
+def test_convert_files_success(mock_files):
+ converter = MarkItDownLoader()
+ file_paths = [mock_files["demo_html"], mock_files["report_pdf"]]
+ results = converter.convert_files(file_paths)
+ assert mock_files["demo_html"] in results
+ assert mock_files["report_pdf"] in results
+ assert all(isinstance(result, str) for result in results.values())
+ assert len(results) == 2
+
+
+def test_convert_files_with_errors(mock_files):
+ converter = MarkItDownLoader()
+ file_paths = [
+ mock_files["demo_html"],
+ mock_files["report_pdf"],
+ "nonexistent.txt",
+ ]
+ results = converter.convert_files(file_paths)
+ assert mock_files["demo_html"] in results
+ assert mock_files["report_pdf"] in results
+ assert "nonexistent.txt" in results
+ assert results["nonexistent.txt"].startswith("Error:")
+ assert len(results) == 3
+
+
+def test_convert_files_skip_failed(mock_files):
+ converter = MarkItDownLoader()
+ file_paths = [
+ mock_files["demo_html"],
+ mock_files["report_pdf"],
+ "nonexistent.txt",
+ ]
+ results = converter.convert_files(file_paths, skip_failed=True)
+ assert mock_files["demo_html"] in results
+ assert mock_files["report_pdf"] in results
+ assert "nonexistent.txt" not in results
+ assert all(isinstance(result, str) for result in results.values())
+ assert len(results) == 2
+
+
+def test_convert_files_parallel(mock_files):
+ converter = MarkItDownLoader()
+ file_paths = [mock_files["demo_html"], mock_files["report_pdf"]]
+ results = converter.convert_files(file_paths, parallel=True)
+ assert mock_files["demo_html"] in results
+ assert mock_files["report_pdf"] in results
+ assert all(isinstance(result, str) for result in results.values())
+ assert len(results) == 2
diff --git a/uv.lock b/uv.lock
index 6ada66245..59c42f0f2 100644
--- a/uv.lock
+++ b/uv.lock
@@ -616,6 +616,7 @@ all = [
{ name = "jupyter-client" },
{ name = "linkup-sdk" },
{ name = "litellm" },
+ { name = "markitdown" },
{ name = "math-verify" },
{ name = "mcp" },
{ name = "mem0ai" },
@@ -753,6 +754,7 @@ document-tools = [
{ name = "docx" },
{ name = "docx2txt" },
{ name = "fpdf" },
+ { name = "markitdown" },
{ name = "numpy" },
{ name = "openapi-spec-validator" },
{ name = "openpyxl" },
@@ -1016,6 +1018,8 @@ requires-dist = [
{ name = "linkup-sdk", marker = "extra == 'web-tools'", specifier = ">=0.2.1,<0.3" },
{ name = "litellm", marker = "extra == 'all'", specifier = ">=1.38.1,<2" },
{ name = "litellm", marker = "extra == 'model-platforms'", specifier = ">=1.38.1,<2" },
+ { name = "markitdown", marker = "extra == 'all'", specifier = "==0.1.1" },
+ { name = "markitdown", marker = "extra == 'document-tools'", specifier = "==0.1.1" },
{ name = "math-verify", marker = "extra == 'all'", specifier = ">=0.7.0,<0.8" },
{ name = "math-verify", marker = "extra == 'data-tools'", specifier = ">=0.7.0,<0.8" },
{ name = "mcp", specifier = ">=1.3.0" },
@@ -1402,6 +1406,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335 },
]
+[[package]]
+name = "coloredlogs"
+version = "15.0.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "humanfriendly" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/cc/c7/eed8f27100517e8c0e6b923d5f0845d0cb99763da6fdee00478f91db7325/coloredlogs-15.0.1.tar.gz", hash = "sha256:7c991aa71a4577af2f82600d8f8f3a89f936baeaf9b50a9c197da014e5bf16b0", size = 278520 }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/a7/06/3d6badcf13db419e25b07041d9c7b4a2c331d3f4e7134445ec5df57714cd/coloredlogs-15.0.1-py2.py3-none-any.whl", hash = "sha256:612ee75c546f53e92e70049c9dbfcc18c935a2b9a53b66085ce9ef6a6e5c0934", size = 46018 },
+]
+
[[package]]
name = "comm"
version = "0.2.2"
@@ -2218,6 +2234,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/b6/6d/b3f92e7a166ebb2033c41b45f5b5fdbc4282e680492eff38f72180bc0510/fish_audio_sdk-2024.12.5-py3-none-any.whl", hash = "sha256:a4ce81c6609c239d5518cc1d78c7a0699da50797e0af4d6b3758a8fb5c12384e", size = 7934 },
]
+[[package]]
+name = "flatbuffers"
+version = "25.2.10"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/e4/30/eb5dce7994fc71a2f685d98ec33cc660c0a5887db5610137e60d8cbc4489/flatbuffers-25.2.10.tar.gz", hash = "sha256:97e451377a41262f8d9bd4295cc836133415cc03d8cb966410a4af92eb00d26e", size = 22170 }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/b8/25/155f9f080d5e4bc0082edfda032ea2bc2b8fab3f4d25d46c1e9dd22a1a89/flatbuffers-25.2.10-py2.py3-none-any.whl", hash = "sha256:ebba5f4d5ea615af3f7fd70fc310636fbb2bbd1f566ac0a23d98dd412de50051", size = 30953 },
+]
+
[[package]]
name = "fonttools"
version = "4.57.0"
@@ -2846,6 +2871,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/93/27/1fb384a841e9661faad1c31cbfa62864f59632e876df5d795234da51c395/huggingface_hub-0.30.2-py3-none-any.whl", hash = "sha256:68ff05969927058cfa41df4f2155d4bb48f5f54f719dd0390103eefa9b191e28", size = 481433 },
]
+[[package]]
+name = "humanfriendly"
+version = "10.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "pyreadline3", marker = "sys_platform == 'win32'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/cc/3f/2c29224acb2e2df4d2046e4c73ee2662023c58ff5b113c4c1adac0886c43/humanfriendly-10.0.tar.gz", hash = "sha256:6b0b831ce8f15f7300721aa49829fc4e83921a9a301cc7f606be6686a2288ddc", size = 360702 }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl", hash = "sha256:1697e1a8a8f550fd43c2865cd84542fc175a61dcb779b6fee18cf6b6ccba1477", size = 86794 },
+]
+
[[package]]
name = "hyperframe"
version = "6.1.0"
@@ -3499,6 +3536,24 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/6b/40/7d49ff503cc90b03253eba0768feec909b47ce92a90591b025c774a29a95/lxml-5.3.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:0a006390834603e5952a2ff74b9a31a6007c7cc74282a087aa6467afb4eea987", size = 3487898 },
]
+[[package]]
+name = "magika"
+version = "0.6.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "click" },
+ { name = "numpy" },
+ { name = "onnxruntime" },
+ { name = "python-dotenv" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/6d/18/ea70f6abd36f455037340f12c8125918c726d08cd6e01f0b76b6884e0c38/magika-0.6.1.tar.gz", hash = "sha256:e3dd22c73936630b1cd79d0f412d6d9a53dc99ba5e3709b1ac53f56bc998e635", size = 3030234 }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/1f/be/c9f7bb9ee94abe8d344b660672001313e459c67b867b24abe32d5c80a9ce/magika-0.6.1-py3-none-any.whl", hash = "sha256:15838d2469f1394d8e9598bc7fceea1ede7f35aebe9675c6b45c6b5c48315931", size = 2968516 },
+ { url = "https://files.pythonhosted.org/packages/3c/b9/016b174520e81faef5edb31b6c7a73966dc84ee33acd23a2e7b775df7ba4/magika-0.6.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:dadd036296a2e4840fd48fa0712848fe122da438e8f607dc8f19ca4663c359dc", size = 12408519 },
+ { url = "https://files.pythonhosted.org/packages/02/b7/e7dfeb235823a82d676c68a748541c24db0249b854f945f6e3cec11c1b7e/magika-0.6.1-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:133c0e1a844361de86ca2dd7c530e38b324e86177d30c52e36fd82101c190b5c", size = 15089294 },
+ { url = "https://files.pythonhosted.org/packages/64/f0/bec5bff0125d08c1bc3baef88beeb910121085249f67b5994ea961615b55/magika-0.6.1-py3-none-win_amd64.whl", hash = "sha256:0342b6230ea9aea7ab4b8fa92e1b46f1cc62e724d452ee8d6821a37f56738d22", size = 12378455 },
+]
+
[[package]]
name = "markdown-it-py"
version = "3.0.0"
@@ -3532,6 +3587,22 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/64/11/b751af7ad41b254a802cf52f7bc1fca7cabe2388132f2ce60a1a6b9b9622/markdownify-1.1.0-py3-none-any.whl", hash = "sha256:32a5a08e9af02c8a6528942224c91b933b4bd2c7d078f9012943776fc313eeef", size = 13901 },
]
+[[package]]
+name = "markitdown"
+version = "0.1.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "beautifulsoup4" },
+ { name = "charset-normalizer" },
+ { name = "magika" },
+ { name = "markdownify" },
+ { name = "requests" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/cb/e8/83669ba97718bbbccd4c432b763d22783df4c8218e770717151acf01e85b/markitdown-0.1.1.tar.gz", hash = "sha256:da97a55a45a3d775ea758e88a344d5cac94ee97115fb0293f99027d32c2fc3f6", size = 31475 }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/0b/8a/c1f85ee609de5d45f80d0213bebf6664f76ab406e9d57709e684a4a436ba/markitdown-0.1.1-py3-none-any.whl", hash = "sha256:98ea8c009fe174b37ef933e00f4364214e8fed35691178b8521b13604d0c4a58", size = 48230 },
+]
+
[[package]]
name = "markupsafe"
version = "3.0.2"
@@ -4308,6 +4379,33 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/17/d3/b64c356a907242d719fc668b71befd73324e47ab46c8ebbbede252c154b2/olefile-0.47-py2.py3-none-any.whl", hash = "sha256:543c7da2a7adadf21214938bb79c83ea12b473a4b6ee4ad4bf854e7715e13d1f", size = 114565 },
]
+[[package]]
+name = "onnxruntime"
+version = "1.21.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "coloredlogs" },
+ { name = "flatbuffers" },
+ { name = "numpy" },
+ { name = "packaging" },
+ { name = "protobuf" },
+ { name = "sympy" },
+]
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/06/72/09d8f206402cd91805828354ad1d7473b1bace60fc54a11971012906d9b7/onnxruntime-1.21.1-cp310-cp310-macosx_13_0_universal2.whl", hash = "sha256:daedb5d33d8963062a25f4a3c788262074587f685a19478ef759a911b4b12c25", size = 33639134 },
+ { url = "https://files.pythonhosted.org/packages/1f/66/31384dc7beea89f21ec7d1582c1b50e9d047d505db38f32cf49693fad1b4/onnxruntime-1.21.1-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a402f9bda0b1cc791d9cf31d23c471e8189a55369b49ef2b9d0854eb11d22c4", size = 14162243 },
+ { url = "https://files.pythonhosted.org/packages/a9/fb/76597b77785b2012317ffdd817101ccfab784e2c125645d002c4c9cd377b/onnxruntime-1.21.1-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:15656a2d0126f4f66295381e39c8812a6d845ccb1bb1f7bf6dd0a46d7d602e7f", size = 16000498 },
+ { url = "https://files.pythonhosted.org/packages/91/83/c7287845f22f2e1d37a54b5997e9589b6931e264cc0f16250d1706eadf79/onnxruntime-1.21.1-cp310-cp310-win_amd64.whl", hash = "sha256:79bbedfd1263065532967a2132fb365a27ffe5f7ed962e16fec55cca741f72aa", size = 12300918 },
+ { url = "https://files.pythonhosted.org/packages/70/ba/13c46c22fb52d8fea53575da163399a7d75fe61223aba685370f047a0882/onnxruntime-1.21.1-cp311-cp311-macosx_13_0_universal2.whl", hash = "sha256:8bee9b5ba7b88ae7bfccb4f97bbe1b4bae801b0fb05d686b28a722cb27c89931", size = 33643424 },
+ { url = "https://files.pythonhosted.org/packages/18/4f/68985138c507b6ad34061aa4f330b8fbd30b0c5c299be53f0c829420528e/onnxruntime-1.21.1-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4b6a29a1767b92d543091349f5397a1c7619eaca746cd1bc47f8b4ec5a9f1a6c", size = 14162437 },
+ { url = "https://files.pythonhosted.org/packages/0f/76/7dfa4b63f95a17eaf881c9c464feaa59a25bbfb578db204fc22d522b5199/onnxruntime-1.21.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:982dcc04a6688e1af9e3da1d4ef2bdeb11417cf3f8dde81f8f721043c1919a4f", size = 16002403 },
+ { url = "https://files.pythonhosted.org/packages/80/85/397406e758d6c30fb6d0d0152041c6b9ee835c3584765837ce54230c8bc9/onnxruntime-1.21.1-cp311-cp311-win_amd64.whl", hash = "sha256:2b6052c04b9125319293abb9bdcce40e806db3e097f15b82242d4cd72d81fd0c", size = 12301824 },
+ { url = "https://files.pythonhosted.org/packages/a5/42/274438bbc259439fa1606d0d6d2eef4171cdbd2d7a1c3b249b4ba440424b/onnxruntime-1.21.1-cp312-cp312-macosx_13_0_universal2.whl", hash = "sha256:f615c05869a523a94d0a4de1f0936d0199a473cf104d630fc26174bebd5759bd", size = 33658457 },
+ { url = "https://files.pythonhosted.org/packages/9c/93/76f629d4f22571b0b3a29a9d375204faae2bd2b07d557043b56df5848779/onnxruntime-1.21.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:79dfb1f47386c4edd115b21015354b2f05f5566c40c98606251f15a64add3cbe", size = 14164881 },
+ { url = "https://files.pythonhosted.org/packages/1b/86/75cbaa4058758fa8ef912dfebba2d5a4e4fd6738615c15b6a2262d076198/onnxruntime-1.21.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2742935d6610fe0f58e1995018d9db7e8239d0201d9ebbdb7964a61386b5390a", size = 16019966 },
+ { url = "https://files.pythonhosted.org/packages/5f/9d/fb8895b2cb38c9965d4b4e0a9aa1398f3e3f16c4acb75cf3b61689780a65/onnxruntime-1.21.1-cp312-cp312-win_amd64.whl", hash = "sha256:a7afdb3fcb162f5536225e13c2b245018068964b1d0eee05303ea6823ca6785e", size = 12302925 },
+]
+
[[package]]
name = "openai"
version = "1.75.0"
@@ -5577,6 +5675,15 @@ version = "1.9.0"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/30/23/2f0a3efc4d6a32f3b63cdff36cd398d9701d26cda58e3ab97ac79fb5e60d/pyperclip-1.9.0.tar.gz", hash = "sha256:b7de0142ddc81bfc5c7507eea19da920b92252b548b96186caf94a5e2527d310", size = 20961 }
+[[package]]
+name = "pyreadline3"
+version = "3.5.4"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/0f/49/4cea918a08f02817aabae639e3d0ac046fef9f9180518a3ad394e22da148/pyreadline3-3.5.4.tar.gz", hash = "sha256:8d57d53039a1c75adba8e50dd3d992b28143480816187ea5efbd5c78e6c885b7", size = 99839 }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/5a/dc/491b7661614ab97483abf2056be1deee4dc2490ecbf7bff9ab5cdbac86e1/pyreadline3-3.5.4-py3-none-any.whl", hash = "sha256:eaf8e6cc3c49bcccf145fc6067ba8643d1df34d604a1ec0eccbf7a18e6d3fae6", size = 83178 },
+]
+
[[package]]
name = "pyrect"
version = "0.2.0"