Python tool for converting files and office documents to Markdown.
Exploring ways to allow Optional dependencies (#1079)
afourneyc5cd659f6381
diff --git a/README.md b/README.mdindex 70f188d6..2563a689 100644--- a/README.md+++ b/README.md@@ -5,7 +5,8 @@[](https://github.com/microsoft/autogen)> [!IMPORTANT]-> MarkItDown 0.0.2 alpha 1 (0.0.2a1) introduces a plugin-based architecture. As much as was possible, command-line and Python interfaces have remained the same as 0.0.1a3 to support backward compatibility. Please report any issues you encounter. Some interface changes may yet occur as we continue to refine MarkItDown to a first non-alpha release.+> Breaking changes between 0.0.1 to 0.0.2:+> * Dependencies are now organized into optional feature-groups (further details below). Use `pip install markitdown[all]` to have backward-compatible behavior.MarkItDown is a utility for converting various files to Markdown (e.g., for indexing, text analysis, etc).It supports:@@ -22,12 +23,12 @@ It supports:- Youtube URLs- ... and more!-To install MarkItDown, use pip: `pip install markitdown`. Alternatively, you can install it from the source:+To install MarkItDown, use pip: `pip install markitdown[all]`. Alternatively, you can install it from the source:```bashgit clone [email protected]:microsoft/markitdown.gitcd markitdown-pip install -e packages/markitdown+pip install -e packages/markitdown[all]```## Usage@@ -50,6 +51,28 @@ You can also pipe content:cat path-to-file.pdf | markitdown```+### Optional Dependencies+MarkItDown has optional dependencies for activating various file formats. Earlier in this document, we installed all optional dependencies with the `[all]` option. However, you can also install them individually for more control. For example:++```bash+pip install markitdown[pdf, docx, pptx]+```++will install only the dependencies for PDF, DOCX, and PPTX files.++At the moment, the following optional dependencies are available:++* `[all]` Installs all optional dependencies+* `[pptx]` Installs dependencies for PowerPoint files+* `[docx]` Installs dependencies for Word files+* `[xlsx]` Installs dependencies for Excel files+* `[xls]` Installs dependencies for older Excel files+* `[pdf]` Installs dependencies for PDF files+* `[outlook]` Installs dependencies for Outlook messages+* `[az-doc-intel]` Installs dependencies for Azure Document Intelligence+* `[audio-transcription]` Installs dependencies for audio transcription of wav and mp3 files+* `[youtube-transcription]` Installs dependencies for fetching YouTube video transcription+### PluginsMarkItDown also supports 3rd-party plugins. Plugins are disabled by default. To list installed plugins:diff --git a/packages/markitdown/README.md b/packages/markitdown/README.mdindex 54453abc..edd27016 100644--- a/packages/markitdown/README.md+++ b/packages/markitdown/README.md@@ -10,7 +10,7 @@From PyPI:```bash-pip install markitdown+pip install markitdown[all]```From source:@@ -18,7 +18,7 @@ From source:```bashgit clone [email protected]:microsoft/markitdown.gitcd markitdown-pip install -e packages/markitdown+pip install -e packages/markitdown[all]```## Usagediff --git a/packages/markitdown/pyproject.toml b/packages/markitdown/pyproject.tomlindex a321fee4..c053c7b2 100644--- a/packages/markitdown/pyproject.toml+++ b/packages/markitdown/pyproject.toml@@ -26,25 +26,36 @@ classifiers = [dependencies = ["beautifulsoup4","requests",- "mammoth","markdownify~=0.14.1",- "numpy",+ "puremagic",+ "pathvalidate",+ "charset-normalizer",+]++[project.optional-dependencies]+all = ["python-pptx",+ "mammoth","pandas","openpyxl","xlrd","pdfminer.six",- "puremagic",- "pydub","olefile",- "youtube-transcript-api",+ "pydub","SpeechRecognition",- "pathvalidate",- "charset-normalizer",- "openai",+ "youtube-transcript-api","azure-ai-documentintelligence","azure-identity"]+pptx = ["python-pptx"]+docx = ["mammoth"]+xlsx = ["pandas", "openpyxl"]+xls = ["pandas", "xlrd"]+pdf = ["pdfminer.six"]+outlook = ["olefile"]+audio-transcription = ["pydub", "SpeechRecognition"]+youtube-transcription = ["youtube-transcript-api"]+az-doc-intel = ["azure-ai-documentintelligence", "azure-identity"][project.urls]Documentation = "https://github.com/microsoft/markitdown#readme"@@ -57,6 +68,15 @@ path = "src/markitdown/__about__.py"[project.scripts]markitdown = "markitdown.__main__:main"+[tool.hatch.envs.default]+features = ["all"]++[tool.hatch.envs.hatch-test]+features = ["all"]+extra-dependencies = [+ "openai",+]+[tool.hatch.envs.types]extra-dependencies = ["mypy>=1.0.0",diff --git a/packages/markitdown/src/markitdown/__init__.py b/packages/markitdown/src/markitdown/__init__.pyindex c6d53633..9f7db16e 100644--- a/packages/markitdown/src/markitdown/__init__.py+++ b/packages/markitdown/src/markitdown/__init__.py@@ -6,7 +6,7 @@ from .__about__ import __version__from ._markitdown import MarkItDownfrom ._exceptions import (MarkItDownException,- ConverterPrerequisiteException,+ MissingDependencyException,FailedConversionAttempt,FileConversionException,UnsupportedFormatException,@@ -19,7 +19,7 @@ __all__ = ["DocumentConverter","DocumentConverterResult","MarkItDownException",- "ConverterPrerequisiteException",+ "MissingDependencyException","FailedConversionAttempt","FileConversionException","UnsupportedFormatException",diff --git a/packages/markitdown/src/markitdown/_exceptions.py b/packages/markitdown/src/markitdown/_exceptions.pyindex 4f443b2c..abfebc6e 100644--- a/packages/markitdown/src/markitdown/_exceptions.py+++ b/packages/markitdown/src/markitdown/_exceptions.py@@ -1,5 +1,12 @@from typing import Optional, List, Any+MISSING_DEPENDENCY_MESSAGE = """{converter} recognized the input as a potential {extension} file, but the dependencies needed to read {extension} files have not been installed. To resolve this error, include the optional dependency [{feature}] or [all] when installing MarkItDown. For example:++* pip install markitdown[{feature}]+* pip install markitdown[all]+* pip install markitdown[{feature}, ...]+* etc."""+class MarkItDownException(Exception):"""@@ -9,15 +16,16 @@ class MarkItDownException(Exception):pass-class ConverterPrerequisiteException(MarkItDownException):+class MissingDependencyException(MarkItDownException):"""- Thrown when instantiating a DocumentConverter in cases where- a required library or dependency is not installed, an API key- is not set, or some other prerequisite is not met.-- This is not necessarily a fatal error. If thrown during- MarkItDown's plugin loading phase, the converter will simply be- skipped, and a warning will be issued.+ Converters shipped with MarkItDown may depend on optional+ dependencies. This exception is thrown when a converter's+ convert() method is called, but the required dependency is not+ installed. This is not necessarily a fatal error, as the converter+ will simply be skipped (an error will bubble up only if no other+ suitable converter is found).++ Error messages should clearly indicate which dependency is missing."""passdiff --git a/packages/markitdown/src/markitdown/_markitdown.py b/packages/markitdown/src/markitdown/_markitdown.pyindex 49b817df..8f1bd463 100644--- a/packages/markitdown/src/markitdown/_markitdown.py+++ b/packages/markitdown/src/markitdown/_markitdown.py@@ -42,7 +42,6 @@ from .converters import (from ._exceptions import (FileConversionException,UnsupportedFormatException,- ConverterPrerequisiteException,FailedConversionAttempt,)diff --git a/packages/markitdown/src/markitdown/converters/_doc_intel_converter.py b/packages/markitdown/src/markitdown/converters/_doc_intel_converter.pyindex ed8aabfb..6fe79c0c 100644--- a/packages/markitdown/src/markitdown/converters/_doc_intel_converter.py+++ b/packages/markitdown/src/markitdown/converters/_doc_intel_converter.py@@ -1,16 +1,24 @@from typing import Any, Unionimport re--# Azure imports-from azure.ai.documentintelligence import DocumentIntelligenceClient-from azure.ai.documentintelligence.models import (- AnalyzeDocumentRequest,- AnalyzeResult,- DocumentAnalysisFeature,-)-from azure.identity import DefaultAzureCredential+import sysfrom ._base import DocumentConverter, DocumentConverterResult+from .._exceptions import MissingDependencyException++# Try loading optional (but in this case, required) dependencies+# Save reporting of any exceptions for later+_dependency_exc_info = None+try:+ from azure.ai.documentintelligence import DocumentIntelligenceClient+ from azure.ai.documentintelligence.models import (+ AnalyzeDocumentRequest,+ AnalyzeResult,+ DocumentAnalysisFeature,+ )+ from azure.identity import DefaultAzureCredential+except ImportError:+ # Preserve the error and stack trace for later+ _dependency_exc_info = sys.exc_info()# TODO: currently, there is a bug in the document intelligence SDK with importing the "ContentFormat" enum.@@ -30,6 +38,16 @@ class DocumentIntelligenceConverter(DocumentConverter):):super().__init__(priority=priority)+ # Raise an error if the dependencies are not available.+ # This is different than other converters since this one isn't even instantiated+ # unless explicitly requested.+ if _dependency_exc_info is not None:+ raise MissingDependencyException(+ "DocumentIntelligenceConverter requires the optional dependency [az-doc-intel] (or [all]) to be installed. E.g., `pip install markitdown[az-doc-intel]`"+ ) from _dependency_exc_info[1].with_traceback(+ _dependency_exc_info[2]+ ) # Restore the original traceback+self.endpoint = endpointself.api_version = api_versionself.doc_intel_client = DocumentIntelligenceClient(diff --git a/packages/markitdown/src/markitdown/converters/_docx_converter.py b/packages/markitdown/src/markitdown/converters/_docx_converter.pyindex 8515f6d7..0866e594 100644--- a/packages/markitdown/src/markitdown/converters/_docx_converter.py+++ b/packages/markitdown/src/markitdown/converters/_docx_converter.py@@ -1,6 +1,6 @@-from typing import Union+import sys-import mammoth+from typing import Unionfrom ._base import (DocumentConverterResult,@@ -8,6 +8,16 @@ from ._base import (from ._base import DocumentConverterfrom ._html_converter import HtmlConverter+from .._exceptions import MissingDependencyException, MISSING_DEPENDENCY_MESSAGE++# Try loading optional (but in this case, required) dependencies+# Save reporting of any exceptions for later+_dependency_exc_info = None+try:+ import mammoth+except ImportError:+ # Preserve the error and stack trace for later+ _dependency_exc_info = sys.exc_info()class DocxConverter(HtmlConverter):@@ -26,6 +36,18 @@ class DocxConverter(HtmlConverter):if extension.lower() != ".docx":return None+ # Check: the dependencies+ if _dependency_exc_info is not None:+ raise MissingDependencyException(+ MISSING_DEPENDENCY_MESSAGE.format(+ converter=type(self).__name__,+ extension=".docx",+ feature="docx",+ )+ ) from _dependency_exc_info[1].with_traceback(+ _dependency_exc_info[2]+ ) # Restore the original traceback+result = Nonewith open(local_path, "rb") as docx_file:style_map = kwargs.get("style_map", None)diff --git a/packages/markitdown/src/markitdown/converters/_image_converter.py b/packages/markitdown/src/markitdown/converters/_image_converter.pyindex 3c848dd4..4eb61552 100644--- a/packages/markitdown/src/markitdown/converters/_image_converter.py+++ b/packages/markitdown/src/markitdown/converters/_image_converter.py@@ -7,7 +7,7 @@ import mimetypesclass ImageConverter(MediaConverter):"""- Converts images to markdown via extraction of metadata (if `exiftool` is installed), OCR (if `easyocr` is installed), and description via a multimodal LLM (if an llm_client is configured).+ Converts images to markdown via extraction of metadata (if `exiftool` is installed), and description via a multimodal LLM (if an llm_client is configured)."""def __init__(diff --git a/packages/markitdown/src/markitdown/converters/_outlook_msg_converter.py b/packages/markitdown/src/markitdown/converters/_outlook_msg_converter.pyindex 6764fc56..eb7a065d 100644--- a/packages/markitdown/src/markitdown/converters/_outlook_msg_converter.py+++ b/packages/markitdown/src/markitdown/converters/_outlook_msg_converter.py@@ -1,6 +1,16 @@-import olefile+import sysfrom typing import Any, Unionfrom ._base import DocumentConverter, DocumentConverterResult+from .._exceptions import MissingDependencyException, MISSING_DEPENDENCY_MESSAGE++# Try loading optional (but in this case, required) dependencies+# Save reporting of any exceptions for later+_dependency_exc_info = None+try:+ import olefile+except ImportError:+ # Preserve the error and stack trace for later+ _dependency_exc_info = sys.exc_info()class OutlookMsgConverter(DocumentConverter):@@ -24,6 +34,18 @@ class OutlookMsgConverter(DocumentConverter):if extension.lower() != ".msg":return None+ # Check: the dependencies+ if _dependency_exc_info is not None:+ raise MissingDependencyException(+ MISSING_DEPENDENCY_MESSAGE.format(+ converter=type(self).__name__,+ extension=".msg",+ feature="outlook",+ )+ ) from _dependency_exc_info[1].with_traceback(+ _dependency_exc_info[2]+ ) # Restore the original traceback+try:msg = olefile.OleFileIO(local_path)# Extract email metadata@@ -59,10 +81,12 @@ class OutlookMsgConverter(DocumentConverter):f"Could not convert MSG file '{local_path}': {str(e)}")- def _get_stream_data(- self, msg: olefile.OleFileIO, stream_path: str- ) -> Union[str, None]:+ def _get_stream_data(self, msg: Any, stream_path: str) -> Union[str, None]:"""Helper to safely extract and decode stream data from the MSG file."""+ assert isinstance(+ msg, olefile.OleFileIO+ ) # Ensure msg is of the correct type (type hinting is not possible with the optional olefile package)+try:if msg.exists(stream_path):data = msg.openstream(stream_path).read()diff --git a/packages/markitdown/src/markitdown/converters/_pdf_converter.py b/packages/markitdown/src/markitdown/converters/_pdf_converter.pyindex 3a2b671d..3c5ecada 100644--- a/packages/markitdown/src/markitdown/converters/_pdf_converter.py+++ b/packages/markitdown/src/markitdown/converters/_pdf_converter.py@@ -1,7 +1,17 @@-import pdfminer-import pdfminer.high_level+import sysfrom typing import Unionfrom ._base import DocumentConverter, DocumentConverterResult+from .._exceptions import MissingDependencyException, MISSING_DEPENDENCY_MESSAGE++# Try loading optional (but in this case, required) dependencies+# Save reporting of any exceptions for later+_dependency_exc_info = None+try:+ import pdfminer+ import pdfminer.high_level+except ImportError:+ # Preserve the error and stack trace for later+ _dependency_exc_info = sys.exc_info()class PdfConverter(DocumentConverter):@@ -20,6 +30,18 @@ class PdfConverter(DocumentConverter):if extension.lower() != ".pdf":return None+ # Check the dependencies+ if _dependency_exc_info is not None:+ raise MissingDependencyException(+ MISSING_DEPENDENCY_MESSAGE.format(+ converter=type(self).__name__,+ extension=".pdf",+ feature="pdf",+ )+ ) from _dependency_exc_info[1].with_traceback(+ _dependency_exc_info[2]+ ) # Restore the original traceback+return DocumentConverterResult(title=None,text_content=pdfminer.high_level.extract_text(local_path),diff --git a/packages/markitdown/src/markitdown/converters/_plain_text_converter.py b/packages/markitdown/src/markitdown/converters/_plain_text_converter.pyindex 75f74a81..b4c9282c 100644--- a/packages/markitdown/src/markitdown/converters/_plain_text_converter.py+++ b/packages/markitdown/src/markitdown/converters/_plain_text_converter.py@@ -6,6 +6,13 @@ from typing import Any, Unionfrom ._base import DocumentConverter, DocumentConverterResult+# Mimetypes to ignore (commonly confused extensions)+IGNORE_MIMETYPES = [+ "text/vnd.in3d.spot", # .spo wich is confused with xls, doc, etc.+ "text/vnd.graphviz", # .dot which is confused with xls, doc, etc.+]++class PlainTextConverter(DocumentConverter):"""Anything with content type text/plain"""@@ -22,6 +29,10 @@ class PlainTextConverter(DocumentConverter):"__placeholder" + kwargs.get("file_extension", ""))+ # Ignore common false positives+ if content_type in IGNORE_MIMETYPES:+ content_type = None+# Only accept text filesif content_type is None:return Nonediff --git a/packages/markitdown/src/markitdown/converters/_pptx_converter.py b/packages/markitdown/src/markitdown/converters/_pptx_converter.pyindex 76c481a7..431b6a02 100644--- a/packages/markitdown/src/markitdown/converters/_pptx_converter.py+++ b/packages/markitdown/src/markitdown/converters/_pptx_converter.py@@ -1,12 +1,22 @@import base64-import pptximport reimport html+import sysfrom typing import Unionfrom ._base import DocumentConverterResult, DocumentConverterfrom ._html_converter import HtmlConverter+from .._exceptions import MissingDependencyException, MISSING_DEPENDENCY_MESSAGE++# Try loading optional (but in this case, required) dependencies+# Save reporting of any exceptions for later+_dependency_exc_info = None+try:+ import pptx+except ImportError:+ # Preserve the error and stack trace for later+ _dependency_exc_info = sys.exc_info()class PptxConverter(HtmlConverter):@@ -54,9 +64,20 @@ class PptxConverter(HtmlConverter):if extension.lower() != ".pptx":return None- md_content = ""+ # Check the dependencies+ if _dependency_exc_info is not None:+ raise MissingDependencyException(+ MISSING_DEPENDENCY_MESSAGE.format(+ converter=type(self).__name__,+ extension=".pptx",+ feature="pptx",+ )+ ) from _dependency_exc_info[1].with_traceback(+ _dependency_exc_info[2]+ ) # Restore the original tracebackpresentation = pptx.Presentation(local_path)+ md_content = ""slide_num = 0for slide in presentation.slides:slide_num += 1diff --git a/packages/markitdown/src/markitdown/converters/_xlsx_converter.py b/packages/markitdown/src/markitdown/converters/_xlsx_converter.pyindex 2bdfd5d1..56398ca8 100644--- a/packages/markitdown/src/markitdown/converters/_xlsx_converter.py+++ b/packages/markitdown/src/markitdown/converters/_xlsx_converter.py@@ -1,9 +1,26 @@-from typing import Union+import sys-import pandas as pd+from typing import Unionfrom ._base import DocumentConverter, DocumentConverterResultfrom ._html_converter import HtmlConverter+from .._exceptions import MissingDependencyException, MISSING_DEPENDENCY_MESSAGE++# Try loading optional (but in this case, required) dependencies+# Save reporting of any exceptions for later+_xlsx_dependency_exc_info = None+try:+ import pandas as pd+ import openpyxl+except ImportError:+ _xlsx_dependency_exc_info = sys.exc_info()++_xls_dependency_exc_info = None+try:+ import pandas as pd+ import xlrd+except ImportError:+ _xls_dependency_exc_info = sys.exc_info()class XlsxConverter(HtmlConverter):@@ -22,6 +39,18 @@ class XlsxConverter(HtmlConverter):if extension.lower() != ".xlsx":return None+ # Check the dependencies+ if _xlsx_dependency_exc_info is not None:+ raise MissingDependencyException(+ MISSING_DEPENDENCY_MESSAGE.format(+ converter=type(self).__name__,+ extension=".xlsx",+ feature="xlsx",+ )+ ) from _xlsx_dependency_exc_info[1].with_traceback(+ _xlsx_dependency_exc_info[2]+ ) # Restore the original traceback+sheets = pd.read_excel(local_path, sheet_name=None, engine="openpyxl")md_content = ""for s in sheets:@@ -46,6 +75,18 @@ class XlsConverter(HtmlConverter):if extension.lower() != ".xls":return None+ # Load the dependencies+ if _xls_dependency_exc_info is not None:+ raise MissingDependencyException(+ MISSING_DEPENDENCY_MESSAGE.format(+ converter=type(self).__name__,+ extension=".xls",+ feature="xls",+ )+ ) from _xls_dependency_exc_info[1].with_traceback(+ _xls_dependency_exc_info[2]+ ) # Restore the original traceback+sheets = pd.read_excel(local_path, sheet_name=None, engine="xlrd")md_content = ""for s in sheets: