The Memory Layer for AI Agents - Drop-in memory infrastructure for AI agents and apps. Context that persists. Built for production.
Python agentic-memoryagentic-memory-systemagentsaiai-agentschatgptgenaillmlong-term-memorymemorymemory-managementpythonragstate-management
[feat]: Add support for XML file format (#757)
Ojuswi Rastogi540a0a368536
diff --git a/docs/data-sources/xml.mdx b/docs/data-sources/xml.mdxnew file mode 100644index 000000000..0398afdc0--- /dev/null+++ b/docs/data-sources/xml.mdx@@ -0,0 +1,13 @@+---+title: 'XML File'+---++### XML file++To add any xml file, use the data_type as `xml`. Eg:++```python+app.add('content/data.xml')+```++Note: Only the text content of the xml file will be added to the app. The tags will be ignored.\ No newline at end of filediff --git a/embedchain/chunkers/xml.py b/embedchain/chunkers/xml.pynew file mode 100644index 000000000..cae519ab9--- /dev/null+++ b/embedchain/chunkers/xml.py@@ -0,0 +1,22 @@+from typing import Optional++from langchain.text_splitter import RecursiveCharacterTextSplitter++from embedchain.chunkers.base_chunker import BaseChunker+from embedchain.config.add_config import ChunkerConfig+from embedchain.helper.json_serializable import register_deserializable+++@register_deserializable+class XmlChunker(BaseChunker):+ """Chunker for XML files."""++ def __init__(self, config: Optional[ChunkerConfig] = None):+ if config is None:+ config = ChunkerConfig(chunk_size=500, chunk_overlap=50, length_function=len)+ text_splitter = RecursiveCharacterTextSplitter(+ chunk_size=config.chunk_size,+ chunk_overlap=config.chunk_overlap,+ length_function=config.length_function,+ )+ super().__init__(text_splitter)diff --git a/embedchain/data_formatter/data_formatter.py b/embedchain/data_formatter/data_formatter.pyindex 108294741..635f4ea76 100644--- a/embedchain/data_formatter/data_formatter.py+++ b/embedchain/data_formatter/data_formatter.py@@ -9,6 +9,7 @@ from embedchain.chunkers.qna_pair import QnaPairChunkerfrom embedchain.chunkers.table import TableChunkerfrom embedchain.chunkers.text import TextChunkerfrom embedchain.chunkers.web_page import WebPageChunker+from embedchain.chunkers.xml import XmlChunkerfrom embedchain.chunkers.youtube_video import YoutubeVideoChunkerfrom embedchain.config import AddConfigfrom embedchain.config.add_config import ChunkerConfig, LoaderConfig@@ -24,6 +25,7 @@ from embedchain.loaders.mdx import MdxLoaderfrom embedchain.loaders.pdf_file import PdfFileLoaderfrom embedchain.loaders.sitemap import SitemapLoaderfrom embedchain.loaders.web_page import WebPageLoader+from embedchain.loaders.xml import XmlLoaderfrom embedchain.loaders.youtube_video import YoutubeVideoLoaderfrom embedchain.models.data_type import DataType@@ -67,6 +69,7 @@ class DataFormatter(JSONSerializable):DataType.TEXT: LocalTextLoader,DataType.DOCX: DocxFileLoader,DataType.SITEMAP: SitemapLoader,+ DataType.XML: XmlLoader,DataType.DOCS_SITE: DocsSiteLoader,DataType.CSV: CsvLoader,DataType.MDX: MdxLoader,@@ -110,6 +113,7 @@ class DataFormatter(JSONSerializable):DataType.CSV: TableChunker,DataType.MDX: MdxChunker,DataType.IMAGES: ImagesChunker,+ DataType.XML: XmlChunker,}if data_type in chunker_classes:chunker_class: type = chunker_classes[data_type]diff --git a/embedchain/loaders/xml.py b/embedchain/loaders/xml.pynew file mode 100644index 000000000..324a8c71e--- /dev/null+++ b/embedchain/loaders/xml.py@@ -0,0 +1,26 @@+import hashlib++from langchain.document_loaders import UnstructuredXMLLoader++from embedchain.helper.json_serializable import register_deserializable+from embedchain.loaders.base_loader import BaseLoader+from embedchain.utils import clean_string+++@register_deserializable+class XmlLoader(BaseLoader):+ def load_data(self, xml_url):+ """Load data from a XML file."""+ loader = UnstructuredXMLLoader(xml_url)+ data = loader.load()+ content = data[0].page_content+ content = clean_string(content)+ meta_data = data[0].metadata+ meta_data["url"] = meta_data["source"]+ del meta_data["source"]+ output = [{"content": content, "meta_data": meta_data}]+ doc_id = hashlib.sha256((content + xml_url).encode()).hexdigest()+ return {+ "doc_id": doc_id,+ "data": output,+ }diff --git a/embedchain/models/data_type.py b/embedchain/models/data_type.pyindex 90d7dd910..566fe6574 100644--- a/embedchain/models/data_type.py+++ b/embedchain/models/data_type.py@@ -18,6 +18,7 @@ class IndirectDataType(Enum):PDF_FILE = "pdf_file"WEB_PAGE = "web_page"SITEMAP = "sitemap"+ XML = "xml"DOCX = "docx"DOCS_SITE = "docs_site"NOTION = "notion"@@ -40,6 +41,7 @@ class DataType(Enum):PDF_FILE = IndirectDataType.PDF_FILE.valueWEB_PAGE = IndirectDataType.WEB_PAGE.valueSITEMAP = IndirectDataType.SITEMAP.value+ XML = IndirectDataType.XML.valueDOCX = IndirectDataType.DOCX.valueDOCS_SITE = IndirectDataType.DOCS_SITE.valueNOTION = IndirectDataType.NOTION.valuediff --git a/embedchain/utils.py b/embedchain/utils.pyindex 43c5ef694..9b5709c3c 100644--- a/embedchain/utils.py+++ b/embedchain/utils.py@@ -128,8 +128,7 @@ def detect_datatype(source: Any) -> DataType:formatted_source = format_source(str(source), 30)if url:- from langchain.document_loaders.youtube import \- ALLOWED_NETLOCK as YOUTUBE_ALLOWED_NETLOCS+ from langchain.document_loaders.youtube import ALLOWED_NETLOCK as YOUTUBE_ALLOWED_NETLOCSif url.netloc in YOUTUBE_ALLOWED_NETLOCS:logging.debug(f"Source of `{formatted_source}` detected as `youtube_video`.")@@ -190,6 +189,10 @@ def detect_datatype(source: Any) -> DataType:logging.debug(f"Source of `{formatted_source}` detected as `csv`.")return DataType.CSV+ if source.endswith(".xml"):+ logging.debug(f"Source of `{formatted_source}` detected as `xml`.")+ return DataType.XML+# If the source is a valid file, that's not detectable as a type, an error is raised.# It does not fallback to text.raise ValueError(diff --git a/pyproject.toml b/pyproject.tomlindex dbc960b07..758d3705e 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -106,6 +106,7 @@ fastapi-poe = { version = "0.0.16", optional = true }discord = { version = "^2.3.2", optional = true }slack-sdk = { version = "3.21.3", optional = true }docx2txt = "^0.8"+unstructured = {extras = ["local-inference"], version = "^0.10.18"}pillow = { version = "10.0.1", optional = true }torchvision = { version = ">=0.15.1, !=0.15.2", optional = true }ftfy = { version = "6.1.1", optional = true }diff --git a/tests/loaders/test_xml.py b/tests/loaders/test_xml.pynew file mode 100644index 000000000..d1ff5daad--- /dev/null+++ b/tests/loaders/test_xml.py@@ -0,0 +1,62 @@+import tempfile++import pytest++from embedchain.loaders.xml import XmlLoader++# Taken from https://github.com/langchain-ai/langchain/blob/master/libs/langchain/tests/integration_tests/examples/factbook.xml+SAMPLE_XML = """<?xml version="1.0" encoding="UTF-8"?>+<factbook>+ <country>+ <name>United States</name>+ <capital>Washington, DC</capital>+ <leader>Joe Biden</leader>+ <sport>Baseball</sport>+ </country>+ <country>+ <name>Canada</name>+ <capital>Ottawa</capital>+ <leader>Justin Trudeau</leader>+ <sport>Hockey</sport>+ </country>+ <country>+ <name>France</name>+ <capital>Paris</capital>+ <leader>Emmanuel Macron</leader>+ <sport>Soccer</sport>+ </country>+ <country>+ <name>Trinidad & Tobado</name>+ <capital>Port of Spain</capital>+ <leader>Keith Rowley</leader>+ <sport>Track & Field</sport>+ </country>+</factbook>"""++[email protected]("xml", [SAMPLE_XML])+def test_load_data(xml: str):+ """+ Test XML loader++ Tests that XML file is loaded, metadata is correct and content is correct+ """+ # Creating temporary XML file+ with tempfile.NamedTemporaryFile(mode="w+") as tmpfile:+ tmpfile.write(xml)++ tmpfile.seek(0)+ filename = tmpfile.name++ # Loading CSV using XmlLoader+ loader = XmlLoader()+ result = loader.load_data(filename)+ data = result["data"]++ # Assertions+ assert len(data) == 1+ assert "United States Washington, DC Joe Biden" in data[0]["content"]+ assert "Canada Ottawa Justin Trudeau" in data[0]["content"]+ assert "France Paris Emmanuel Macron" in data[0]["content"]+ assert "Trinidad & Tobado Port of Spain Keith Rowley" in data[0]["content"]+ assert data[0]["meta_data"]["url"] == filename