The Memory Layer for AI Agents - Drop-in memory infrastructure for AI agents and apps. Context that persists. Built for production.
Python agentic-memoryagentic-memory-systemagentsaiai-agentschatgptgenaillmlong-term-memorymemorymemory-managementpythonragstate-management
[Misc] Minor fixes and refactor utils code (#1117)
Deshraj Yadav5c1f78879f82
diff --git a/embedchain/app.py b/embedchain/app.pyindex 288e5650f..ad8afc3f0 100644--- a/embedchain/app.py+++ b/embedchain/app.py@@ -23,7 +23,7 @@ from embedchain.helpers.json_serializable import register_deserializablefrom embedchain.llm.base import BaseLlmfrom embedchain.llm.openai import OpenAILlmfrom embedchain.telemetry.posthog import AnonymousTelemetry-from embedchain.utils import validate_config+from embedchain.utils.misc import validate_configfrom embedchain.vectordb.base import BaseVectorDBfrom embedchain.vectordb.chroma import ChromaDBdiff --git a/embedchain/cli.py b/embedchain/cli.pyindex 4037b65fb..e6cca4c11 100644--- a/embedchain/cli.py+++ b/embedchain/cli.py@@ -506,19 +506,6 @@ def runserver():return# Step 5: Install UI requirements and start the UI server- try:- os.chdir("ui")- subprocess.run(["yarn"], check=True)- ui_process = subprocess.Popen(["yarn", "dev"], stdout=None, stderr=None)- console.print("✅ [bold green]UI server started successfully.[/bold green]")- except Exception as e:- console.print(f"❌ [bold red]Failed to start the UI server: {e}[/bold red]")-- # Wait for the subprocesses to complete- api_process.wait()- ui_process.wait()-- # Step 6: Install UI requirements and start the UI servertry:os.chdir("ui")subprocess.run(["yarn"], check=True)diff --git a/embedchain/embedchain.py b/embedchain/embedchain.pyindex bc74d5694..190d2f555 100644--- a/embedchain/embedchain.py+++ b/embedchain/embedchain.py@@ -19,7 +19,7 @@ from embedchain.llm.base import BaseLlmfrom embedchain.loaders.base_loader import BaseLoaderfrom embedchain.models.data_type import DataType, DirectDataType, IndirectDataType, SpecialDataTypefrom embedchain.telemetry.posthog import AnonymousTelemetry-from embedchain.utils import detect_datatype, is_valid_json_string+from embedchain.utils.misc import detect_datatype, is_valid_json_stringfrom embedchain.vectordb.base import BaseVectorDBload_dotenv()diff --git a/embedchain/embedder/base.py b/embedchain/embedder/base.pyindex fdad3b2e1..60c53c030 100644--- a/embedchain/embedder/base.py+++ b/embedchain/embedder/base.py@@ -5,7 +5,7 @@ from embedchain.config.embedder.base import BaseEmbedderConfigtry:from chromadb.api.types import Embeddable, EmbeddingFunction, Embeddingsexcept RuntimeError:- from embedchain.utils import use_pysqlite3+ from embedchain.utils.misc import use_pysqlite3use_pysqlite3()from chromadb.api.types import Embeddable, EmbeddingFunction, Embeddingsdiff --git a/embedchain/loaders/beehiiv.py b/embedchain/loaders/beehiiv.pyindex f9cc920f3..5169f7aed 100644--- a/embedchain/loaders/beehiiv.py+++ b/embedchain/loaders/beehiiv.py@@ -7,7 +7,7 @@ import requestsfrom embedchain.helpers.json_serializable import register_deserializablefrom embedchain.loaders.base_loader import BaseLoader-from embedchain.utils import is_readable+from embedchain.utils.misc import is_readable@register_deserializablediff --git a/embedchain/loaders/directory_loader.py b/embedchain/loaders/directory_loader.pyindex 9a4bc48cc..2790724ea 100644--- a/embedchain/loaders/directory_loader.py+++ b/embedchain/loaders/directory_loader.py@@ -8,7 +8,7 @@ from embedchain.data_formatter.data_formatter import DataFormatterfrom embedchain.helpers.json_serializable import register_deserializablefrom embedchain.loaders.base_loader import BaseLoaderfrom embedchain.loaders.text_file import TextFileLoader-from embedchain.utils import detect_datatype+from embedchain.utils.misc import detect_datatype@register_deserializablediff --git a/embedchain/loaders/discourse.py b/embedchain/loaders/discourse.pyindex d5b4da80d..363bcb8ad 100644--- a/embedchain/loaders/discourse.py+++ b/embedchain/loaders/discourse.py@@ -6,7 +6,7 @@ from typing import Any, Dict, Optionalimport requestsfrom embedchain.loaders.base_loader import BaseLoader-from embedchain.utils import clean_string+from embedchain.utils.misc import clean_stringclass DiscourseLoader(BaseLoader):diff --git a/embedchain/loaders/github.py b/embedchain/loaders/github.pyindex 7c54a59c8..bae36973c 100644--- a/embedchain/loaders/github.py+++ b/embedchain/loaders/github.py@@ -9,7 +9,7 @@ from typing import Any, Dict, Optionalfrom tqdm import tqdmfrom embedchain.loaders.base_loader import BaseLoader-from embedchain.utils import clean_string+from embedchain.utils.misc import clean_stringGITHUB_URL = "https://github.com"GITHUB_API_URL = "https://api.github.com"diff --git a/embedchain/loaders/gmail.py b/embedchain/loaders/gmail.pyindex e87819e6d..3487a20ba 100644--- a/embedchain/loaders/gmail.py+++ b/embedchain/loaders/gmail.py@@ -20,7 +20,7 @@ except ImportError:) from Nonefrom embedchain.loaders.base_loader import BaseLoader-from embedchain.utils import clean_string+from embedchain.utils.misc import clean_stringclass GmailReader:diff --git a/embedchain/loaders/google_drive.py b/embedchain/loaders/google_drive.pyindex dde405aa3..5db6c8b84 100644--- a/embedchain/loaders/google_drive.py+++ b/embedchain/loaders/google_drive.py@@ -8,7 +8,8 @@ except ImportError:"Google Drive requires extra dependencies. Install with `pip install embedchain[googledrive]`") from None-from langchain.document_loaders import GoogleDriveLoader as Loader, UnstructuredFileIOLoader+from langchain.document_loaders import GoogleDriveLoader as Loader+from langchain.document_loaders import UnstructuredFileIOLoaderfrom embedchain.helpers.json_serializable import register_deserializablefrom embedchain.loaders.base_loader import BaseLoaderdiff --git a/embedchain/loaders/json.py b/embedchain/loaders/json.pyindex 51cdd79eb..2d090f684 100644--- a/embedchain/loaders/json.py+++ b/embedchain/loaders/json.py@@ -7,7 +7,7 @@ from typing import Dict, List, Unionimport requestsfrom embedchain.loaders.base_loader import BaseLoader-from embedchain.utils import clean_string, is_valid_json_string+from embedchain.utils.misc import clean_string, is_valid_json_stringclass JSONReader:diff --git a/embedchain/loaders/mysql.py b/embedchain/loaders/mysql.pyindex 63cfc0407..3574b6c02 100644--- a/embedchain/loaders/mysql.py+++ b/embedchain/loaders/mysql.py@@ -3,7 +3,7 @@ import loggingfrom typing import Any, Dict, Optionalfrom embedchain.loaders.base_loader import BaseLoader-from embedchain.utils import clean_string+from embedchain.utils.misc import clean_stringclass MySQLLoader(BaseLoader):diff --git a/embedchain/loaders/notion.py b/embedchain/loaders/notion.pyindex 4b45fc8bd..d51753b2f 100644--- a/embedchain/loaders/notion.py+++ b/embedchain/loaders/notion.py@@ -7,7 +7,7 @@ import requestsfrom embedchain.helpers.json_serializable import register_deserializablefrom embedchain.loaders.base_loader import BaseLoader-from embedchain.utils import clean_string+from embedchain.utils.misc import clean_stringclass NotionDocument:diff --git a/embedchain/loaders/pdf_file.py b/embedchain/loaders/pdf_file.pyindex 03495edbe..fb97f324f 100644--- a/embedchain/loaders/pdf_file.py+++ b/embedchain/loaders/pdf_file.py@@ -8,7 +8,7 @@ except ImportError:) from Nonefrom embedchain.helpers.json_serializable import register_deserializablefrom embedchain.loaders.base_loader import BaseLoader-from embedchain.utils import clean_string+from embedchain.utils.misc import clean_string@register_deserializablediff --git a/embedchain/loaders/slack.py b/embedchain/loaders/slack.pyindex cb785efb1..44de53b5a 100644--- a/embedchain/loaders/slack.py+++ b/embedchain/loaders/slack.py@@ -7,7 +7,7 @@ from typing import Any, Dict, Optionalimport certififrom embedchain.loaders.base_loader import BaseLoader-from embedchain.utils import clean_string+from embedchain.utils.misc import clean_stringSLACK_API_BASE_URL = "https://www.slack.com/api/"diff --git a/embedchain/loaders/substack.py b/embedchain/loaders/substack.pyindex 40278a6b4..77903d173 100644--- a/embedchain/loaders/substack.py+++ b/embedchain/loaders/substack.py@@ -7,7 +7,7 @@ import requestsfrom embedchain.helpers.json_serializable import register_deserializablefrom embedchain.loaders.base_loader import BaseLoader-from embedchain.utils import is_readable+from embedchain.utils.misc import is_readable@register_deserializablediff --git a/embedchain/loaders/unstructured_file.py b/embedchain/loaders/unstructured_file.pyindex 294c596c2..c9b9f7e82 100644--- a/embedchain/loaders/unstructured_file.py+++ b/embedchain/loaders/unstructured_file.py@@ -2,7 +2,7 @@ import hashlibfrom embedchain.helpers.json_serializable import register_deserializablefrom embedchain.loaders.base_loader import BaseLoader-from embedchain.utils import clean_string+from embedchain.utils.misc import clean_string@register_deserializablediff --git a/embedchain/loaders/web_page.py b/embedchain/loaders/web_page.pyindex ecf03e9df..0995fc868 100644--- a/embedchain/loaders/web_page.py+++ b/embedchain/loaders/web_page.py@@ -12,7 +12,7 @@ except ImportError:from embedchain.helpers.json_serializable import register_deserializablefrom embedchain.loaders.base_loader import BaseLoader-from embedchain.utils import clean_string+from embedchain.utils.misc import clean_string@register_deserializablediff --git a/embedchain/loaders/xml.py b/embedchain/loaders/xml.pyindex 00fe47701..5436d7495 100644--- a/embedchain/loaders/xml.py+++ b/embedchain/loaders/xml.py@@ -8,7 +8,7 @@ except ImportError:) from Nonefrom embedchain.helpers.json_serializable import register_deserializablefrom embedchain.loaders.base_loader import BaseLoader-from embedchain.utils import clean_string+from embedchain.utils.misc import clean_string@register_deserializablediff --git a/embedchain/loaders/youtube_video.py b/embedchain/loaders/youtube_video.pyindex 2aa080247..4c1a8453a 100644--- a/embedchain/loaders/youtube_video.py+++ b/embedchain/loaders/youtube_video.py@@ -8,7 +8,7 @@ except ImportError:) from Nonefrom embedchain.helpers.json_serializable import register_deserializablefrom embedchain.loaders.base_loader import BaseLoader-from embedchain.utils import clean_string+from embedchain.utils.misc import clean_string@register_deserializablediff --git a/embedchain/store/assistants.py b/embedchain/store/assistants.pyindex 87f6cb28e..3c77d577e 100644--- a/embedchain/store/assistants.py+++ b/embedchain/store/assistants.py@@ -15,7 +15,7 @@ from embedchain.config import AddConfigfrom embedchain.data_formatter import DataFormatterfrom embedchain.models.data_type import DataTypefrom embedchain.telemetry.posthog import AnonymousTelemetry-from embedchain.utils import detect_datatype+from embedchain.utils.misc import detect_datatypelogging.basicConfig(level=logging.WARN)diff --git a/embedchain/utils/__init__.py b/embedchain/utils/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/embedchain/utils.py b/embedchain/utils/misc.pysimilarity index 99%rename from embedchain/utils.pyrename to embedchain/utils/misc.pyindex 43665d54c..03ed5bb35 100644--- a/embedchain/utils.py+++ b/embedchain/utils/misc.py@@ -201,7 +201,8 @@ def detect_datatype(source: Any) -> DataType:formatted_source = format_source(str(source), 30)if url:- from langchain.document_loaders.youtube import ALLOWED_NETLOCK as YOUTUBE_ALLOWED_NETLOCS+ from langchain.document_loaders.youtube import \+ ALLOWED_NETLOCK as YOUTUBE_ALLOWED_NETLOCSif url.netloc in YOUTUBE_ALLOWED_NETLOCS:logging.debug(f"Source of `{formatted_source}` detected as `youtube_video`.")diff --git a/embedchain/vectordb/chroma.py b/embedchain/vectordb/chroma.pyindex 7763c207f..27156ab7b 100644--- a/embedchain/vectordb/chroma.py+++ b/embedchain/vectordb/chroma.py@@ -14,7 +14,7 @@ try:from chromadb.config import Settingsfrom chromadb.errors import InvalidDimensionExceptionexcept RuntimeError:- from embedchain.utils import use_pysqlite3+ from embedchain.utils.misc import use_pysqlite3use_pysqlite3()import chromadbdiff --git a/embedchain/vectordb/elasticsearch.py b/embedchain/vectordb/elasticsearch.pyindex 62744f5d4..c38e8d5e3 100644--- a/embedchain/vectordb/elasticsearch.py+++ b/embedchain/vectordb/elasticsearch.py@@ -11,7 +11,7 @@ except ImportError:from embedchain.config import ElasticsearchDBConfigfrom embedchain.helpers.json_serializable import register_deserializable-from embedchain.utils import chunks+from embedchain.utils.misc import chunksfrom embedchain.vectordb.base import BaseVectorDBdiff --git a/embedchain/vectordb/pinecone.py b/embedchain/vectordb/pinecone.pyindex dd3da4ac6..336967d68 100644--- a/embedchain/vectordb/pinecone.py+++ b/embedchain/vectordb/pinecone.py@@ -10,7 +10,7 @@ except ImportError:from embedchain.config.vectordb.pinecone import PineconeDBConfigfrom embedchain.helpers.json_serializable import register_deserializable-from embedchain.utils import chunks+from embedchain.utils.misc import chunksfrom embedchain.vectordb.base import BaseVectorDBdiff --git a/pyproject.toml b/pyproject.tomlindex e642799e3..34021617b 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -1,6 +1,6 @@[tool.poetry]name = "embedchain"-version = "0.1.52"+version = "0.1.53"description = "Data platform for LLMs - Load, index, retrieve and sync any unstructured data"authors = ["Taranjeet Singh <[email protected]>",diff --git a/tests/embedchain/test_utils.py b/tests/embedchain/test_utils.pyindex 80365abd8..22806b79f 100644--- a/tests/embedchain/test_utils.py+++ b/tests/embedchain/test_utils.py@@ -3,7 +3,7 @@ import unittestfrom unittest.mock import patchfrom embedchain.models.data_type import DataType-from embedchain.utils import detect_datatype+from embedchain.utils.misc import detect_datatypeclass TestApp(unittest.TestCase):diff --git a/tests/test_utils.py b/tests/test_utils.pyindex fa8106431..3e50e1e16 100644--- a/tests/test_utils.py+++ b/tests/test_utils.py@@ -1,6 +1,6 @@import yaml-from embedchain.utils import validate_config+from embedchain.utils.misc import validate_configCONFIG_YAMLS = ["configs/anthropic.yaml",