Skip to content

armgit / mem0

Public Imported from GitHub
0

The Memory Layer for AI Agents - Drop-in memory infrastructure for AI agents and apps. Context that persists. Built for production.

[Misc] Minor fixes and refactor utils code (#1117)

Deshraj Yadav5c1f78879f82
diff --git a/embedchain/app.py b/embedchain/app.py
index 288e5650f..ad8afc3f0 100644
--- a/embedchain/app.py
+++ b/embedchain/app.py
@@ -23,7 +23,7 @@ from embedchain.helpers.json_serializable import register_deserializable
from embedchain.llm.base import BaseLlm
from embedchain.llm.openai import OpenAILlm
from embedchain.telemetry.posthog import AnonymousTelemetry
-from embedchain.utils import validate_config
+from embedchain.utils.misc import validate_config
from embedchain.vectordb.base import BaseVectorDB
from embedchain.vectordb.chroma import ChromaDB
diff --git a/embedchain/cli.py b/embedchain/cli.py
index 4037b65fb..e6cca4c11 100644
--- a/embedchain/cli.py
+++ b/embedchain/cli.py
@@ -506,19 +506,6 @@ def runserver():
return
# Step 5: Install UI requirements and start the UI server
- try:
- os.chdir("ui")
- subprocess.run(["yarn"], check=True)
- ui_process = subprocess.Popen(["yarn", "dev"], stdout=None, stderr=None)
- console.print("✅ [bold green]UI server started successfully.[/bold green]")
- except Exception as e:
- console.print(f"❌ [bold red]Failed to start the UI server: {e}[/bold red]")
-
- # Wait for the subprocesses to complete
- api_process.wait()
- ui_process.wait()
-
- # Step 6: Install UI requirements and start the UI server
try:
os.chdir("ui")
subprocess.run(["yarn"], check=True)
diff --git a/embedchain/embedchain.py b/embedchain/embedchain.py
index bc74d5694..190d2f555 100644
--- a/embedchain/embedchain.py
+++ b/embedchain/embedchain.py
@@ -19,7 +19,7 @@ from embedchain.llm.base import BaseLlm
from embedchain.loaders.base_loader import BaseLoader
from embedchain.models.data_type import DataType, DirectDataType, IndirectDataType, SpecialDataType
from embedchain.telemetry.posthog import AnonymousTelemetry
-from embedchain.utils import detect_datatype, is_valid_json_string
+from embedchain.utils.misc import detect_datatype, is_valid_json_string
from embedchain.vectordb.base import BaseVectorDB
load_dotenv()
diff --git a/embedchain/embedder/base.py b/embedchain/embedder/base.py
index fdad3b2e1..60c53c030 100644
--- a/embedchain/embedder/base.py
+++ b/embedchain/embedder/base.py
@@ -5,7 +5,7 @@ from embedchain.config.embedder.base import BaseEmbedderConfig
try:
from chromadb.api.types import Embeddable, EmbeddingFunction, Embeddings
except RuntimeError:
- from embedchain.utils import use_pysqlite3
+ from embedchain.utils.misc import use_pysqlite3
use_pysqlite3()
from chromadb.api.types import Embeddable, EmbeddingFunction, Embeddings
diff --git a/embedchain/loaders/beehiiv.py b/embedchain/loaders/beehiiv.py
index f9cc920f3..5169f7aed 100644
--- a/embedchain/loaders/beehiiv.py
+++ b/embedchain/loaders/beehiiv.py
@@ -7,7 +7,7 @@ import requests
from embedchain.helpers.json_serializable import register_deserializable
from embedchain.loaders.base_loader import BaseLoader
-from embedchain.utils import is_readable
+from embedchain.utils.misc import is_readable
@register_deserializable
diff --git a/embedchain/loaders/directory_loader.py b/embedchain/loaders/directory_loader.py
index 9a4bc48cc..2790724ea 100644
--- a/embedchain/loaders/directory_loader.py
+++ b/embedchain/loaders/directory_loader.py
@@ -8,7 +8,7 @@ from embedchain.data_formatter.data_formatter import DataFormatter
from embedchain.helpers.json_serializable import register_deserializable
from embedchain.loaders.base_loader import BaseLoader
from embedchain.loaders.text_file import TextFileLoader
-from embedchain.utils import detect_datatype
+from embedchain.utils.misc import detect_datatype
@register_deserializable
diff --git a/embedchain/loaders/discourse.py b/embedchain/loaders/discourse.py
index d5b4da80d..363bcb8ad 100644
--- a/embedchain/loaders/discourse.py
+++ b/embedchain/loaders/discourse.py
@@ -6,7 +6,7 @@ from typing import Any, Dict, Optional
import requests
from embedchain.loaders.base_loader import BaseLoader
-from embedchain.utils import clean_string
+from embedchain.utils.misc import clean_string
class DiscourseLoader(BaseLoader):
diff --git a/embedchain/loaders/github.py b/embedchain/loaders/github.py
index 7c54a59c8..bae36973c 100644
--- a/embedchain/loaders/github.py
+++ b/embedchain/loaders/github.py
@@ -9,7 +9,7 @@ from typing import Any, Dict, Optional
from tqdm import tqdm
from embedchain.loaders.base_loader import BaseLoader
-from embedchain.utils import clean_string
+from embedchain.utils.misc import clean_string
GITHUB_URL = "https://github.com"
GITHUB_API_URL = "https://api.github.com"
diff --git a/embedchain/loaders/gmail.py b/embedchain/loaders/gmail.py
index e87819e6d..3487a20ba 100644
--- a/embedchain/loaders/gmail.py
+++ b/embedchain/loaders/gmail.py
@@ -20,7 +20,7 @@ except ImportError:
) from None
from embedchain.loaders.base_loader import BaseLoader
-from embedchain.utils import clean_string
+from embedchain.utils.misc import clean_string
class GmailReader:
diff --git a/embedchain/loaders/google_drive.py b/embedchain/loaders/google_drive.py
index dde405aa3..5db6c8b84 100644
--- a/embedchain/loaders/google_drive.py
+++ b/embedchain/loaders/google_drive.py
@@ -8,7 +8,8 @@ except ImportError:
"Google Drive requires extra dependencies. Install with `pip install embedchain[googledrive]`"
) from None
-from langchain.document_loaders import GoogleDriveLoader as Loader, UnstructuredFileIOLoader
+from langchain.document_loaders import GoogleDriveLoader as Loader
+from langchain.document_loaders import UnstructuredFileIOLoader
from embedchain.helpers.json_serializable import register_deserializable
from embedchain.loaders.base_loader import BaseLoader
diff --git a/embedchain/loaders/json.py b/embedchain/loaders/json.py
index 51cdd79eb..2d090f684 100644
--- a/embedchain/loaders/json.py
+++ b/embedchain/loaders/json.py
@@ -7,7 +7,7 @@ from typing import Dict, List, Union
import requests
from embedchain.loaders.base_loader import BaseLoader
-from embedchain.utils import clean_string, is_valid_json_string
+from embedchain.utils.misc import clean_string, is_valid_json_string
class JSONReader:
diff --git a/embedchain/loaders/mysql.py b/embedchain/loaders/mysql.py
index 63cfc0407..3574b6c02 100644
--- a/embedchain/loaders/mysql.py
+++ b/embedchain/loaders/mysql.py
@@ -3,7 +3,7 @@ import logging
from typing import Any, Dict, Optional
from embedchain.loaders.base_loader import BaseLoader
-from embedchain.utils import clean_string
+from embedchain.utils.misc import clean_string
class MySQLLoader(BaseLoader):
diff --git a/embedchain/loaders/notion.py b/embedchain/loaders/notion.py
index 4b45fc8bd..d51753b2f 100644
--- a/embedchain/loaders/notion.py
+++ b/embedchain/loaders/notion.py
@@ -7,7 +7,7 @@ import requests
from embedchain.helpers.json_serializable import register_deserializable
from embedchain.loaders.base_loader import BaseLoader
-from embedchain.utils import clean_string
+from embedchain.utils.misc import clean_string
class NotionDocument:
diff --git a/embedchain/loaders/pdf_file.py b/embedchain/loaders/pdf_file.py
index 03495edbe..fb97f324f 100644
--- a/embedchain/loaders/pdf_file.py
+++ b/embedchain/loaders/pdf_file.py
@@ -8,7 +8,7 @@ except ImportError:
) from None
from embedchain.helpers.json_serializable import register_deserializable
from embedchain.loaders.base_loader import BaseLoader
-from embedchain.utils import clean_string
+from embedchain.utils.misc import clean_string
@register_deserializable
diff --git a/embedchain/loaders/slack.py b/embedchain/loaders/slack.py
index cb785efb1..44de53b5a 100644
--- a/embedchain/loaders/slack.py
+++ b/embedchain/loaders/slack.py
@@ -7,7 +7,7 @@ from typing import Any, Dict, Optional
import certifi
from embedchain.loaders.base_loader import BaseLoader
-from embedchain.utils import clean_string
+from embedchain.utils.misc import clean_string
SLACK_API_BASE_URL = "https://www.slack.com/api/"
diff --git a/embedchain/loaders/substack.py b/embedchain/loaders/substack.py
index 40278a6b4..77903d173 100644
--- a/embedchain/loaders/substack.py
+++ b/embedchain/loaders/substack.py
@@ -7,7 +7,7 @@ import requests
from embedchain.helpers.json_serializable import register_deserializable
from embedchain.loaders.base_loader import BaseLoader
-from embedchain.utils import is_readable
+from embedchain.utils.misc import is_readable
@register_deserializable
diff --git a/embedchain/loaders/unstructured_file.py b/embedchain/loaders/unstructured_file.py
index 294c596c2..c9b9f7e82 100644
--- a/embedchain/loaders/unstructured_file.py
+++ b/embedchain/loaders/unstructured_file.py
@@ -2,7 +2,7 @@ import hashlib
from embedchain.helpers.json_serializable import register_deserializable
from embedchain.loaders.base_loader import BaseLoader
-from embedchain.utils import clean_string
+from embedchain.utils.misc import clean_string
@register_deserializable
diff --git a/embedchain/loaders/web_page.py b/embedchain/loaders/web_page.py
index ecf03e9df..0995fc868 100644
--- a/embedchain/loaders/web_page.py
+++ b/embedchain/loaders/web_page.py
@@ -12,7 +12,7 @@ except ImportError:
from embedchain.helpers.json_serializable import register_deserializable
from embedchain.loaders.base_loader import BaseLoader
-from embedchain.utils import clean_string
+from embedchain.utils.misc import clean_string
@register_deserializable
diff --git a/embedchain/loaders/xml.py b/embedchain/loaders/xml.py
index 00fe47701..5436d7495 100644
--- a/embedchain/loaders/xml.py
+++ b/embedchain/loaders/xml.py
@@ -8,7 +8,7 @@ except ImportError:
) from None
from embedchain.helpers.json_serializable import register_deserializable
from embedchain.loaders.base_loader import BaseLoader
-from embedchain.utils import clean_string
+from embedchain.utils.misc import clean_string
@register_deserializable
diff --git a/embedchain/loaders/youtube_video.py b/embedchain/loaders/youtube_video.py
index 2aa080247..4c1a8453a 100644
--- a/embedchain/loaders/youtube_video.py
+++ b/embedchain/loaders/youtube_video.py
@@ -8,7 +8,7 @@ except ImportError:
) from None
from embedchain.helpers.json_serializable import register_deserializable
from embedchain.loaders.base_loader import BaseLoader
-from embedchain.utils import clean_string
+from embedchain.utils.misc import clean_string
@register_deserializable
diff --git a/embedchain/store/assistants.py b/embedchain/store/assistants.py
index 87f6cb28e..3c77d577e 100644
--- a/embedchain/store/assistants.py
+++ b/embedchain/store/assistants.py
@@ -15,7 +15,7 @@ from embedchain.config import AddConfig
from embedchain.data_formatter import DataFormatter
from embedchain.models.data_type import DataType
from embedchain.telemetry.posthog import AnonymousTelemetry
-from embedchain.utils import detect_datatype
+from embedchain.utils.misc import detect_datatype
logging.basicConfig(level=logging.WARN)
diff --git a/embedchain/utils/__init__.py b/embedchain/utils/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/embedchain/utils.py b/embedchain/utils/misc.py
similarity index 99%
rename from embedchain/utils.py
rename to embedchain/utils/misc.py
index 43665d54c..03ed5bb35 100644
--- a/embedchain/utils.py
+++ b/embedchain/utils/misc.py
@@ -201,7 +201,8 @@ def detect_datatype(source: Any) -> DataType:
formatted_source = format_source(str(source), 30)
if url:
- from langchain.document_loaders.youtube import ALLOWED_NETLOCK as YOUTUBE_ALLOWED_NETLOCS
+ from langchain.document_loaders.youtube import \
+ ALLOWED_NETLOCK as YOUTUBE_ALLOWED_NETLOCS
if url.netloc in YOUTUBE_ALLOWED_NETLOCS:
logging.debug(f"Source of `{formatted_source}` detected as `youtube_video`.")
diff --git a/embedchain/vectordb/chroma.py b/embedchain/vectordb/chroma.py
index 7763c207f..27156ab7b 100644
--- a/embedchain/vectordb/chroma.py
+++ b/embedchain/vectordb/chroma.py
@@ -14,7 +14,7 @@ try:
from chromadb.config import Settings
from chromadb.errors import InvalidDimensionException
except RuntimeError:
- from embedchain.utils import use_pysqlite3
+ from embedchain.utils.misc import use_pysqlite3
use_pysqlite3()
import chromadb
diff --git a/embedchain/vectordb/elasticsearch.py b/embedchain/vectordb/elasticsearch.py
index 62744f5d4..c38e8d5e3 100644
--- a/embedchain/vectordb/elasticsearch.py
+++ b/embedchain/vectordb/elasticsearch.py
@@ -11,7 +11,7 @@ except ImportError:
from embedchain.config import ElasticsearchDBConfig
from embedchain.helpers.json_serializable import register_deserializable
-from embedchain.utils import chunks
+from embedchain.utils.misc import chunks
from embedchain.vectordb.base import BaseVectorDB
diff --git a/embedchain/vectordb/pinecone.py b/embedchain/vectordb/pinecone.py
index dd3da4ac6..336967d68 100644
--- a/embedchain/vectordb/pinecone.py
+++ b/embedchain/vectordb/pinecone.py
@@ -10,7 +10,7 @@ except ImportError:
from embedchain.config.vectordb.pinecone import PineconeDBConfig
from embedchain.helpers.json_serializable import register_deserializable
-from embedchain.utils import chunks
+from embedchain.utils.misc import chunks
from embedchain.vectordb.base import BaseVectorDB
diff --git a/pyproject.toml b/pyproject.toml
index e642799e3..34021617b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
[tool.poetry]
name = "embedchain"
-version = "0.1.52"
+version = "0.1.53"
description = "Data platform for LLMs - Load, index, retrieve and sync any unstructured data"
authors = [
"Taranjeet Singh <[email protected]>",
diff --git a/tests/embedchain/test_utils.py b/tests/embedchain/test_utils.py
index 80365abd8..22806b79f 100644
--- a/tests/embedchain/test_utils.py
+++ b/tests/embedchain/test_utils.py
@@ -3,7 +3,7 @@ import unittest
from unittest.mock import patch
from embedchain.models.data_type import DataType
-from embedchain.utils import detect_datatype
+from embedchain.utils.misc import detect_datatype
class TestApp(unittest.TestCase):
diff --git a/tests/test_utils.py b/tests/test_utils.py
index fa8106431..3e50e1e16 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -1,6 +1,6 @@
import yaml
-from embedchain.utils import validate_config
+from embedchain.utils.misc import validate_config
CONFIG_YAMLS = [
"configs/anthropic.yaml",