Skip to content

armgit / mem0

Public Imported from GitHub
0

The Memory Layer for AI Agents - Drop-in memory infrastructure for AI agents and apps. Context that persists. Built for production.

Clean json data before loading (#895)

Deven Pateldf314dc6d118
diff --git a/embedchain/loaders/json.py b/embedchain/loaders/json.py
index 23d094b63..a76d0a0c3 100644
--- a/embedchain/loaders/json.py
+++ b/embedchain/loaders/json.py
@@ -6,6 +6,7 @@ import re
import requests
from embedchain.loaders.base_loader import BaseLoader
+from embedchain.utils import clean_string
VALID_URL_PATTERN = "^https:\/\/[0-9A-z.]+.[0-9A-z.]+.[a-z]+\/.*\.json$"
@@ -49,7 +50,7 @@ class JSONLoader(BaseLoader):
docs = loader.load_data(json_data)
for doc in docs:
- doc_content = doc.text
+ doc_content = clean_string(doc.text)
data.append({"content": doc_content, "meta_data": {"url": content}})
data_content.append(doc_content)
doc_id = hashlib.sha256((content + ", ".join(data_content)).encode()).hexdigest()