The Memory Layer for AI Agents - Drop-in memory infrastructure for AI agents and apps. Context that persists. Built for production.
Python agentic-memoryagentic-memory-systemagentsaiai-agentschatgptgenaillmlong-term-memorymemorymemory-managementpythonragstate-management
[Feature]: Add posthog anonymous telemetry and update docs (#867)
Deshraj Yadav81336668b3c3
diff --git a/docs/components/embedding-models.mdx b/docs/components/embedding-models.mdxindex 0f030bdcd..f1dab6865 100644--- a/docs/components/embedding-models.mdx+++ b/docs/components/embedding-models.mdx@@ -24,7 +24,7 @@ Once you have obtained the key, you can use it like this:```python main.pyimport os-from embedchain import App+from embedchain import Pipeline as Appos.environ['OPENAI_API_KEY'] = 'xxx'@@ -52,7 +52,7 @@ To use Azure OpenAI embedding model, you have to set some of the azure openai re```python main.pyimport os-from embedchain import App+from embedchain import Pipeline as Appos.environ["OPENAI_API_TYPE"] = "azure"os.environ["OPENAI_API_BASE"] = "https://xxx.openai.azure.com/"@@ -90,7 +90,7 @@ GPT4All supports generating high quality embeddings of arbitrary length document<CodeGroup>```python main.py-from embedchain import App+from embedchain import Pipeline as App# load embedding model configuration from config.yaml fileapp = App.from_config(yaml_path="config.yaml")@@ -119,7 +119,7 @@ Hugging Face supports generating embeddings of arbitrary length documents of tex<CodeGroup>```python main.py-from embedchain import App+from embedchain import Pipeline as App# load embedding model configuration from config.yaml fileapp = App.from_config(yaml_path="config.yaml")@@ -150,7 +150,7 @@ Embedchain supports Google's VertexAI embeddings model through a simple interfac<CodeGroup>```python main.py-from embedchain import App+from embedchain import Pipeline as App# load embedding model configuration from config.yaml fileapp = App.from_config(yaml_path="config.yaml")diff --git a/docs/components/llms.mdx b/docs/components/llms.mdxindex 49b5d7cd1..031a90a4b 100644--- a/docs/components/llms.mdx+++ b/docs/components/llms.mdx@@ -26,7 +26,7 @@ Once you have obtained the key, you can use it like this:```pythonimport os-from embedchain import App+from embedchain import Pipeline as Appos.environ['OPENAI_API_KEY'] = 'xxx'@@ -41,7 +41,7 @@ If you are looking to configure the different parameters of the LLM, you can do```python main.pyimport os-from embedchain import App+from embedchain import Pipeline as Appos.environ['OPENAI_API_KEY'] = 'xxx'@@ -71,7 +71,7 @@ To use Azure OpenAI model, you have to set some of the azure openai related envi```python main.pyimport os-from embedchain import App+from embedchain import Pipeline as Appos.environ["OPENAI_API_TYPE"] = "azure"os.environ["OPENAI_API_BASE"] = "https://xxx.openai.azure.com/"@@ -110,7 +110,7 @@ To use anthropic's model, please set the `ANTHROPIC_API_KEY` which you find on t```python main.pyimport os-from embedchain import App+from embedchain import Pipeline as Appos.environ["ANTHROPIC_API_KEY"] = "xxx"@@ -147,7 +147,7 @@ Once you have the API key, you are all set to use it with Embedchain.```python main.pyimport os-from embedchain import App+from embedchain import Pipeline as Appos.environ["COHERE_API_KEY"] = "xxx"@@ -180,7 +180,7 @@ GPT4all is a free-to-use, locally running, privacy-aware chatbot. No GPU or inte<CodeGroup>```python main.py-from embedchain import App+from embedchain import Pipeline as App# load llm configuration from config.yaml fileapp = App.from_config(yaml_path="config.yaml")@@ -212,7 +212,7 @@ Once you have the key, load the app using the config yaml file:```python main.pyimport os-from embedchain import App+from embedchain import Pipeline as Appos.environ["JINACHAT_API_KEY"] = "xxx"# load llm configuration from config.yaml file@@ -248,7 +248,7 @@ Once you have the token, load the app using the config yaml file:```python main.pyimport os-from embedchain import App+from embedchain import Pipeline as Appos.environ["HUGGINGFACE_ACCESS_TOKEN"] = "xxx"@@ -278,7 +278,7 @@ Once you have the token, load the app using the config yaml file:```python main.pyimport os-from embedchain import App+from embedchain import Pipeline as Appos.environ["REPLICATE_API_TOKEN"] = "xxx"@@ -305,7 +305,7 @@ Setup Google Cloud Platform application credentials by following the instruction<CodeGroup>```python main.py-from embedchain import App+from embedchain import Pipeline as App# load llm configuration from config.yaml fileapp = App.from_config(yaml_path="config.yaml")diff --git a/docs/components/vector-databases.mdx b/docs/components/vector-databases.mdxindex 94e965d37..ccb5dc668 100644--- a/docs/components/vector-databases.mdx+++ b/docs/components/vector-databases.mdx@@ -22,7 +22,7 @@ Utilizing a vector database alongside Embedchain is a seamless process. All you<CodeGroup>```python main.py-from embedchain import App+from embedchain import Pipeline as App# load chroma configuration from yaml fileapp = App.from_config(yaml_path="config1.yaml")@@ -61,7 +61,7 @@ pip install --upgrade 'embedchain[elasticsearch]'<CodeGroup>```python main.py-from embedchain import App+from embedchain import Pipeline as App# load elasticsearch configuration from yaml fileapp = App.from_config(yaml_path="config.yaml")@@ -89,7 +89,7 @@ pip install --upgrade 'embedchain[opensearch]'<CodeGroup>```python main.py-from embedchain import App+from embedchain import Pipeline as App# load opensearch configuration from yaml fileapp = App.from_config(yaml_path="config.yaml")@@ -125,7 +125,7 @@ Set the Zilliz environment variables `ZILLIZ_CLOUD_URI` and `ZILLIZ_CLOUD_TOKEN````python main.pyimport os-from embedchain import App+from embedchain import Pipeline as Appos.environ['ZILLIZ_CLOUD_URI'] = 'https://xxx.zillizcloud.com'os.environ['ZILLIZ_CLOUD_TOKEN'] = 'xxx'@@ -164,7 +164,7 @@ In order to use Pinecone as vector database, set the environment variables `PINE<CodeGroup>```python main.py-from embedchain import App+from embedchain import Pipeline as App# load pinecone configuration from yaml fileapp = App.from_config(yaml_path="config.yaml")@@ -187,7 +187,7 @@ In order to use Qdrant as a vector database, set the environment variables `QDRA<CodeGroup>```python main.py-from embedchain import App+from embedchain import Pipeline as App# load qdrant configuration from yaml fileapp = App.from_config(yaml_path="config.yaml")@@ -207,7 +207,7 @@ In order to use Weaviate as a vector database, set the environment variables `WE<CodeGroup>```python main.py-from embedchain import App+from embedchain import Pipeline as App# load weaviate configuration from yaml fileapp = App.from_config(yaml_path="config.yaml")diff --git a/docs/data-sources/csv.mdx b/docs/data-sources/csv.mdxindex b56453ecb..add89bc7c 100644--- a/docs/data-sources/csv.mdx+++ b/docs/data-sources/csv.mdx@@ -5,7 +5,7 @@ title: 'π CSV'To add any csv file, use the data_type as `csv`. `csv` allows remote urls and conventional file paths. Headers are included for each line, so if you have an `age` column, `18` will be added as `age: 18`. Eg:```python-from embedchain import App+from embedchain import Pipeline as Appapp = App()app.add('https://people.sc.fsu.edu/~jburkardt/data/csv/airtravel.csv', data_type="csv")diff --git a/docs/data-sources/data-type-handling.mdx b/docs/data-sources/data-type-handling.mdxindex bc2ed3dc0..b6798e01d 100644--- a/docs/data-sources/data-type-handling.mdx+++ b/docs/data-sources/data-type-handling.mdx@@ -35,7 +35,7 @@ Default behavior is to create a persistent vector db in the directory **./db**.Create a local index:```python-from embedchain import App+from embedchain import Pipeline as Appnaval_chat_bot = App()naval_chat_bot.add("https://www.youtube.com/watch?v=3qHkcs3kG44")@@ -45,7 +45,7 @@ naval_chat_bot.add("https://navalmanack.s3.amazonaws.com/Eric-Jorgenson_The-AlmaYou can reuse the local index with the same code, but without adding new documents:```python-from embedchain import App+from embedchain import Pipeline as Appnaval_chat_bot = App()print(naval_chat_bot.query("What unique capacity does Naval argue humans possess when it comes to understanding explanations or concepts?"))diff --git a/docs/data-sources/docs-site.mdx b/docs/data-sources/docs-site.mdxindex 206ef2090..cb190536d 100644--- a/docs/data-sources/docs-site.mdx+++ b/docs/data-sources/docs-site.mdx@@ -5,7 +5,7 @@ title: 'ππ Code documentation'To add any code documentation website as a loader, use the data_type as `docs_site`. Eg:```python-from embedchain import App+from embedchain import Pipeline as Appapp = App()app.add("https://docs.embedchain.ai/", data_type="docs_site")diff --git a/docs/data-sources/docx.mdx b/docs/data-sources/docx.mdxindex cc459621f..bee467304 100644--- a/docs/data-sources/docx.mdx+++ b/docs/data-sources/docx.mdx@@ -7,7 +7,7 @@ title: 'π Docx file'To add any doc/docx file, use the data_type as `docx`. `docx` allows remote urls and conventional file paths. Eg:```python-from embedchain import App+from embedchain import Pipeline as Appapp = App()app.add('https://example.com/content/intro.docx', data_type="docx")diff --git a/docs/data-sources/mdx.mdx b/docs/data-sources/mdx.mdxindex c59569e50..4e30235ae 100644--- a/docs/data-sources/mdx.mdx+++ b/docs/data-sources/mdx.mdx@@ -5,7 +5,7 @@ title: 'π Mdx file'To add any `.mdx` file to your app, use the data_type (first argument to `.add()` method) as `mdx`. Note that this supports support mdx file present on machine, so this should be a file path. Eg:```python-from embedchain import App+from embedchain import Pipeline as Appapp = App()app.add('path/to/file.mdx', data_type='mdx')diff --git a/docs/data-sources/notion.mdx b/docs/data-sources/notion.mdxindex d6c616df8..04e232c7a 100644--- a/docs/data-sources/notion.mdx+++ b/docs/data-sources/notion.mdx@@ -8,7 +8,7 @@ To load a notion page, use the data_type as `notion`. Since it is hard to automaThe next argument must **end** with the `notion page id`. The id is a 32-character string. Eg:```python-from embedchain import App+from embedchain import Pipeline as Appapp = App()diff --git a/docs/data-sources/pdf-file.mdx b/docs/data-sources/pdf-file.mdxindex fe8b8884b..637463a2f 100644--- a/docs/data-sources/pdf-file.mdx+++ b/docs/data-sources/pdf-file.mdx@@ -5,7 +5,7 @@ title: 'π° PDF file'To add any pdf file, use the data_type as `pdf_file`. Eg:```python-from embedchain import App+from embedchain import Pipeline as Appapp = App()diff --git a/docs/data-sources/qna.mdx b/docs/data-sources/qna.mdxindex f7a034d90..122d234c5 100644--- a/docs/data-sources/qna.mdx+++ b/docs/data-sources/qna.mdx@@ -5,7 +5,7 @@ title: 'βπ¬ Queston and answer pair'QnA pair is a local data type. To supply your own QnA pair, use the data_type as `qna_pair` and enter a tuple. Eg:```python-from embedchain import App+from embedchain import Pipeline as Appapp = App()diff --git a/docs/data-sources/sitemap.mdx b/docs/data-sources/sitemap.mdxindex 96b47ef1c..e0c100584 100644--- a/docs/data-sources/sitemap.mdx+++ b/docs/data-sources/sitemap.mdx@@ -5,7 +5,7 @@ title: 'πΊοΈ Sitemap'Add all web pages from an xml-sitemap. Filters non-text files. Use the data_type as `sitemap`. Eg:```python-from embedchain import App+from embedchain import Pipeline as Appapp = App()diff --git a/docs/data-sources/text.mdx b/docs/data-sources/text.mdxindex 0fda6f573..78d85f7b3 100644--- a/docs/data-sources/text.mdx+++ b/docs/data-sources/text.mdx@@ -7,7 +7,7 @@ title: 'π Text'Text is a local data type. To supply your own text, use the data_type as `text` and enter a string. The text is not processed, this can be very versatile. Eg:```python-from embedchain import App+from embedchain import Pipeline as Appapp = App()diff --git a/docs/data-sources/web-page.mdx b/docs/data-sources/web-page.mdxindex a30b836c9..09144ccca 100644--- a/docs/data-sources/web-page.mdx+++ b/docs/data-sources/web-page.mdx@@ -5,7 +5,7 @@ title: 'ππ Web page'To add any web page, use the data_type as `web_page`. Eg:```python-from embedchain import App+from embedchain import Pipeline as Appapp = App()diff --git a/docs/data-sources/xml.mdx b/docs/data-sources/xml.mdxindex afe9a4124..e8f91844e 100644--- a/docs/data-sources/xml.mdx+++ b/docs/data-sources/xml.mdx@@ -7,7 +7,7 @@ title: 'π§Ύ XML file'To add any xml file, use the data_type as `xml`. Eg:```python-from embedchain import App+from embedchain import Pipeline as Appapp = App()diff --git a/docs/data-sources/youtube-video.mdx b/docs/data-sources/youtube-video.mdxindex 62a769ceb..5baf2f9ad 100644--- a/docs/data-sources/youtube-video.mdx+++ b/docs/data-sources/youtube-video.mdx@@ -6,7 +6,7 @@ title: 'π₯πΊ Youtube video'To add any youtube video to your app, use the data_type (first argument to `.add()` method) as `youtube_video`. Eg:```python-from embedchain import App+from embedchain import Pipeline as Appapp = App()app.add('a_valid_youtube_url_here', data_type='youtube_video')diff --git a/docs/get-started/faq.mdx b/docs/get-started/faq.mdxindex 7547670b9..93da577a9 100644--- a/docs/get-started/faq.mdx+++ b/docs/get-started/faq.mdx@@ -9,7 +9,7 @@ description: 'Collections of all the frequently asked questions'```python main.pyimport os-from embedchain import App+from embedchain import Pipeline as Appos.environ['OPENAI_API_KEY'] = 'xxx'@@ -36,7 +36,7 @@ llm:```python main.pyimport os-from embedchain import App+from embedchain import Pipeline as Appos.environ['OPENAI_API_KEY'] = 'xxx'diff --git a/docs/get-started/introduction.mdx b/docs/get-started/introduction.mdxindex 4a78c361d..3c278efba 100644--- a/docs/get-started/introduction.mdx+++ b/docs/get-started/introduction.mdx@@ -12,7 +12,7 @@ You can add data from different data sources using the `.add()` method. Then, siIf you want to create a Naval Ravikant bot with a YouTube video, a book in PDF format, two blog posts, and a question and answer pair, all you need to do is add the respective links. Embedchain will take care of the rest, creating a bot for you.```python-from embedchain import App+from embedchain import Pipeline as Appnaval_bot = App()# Add online datadiff --git a/docs/get-started/quickstart.mdx b/docs/get-started/quickstart.mdxindex c6344ebfa..6ae4e9c68 100644--- a/docs/get-started/quickstart.mdx+++ b/docs/get-started/quickstart.mdx@@ -16,22 +16,22 @@ Creating an app involves 3 steps:<Steps><Step title="βοΈ Import app instance">```python-from embedchain import App+from embedchain import Pipeline as Appapp = App()```</Step><Step title="ποΈ Add data sources">```python# Add different data sources-elon_bot.add("https://en.wikipedia.org/wiki/Elon_Musk")-elon_bot.add("https://www.forbes.com/profile/elon-musk")+app.add("https://en.wikipedia.org/wiki/Elon_Musk")+app.add("https://www.forbes.com/profile/elon-musk")# You can also add local data sources such as pdf, csv files etc.-# elon_bot.add("/path/to/file.pdf")+# app.add("/path/to/file.pdf")```</Step>- <Step title="π¬ Query or chat on your data and get answers">+ <Step title="π¬ Query or chat or search context on your data">```python-elon_bot.query("What is the net worth of Elon Musk today?")+app.query("What is the net worth of Elon Musk today?")# Answer: The net worth of Elon Musk today is $258.7 billion.```</Step>@@ -41,18 +41,18 @@ Putting it together, you can run your first app using the following code. Make s```pythonimport os-from embedchain import App+from embedchain import Pipeline as Appos.environ["OPENAI_API_KEY"] = "xxx"-elon_bot = App()+app = App()# Add different data sources-elon_bot.add("https://en.wikipedia.org/wiki/Elon_Musk")-elon_bot.add("https://www.forbes.com/profile/elon-musk")+app.add("https://en.wikipedia.org/wiki/Elon_Musk")+app.add("https://www.forbes.com/profile/elon-musk")# You can also add local data sources such as pdf, csv files etc.-# elon_bot.add("/path/to/file.pdf")+# app.add("/path/to/file.pdf")-response = elon_bot.query("What is the net worth of Elon Musk today?")+response = app.query("What is the net worth of Elon Musk today?")print(response)# Answer: The net worth of Elon Musk today is $258.7 billion.```diff --git a/docs/integration/langsmith.mdx b/docs/integration/langsmith.mdxindex 5f12717b2..fb9e2a9fb 100644--- a/docs/integration/langsmith.mdx+++ b/docs/integration/langsmith.mdx@@ -39,7 +39,7 @@ os.environ['LANGCHAIN_PROJECT] = <your-project>```python-from embedchain import App+from embedchain import Pipeline as Appapp = App()app.add("https://en.wikipedia.org/wiki/Elon_Musk")diff --git a/docs/mint.json b/docs/mint.jsonindex 137a10cd4..0aee9c780 100644--- a/docs/mint.json+++ b/docs/mint.json@@ -71,10 +71,6 @@"group": "Examples","pages": ["examples/full_stack", "examples/api_server", "examples/discord_bot", "examples/slack_bot", "examples/telegram_bot", "examples/whatsapp_bot", "examples/poe_bot"]},- {- "group": "Pipelines",- "pages": ["pipelines/quickstart"]- },{"group": "Community","pages": [diff --git a/docs/pipelines/quickstart.mdx b/docs/pipelines/quickstart.mdxdeleted file mode 100644index 660ca8a6e..000000000--- a/docs/pipelines/quickstart.mdx+++ /dev/null@@ -1,44 +0,0 @@-----title: 'π Pipelines'-description: 'π‘ Start building LLM powered data pipelines in 1 minute'------Embedchain lets you build data pipelines on your own data sources and deploy it in production in less than a minute. It can load, index, retrieve, and sync any unstructured data.--Install embedchain python package:--```bash-pip install embedchain-```--Creating a pipeline involves 3 steps:--<Steps>- <Step title="βοΈ Import pipeline instance">-```python-from embedchain import Pipeline-p = Pipeline(name="Elon Musk")-```- </Step>-- <Step title="ποΈ Add data sources">-```python-# Add different data sources-p.add("https://en.wikipedia.org/wiki/Elon_Musk")-p.add("https://www.forbes.com/profile/elon-musk")-# You can also add local data sources such as pdf, csv files etc.-# p.add("/path/to/file.pdf")-```- </Step>- <Step title="π¬ Deploy your pipeline to Embedchain platform">-```python-p.deploy()-```- </Step>-</Steps>--That's it. Now, head to the [Embedchain platform](https://app.embedchain.ai) and your pipeline is available there. Make sure to set the `OPENAI_API_KEY` π environment variable in the code.--After you deploy your pipeline to Embedchain platform, you can still add more data sources and update the pipeline multiple times.--Here is a Google Colab notebook for you to get started: [](https://colab.research.google.com/drive/1YVXaBO4yqlHZY4ho67GCJ6aD4CHNiScD?usp=sharing)diff --git a/embedchain/config/apps/app_config.py b/embedchain/config/apps/app_config.pyindex 2aee20e5a..f0efaff9f 100644--- a/embedchain/config/apps/app_config.py+++ b/embedchain/config/apps/app_config.py@@ -15,7 +15,7 @@ class AppConfig(BaseAppConfig):self,log_level: str = "WARNING",id: Optional[str] = None,- collect_metrics: Optional[bool] = None,+ collect_metrics: Optional[bool] = True,collection_name: Optional[str] = None,):"""diff --git a/embedchain/config/pipeline_config.py b/embedchain/config/pipeline_config.pyindex 162a35496..e46456a1b 100644--- a/embedchain/config/pipeline_config.py+++ b/embedchain/config/pipeline_config.py@@ -16,7 +16,7 @@ class PipelineConfig(BaseAppConfig):log_level: str = "WARNING",id: Optional[str] = None,name: Optional[str] = None,- collect_metrics: Optional[bool] = False,+ collect_metrics: Optional[bool] = True,):"""Initializes a configuration class instance for an App. This is the simplest form of an embedchain app.diff --git a/embedchain/embedchain.py b/embedchain/embedchain.pyindex 106366df7..349332038 100644--- a/embedchain/embedchain.py+++ b/embedchain/embedchain.py@@ -1,18 +1,13 @@import hashlib-import importlib.metadataimport jsonimport loggingimport osimport sqlite3-import threading-import uuidfrom pathlib import Pathfrom typing import Any, Dict, List, Optional-import requestsfrom dotenv import load_dotenvfrom langchain.docstore.document import Document-from tenacity import retry, stop_after_attempt, wait_fixedfrom embedchain.chunkers.base_chunker import BaseChunkerfrom embedchain.config import AddConfig, BaseLlmConfig@@ -24,6 +19,7 @@ from embedchain.llm.base import BaseLlmfrom embedchain.loaders.base_loader import BaseLoaderfrom embedchain.models.data_type import (DataType, DirectDataType,IndirectDataType, SpecialDataType)+from embedchain.telemetry.posthog import AnonymousTelemetryfrom embedchain.utils import detect_datatypefrom embedchain.vectordb.base import BaseVectorDB@@ -89,9 +85,8 @@ class EmbedChain(JSONSerializable):self.user_asks = []# Send anonymous telemetry- self.s_id = self.config.id if self.config.id else str(uuid.uuid4())- self.u_id = self._load_or_generate_user_id()-+ self._telemetry_props = {"class": self.__class__.__name__}+ self.telemetry = AnonymousTelemetry(enabled=self.config.collect_metrics)# Establish a connection to the SQLite databaseself.connection = sqlite3.connect(SQLITE_PATH)self.cursor = self.connection.cursor()@@ -111,12 +106,8 @@ class EmbedChain(JSONSerializable):""")self.connection.commit()-- # NOTE: Uncomment the next two lines when running tests to see if any test fires a telemetry event.- # if (self.config.collect_metrics):- # raise ConnectionRefusedError("Collection of metrics should not be allowed.")- thread_telemetry = threading.Thread(target=self._send_telemetry_event, args=("init",))- thread_telemetry.start()+ # Send anonymous telemetry+ self.telemetry.capture(event_name="init", properties=self._telemetry_props)@propertydef collect_metrics(self):@@ -138,29 +129,6 @@ class EmbedChain(JSONSerializable):raise ValueError(f"Boolean value expected but got {type(value)}.")self.llm.online = value- def _load_or_generate_user_id(self) -> str:- """- Loads the user id from the config file if it exists, otherwise generates a new- one and saves it to the config file.-- :return: user id- :rtype: str- """- if not os.path.exists(CONFIG_DIR):- os.makedirs(CONFIG_DIR)-- if os.path.exists(CONFIG_FILE):- with open(CONFIG_FILE, "r") as f:- data = json.load(f)- if "user_id" in data:- return data["user_id"]-- u_id = str(uuid.uuid4())- with open(CONFIG_FILE, "w") as f:- json.dump({"user_id": u_id}, f)-- return u_id-def add(self,source: Any,@@ -259,9 +227,14 @@ class EmbedChain(JSONSerializable):# it's quicker to check the variable twice than to count words when they won't be submitted.word_count = data_formatter.chunker.get_word_count(documents)- extra_metadata = {"data_type": data_type.value, "word_count": word_count, "chunks_count": new_chunks}- thread_telemetry = threading.Thread(target=self._send_telemetry_event, args=("add", extra_metadata))- thread_telemetry.start()+ # Send anonymous telemetry+ event_properties = {+ **self._telemetry_props,+ "data_type": data_type.value,+ "word_count": word_count,+ "chunks_count": new_chunks,+ }+ self.telemetry.capture(event_name="add", properties=event_properties)return source_hash@@ -535,9 +508,7 @@ class EmbedChain(JSONSerializable):answer = self.llm.query(input_query=input_query, contexts=contexts, config=config, dry_run=dry_run)# Send anonymous telemetry- thread_telemetry = threading.Thread(target=self._send_telemetry_event, args=("query",))- thread_telemetry.start()-+ self.telemetry.capture(event_name="query", properties=self._telemetry_props)return answerdef chat(@@ -569,10 +540,8 @@ class EmbedChain(JSONSerializable):"""contexts = self.retrieve_from_database(input_query=input_query, config=config, where=where)answer = self.llm.chat(input_query=input_query, contexts=contexts, config=config, dry_run=dry_run)-# Send anonymous telemetry- thread_telemetry = threading.Thread(target=self._send_telemetry_event, args=("chat",))- thread_telemetry.start()+ self.telemetry.capture(event_name="chat", properties=self._telemetry_props)return answer@@ -608,34 +577,8 @@ class EmbedChain(JSONSerializable):Resets the database. Deletes all embeddings irreversibly.`App` does not have to be reinitialized after using this method."""- # Send anonymous telemetry- thread_telemetry = threading.Thread(target=self._send_telemetry_event, args=("reset",))- thread_telemetry.start()-self.db.reset()self.cursor.execute("DELETE FROM data_sources WHERE pipeline_id = ?", (self.config.id,))self.connection.commit()-- @retry(stop=stop_after_attempt(3), wait=wait_fixed(1))- def _send_telemetry_event(self, method: str, extra_metadata: Optional[dict] = None):- """- Send telemetry event to the embedchain server. This is anonymous. It can be toggled off in `AppConfig`.- """- if not self.config.collect_metrics:- return-- with threading.Lock():- url = "https://api.embedchain.ai/api/v1/telemetry/"- metadata = {- "s_id": self.s_id,- "version": importlib.metadata.version(__package__ or __name__),- "method": method,- "language": "py",- "u_id": self.u_id,- }- if extra_metadata:- metadata.update(extra_metadata)-- response = requests.post(url, json={"metadata": metadata})- if response.status_code != 200:- logging.warning(f"Telemetry event failed with status code {response.status_code}")+ # Send anonymous telemetry+ self.telemetry.capture(event_name="reset", properties=self._telemetry_props)diff --git a/embedchain/pipeline.py b/embedchain/pipeline.pyindex 420eaf7fe..6e9ba29e6 100644--- a/embedchain/pipeline.py+++ b/embedchain/pipeline.py@@ -18,6 +18,7 @@ from embedchain.factory import EmbedderFactory, LlmFactory, VectorDBFactoryfrom embedchain.helper.json_serializable import register_deserializablefrom embedchain.llm.base import BaseLlmfrom embedchain.llm.openai import OpenAILlm+from embedchain.telemetry.posthog import AnonymousTelemetryfrom embedchain.vectordb.base import BaseVectorDBfrom embedchain.vectordb.chroma import ChromaDB@@ -109,8 +110,9 @@ class Pipeline(EmbedChain):self.llm = llm or OpenAILlm()self._init_db()- # setup user id and directory- self.u_id = self._load_or_generate_user_id()+ # Send anonymous telemetry+ self._telemetry_props = {"class": self.__class__.__name__}+ self.telemetry = AnonymousTelemetry(enabled=self.config.collect_metrics)# Establish a connection to the SQLite databaseself.connection = sqlite3.connect(SQLITE_PATH)@@ -131,8 +133,10 @@ class Pipeline(EmbedChain):""")self.connection.commit()+ # Send anonymous telemetry+ self.telemetry.capture(event_name="init", properties=self._telemetry_props)- self.user_asks = [] # legacy defaults+ self.user_asks = []if self.auto_deploy:self.deploy()@@ -219,6 +223,9 @@ class Pipeline(EmbedChain):"""Search for similar documents related to the query in the vector database."""+ # Send anonymous telemetry+ self.telemetry.capture(event_name="search", properties=self._telemetry_props)+# TODO: Search will call the endpoint rather than fetching the data from the db itself when deploy=True.if self.id is None:where = {"app_id": self.local_id}@@ -312,6 +319,9 @@ class Pipeline(EmbedChain):data_hash, data_type, data_value = result[1], result[2], result[3]self._process_and_upload_data(data_hash, data_type, data_value)+ # Send anonymous telemetry+ self.telemetry.capture(event_name="deploy", properties=self._telemetry_props)+@classmethoddef from_config(cls, yaml_path: str, auto_deploy: bool = False):"""@@ -347,6 +357,11 @@ class Pipeline(EmbedChain):embedding_model = EmbedderFactory.create(embedding_model_provider, embedding_model_config_data.get("config", {}))++ # Send anonymous telemetry+ event_properties = {"init_type": "yaml_config"}+ AnonymousTelemetry().capture(event_name="init", properties=event_properties)+return cls(config=pipeline_config,llm=llm,diff --git a/embedchain/telemetry/__init__.py b/embedchain/telemetry/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/embedchain/telemetry/posthog.py b/embedchain/telemetry/posthog.pynew file mode 100644index 000000000..cd4219893--- /dev/null+++ b/embedchain/telemetry/posthog.py@@ -0,0 +1,67 @@+import json+import logging+import os+import uuid+from pathlib import Path++from posthog import Posthog++import embedchain++HOME_DIR = str(Path.home())+CONFIG_DIR = os.path.join(HOME_DIR, ".embedchain")+CONFIG_FILE = os.path.join(CONFIG_DIR, "config.json")++logger = logging.getLogger(__name__)+++class AnonymousTelemetry:+ def __init__(self, host="https://app.posthog.com", enabled=True):+ self.project_api_key = "phc_XnMmNHzwxE7PVHX4mD2r8K6nfxVM48a2sq2U3N1p2lO"+ self.host = host+ self.posthog = Posthog(project_api_key=self.project_api_key, host=self.host)+ self.user_id = self.get_user_id()+ self.enabled = enabled++ # Check if telemetry tracking is disabled via environment variable+ if "EC_TELEMETRY" in os.environ and os.environ["EC_TELEMETRY"].lower() not in [+ "1",+ "true",+ "yes",+ ]:+ self.enabled = False++ if not self.enabled:+ self.posthog.disabled = True++ # Silence posthog logging+ posthog_logger = logging.getLogger("posthog")+ posthog_logger.disabled = True++ def get_user_id(self):+ if not os.path.exists(CONFIG_DIR):+ os.makedirs(CONFIG_DIR)++ if os.path.exists(CONFIG_FILE):+ with open(CONFIG_FILE, "r") as f:+ data = json.load(f)+ if "user_id" in data:+ return data["user_id"]++ user_id = str(uuid.uuid4())+ with open(CONFIG_FILE, "w") as f:+ json.dump({"user_id": user_id}, f)+ return user_id++ def capture(self, event_name, properties=None):+ default_properties = {+ "version": embedchain.__version__,+ "language": "python",+ "pid": os.getpid(),+ }+ properties.update(default_properties)++ try:+ self.posthog.capture(self.user_id, event_name, properties)+ except Exception:+ logger.exception(f"Failed to send telemetry {event_name=}")diff --git a/embedchain/vectordb/chroma.py b/embedchain/vectordb/chroma.pyindex c8d2194d6..90195459a 100644--- a/embedchain/vectordb/chroma.py+++ b/embedchain/vectordb/chroma.py@@ -38,7 +38,7 @@ class ChromaDB(BaseVectorDB):else:self.config = ChromaDbConfig()- self.settings = Settings()+ self.settings = Settings(anonymized_telemetry=False)self.settings.allow_reset = self.config.allow_reset if hasattr(self.config, "allow_reset") else Falseif self.config.chroma_settings:for key, value in self.config.chroma_settings.items():diff --git a/poetry.lock b/poetry.lockindex 8b636f024..3edd20040 100644--- a/poetry.lock+++ b/poetry.lock@@ -7196,4 +7196,4 @@ whatsapp = ["flask", "twilio"][metadata]lock-version = "2.0"python-versions = ">=3.9,<3.13"-content-hash = "4021d63d76a9128c8d7342bc482dfe90fa9878fde1359c2f0730fa9188e503af"+content-hash = "fad388baca92d2669530c2d928920e9f7abbe0ad74786b34dc915603b4a08a17"diff --git a/pyproject.toml b/pyproject.tomlindex afb25aced..215f9660f 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -1,6 +1,6 @@[tool.poetry]name = "embedchain"-version = "0.0.82"+version = "0.0.83"description = "Data platform for LLMs - Load, index, retrieve and sync any unstructured data"authors = ["Taranjeet Singh <[email protected]>",@@ -93,13 +93,14 @@ python-dotenv = "^1.0.0"langchain = "^0.0.279"requests = "^2.31.0"openai = ">=0.28.0"-tiktoken = { version="^0.4.0", optional=true }-chromadb ="^0.4.8"-youtube-transcript-api = { version="^0.6.1", optional=true }-beautifulsoup4 = { version="^4.12.2", optional=true }-pypdf = { version="^3.11.0", optional=true }-pytube = { version="^15.0.0", optional=true }-duckduckgo-search = { version="^3.8.5", optional=true }+chromadb = "^0.4.8"+posthog = "^3.0.2"+tiktoken = { version = "^0.4.0", optional = true }+youtube-transcript-api = { version = "^0.6.1", optional = true }+beautifulsoup4 = { version = "^4.12.2", optional = true }+pypdf = { version = "^3.11.0", optional = true }+pytube = { version = "^15.0.0", optional = true }+duckduckgo-search = { version = "^3.8.5", optional = true }llama-hub = { version = "^0.0.29", optional = true }sentence-transformers = { version = "^2.2.2", optional = true }torch = { version = "2.0.0", optional = true }@@ -113,12 +114,12 @@ twilio = { version = "^8.5.0", optional = true }fastapi-poe = { version = "0.0.16", optional = true }discord = { version = "^2.3.2", optional = true }slack-sdk = { version = "3.21.3", optional = true }-cohere = { version = "^4.27", optional= true }-weaviate-client = { version = "^3.24.1", optional= true }-docx2txt = { version="^0.8", optional=true }+cohere = { version = "^4.27", optional = true }+weaviate-client = { version = "^3.24.1", optional = true }+docx2txt = { version = "^0.8", optional = true }pinecone-client = { version = "^2.2.4", optional = true }qdrant-client = { version = "1.6.3", optional = true }-unstructured = {extras = ["local-inference"], version = "^0.10.18", optional=true}+unstructured = {extras = ["local-inference"], version = "^0.10.18", optional = true}pillow = { version = "10.0.1", optional = true }torchvision = { version = ">=0.15.1, !=0.15.2", optional = true }ftfy = { version = "6.1.1", optional = true }@@ -127,7 +128,7 @@ huggingface_hub = { version = "^0.17.3", optional = true }pymilvus = { version = "2.3.1", optional = true }google-cloud-aiplatform = { version = "^1.26.1", optional = true }replicate = { version = "^0.15.4", optional = true }-jq = { version=">=1.6.0", optional = true}+jq = { version = ">=1.6.0", optional = true}[tool.poetry.group.dev.dependencies]black = "^23.3.0"diff --git a/tests/conftest.py b/tests/conftest.pyindex 2465fa724..962d6af52 100644--- a/tests/conftest.py+++ b/tests/conftest.py@@ -14,3 +14,10 @@ def setup():clean_db()yieldclean_db()++[email protected](autouse=True)+def disable_telemetry():+ os.environ["EC_TELEMETRY"] = "false"+ yield+ del os.environ["EC_TELEMETRY"]diff --git a/tests/telemetry/test_posthog.py b/tests/telemetry/test_posthog.pynew file mode 100644index 000000000..c85af370c--- /dev/null+++ b/tests/telemetry/test_posthog.py@@ -0,0 +1,62 @@+import os+import logging+from embedchain.telemetry.posthog import AnonymousTelemetry+++class TestAnonymousTelemetry:+ def test_init(self, mocker):+ # Enable telemetry specifically for this test+ os.environ["EC_TELEMETRY"] = "true"+ mock_posthog = mocker.patch("embedchain.telemetry.posthog.Posthog")+ telemetry = AnonymousTelemetry()+ assert telemetry.project_api_key == "phc_XnMmNHzwxE7PVHX4mD2r8K6nfxVM48a2sq2U3N1p2lO"+ assert telemetry.host == "https://app.posthog.com"+ assert telemetry.enabled is True+ assert telemetry.user_id+ mock_posthog.assert_called_once_with(project_api_key=telemetry.project_api_key, host=telemetry.host)++ def test_init_with_disabled_telemetry(self, mocker, monkeypatch):+ mocker.patch("embedchain.telemetry.posthog.Posthog")+ telemetry = AnonymousTelemetry()+ assert telemetry.enabled is False+ assert telemetry.posthog.disabled is True++ def test_get_user_id(self, mocker, tmpdir):+ mock_uuid = mocker.patch("embedchain.telemetry.posthog.uuid.uuid4")+ mock_uuid.return_value = "unique_user_id"+ config_file = tmpdir.join("config.json")+ mocker.patch("embedchain.telemetry.posthog.CONFIG_FILE", str(config_file))+ telemetry = AnonymousTelemetry()++ user_id = telemetry.get_user_id()+ assert user_id == "unique_user_id"+ assert config_file.read() == '{"user_id": "unique_user_id"}'++ def test_capture(self, mocker):+ # Enable telemetry specifically for this test+ os.environ["EC_TELEMETRY"] = "true"+ mock_posthog = mocker.patch("embedchain.telemetry.posthog.Posthog")+ telemetry = AnonymousTelemetry()+ event_name = "test_event"+ properties = {"key": "value"}+ telemetry.capture(event_name, properties)++ mock_posthog.assert_called_once_with(+ project_api_key=telemetry.project_api_key,+ host=telemetry.host,+ )+ mock_posthog.return_value.capture.assert_called_once_with(+ telemetry.user_id,+ event_name,+ properties,+ )++ def test_capture_with_exception(self, mocker, caplog):+ mock_posthog = mocker.patch("embedchain.telemetry.posthog.Posthog")+ mock_posthog.return_value.capture.side_effect = Exception("Test Exception")+ telemetry = AnonymousTelemetry()+ event_name = "test_event"+ properties = {"key": "value"}+ with caplog.at_level(logging.ERROR):+ telemetry.capture(event_name, properties)+ assert "Failed to send telemetry event" in caplog.textdiff --git a/tests/vectordb/test_qdrant.py b/tests/vectordb/test_qdrant.pyindex 47b54504c..b872ab1d6 100644--- a/tests/vectordb/test_qdrant.py+++ b/tests/vectordb/test_qdrant.py@@ -76,7 +76,7 @@ class TestQdrantDB(unittest.TestCase):qdrant_client_mock.return_value.upsert.assert_called_once_with(collection_name="embedchain-store-1526",points=Batch(- ids=["def", "ghi"],+ ids=["abc", "def"],payloads=[{"identifier": "123",