💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Update NLTK model downloads, closes #760
davidmezzettia014c3bcda13
diff --git a/.github/workflows/build.yml b/.github/workflows/build.ymlindex cf9def6c..0542b413 100644--- a/.github/workflows/build.yml+++ b/.github/workflows/build.yml@@ -54,7 +54,7 @@ jobs:run: |pip install -U wheelpip install .[all,dev] fasttext==0.9.2 faiss-cpu==1.8.0 "numpy < 2.0.0"- python -c "import nltk; nltk.download('punkt')"+ python -c "import nltk; nltk.download(['punkt', 'punkt_tab', 'averaged_perceptron_tagger_eng'])"python --versionmake data coveragediff --git a/docker/base/Dockerfile b/docker/base/Dockerfileindex 42eaa762..3cad841d 100644--- a/docker/base/Dockerfile+++ b/docker/base/Dockerfile@@ -29,7 +29,7 @@ RUN \python -m pip install --no-cache-dir -U pip wheel setuptools && \if [ -z ${GPU} ] && { [ -z ${TARGETARCH} ] || [ ${TARGETARCH} = "amd64" ] ;}; then pip install --no-cache-dir torch==2.3.1+cpu torchvision==0.18.1+cpu -f https://download.pytorch.org/whl/torch_stable.html; fi && \python -m pip install --no-cache-dir txtai${COMPONENTS} && \- python -c "import sys, importlib.util as util; 1 if util.find_spec('nltk') else sys.exit(); import nltk; nltk.download('punkt')" && \+ python -c "import sys, importlib.util as util; 1 if util.find_spec('nltk') else sys.exit(); import nltk; nltk.download(['punkt', 'punkt_tab', 'averaged_perceptron_tagger_eng'])" && \\# Cleanup build packagesapt-get -y purge gcc g++ python${PYTHON_VERSION}-dev && apt-get -y autoremovediff --git a/examples/10_Extract_text_from_documents.ipynb b/examples/10_Extract_text_from_documents.ipynbindex e6463a04..b44f8a7d 100644--- a/examples/10_Extract_text_from_documents.ipynb+++ b/examples/10_Extract_text_from_documents.ipynb@@ -50,7 +50,7 @@"\n","# Install NLTK\n","import nltk\n",- "nltk.download('punkt')"+ "nltk.download(['punkt', 'punkt_tab'])"],"execution_count": 19,"outputs": []diff --git a/examples/40_Text_to_Speech_Generation.ipynb b/examples/40_Text_to_Speech_Generation.ipynbindex 4f884b4f..215d8cab 100644--- a/examples/40_Text_to_Speech_Generation.ipynb+++ b/examples/40_Text_to_Speech_Generation.ipynb@@ -52,7 +52,11 @@},"source": ["%%capture\n",- "!pip install git+https://github.com/neuml/txtai#egg=txtai[pipeline] onnxruntime-gpu librosa"+ "!pip install git+https://github.com/neuml/txtai#egg=txtai[pipeline] onnxruntime-gpu librosa\n",+ "\n",+ "# Install NLTK\n",+ "import nltk\n",+ "nltk.download('averaged_perceptron_tagger_eng')"],"execution_count": 1,"outputs": []diff --git a/examples/52_Build_RAG_pipelines_with_txtai.ipynb b/examples/52_Build_RAG_pipelines_with_txtai.ipynbindex 77eb90a0..d1cc9c94 100644--- a/examples/52_Build_RAG_pipelines_with_txtai.ipynb+++ b/examples/52_Build_RAG_pipelines_with_txtai.ipynb@@ -63,7 +63,7 @@"\n","# Install NLTK\n","import nltk\n",- "nltk.download('punkt')"+ "nltk.download(['punkt', 'punkt_tab'])"]},{