DSPy: The framework for programming—not prompting—language models
Python
use faiss-gpu instead
Jasper Xianc5958fb05642
diff --git a/pyserini.ipynb b/pyserini.ipynbindex 2a460d610..e584c129d 100644--- a/pyserini.ipynb+++ b/pyserini.ipynb@@ -19,15 +19,15 @@},{"cell_type": "code",- "execution_count": 5,+ "execution_count": 1,"metadata": {},"outputs": [{- "name": "stdout",+ "name": "stderr","output_type": "stream","text": [- "The autoreload extension is already loaded. To reload it, use:\n",- " %reload_ext autoreload\n"+ "/tmp/ipykernel_15686/3452029479.py:5: DeprecationWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html\n",+ " import pkg_resources\n"]}],@@ -48,8 +48,8 @@" !pip install pyserini\n"," if not \"torch\" in {pkg.key for pkg in pkg_resources.working_set}:\n"," !pip install torch\n",- " if not \"faiss-cpu\" in {pkg.key for pkg in pkg_resources.working_set}:\n",- " !pip install faiss-cpu\n",+ " if not \"faiss-gpu\" in {pkg.key for pkg in pkg_resources.working_set}:\n",+ " !pip install faiss-gpu\n","except:\n"," repo_path = '.'\n"," # Install the package if it's not installed\n",@@ -73,7 +73,7 @@},{"cell_type": "code",- "execution_count": 6,+ "execution_count": 2,"metadata": {},"outputs": [{@@ -167,7 +167,7 @@},{"cell_type": "code",- "execution_count": 10,+ "execution_count": 5,"metadata": {},"outputs": [{@@ -178,8 +178,8 @@"To disable this warning, you can either:\n","\t- Avoid using `tokenizers` before the fork if possible\n","\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",- "3633it [00:00, 106850.95it/s]\n",- "100%|███████████████████████████████████████████| 57/57 [08:52<00:00, 9.35s/it]\n"+ "3633it [00:00, 78339.56it/s]\n",+ "100%|███████████████████████████████████████████| 57/57 [00:17<00:00, 3.33it/s]\n"]}],@@ -187,10 +187,10 @@"!python -m pyserini.encode \\\n"," input --corpus collections/nfcorpus/corpus.jsonl \\\n"," --fields title text \\\n",- " output --embeddings indexes/faiss.nfcorpus.contriever-msmacro \\\n",+ " output --embeddings indexes/faiss.nfcorpus.contriever-msmarco \\\n"," --to-faiss \\\n"," encoder --encoder facebook/contriever-msmarco \\\n",- " --device cpu \\\n",+ " --device cuda:0 \\\n"," --pooling mean \\\n"," --fields title text"]@@ -204,51 +204,9 @@},{"cell_type": "code",- "execution_count": 11,+ "execution_count": 6,"metadata": {},"outputs": [- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "ad31b8b0072648d1913010f5c3373164",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Downloading data files: 0%| | 0/1 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "3424554a6fb54fd381c21ddf8d76d83d",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Extracting data files: 0%| | 0/1 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "0beb45f6707943bfb0182786fd32fc97",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Generating train split: 0 examples [00:00, ? examples/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },{"name": "stdout","output_type": "stream",@@ -270,7 +228,7 @@"\n","dataset = load_dataset(path='json', data_files='collections/nfcorpus/corpus.jsonl', split='train')\n","\n",- "pys_ret_local = dspy.Pyserini(index='indexes/faiss.nfcorpus.contriever-msmacro', query_encoder='facebook/contriever-msmarco', dataset=dataset, id_field='_id', text_fields=['title', 'text'])\n",+ "pys_ret_local = dspy.Pyserini(index='indexes/faiss.nfcorpus.contriever-msmarco', query_encoder='facebook/contriever-msmarco', dataset=dataset, id_field='_id', text_fields=['title', 'text'])\n","\n","dspy.settings.configure(rm=pys_ret_local)\n","\n",