Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

use faiss-gpu instead

Jasper Xianc5958fb05642
diff --git a/pyserini.ipynb b/pyserini.ipynb
index 2a460d610..e584c129d 100644
--- a/pyserini.ipynb
+++ b/pyserini.ipynb
@@ -19,15 +19,15 @@
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": 1,
"metadata": {},
"outputs": [
{
- "name": "stdout",
+ "name": "stderr",
"output_type": "stream",
"text": [
- "The autoreload extension is already loaded. To reload it, use:\n",
- " %reload_ext autoreload\n"
+ "/tmp/ipykernel_15686/3452029479.py:5: DeprecationWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html\n",
+ " import pkg_resources\n"
]
}
],
@@ -48,8 +48,8 @@
" !pip install pyserini\n",
" if not \"torch\" in {pkg.key for pkg in pkg_resources.working_set}:\n",
" !pip install torch\n",
- " if not \"faiss-cpu\" in {pkg.key for pkg in pkg_resources.working_set}:\n",
- " !pip install faiss-cpu\n",
+ " if not \"faiss-gpu\" in {pkg.key for pkg in pkg_resources.working_set}:\n",
+ " !pip install faiss-gpu\n",
"except:\n",
" repo_path = '.'\n",
" # Install the package if it's not installed\n",
@@ -73,7 +73,7 @@
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": 2,
"metadata": {},
"outputs": [
{
@@ -167,7 +167,7 @@
},
{
"cell_type": "code",
- "execution_count": 10,
+ "execution_count": 5,
"metadata": {},
"outputs": [
{
@@ -178,8 +178,8 @@
"To disable this warning, you can either:\n",
"\t- Avoid using `tokenizers` before the fork if possible\n",
"\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
- "3633it [00:00, 106850.95it/s]\n",
- "100%|███████████████████████████████████████████| 57/57 [08:52<00:00, 9.35s/it]\n"
+ "3633it [00:00, 78339.56it/s]\n",
+ "100%|███████████████████████████████████████████| 57/57 [00:17<00:00, 3.33it/s]\n"
]
}
],
@@ -187,10 +187,10 @@
"!python -m pyserini.encode \\\n",
" input --corpus collections/nfcorpus/corpus.jsonl \\\n",
" --fields title text \\\n",
- " output --embeddings indexes/faiss.nfcorpus.contriever-msmacro \\\n",
+ " output --embeddings indexes/faiss.nfcorpus.contriever-msmarco \\\n",
" --to-faiss \\\n",
" encoder --encoder facebook/contriever-msmarco \\\n",
- " --device cpu \\\n",
+ " --device cuda:0 \\\n",
" --pooling mean \\\n",
" --fields title text"
]
@@ -204,51 +204,9 @@
},
{
"cell_type": "code",
- "execution_count": 11,
+ "execution_count": 6,
"metadata": {},
"outputs": [
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "ad31b8b0072648d1913010f5c3373164",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Downloading data files: 0%| | 0/1 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "3424554a6fb54fd381c21ddf8d76d83d",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Extracting data files: 0%| | 0/1 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "0beb45f6707943bfb0182786fd32fc97",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Generating train split: 0 examples [00:00, ? examples/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
{
"name": "stdout",
"output_type": "stream",
@@ -270,7 +228,7 @@
"\n",
"dataset = load_dataset(path='json', data_files='collections/nfcorpus/corpus.jsonl', split='train')\n",
"\n",
- "pys_ret_local = dspy.Pyserini(index='indexes/faiss.nfcorpus.contriever-msmacro', query_encoder='facebook/contriever-msmarco', dataset=dataset, id_field='_id', text_fields=['title', 'text'])\n",
+ "pys_ret_local = dspy.Pyserini(index='indexes/faiss.nfcorpus.contriever-msmarco', query_encoder='facebook/contriever-msmarco', dataset=dataset, id_field='_id', text_fields=['title', 'text'])\n",
"\n",
"dspy.settings.configure(rm=pys_ret_local)\n",
"\n",