DSPy: The framework for programming—not prompting—language models
Python
initial commit
Jasper Xian2681ed94bd81
diff --git a/dsp/modules/__init__.py b/dsp/modules/__init__.pyindex d0d9e7583..482b53492 100644--- a/dsp/modules/__init__.py+++ b/dsp/modules/__init__.py@@ -5,5 +5,6 @@ from .colbertv2 import ColBERTv2from .sentence_vectorizer import *from .cohere import *from .sbert import *+from .pyserini import *from .hf_client import HFClientTGI\ No newline at end of filediff --git a/dsp/modules/pyserini.py b/dsp/modules/pyserini.pynew file mode 100644index 000000000..0adf07337--- /dev/null+++ b/dsp/modules/pyserini.py@@ -0,0 +1,73 @@+from typing import Union+import json+from datasets import Dataset++from dsp.utils import dotdict+from pyserini.search import FaissSearcher+from pyserini.prebuilt_index_info import TF_INDEX_INFO, FAISS_INDEX_INFO, IMPACT_INDEX_INFO+++class PyseriniRetriever:+ """Wrapper for retrieval with Pyserini. Supports using either pyserini prebuilt faiss indexes or your own faiss index."""++ def __init__(self,+ query_encoder: str = 'castorini/dkrr-dpr-nq-retriever',+ index: str = 'wikipedia-dpr-dkrr-nq',+ dataset: Dataset = None,+ id_field: str = 'id',+ text_fields: list[str] = ['text']) -> None:+ """+ Args:++ query_encoder (`str`):+ Huggingface model to encode queries+ index (`str`):+ Either a prebuilt index from pyserini or a local path to a faiss index+ dataset (`Dataset`):+ Only required when using a local faiss index. The dataset should be the one that has been put into the faiss index.+ id_field (`str`):+ The name of the id field of the dataset used for retrieval.+ text_fields (`list[str]`):+ A list of the names of the text fields for the dataset used for retrieval.+ """++ self.encoder = FaissSearcher._init_encoder_from_str(query_encoder)+ self.dataset = dataset+ self.id_field = id_field+ self.text_fields = text_fields++ if index in TF_INDEX_INFO or index in FAISS_INDEX_INFO or index in IMPACT_INDEX_INFO:+ self.searcher = FaissSearcher.from_prebuilt_index(index, self.encoder)+ else:+ self.searcher = FaissSearcher(index_dir=index, query_encoder=self.encoder)++ def __call__(+ self, query: str, k: int = 10, simplify: bool = False, threads: int = 16,+ ) -> Union[list[str], list[dotdict]]:+ hits = self.searcher.search(query, k=k, threads=threads)++ topk = []+ for rank, hit in enumerate(hits, start=1):+ if self.dataset is not None:+ # Search through user's dataset for the document (could be slow), maybe room for improvement?+ row = self.dataset[self.id_field].index(hit.docid)+ text = ' '.join(self.dataset[field][row] for field in self.text_fields)+ pid = self.dataset[self.id_field][row]+ else:+ # Pyserini prebuilt faiss indexes can perform docid lookup+ psg = json.loads(self.searcher.doc(hit.docid).raw())+ text = ' '.join(psg[field] for field in self.text_fields)+ pid = psg[self.id_field]++ topk.append({+ 'text': text,+ 'long_text': text,+ 'pid': pid,+ 'score': hit.score,+ 'rank': rank,+ })++ if simplify:+ return [psg["long_text"] for psg in topk]++ return [dotdict(psg) for psg in topk]\ No newline at end of filediff --git a/dspy/__init__.py b/dspy/__init__.pyindex 3c46b8cea..84d1487f1 100644--- a/dspy/__init__.py+++ b/dspy/__init__.py@@ -16,5 +16,6 @@ settings = dsp.settingsOpenAI = dsp.GPT3ColBERTv2 = dsp.ColBERTv2+Pyserini = dsp.PyseriniRetrieverHFClientTGI = dsp.HFClientTGIChatModuleClient = ChatModuleClient\ No newline at end of filediff --git a/pyserini.ipynb b/pyserini.ipynbnew file mode 100644index 000000000..0fd8eecc8--- /dev/null+++ b/pyserini.ipynb@@ -0,0 +1,223 @@+{+ "cells": [+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "# An Introduction to Using [Pyserini](https://pyserini.io/) for DSPy\n",+ "\n",+ "Pyserini is a tool maintained by the Data Systems Group at the University of Waterloo, and you can use it to incorporate your own data into `dspy.Retrieve`. Currently, `dspy.Pyserini` supports using your own Faiss index one of pyserini's [prebuilt indexes](https://github.com/castorini/pyserini/blob/master/pyserini/prebuilt_index_info.py) to perform retrieval."+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "## 1. Installation/Setup\n",+ "Using `dspy.Pyserini` will require installing pyserini, Pytorch, and Faiss. Pyserini can be installed with `pip install pyserini`, and we'll leave it to you to decide the right versions of Pytorch and Faiss to install."+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "## 2. Using Pyserini's prebuilt indexes"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 9,+ "metadata": {},+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Attempting to initialize pre-built index beir-v1.0.0-nfcorpus.contriever-msmarco.\n",+ "/store2/scratch/j5xian/cache/pyserini/indexes/faiss.beir-v1.0.0-nfcorpus.contriever-msmarco.20230124.657649d19fafd06cb031c6b11868d7f9 already exists, skipping download.\n",+ "Initializing beir-v1.0.0-nfcorpus.contriever-msmarco...\n",+ "Top 3 passages for question: How Curry Can Kill Cancer Cells \n",+ " ------------------------------ \n",+ "\n",+ "1] Curcumin and Cancer Cells: How Many Ways Can Curry Kill Tumor Cells Selectively? Cancer is a hyperproliferative disorder that is usually treated by chemotherapeutic agents that are toxic not only to tumor cells but also to normal cells, so these agents produce major side effects. In addition, these agents are highly expensive and thus not affordable for most. Moreover, such agents cannot be used for cancer prevention. Traditional medicines are generally free of the deleterious side effects and usually inexpensive. Curcumin, a component of turmeric (Curcuma longa), is one such agent that is safe, affordable, and efficacious. How curcumin kills tumor cells is the focus of this review. We show that curcumin modulates growth of tumor cells through regulation of multiple cell signaling pathways including cell proliferation pathway (cyclin D1, c-myc), cell survival pathway (Bcl-2, Bcl-xL, cFLIP, XIAP, c-IAP1), caspase activation pathway (caspase-8, 3, 9), tumor suppressor pathway (p53, p21) death receptor pathway (DR4, DR5), mitochondrial pathways, and protein kinase pathway (JNK, Akt, and AMPK). How curcumin selectively kills tumor cells, and not normal cells, is also described in detail. \n",+ "\n",+ "2] Triphala, Ayurvedic formulation for treating and preventing cancer: a review. BACKGROUND: Triphala (Sanskrit tri = three and phala = fruits), composed of the three medicinal fruits Phyllanthus emblica L. or Emblica officinalis Gaertn., Terminalia chebula Retz., and Terminalia belerica Retz. is an important herbal preparation in the traditional Indian system of medicine, Ayurveda. Triphala is an antioxidant-rich herbal formulation and possesses diverse beneficial properties. It is a widely prescribed Ayurvedic drug and is used as a colon cleanser, digestive, diuretic, and laxative. Cancer is a major cause of death, and globally studies are being conducted to prevent cancer or to develop effective nontoxic therapeutic agents. Experimental studies in the past decade have shown that Triphala is useful in the prevention of cancer and that it also possesses antineoplastic, radioprotective and chemoprotective effects. CONCLUSIONS: This review for the first time summarizes these results, with emphasis on published observations. Furthermore, the possible mechanisms responsible for the beneficial effects and lacunas in the existing knowledge that need to be bridged are also discussed. \n",+ "\n",+ "3] Dietary turmeric potentially reduces the risk of cancer. Turmeric, a plant rhizome that is often dried, ground and used as a cooking spice, has also been used medicinally for several thousand years. Curcumin, the phytochemical that gives turmeric its golden color, is responsible for most of the therapeutic effects of turmeric. In recent years curcumin has been studied for its effects on chronic diseases such as diabetes, Alzheimer's, and cancer. Though many researchers are investigating turmeric/curcumin in cancer therapy, there is little epidemiologic information on the effects of turmeric consumption. With limited availability of pharmacologic interventions in many areas of the world, use of turmeric in the diet may help to alleviate some of the disease burden through prevention. Here we provide a brief overview of turmeric consumption in different parts of the world, cancer rates in those regions, possible biochemical mechanisms by which turmeric acts and practical recommendations based on the information available. \n",+ "\n"+ ]+ }+ ],+ "source": [+ "import dspy\n",+ "pys_ret_prebuilt = dspy.Pyserini(index='beir-v1.0.0-nfcorpus.contriever-msmarco', query_encoder='facebook/contriever-msmarco', id_field='_id', text_fields=['title', 'text'])\n",+ "\n",+ "dspy.settings.configure(rm=pys_ret_prebuilt)\n",+ "\n",+ "example_question = \"How Curry Can Kill Cancer Cells\"\n",+ "\n",+ "retrieve = dspy.Retrieve(k=3)\n",+ "topK_passages = retrieve(example_question).passages\n",+ "\n",+ "print(f\"Top {retrieve.k} passages for question: {example_question} \\n\", '-' * 30, '\\n')\n",+ "\n",+ "for idx, passage in enumerate(topK_passages):\n",+ " print(f'{idx+1}]', passage, '\\n')\n"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "## 3. Using your own data\n",+ "As an example, we'll be using [NFCorpus](https://www.cl.uni-heidelberg.de/statnlpgroup/nfcorpus/), a full-text learning to rank dataset for medical information retrieval. This corpus is quite small so encoding, indexing, and retrieval should be tolerable on CPU. \n",+ "\n",+ "First, let's fetch the data:"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 4,+ "metadata": {},+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",+ "To disable this warning, you can either:\n",+ "\t- Avoid using `tokenizers` before the fork if possible\n",+ "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",+ "--2023-09-15 20:57:57-- https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nfcorpus.zip\n",+ "Resolving public.ukp.informatik.tu-darmstadt.de (public.ukp.informatik.tu-darmstadt.de)... 130.83.167.186\n",+ "Connecting to public.ukp.informatik.tu-darmstadt.de (public.ukp.informatik.tu-darmstadt.de)|130.83.167.186|:443... connected.\n",+ "HTTP request sent, awaiting response... 200 OK\n",+ "Length: 2448432 (2.3M) [application/zip]\n",+ "Saving to: ‘collections/nfcorpus.zip’\n",+ "\n",+ "nfcorpus.zip 100%[===================>] 2.33M 3.33MB/s in 0.7s \n",+ "\n",+ "2023-09-15 20:57:59 (3.33 MB/s) - ‘collections/nfcorpus.zip’ saved [2448432/2448432]\n",+ "\n",+ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",+ "To disable this warning, you can either:\n",+ "\t- Avoid using `tokenizers` before the fork if possible\n",+ "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",+ "/bin/bash: unzip: command not found\n"+ ]+ }+ ],+ "source": [+ "!wget https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nfcorpus.zip -P collections\n",+ "!unzip collections/nfcorpus.zip -d collections"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "Next, we can use pyserini to encode and pack up our data into a Faiss index:"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 5,+ "metadata": {},+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",+ "To disable this warning, you can either:\n",+ "\t- Avoid using `tokenizers` before the fork if possible\n",+ "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",+ "3633it [00:00, 101455.50it/s]\n",+ "100%|███████████████████████████████████████████| 57/57 [08:56<00:00, 9.41s/it]\n"+ ]+ }+ ],+ "source": [+ "!python -m pyserini.encode \\\n",+ " input --corpus collections/nfcorpus/corpus.jsonl \\\n",+ " --fields title text \\\n",+ " output --embeddings indexes/faiss.nfcorpus.contriever-msmacro \\\n",+ " --to-faiss \\\n",+ " encoder --encoder facebook/contriever-msmarco \\\n",+ " --device cpu \\\n",+ " --pooling mean \\\n",+ " --fields title text"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "Now, we can use `dspy.Pyserini` to read our local Faiss index and perform retrieval. Note that using a local index requires passing in a Huggingface `Dataset` for document lookup."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 8,+ "metadata": {},+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Top 3 passages for question: How Curry Can Kill Cancer Cells \n",+ " ------------------------------ \n",+ "\n",+ "1] Curcumin and Cancer Cells: How Many Ways Can Curry Kill Tumor Cells Selectively? Cancer is a hyperproliferative disorder that is usually treated by chemotherapeutic agents that are toxic not only to tumor cells but also to normal cells, so these agents produce major side effects. In addition, these agents are highly expensive and thus not affordable for most. Moreover, such agents cannot be used for cancer prevention. Traditional medicines are generally free of the deleterious side effects and usually inexpensive. Curcumin, a component of turmeric (Curcuma longa), is one such agent that is safe, affordable, and efficacious. How curcumin kills tumor cells is the focus of this review. We show that curcumin modulates growth of tumor cells through regulation of multiple cell signaling pathways including cell proliferation pathway (cyclin D1, c-myc), cell survival pathway (Bcl-2, Bcl-xL, cFLIP, XIAP, c-IAP1), caspase activation pathway (caspase-8, 3, 9), tumor suppressor pathway (p53, p21) death receptor pathway (DR4, DR5), mitochondrial pathways, and protein kinase pathway (JNK, Akt, and AMPK). How curcumin selectively kills tumor cells, and not normal cells, is also described in detail. \n",+ "\n",+ "2] Curcumin and cancer: an \"old-age\" disease with an \"age-old\" solution. Cancer is primarily a disease of old age, and that life style plays a major role in the development of most cancers is now well recognized. While plant-based formulations have been used to treat cancer for centuries, current treatments usually involve poisonous mustard gas, chemotherapy, radiation, and targeted therapies. While traditional plant-derived medicines are safe, what are the active principles in them and how do they mediate their effects against cancer is perhaps best illustrated by curcumin, a derivative of turmeric used for centuries to treat a wide variety of inflammatory conditions. Curcumin is a diferuloylmethane derived from the Indian spice, turmeric (popularly called \"curry powder\") that has been shown to interfere with multiple cell signaling pathways, including cell cycle (cyclin D1 and cyclin E), apoptosis (activation of caspases and down-regulation of antiapoptotic gene products), proliferation (HER-2, EGFR, and AP-1), survival (PI3K/AKT pathway), invasion (MMP-9 and adhesion molecules), angiogenesis (VEGF), metastasis (CXCR-4) and inflammation (NF-kappaB, TNF, IL-6, IL-1, COX-2, and 5-LOX). The activity of curcumin reported against leukemia and lymphoma, gastrointestinal cancers, genitourinary cancers, breast cancer, ovarian cancer, head and neck squamous cell carcinoma, lung cancer, melanoma, neurological cancers, and sarcoma reflects its ability to affect multiple targets. Thus an \"old-age\" disease such as cancer requires an \"age-old\" treatment. \n",+ "\n",+ "3] Triphala, Ayurvedic formulation for treating and preventing cancer: a review. BACKGROUND: Triphala (Sanskrit tri = three and phala = fruits), composed of the three medicinal fruits Phyllanthus emblica L. or Emblica officinalis Gaertn., Terminalia chebula Retz., and Terminalia belerica Retz. is an important herbal preparation in the traditional Indian system of medicine, Ayurveda. Triphala is an antioxidant-rich herbal formulation and possesses diverse beneficial properties. It is a widely prescribed Ayurvedic drug and is used as a colon cleanser, digestive, diuretic, and laxative. Cancer is a major cause of death, and globally studies are being conducted to prevent cancer or to develop effective nontoxic therapeutic agents. Experimental studies in the past decade have shown that Triphala is useful in the prevention of cancer and that it also possesses antineoplastic, radioprotective and chemoprotective effects. CONCLUSIONS: This review for the first time summarizes these results, with emphasis on published observations. Furthermore, the possible mechanisms responsible for the beneficial effects and lacunas in the existing knowledge that need to be bridged are also discussed. \n",+ "\n"+ ]+ }+ ],+ "source": [+ "from datasets import load_dataset\n",+ "\n",+ "dataset = load_dataset(path='json', data_files='collections/nfcorpus/corpus.jsonl', split='train')\n",+ "\n",+ "pys_ret_local = dspy.Pyserini(index='indexes/faiss.nfcorpus.contriever-msmacro', query_encoder='facebook/contriever-msmarco', dataset=dataset, id_field='_id', text_fields=['title', 'text'])\n",+ "\n",+ "dspy.settings.configure(rm=pys_ret_local)\n",+ "\n",+ "dev_example = \"How Curry Can Kill Cancer Cells\"\n",+ "\n",+ "retrieve = dspy.Retrieve(k=3)\n",+ "topK_passages = retrieve(dev_example).passages\n",+ "\n",+ "print(f\"Top {retrieve.k} passages for question: {dev_example} \\n\", '-' * 30, '\\n')\n",+ "\n",+ "for idx, passage in enumerate(topK_passages):\n",+ " print(f'{idx+1}]', passage, '\\n')\n"+ ]+ }+ ],+ "metadata": {+ "kernelspec": {+ "display_name": "dspy-pys",+ "language": "python",+ "name": "python3"+ },+ "language_info": {+ "codemirror_mode": {+ "name": "ipython",+ "version": 3+ },+ "file_extension": ".py",+ "mimetype": "text/x-python",+ "name": "python",+ "nbconvert_exporter": "python",+ "pygments_lexer": "ipython3",+ "version": "3.10.4"+ },+ "orig_nbformat": 4+ },+ "nbformat": 4,+ "nbformat_minor": 2+}