Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Remove offline modules

Isaac Millerdd0695992a53
diff --git a/dsp/__init__.py b/dsp/__init__.py
index 3018ac82a..f71e1bc2c 100644
--- a/dsp/__init__.py
+++ b/dsp/__init__.py
@@ -2,7 +2,7 @@ from .modules import * # noqa
from .primitives import * # noqa
from .adapters import * # noqa
from .utils import settings # noqa
-from .deploy_dspy import *
+# from .deploy_dspy import *
"""
TODO:
diff --git a/dsp/modules/__init__.py b/dsp/modules/__init__.py
index 852aba423..2105dc3ff 100644
--- a/dsp/modules/__init__.py
+++ b/dsp/modules/__init__.py
@@ -29,8 +29,8 @@ from .sentence_vectorizer import *
from .snowflake import *
from .tensorrt_llm import TensorRTModel
from .trainable_anyscale import TrainableAnyscale
-from .vllm_engine import VLLMOfflineEngine
-from .vllm_engine_2 import VLLMOfflineEngine2
-from .vllm_engine_3 import VLLMOfflineEngine3
+# from .vllm_engine import VLLMOfflineEngine
+# from .vllm_engine_2 import VLLMOfflineEngine2
+# from .vllm_engine_3 import VLLMOfflineEngine3
from .watsonx import *
from .you import You
diff --git a/dspy/__init__.py b/dspy/__init__.py
index e78f7e7ef..b5d8fd5c8 100644
--- a/dspy/__init__.py
+++ b/dspy/__init__.py
@@ -54,9 +54,9 @@ AWSMeta = dsp.AWSMeta
Watsonx = dsp.Watsonx
PremAI = dsp.PremAI
-VLLMOfflineEngine = dsp.VLLMOfflineEngine
-VLLMOfflineEngine2 = dsp.VLLMOfflineEngine2
-VLLMOfflineEngine3 = dsp.VLLMOfflineEngine3
+# VLLMOfflineEngine = dsp.VLLMOfflineEngine
+# VLLMOfflineEngine2 = dsp.VLLMOfflineEngine2
+# VLLMOfflineEngine3 = dsp.VLLMOfflineEngine3
You = dsp.You
configure = settings.configure
diff --git a/examples/anyscale/e2e/70b_genconfig.png b/examples/anyscale/e2e/70b_genconfig.png
new file mode 100644
index 000000000..777f4bb99
Binary files /dev/null and b/examples/anyscale/e2e/70b_genconfig.png differ
diff --git a/examples/anyscale/e2e/8b_genconfig.png b/examples/anyscale/e2e/8b_genconfig.png
new file mode 100644
index 000000000..7283feb26
Binary files /dev/null and b/examples/anyscale/e2e/8b_genconfig.png differ
diff --git a/examples/anyscale/e2e/basicmh_70b_31_bfrs_3_3_6.json b/examples/anyscale/e2e/basicmh_70b_31_bfrs_3_3_6.json
new file mode 100644
index 000000000..d6d6437c6
--- /dev/null
+++ b/examples/anyscale/e2e/basicmh_70b_31_bfrs_3_3_6.json
@@ -0,0 +1,92 @@
+{
+ "retrieve": {
+ "k": 3
+ },
+ "generate_query[0]": {
+ "lm": null,
+ "traces": [],
+ "train": [],
+ "demos": [
+ {
+ "question": "Is it true that both Cycads and Carpentarias are both plants that grow in different regions?",
+ "answer": "yes",
+ "dspy_uuid": "ef7b6cb9-6258-4796-9019-4c50be052cb3",
+ "dspy_split": "train"
+ },
+ {
+ "question": "when was the English actress, who starred in Frasier born ",
+ "answer": "18 April 1961",
+ "dspy_uuid": "941573de-63ce-411b-8817-7fe52e2cd5c0",
+ "dspy_split": "train"
+ },
+ {
+ "question": "Which airport is closer to a city, Monterey Regional Airport or Gulfport\u2013Biloxi International Airport?",
+ "answer": "Monterey Regional Airport",
+ "dspy_uuid": "076806c3-4542-4f68-9ad8-3a6f117febc9",
+ "dspy_split": "train"
+ }
+ ],
+ "signature_instructions": "Given the fields `context`, `question`, produce the fields `search_query`.",
+ "signature_prefix": "Search Query:",
+ "extended_signature_instructions": "Given the fields `context`, `question`, produce the fields `search_query`.",
+ "extended_signature_prefix": "Search Query:"
+ },
+ "generate_query[1]": {
+ "lm": null,
+ "traces": [],
+ "train": [],
+ "demos": [
+ {
+ "question": "The sports commentator who performed play-by-plays for the Showtime broadcaster for a fight known as \"The Money Fight\" is currently signed as an announcer with which MMA organization?",
+ "answer": "Bellator",
+ "dspy_uuid": "a8bb5098-daf7-4c92-aafc-bf7fd180bd38",
+ "dspy_split": "train"
+ },
+ {
+ "question": "Which is longer, the Black River Canal or Gowanus Canal?",
+ "answer": "Black River Canal",
+ "dspy_uuid": "09add453-3a74-41f0-9fef-72aa456e2c88",
+ "dspy_split": "train"
+ },
+ {
+ "question": "What do both Aealo and Rotting Christ have in common?",
+ "answer": "Greek extreme metal band",
+ "dspy_uuid": "cb830396-8fcb-4e13-88aa-688caa3f1daa",
+ "dspy_split": "train"
+ }
+ ],
+ "signature_instructions": "Given the fields `context`, `question`, produce the fields `search_query`.",
+ "signature_prefix": "Search Query:",
+ "extended_signature_instructions": "Given the fields `context`, `question`, produce the fields `search_query`.",
+ "extended_signature_prefix": "Search Query:"
+ },
+ "generate_answer": {
+ "lm": null,
+ "traces": [],
+ "train": [],
+ "demos": [
+ {
+ "question": "Who wrote the opera that was described as A New Grand Opera Bouffe Feerie, in Four Acts and Nine Tableaux, and was English Folklore?",
+ "answer": "Jacques Offenbach",
+ "dspy_uuid": "2fcb0414-b1dc-4314-baf6-26db9e3ee8cb",
+ "dspy_split": "train"
+ },
+ {
+ "question": "when was a narrative poem most known from an American poet born July 18, 1850 written?",
+ "answer": "1867",
+ "dspy_uuid": "a8c9fc63-1cdb-4f03-8914-f738856628a0",
+ "dspy_split": "train"
+ },
+ {
+ "question": "What is the name of the hedge fund whose co-CEO has a daughter named Rebekah?",
+ "answer": "Renaissance Technologies",
+ "dspy_uuid": "27661e2e-98b2-4037-b596-7a0fb99b89b3",
+ "dspy_split": "train"
+ }
+ ],
+ "signature_instructions": "Given the fields `context`, `question`, produce the fields `answer`.",
+ "signature_prefix": "Answer:",
+ "extended_signature_instructions": "Given the fields `context`, `question`, produce the fields `answer`.",
+ "extended_signature_prefix": "Answer:"
+ }
+}
\ No newline at end of file
diff --git a/examples/anyscale/e2e/e2e_dspy.ipynb b/examples/anyscale/e2e/e2e_dspy.ipynb
index e59704c44..4044f55fa 100644
--- a/examples/anyscale/e2e/e2e_dspy.ipynb
+++ b/examples/anyscale/e2e/e2e_dspy.ipynb
@@ -53,9 +53,18 @@
},
{
"cell_type": "code",
- "execution_count": 1,
+ "execution_count": 7,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "The autoreload extension is already loaded. To reload it, use:\n",
+ " %reload_ext autoreload\n"
+ ]
+ }
+ ],
"source": [
"%load_ext autoreload\n",
"%autoreload 2"
@@ -63,7 +72,7 @@
},
{
"cell_type": "code",
- "execution_count": 2,
+ "execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
@@ -83,24 +92,16 @@
},
{
"cell_type": "code",
- "execution_count": 3,
+ "execution_count": 9,
"metadata": {},
"outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "WARNING 09-24 09:57:06 cuda.py:22] You are using a deprecated `pynvml` package. Please install `nvidia-ml-py` instead, and make sure to uninstall `pynvml`. When both of them are installed, `pynvml` will take precedence and cause errors. See https://pypi.org/project/pynvml for more information.\n",
- "unable to import apply_chat_template\n"
- ]
- },
{
"data": {
"text/plain": [
"True"
]
},
- "execution_count": 3,
+ "execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
@@ -124,7 +125,7 @@
},
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
@@ -140,33 +141,9 @@
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": 11,
"metadata": {},
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "2024-09-24 09:57:10,536\tINFO worker.py:1603 -- Connecting to existing Ray cluster at address: 10.0.15.220:6379...\n",
- "2024-09-24 09:57:10,544\tINFO worker.py:1779 -- Connected to Ray cluster. View the dashboard at https://session-fkvdirx4bzefi53sjl55m7asad.i.anyscaleuserdata.com \n",
- "2024-09-24 09:57:10,566\tINFO packaging.py:530 -- Creating a file package for local directory '/home/ray/default/dspy/dspy'.\n",
- "2024-09-24 09:57:10,587\tINFO packaging.py:358 -- Pushing file package 'gcs://_ray_pkg_8ace7ac78ed16883.zip' (0.78MiB) to Ray cluster...\n",
- "2024-09-24 09:57:10,591\tINFO packaging.py:371 -- Successfully pushed file package 'gcs://_ray_pkg_8ace7ac78ed16883.zip'.\n",
- "2024-09-24 09:57:10,603\tINFO packaging.py:530 -- Creating a file package for local directory '/home/ray/default/dspy/dsp'.\n",
- "2024-09-24 09:57:10,622\tINFO packaging.py:358 -- Pushing file package 'gcs://_ray_pkg_956d9051ac54fdb1.zip' (0.86MiB) to Ray cluster...\n",
- "2024-09-24 09:57:10,625\tINFO packaging.py:371 -- Successfully pushed file package 'gcs://_ray_pkg_956d9051ac54fdb1.zip'.\n",
- "2024-09-24 09:57:10,628\tINFO packaging.py:358 -- Pushing file package 'gcs://_ray_pkg_4b07a158880c2a5c4ab67b90b0f259e5bcb2b7e2.zip' (1.37MiB) to Ray cluster...\n",
- "2024-09-24 09:57:10,633\tINFO packaging.py:371 -- Successfully pushed file package 'gcs://_ray_pkg_4b07a158880c2a5c4ab67b90b0f259e5bcb2b7e2.zip'.\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "(autoscaler +1m41s) Tip: use `ray status` to view detailed cluster status. To disable these messages, set RAY_SCHEDULER_EVENTS=0.\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"import ray\n",
"\n",
@@ -183,7 +160,7 @@
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
@@ -201,7 +178,7 @@
},
{
"cell_type": "code",
- "execution_count": 7,
+ "execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
@@ -304,7 +281,7 @@
{
"data": {
"application/vnd.jupyter.widget-view+json": {
- "model_id": "b67a4521d2c247e98ff2a50ca10a51f1",
+ "model_id": "798f0e32df5c429babfb30aad63bf4f8",
"version_major": 2,
"version_minor": 0
},
@@ -318,7 +295,7 @@
{
"data": {
"application/vnd.jupyter.widget-view+json": {
- "model_id": "ed39552f92b6494bb11d6ade994cd8a2",
+ "model_id": "8f152428c0bb414ba99dbd266509e99f",
"version_major": 2,
"version_minor": 0
},
@@ -332,21 +309,7 @@
{
"data": {
"application/vnd.jupyter.widget-view+json": {
- "model_id": "e474b7115b0d45b49d09c8f0465505f4",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Downloading data files: 0%| | 0/3 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "543cc7fc8e01423e966be5fcc107d0bb",
+ "model_id": "a1919a7c0acf45a89947dc51238838fd",
"version_major": 2,
"version_minor": 0
},
@@ -356,76 +319,6 @@
},
"metadata": {},
"output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "c79d8e5a246e40dfbea4294066ddd520",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Downloading data: 0%| | 0.00/47.5M [00:00<?, ?B/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "5f70ef69c1b14c3997b3a87ab38bc420",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Downloading data: 0%| | 0.00/46.2M [00:00<?, ?B/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "70edaffbb578493fbc4288479ba0aaf2",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Generating train split: 0%| | 0/90447 [00:00<?, ? examples/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "67e6b713acf94a06a46ef3c59da93448",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Generating validation split: 0%| | 0/7405 [00:00<?, ? examples/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "0587220093ac4b28989f265fe22f8b06",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Generating test split: 0%| | 0/7405 [00:00<?, ? examples/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
}
],
"source": [
@@ -508,6 +401,15 @@
"vanilla_program = BasicMH()"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Note: Run above this to do all setup without launching any models"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -551,9 +453,17 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 12,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "in multiopenai 0\n"
+ ]
+ }
+ ],
"source": [
"# TODO: switch to local model\n",
"llama_8b = dspy.MultiOpenAI(model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\", **MODEL_PARAMETERS, **LOCAL_API_PARAMETERS)"
@@ -561,9 +471,17 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 14,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Paris\n"
+ ]
+ }
+ ],
"source": [
"# Quick sanity check to see if the program is working\n",
"with dspy.context(lm=llama_8b, rm=retriever):\n",
@@ -573,9 +491,24 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 15,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Evaluating the vanilla program on the devset using the model to be trained (llama 8B)...\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 686 / 1500 (45.7): 100%|██████████| 1500/1500 [14:11<00:00, 1.76it/s]\n"
+ ]
+ }
+ ],
"source": [
"with dspy.context(lm=llama_8b, rm=retriever):\n",
" print(\"Evaluating the vanilla program on the devset using the model to be trained (llama 8B)...\")\n",
@@ -623,6 +556,13 @@
"This configuration allows us to run the 70B model efficiently across our available GPU resources."
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Note that I needed to add the HF_HOME var to my serve config"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -636,30 +576,61 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 12,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "in multiopenai 0\n"
+ ]
+ }
+ ],
"source": [
"llama_70b = dspy.MultiOpenAI(model=\"meta-llama/Meta-Llama-3.1-70B-Instruct\", **MODEL_PARAMETERS, **LOCAL_API_PARAMETERS)"
]
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 13,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Moscow\n"
+ ]
+ }
+ ],
"source": [
"# Another sanity check\n",
"with dspy.context(lm=llama_70b, rm=retriever):\n",
" test_predictor = BasicMH()\n",
- " print(test_predictor(question=\"What is the capital of France?\").answer)"
+ " print(test_predictor(question=\"What is the capital of Russia?\").answer)"
]
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 14,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Evaluating the vanilla program on the devset using llama 70B...\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 770 / 1500 (51.3): 100%|██████████| 1500/1500 [09:08<00:00, 2.73it/s]\n"
+ ]
+ }
+ ],
"source": [
"with dspy.context(lm=llama_70b, rm=retriever):\n",
" print(\"Evaluating the vanilla program on the devset using llama 70B...\")\n",
@@ -703,7 +674,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 16,
"metadata": {},
"outputs": [],
"source": [
@@ -720,21 +691,9 @@
},
{
"cell_type": "code",
- "execution_count": 12,
+ "execution_count": 17,
"metadata": {},
- "outputs": [
- {
- "ename": "NameError",
- "evalue": "name 'OPTIMIZER_NUM_TRAIN' is not defined",
- "output_type": "error",
- "traceback": [
- "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
- "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)",
- "Cell \u001b[0;32mIn[12], line 5\u001b[0m\n\u001b[1;32m 3\u001b[0m shuffled_trainset \u001b[38;5;241m=\u001b[39m [d \u001b[38;5;28;01mfor\u001b[39;00m d \u001b[38;5;129;01min\u001b[39;00m trainset]\n\u001b[1;32m 4\u001b[0m rng\u001b[38;5;241m.\u001b[39mshuffle(shuffled_trainset)\n\u001b[0;32m----> 5\u001b[0m optimizer_trainset \u001b[38;5;241m=\u001b[39m shuffled_trainset[:\u001b[43mOPTIMIZER_NUM_TRAIN\u001b[49m]\n\u001b[1;32m 6\u001b[0m optimizer_valset \u001b[38;5;241m=\u001b[39m shuffled_trainset[OPTIMIZER_NUM_TRAIN:OPTIMIZER_NUM_TRAIN\u001b[38;5;241m+\u001b[39mOPTIMIZER_NUM_VAL]\n",
- "\u001b[0;31mNameError\u001b[0m: name 'OPTIMIZER_NUM_TRAIN' is not defined"
- ]
- }
- ],
+ "outputs": [],
"source": [
"# Prepare the training and validation sets for the optimizer using the original\n",
"# trainset. This ensures that our actual devset is left untouched.\n",
@@ -746,9 +705,59 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 18,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Going to sample between 1 and 3 traces per predictor.\n",
+ "Will attempt to bootstrap 6 candidate sets.\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 79 / 150 (52.7): 100%|██████████| 150/150 [04:36<00:00, 1.84s/it]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Score: 52.67 for set: [0, 0, 0]\n",
+ "New best sscore: 52.67 for seed -3\n",
+ "Scores so far: [52.67]\n",
+ "Best score: 52.67\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 86 / 150 (57.3): 100%|██████████| 150/150 [03:24<00:00, 1.36s/it]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Score: 57.33 for set: [3, 3, 3]\n",
+ "New best sscore: 57.33 for seed -2\n",
+ "Scores so far: [52.67, 57.33]\n",
+ "Best score: 57.33\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ " 1%| | 1/100 [00:14<24:33, 14.88s/it]"
+ ]
+ }
+ ],
"source": [
"# Initialize the optimizer\n",
"bfrs_optimizer = BootstrapFewShotWithRandomSearch(\n",
@@ -760,17 +769,17 @@
")\n",
"\n",
"# We have added this flag to save you some compute and time while running the notebook\n",
- "COMPILE_PROGRAM = False\n",
+ "COMPILE_PROGRAM = True\n",
"\n",
"# Compile the optimizer and evaluate\n",
"with dspy.context(lm=llama_70b, rm=retriever):\n",
" vanilla_program = BasicMH()\n",
" if COMPILE_PROGRAM:\n",
" bfrs_base_program = bfrs_optimizer.compile(vanilla_program, trainset=optimizer_trainset, valset=optimizer_valset)\n",
- " bfrs_base_program.save(f\"basicmh_70b_bfrs_{MAX_BOOTSTRAPPED_DEMOS}_{MAX_LABELED_DEMOS}_{NUM_CANDIDATE_PROGRAMS}.json\")\n",
+ " bfrs_base_program.save(f\"basicmh_70b_31_bfrs_{MAX_BOOTSTRAPPED_DEMOS}_{MAX_LABELED_DEMOS}_{NUM_CANDIDATE_PROGRAMS}.json\")\n",
" else:\n",
" bfrs_base_program = BasicMH()\n",
- " bfrs_base_program.load(f\"basicmh_70b_bfrs_{MAX_BOOTSTRAPPED_DEMOS}_{MAX_LABELED_DEMOS}_{NUM_CANDIDATE_PROGRAMS}.json\")\n",
+ " bfrs_base_program.load(f\"basicmh_70b_31_bfrs_{MAX_BOOTSTRAPPED_DEMOS}_{MAX_LABELED_DEMOS}_{NUM_CANDIDATE_PROGRAMS}.json\")\n",
" \n",
" llama_70b_bfrs_eval = evaluate_devset(bfrs_base_program)"
]
@@ -790,10 +799,10 @@
"if COMPILE_PROGRAM:\n",
" with dspy.context(lm=llama_70b, rm=retriever):\n",
" compiled_program = teleprompter.compile(vanilla_program, trainset=optimizer_trainset, valset=optimizer_valset, num_batches=30, max_bootstrapped_demos=MAX_BOOTSTRAPPED_DEMOS,max_labeled_demos=MAX_LABELED_DEMOS, eval_kwargs=eval_kwargs, requires_permission_to_run=False)\n",
- " compiled_program.save(f\"basicmh_70b_MIPROv2_{MAX_BOOTSTRAPPED_DEMOS}_{MAX_LABELED_DEMOS}.json\")\n",
+ " compiled_program.save(f\"basicmh_70b_31_MIPROv2_{MAX_BOOTSTRAPPED_DEMOS}_{MAX_LABELED_DEMOS}.json\")\n",
"else:\n",
" compiled_program = BasicMH()\n",
- " compiled_program.load(f\"basicmh_70b_MIPROv2_{MAX_BOOTSTRAPPED_DEMOS}_{MAX_LABELED_DEMOS}.json\")\n",
+ " compiled_program.load(f\"basicmh_70b_31_MIPROv2_{MAX_BOOTSTRAPPED_DEMOS}_{MAX_LABELED_DEMOS}.json\")\n",
"\n",
"with dspy.context(lm=llama_70b, rm=retriever):\n",
" llama_70b_mipro_eval = evaluate_devset(compiled_program)"
@@ -847,7 +856,7 @@
" print(\"Bootstrapping and writing data to\", filename)\n",
" correct_data = []\n",
" unsolved_examples = data.copy()\n",
- " sampling_temperature = 0.901\n",
+ " sampling_temperature = 0.902\n",
" sampling_temperature_delta = 0.0001\n",
" \n",
" for i in range(3):\n",
@@ -9353,7 +9362,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.11.9"
+ "version": "3.9.19"
}
},
"nbformat": 4,