Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Updated notebook

Thomas D Ahle1ac3687a1d51
diff --git a/examples/functional/signature_opt_typed.ipynb b/examples/functional/signature_opt_typed.ipynb
index 32db183bd..7447a965e 100644
--- a/examples/functional/signature_opt_typed.ipynb
+++ b/examples/functional/signature_opt_typed.ipynb
@@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
- "execution_count": 1,
+ "execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
@@ -17,18 +17,9 @@
},
{
"cell_type": "code",
- "execution_count": 2,
+ "execution_count": 3,
"metadata": {},
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "/opt/homebrew/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
- " from .autonotebook import tqdm as notebook_tqdm\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"import dspy\n",
"turbo = dspy.OpenAI(model='gpt-3.5-turbo', max_tokens=4000)\n",
@@ -38,7 +29,7 @@
},
{
"cell_type": "code",
- "execution_count": 3,
+ "execution_count": 4,
"metadata": {},
"outputs": [
{
@@ -49,7 +40,7 @@
")"
]
},
- "execution_count": 3,
+ "execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
@@ -60,7 +51,7 @@
},
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": 5,
"metadata": {},
"outputs": [
{
@@ -69,7 +60,7 @@
"(20, 50)"
]
},
- "execution_count": 4,
+ "execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
@@ -89,7 +80,7 @@
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
@@ -102,7 +93,7 @@
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": 7,
"metadata": {},
"outputs": [
{
@@ -110,7 +101,7 @@
"output_type": "stream",
"text": [
"Found 1 typed predictors to optimize.\n",
- "Generating 2 initial signatures for base...\n",
+ "Generating 4 initial signatures for base...\n",
"\n",
"================================================================================\n",
"Running eval iteration 0...\n"
@@ -120,7 +111,7 @@
"name": "stderr",
"output_type": "stream",
"text": [
- "Average Metric: 16 / 50 (32.0): 100%|██████████| 50/50 [00:00<00:00, 2233.25it/s]\n",
+ "Average Metric: 16 / 50 (32.0): 100%|██████████| 50/50 [00:00<00:00, 4290.32it/s]\n",
"/Users/ahle/repos/dspy/dspy/evaluate/evaluate.py:142: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n",
" df = df.applymap(truncate_cell)\n"
]
@@ -139,14 +130,14 @@
"name": "stderr",
"output_type": "stream",
"text": [
- "Average Metric: 14 / 50 (28.0): 100%|██████████| 50/50 [00:02<00:00, 24.02it/s]\n"
+ "Average Metric: 16 / 50 (32.0): 100%|██████████| 50/50 [00:02<00:00, 22.35it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
- "Average Metric: 14 / 50 (28.0%)\n",
+ "Average Metric: 16 / 50 (32.0%)\n",
"\n",
"================================================================================\n",
"Running eval iteration 2...\n"
@@ -156,7 +147,41 @@
"name": "stderr",
"output_type": "stream",
"text": [
- "Average Metric: 15 / 50 (30.0): 100%|██████████| 50/50 [00:02<00:00, 20.98it/s]\n"
+ "Average Metric: 19 / 50 (38.0): 100%|██████████| 50/50 [00:04<00:00, 10.28it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 19 / 50 (38.0%)\n",
+ "\n",
+ "================================================================================\n",
+ "Running eval iteration 3...\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 11 / 50 (22.0): 100%|██████████| 50/50 [00:05<00:00, 8.63it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 11 / 50 (22.0%)\n",
+ "\n",
+ "================================================================================\n",
+ "Running eval iteration 4...\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 15 / 50 (30.0): 100%|██████████| 50/50 [00:02<00:00, 24.53it/s]\n"
]
},
{
@@ -167,32 +192,64 @@
"Generating new signature for base...\n",
"\n",
"================================================================================\n",
- "Running eval iteration 3...\n"
+ "Running eval iteration 5...\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
- "Average Metric: 7 / 50 (14.0): 100%|██████████| 50/50 [00:36<00:00, 1.36it/s]\n"
+ "Average Metric: 18 / 50 (36.0): 100%|██████████| 50/50 [00:02<00:00, 21.89it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
- "Average Metric: 7 / 50 (14.0%)\n",
+ "Average Metric: 18 / 50 (36.0%)\n",
"Generating new signature for base...\n",
"\n",
"================================================================================\n",
- "Running eval iteration 4...\n"
+ "Running eval iteration 6...\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
- "Average Metric: 2 / 5 (40.0): 8%|▊ | 4/50 [00:00<00:04, 9.85it/s]"
+ "Average Metric: 6 / 50 (12.0): 100%|██████████| 50/50 [00:03<00:00, 13.65it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 6 / 50 (12.0%)\n",
+ "Generating new signature for base...\n",
+ "\n",
+ "================================================================================\n",
+ "Running eval iteration 7...\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 17 / 50 (34.0): 100%|██████████| 50/50 [00:02<00:00, 19.56it/s]"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 17 / 50 (34.0%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "\n"
]
}
],
@@ -207,7 +264,7 @@
"program = optimize_signature(\n",
" student=TypedPredictor(BasicQA),\n",
" evaluator=evaluator,\n",
- " initial_prompts=2,\n",
+ " initial_prompts=4,\n",
" n_iterations=8,\n",
" verbose=True,\n",
" prompt_model=gpt4,\n",
@@ -216,463 +273,25 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 16,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
- "\n",
- "\n",
- "\n",
- "\n",
- "Given the fields `basic_signature`, produce the fields `proposed_signatures`.\n",
- "\n",
- "---\n",
- "\n",
- "Follow the following format.\n",
- "\n",
- "Basic Signature: ${basic_signature}\n",
- "Reasoning: Let's think step by step in order to ${produce the proposed_signatures}. We ...\n",
- "Proposed Signatures: A list of 2 very different variations of the basic signature. Respond with a single JSON object. JSON Schema: {\"$defs\": {\"SignatureInfo_BasicQA_\": {\"properties\": {\"instructions\": {\"description\": \"The instructions for the task\", \"title\": \"Instructions\", \"type\": \"string\"}, \"question_prefix\": {\"description\": \"The prefix for question\", \"title\": \"Question Prefix\", \"type\": \"string\"}, \"question_desc\": {\"description\": \"The description for question\", \"title\": \"Question Desc\", \"type\": \"string\"}, \"answer_prefix\": {\"description\": \"The prefix for answer\", \"title\": \"Answer Prefix\", \"type\": \"string\"}, \"answer_desc\": {\"description\": \"The description for answer\", \"title\": \"Answer Desc\", \"type\": \"string\"}}, \"required\": [\"instructions\", \"question_prefix\", \"question_desc\", \"answer_prefix\", \"answer_desc\"], \"title\": \"SignatureInfo[BasicQA]\", \"type\": \"object\"}}, \"properties\": {\"value\": {\"items\": {\"$ref\": \"#/$defs/SignatureInfo_BasicQA_\"}, \"title\": \"Value\", \"type\": \"array\"}}, \"required\": [\"value\"], \"title\": \"Output\", \"type\": \"object\"}\n",
- "\n",
- "---\n",
- "\n",
- "Basic Signature: {\"instructions\":\"Answer questions with short factoid answers.\",\"question_prefix\":\"Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 1 and 5 words\"}\n",
- "Reasoning: Let's think step by step in order to\u001b[32m produce the proposed_signatures. We can modify the instructions to specify the type of questions to be answered. We can also change the answer description to specify the length of the answer in terms of sentences instead of words. \n",
- "Proposed Signatures: \n",
- "{\n",
- " \"value\": [\n",
- " {\n",
- " \"instructions\": \"Answer trivia questions with short factoid answers.\",\n",
- " \"question_prefix\": \"Trivia Question:\",\n",
- " \"question_desc\": \"${question}\",\n",
- " \"answer_prefix\": \"Answer:\",\n",
- " \"answer_desc\": \"often a single sentence\"\n",
- " },\n",
- " {\n",
- " \"instructions\": \"Answer general knowledge questions with short factoid answers.\",\n",
- " \"question_prefix\": \"General Knowledge Question:\",\n",
- " \"question_desc\": \"${question}\",\n",
- " \"answer_prefix\": \"Answer:\",\n",
- " \"answer_desc\": \"often between 1 and 2 sentences\"\n",
- " }\n",
- " ]\n",
- "}\u001b[0m\n",
- "\n",
- "\n",
- "\n",
- "\n",
- "\n",
- "\n",
- "\n",
- "Given the fields , produce the fields `analysis`, `proposed_signature`, `score`.\n",
- "\n",
- "---\n",
- "\n",
- "Follow the following format.\n",
- "\n",
- "Analysis: Consider what made the previous instructions good or bad.\n",
- "Proposed Signature: A signature that will likely lead to a high score.. Respond with a single JSON object. JSON Schema: {\"properties\": {\"instructions\": {\"description\": \"The instructions for the task\", \"title\": \"Instructions\", \"type\": \"string\"}, \"question_prefix\": {\"description\": \"The prefix for question\", \"title\": \"Question Prefix\", \"type\": \"string\"}, \"question_desc\": {\"description\": \"The description for question\", \"title\": \"Question Desc\", \"type\": \"string\"}, \"answer_prefix\": {\"description\": \"The prefix for answer\", \"title\": \"Answer Prefix\", \"type\": \"string\"}, \"answer_desc\": {\"description\": \"The description for answer\", \"title\": \"Answer Desc\", \"type\": \"string\"}}, \"required\": [\"instructions\", \"question_prefix\", \"question_desc\", \"answer_prefix\", \"answer_desc\"], \"title\": \"SignatureInfo[BasicQA]\", \"type\": \"object\"}\n",
- "Score: The expected score for the new signature. Don't write anything after this number. (Respond with a single float value)\n",
- "\n",
- "---\n",
- "\n",
- "Proposed Signature: {\"instructions\":\"Answer general knowledge questions with short factoid answers.\",\"question_prefix\":\"General Knowledge Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 1 and 2 sentences\"}\n",
- "Score: 8.0\n",
- "\n",
- "---\n",
- "\n",
- "Proposed Signature: {\"instructions\":\"Answer trivia questions with short factoid answers.\",\"question_prefix\":\"Trivia Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence\"}\n",
- "Score: 22.0\n",
- "\n",
- "---\n",
- "\n",
- "Proposed Signature: {\"instructions\":\"Answer questions with short factoid answers.\",\"question_prefix\":\"Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 1 and 5 words\"}\n",
- "Score: 32.0\n",
- "\n",
- "---\n",
- "\n",
- "Analysis:\u001b[32m The previous instructions were clear and concise, providing a straightforward task for the user. The use of placeholders for the question and answer descriptions allows for flexibility and adaptability. The instructions also specify the expected length of the answer, which can help guide the user in providing an appropriate response.\n",
- "\n",
- "Proposed Signature: {\"instructions\":\"Answer historical questions with short factoid answers.\",\"question_prefix\":\"History Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence or a specific date\"}\n",
- "\n",
- "Score: 28.0\u001b[0m\n",
- "\n",
- "\n",
- "\n",
- "\n",
- "\n",
- "\n",
- "\n",
- "Given the fields , produce the fields `analysis`, `proposed_signature`, `score`.\n",
- "\n",
- "---\n",
- "\n",
- "Follow the following format.\n",
- "\n",
- "Analysis: Consider what made the previous instructions good or bad.\n",
- "Proposed Signature: A signature that will likely lead to a high score.. Respond with a single JSON object. JSON Schema: {\"properties\": {\"instructions\": {\"description\": \"The instructions for the task\", \"title\": \"Instructions\", \"type\": \"string\"}, \"question_prefix\": {\"description\": \"The prefix for question\", \"title\": \"Question Prefix\", \"type\": \"string\"}, \"question_desc\": {\"description\": \"The description for question\", \"title\": \"Question Desc\", \"type\": \"string\"}, \"answer_prefix\": {\"description\": \"The prefix for answer\", \"title\": \"Answer Prefix\", \"type\": \"string\"}, \"answer_desc\": {\"description\": \"The description for answer\", \"title\": \"Answer Desc\", \"type\": \"string\"}}, \"required\": [\"instructions\", \"question_prefix\", \"question_desc\", \"answer_prefix\", \"answer_desc\"], \"title\": \"SignatureInfo[BasicQA]\", \"type\": \"object\"}\n",
- "Score: The expected score for the new signature. Don't write anything after this number. (Respond with a single float value)\n",
- "\n",
- "---\n",
- "\n",
- "Proposed Signature: {\"instructions\":\"Answer general knowledge questions with short factoid answers.\",\"question_prefix\":\"General Knowledge Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 1 and 2 sentences\"}\n",
- "Score: 8.0\n",
- "\n",
- "---\n",
- "\n",
- "Proposed Signature: {\"instructions\":\"Answer trivia questions with short factoid answers.\",\"question_prefix\":\"Trivia Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence\"}\n",
- "Score: 22.0\n",
- "\n",
- "---\n",
- "\n",
- "Proposed Signature: {\"instructions\":\"Answer historical questions with short factoid answers.\",\"question_prefix\":\"History Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence or a specific date\"}\n",
- "Score: 22.0\n",
- "\n",
- "---\n",
- "\n",
- "Proposed Signature: {\"instructions\":\"Answer questions with short factoid answers.\",\"question_prefix\":\"Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 1 and 5 words\"}\n",
- "Score: 32.0\n",
- "\n",
- "---\n",
- "\n",
- "Analysis:\u001b[32m The previous instructions were clear and concise, providing a specific context for the type of questions to be answered (general knowledge, trivia, history). They also provided guidance on the format of the answer, specifying the length and type of response expected. However, the score could be improved by providing more detailed instructions and specifying the complexity of the questions and answers.\n",
- "\n",
- "Proposed Signature: {\"instructions\":\"Answer complex scientific questions with detailed factoid answers. Use reliable sources and provide references if possible.\",\"question_prefix\":\"Science Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 2 and 3 sentences, including specific scientific terms and concepts\"}\n",
- "\n",
- "Score: 35.0\u001b[0m\n",
- "\n",
- "\n",
- "\n",
- "\n",
- "\n",
- "\n",
- "\n",
- "Given the fields , produce the fields `analysis`, `proposed_signature`, `score`.\n",
- "\n",
- "---\n",
- "\n",
- "Follow the following format.\n",
- "\n",
- "Analysis: Consider what made the previous instructions good or bad.\n",
- "Proposed Signature: A signature that will likely lead to a high score.. Respond with a single JSON object. JSON Schema: {\"properties\": {\"instructions\": {\"description\": \"The instructions for the task\", \"title\": \"Instructions\", \"type\": \"string\"}, \"question_prefix\": {\"description\": \"The prefix for question\", \"title\": \"Question Prefix\", \"type\": \"string\"}, \"question_desc\": {\"description\": \"The description for question\", \"title\": \"Question Desc\", \"type\": \"string\"}, \"answer_prefix\": {\"description\": \"The prefix for answer\", \"title\": \"Answer Prefix\", \"type\": \"string\"}, \"answer_desc\": {\"description\": \"The description for answer\", \"title\": \"Answer Desc\", \"type\": \"string\"}}, \"required\": [\"instructions\", \"question_prefix\", \"question_desc\", \"answer_prefix\", \"answer_desc\"], \"title\": \"SignatureInfo[BasicQA]\", \"type\": \"object\"}\n",
- "Score: The expected score for the new signature. Don't write anything after this number. (Respond with a single float value)\n",
- "\n",
- "---\n",
- "\n",
- "Proposed Signature: {\"instructions\":\"Answer complex scientific questions with detailed factoid answers. Use reliable sources and provide references if possible.\",\"question_prefix\":\"Science Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 2 and 3 sentences, including specific scientific terms and concepts\"}\n",
- "Score: 0.0\n",
- "\n",
- "---\n",
- "\n",
- "Proposed Signature: {\"instructions\":\"Answer general knowledge questions with short factoid answers.\",\"question_prefix\":\"General Knowledge Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 1 and 2 sentences\"}\n",
- "Score: 8.0\n",
- "\n",
- "---\n",
- "\n",
- "Proposed Signature: {\"instructions\":\"Answer trivia questions with short factoid answers.\",\"question_prefix\":\"Trivia Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence\"}\n",
- "Score: 22.0\n",
- "\n",
- "---\n",
- "\n",
- "Proposed Signature: {\"instructions\":\"Answer historical questions with short factoid answers.\",\"question_prefix\":\"History Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence or a specific date\"}\n",
- "Score: 22.0\n",
- "\n",
- "---\n",
- "\n",
- "Proposed Signature: {\"instructions\":\"Answer questions with short factoid answers.\",\"question_prefix\":\"Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 1 and 5 words\"}\n",
- "Score: 32.0\n",
- "\n",
- "---\n",
- "\n",
- "Analysis:\u001b[32m The previous instructions were good as they were clear and concise. They provided a clear context for the type of questions to be answered (e.g., science, general knowledge, trivia, history) and also gave guidance on the expected length and format of the answers. However, the first proposed signature scored 0.0 because it asked for complex scientific answers with references, which may not be feasible in a short-answer format.\n",
- "\n",
- "Proposed Signature: {\"instructions\":\"Answer geography questions with short factoid answers.\",\"question_prefix\":\"Geography Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence or a specific place\"}\n",
- "\n",
- "Score: 22.0\u001b[0m\n",
- "\n",
- "\n",
- "\n",
- "\n",
- "\n",
- "\n",
- "\n",
- "Given the fields , produce the fields `analysis`, `proposed_signature`, `score`.\n",
- "\n",
- "---\n",
- "\n",
- "Follow the following format.\n",
- "\n",
- "Analysis: Consider what made the previous instructions good or bad.\n",
- "Proposed Signature: A signature that will likely lead to a high score.. Respond with a single JSON object. JSON Schema: {\"properties\": {\"instructions\": {\"description\": \"The instructions for the task\", \"title\": \"Instructions\", \"type\": \"string\"}, \"question_prefix\": {\"description\": \"The prefix for question\", \"title\": \"Question Prefix\", \"type\": \"string\"}, \"question_desc\": {\"description\": \"The description for question\", \"title\": \"Question Desc\", \"type\": \"string\"}, \"answer_prefix\": {\"description\": \"The prefix for answer\", \"title\": \"Answer Prefix\", \"type\": \"string\"}, \"answer_desc\": {\"description\": \"The description for answer\", \"title\": \"Answer Desc\", \"type\": \"string\"}}, \"required\": [\"instructions\", \"question_prefix\", \"question_desc\", \"answer_prefix\", \"answer_desc\"], \"title\": \"SignatureInfo[BasicQA]\", \"type\": \"object\"}\n",
- "Score: The expected score for the new signature. Don't write anything after this number. (Respond with a single float value)\n",
- "\n",
- "---\n",
- "\n",
- "Proposed Signature: {\"instructions\":\"Answer complex scientific questions with detailed factoid answers. Use reliable sources and provide references if possible.\",\"question_prefix\":\"Science Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 2 and 3 sentences, including specific scientific terms and concepts\"}\n",
- "Score: 0.0\n",
- "\n",
- "---\n",
- "\n",
- "Proposed Signature: {\"instructions\":\"Answer general knowledge questions with short factoid answers.\",\"question_prefix\":\"General Knowledge Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 1 and 2 sentences\"}\n",
- "Score: 8.0\n",
- "\n",
- "---\n",
- "\n",
- "Proposed Signature: {\"instructions\":\"Answer geography questions with short factoid answers.\",\"question_prefix\":\"Geography Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence or a specific place\"}\n",
- "Score: 16.0\n",
- "\n",
- "---\n",
- "\n",
- "Proposed Signature: {\"instructions\":\"Answer trivia questions with short factoid answers.\",\"question_prefix\":\"Trivia Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence\"}\n",
- "Score: 22.0\n",
- "\n",
- "---\n",
- "\n",
- "Proposed Signature: {\"instructions\":\"Answer historical questions with short factoid answers.\",\"question_prefix\":\"History Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence or a specific date\"}\n",
- "Score: 22.0\n",
- "\n",
- "---\n",
- "\n",
- "Proposed Signature: {\"instructions\":\"Answer questions with short factoid answers.\",\"question_prefix\":\"Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 1 and 5 words\"}\n",
- "Score: 32.0\n",
- "\n",
- "---\n",
- "\n",
- "Analysis:\u001b[32m The previous instructions were good as they were clear, concise, and specific. They provided a clear context (e.g., science, general knowledge, geography, trivia, history) for the type of questions to be answered, which can help in generating more accurate and relevant responses. The instructions also specified the format of the answers (e.g., short factoid answers, often a single sentence or a specific place/date), which can help in maintaining consistency in the responses. However, the first proposed signature had a score of 0.0, possibly because it asked for complex scientific questions to be answered with detailed factoid answers and references, which may not be feasible or practical in many cases.\n",
- "\n",
- "Proposed Signature: {\"instructions\":\"Answer general knowledge questions with concise factoid answers.\",\"question_prefix\":\"General Knowledge Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence or a specific fact\"}\n",
- "\n",
- "Score: 25.0\u001b[0m\n",
- "\n",
- "\n",
- "\n",
- "\n",
- "\n",
- "\n",
- "\n",
- "Given the fields , produce the fields `analysis`, `proposed_signature`, `score`.\n",
- "\n",
- "---\n",
- "\n",
- "Follow the following format.\n",
- "\n",
- "Analysis: Consider what made the previous instructions good or bad.\n",
- "Proposed Signature: A signature that will likely lead to a high score.. Respond with a single JSON object. JSON Schema: {\"properties\": {\"instructions\": {\"description\": \"The instructions for the task\", \"title\": \"Instructions\", \"type\": \"string\"}, \"question_prefix\": {\"description\": \"The prefix for question\", \"title\": \"Question Prefix\", \"type\": \"string\"}, \"question_desc\": {\"description\": \"The description for question\", \"title\": \"Question Desc\", \"type\": \"string\"}, \"answer_prefix\": {\"description\": \"The prefix for answer\", \"title\": \"Answer Prefix\", \"type\": \"string\"}, \"answer_desc\": {\"description\": \"The description for answer\", \"title\": \"Answer Desc\", \"type\": \"string\"}}, \"required\": [\"instructions\", \"question_prefix\", \"question_desc\", \"answer_prefix\", \"answer_desc\"], \"title\": \"SignatureInfo[BasicQA]\", \"type\": \"object\"}\n",
- "Score: The expected score for the new signature. Don't write anything after this number. (Respond with a single float value)\n",
- "\n",
- "---\n",
- "\n",
- "Proposed Signature: {\"instructions\":\"Answer complex scientific questions with detailed factoid answers. Use reliable sources and provide references if possible.\",\"question_prefix\":\"Science Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 2 and 3 sentences, including specific scientific terms and concepts\"}\n",
- "Score: 0.0\n",
- "\n",
- "---\n",
- "\n",
- "Proposed Signature: {\"instructions\":\"Answer general knowledge questions with short factoid answers.\",\"question_prefix\":\"General Knowledge Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 1 and 2 sentences\"}\n",
- "Score: 8.0\n",
- "\n",
- "---\n",
- "\n",
- "Proposed Signature: {\"instructions\":\"Answer general knowledge questions with concise factoid answers.\",\"question_prefix\":\"General Knowledge Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence or a specific fact\"}\n",
- "Score: 14.0\n",
- "\n",
- "---\n",
- "\n",
- "Proposed Signature: {\"instructions\":\"Answer geography questions with short factoid answers.\",\"question_prefix\":\"Geography Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence or a specific place\"}\n",
- "Score: 16.0\n",
- "\n",
- "---\n",
- "\n",
- "Proposed Signature: {\"instructions\":\"Answer trivia questions with short factoid answers.\",\"question_prefix\":\"Trivia Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence\"}\n",
- "Score: 22.0\n",
- "\n",
- "---\n",
- "\n",
- "Proposed Signature: {\"instructions\":\"Answer historical questions with short factoid answers.\",\"question_prefix\":\"History Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence or a specific date\"}\n",
- "Score: 22.0\n",
- "\n",
- "---\n",
- "\n",
- "Proposed Signature: {\"instructions\":\"Answer questions with short factoid answers.\",\"question_prefix\":\"Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 1 and 5 words\"}\n",
- "Score: 32.0\n",
- "\n",
- "---\n",
- "\n",
- "Analysis:\u001b[32m The previous instructions were good because they were clear, concise, and specific. They provided a clear context (e.g., science, general knowledge, geography, trivia, history) for the type of questions to be answered, and they specified the format of the answer (e.g., short factoid, single sentence, specific fact or date). However, the first proposed signature had a score of 0.0 because it was too complex and required references, which may not be feasible for all questions.\n",
- "\n",
- "Proposed Signature: {\"instructions\":\"Answer questions about literature with short factoid answers.\",\"question_prefix\":\"Literature Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence or a specific fact\"}\n",
- "\n",
- "Score: 20.0\u001b[0m\n",
- "\n",
- "\n",
- "\n"
+ "StringSignature(question -> answer\n",
+ " instructions='You are highly intelligent. Please provide short, factual answers to the following questions.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Inquiry:', 'desc': '${question}'})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'usually between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Reply:'})\n",
+ ")\n"
]
}
],
"source": [
- "gpt4.inspect_history(n=10)"
+ "print(program.signature)"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "turbo.inspect_history(n=10)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "\"You are an instruction optimizer for large language models.\\n\\n I will give some task instructions I've tried, along with their corresponding validation scores.\\n - The instructions are arranged in order based on their scores, where higher scores indicate better quality.\\n - Your task is to propose a new instruction that will lead a good language model to perform the task even better.\\n - Be creative, and think out of the box.\\n - Don't repeat instructions, descriptions and prefixes that have already been attempted.\\n \""
- ]
- },
- "execution_count": 12,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "from dspy.teleprompt.signature_opt_typed import GenerateSignature\n",
- "GenerateSignature.instructions"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "Prediction(\n",
- " analysis='The previous instructions were clear and provided a specific format to follow for the response.',\n",
- " proposed_signature=BasicQA(question='What are the fields to produce?', answer='analysis, proposed_signature, score'),\n",
- " score=4.5\n",
- ")"
- ]
- },
- "execution_count": 14,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "dspy.TypedPredictor(GenerateSignature[BasicQA])()"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "\n",
- "\n",
- "\n",
- "\n",
- "Given the fields , produce the fields `analysis`, `proposed_signature`, `score`.\n",
- "\n",
- "---\n",
- "\n",
- "Follow the following format.\n",
- "\n",
- "Analysis: Consider what made the previous instructions good or bad.\n",
- "Proposed Signature: A signature that will likely lead to a high score.. Respond with a single JSON object. JSON Schema: {\"description\": \"Answer questions with short factoid answers.\", \"properties\": {\"question\": {\"__dspy_field_type\": \"input\", \"desc\": \"${question}\", \"prefix\": \"Question:\", \"title\": \"Question\", \"type\": \"string\"}, \"answer\": {\"__dspy_field_type\": \"output\", \"desc\": \"often between 1 and 5 words\", \"prefix\": \"Answer:\", \"title\": \"Answer\", \"type\": \"string\"}}, \"required\": [\"question\", \"answer\"], \"title\": \"BasicQA\", \"type\": \"object\"}\n",
- "Score: The expected score for the new signature. Don't write anything after this number. (Respond with a single float value)\n",
- "\n",
- "---\n",
- "\n",
- "Analysis:\u001b[32m The previous instructions were clear and provided a specific format to follow for the response.\n",
- "\n",
- "Proposed Signature:\n",
- "```json\n",
- "{\n",
- " \"question\": \"What are the fields to produce?\",\n",
- " \"answer\": \"analysis, proposed_signature, score\"\n",
- "}\n",
- "```\n",
- "\n",
- "Score:\n",
- "4.5\u001b[0m\n",
- "\n",
- "\n",
- "\n"
- ]
- }
- ],
- "source": [
- "turbo.inspect_history(n=1)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "GenerateSignature[BasicQA]( -> analysis, proposed_signature, score\n",
- " instructions='Given the fields , produce the fields `analysis`, `proposed_signature`, `score`.'\n",
- " analysis = Field(annotation=str required=True json_schema_extra={'desc': 'Consider what made the previous instructions good or bad.', '__dspy_field_type': 'output', 'prefix': 'Analysis:'})\n",
- " proposed_signature = Field(annotation=BasicQA required=True json_schema_extra={'desc': 'A signature that will likely lead to a high score.', '__dspy_field_type': 'output', 'prefix': 'Proposed Signature:'})\n",
- " score = Field(annotation=float required=True json_schema_extra={'desc': \"The expected score for the new signature. Don't write anything after this number.\", '__dspy_field_type': 'output', 'prefix': 'Score:'})\n",
- ")"
- ]
- },
- "execution_count": 16,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "GenerateSignature[BasicQA]"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "GenerateSignature( -> analysis, proposed_signature, score\n",
- " instructions=\"You are an instruction optimizer for large language models.\\n\\n I will give some task instructions I've tried, along with their corresponding validation scores.\\n - The instructions are arranged in order based on their scores, where higher scores indicate better quality.\\n - Your task is to propose a new instruction that will lead a good language model to perform the task even better.\\n - Be creative, and think out of the box.\\n - Don't repeat instructions, descriptions and prefixes that have already been attempted.\\n \"\n",
- " analysis = Field(annotation=str required=True json_schema_extra={'desc': 'Consider what made the previous instructions good or bad.', '__dspy_field_type': 'output', 'prefix': 'Analysis:'})\n",
- " proposed_signature = Field(annotation=~T required=True json_schema_extra={'desc': 'A signature that will likely lead to a high score.', '__dspy_field_type': 'output', 'prefix': 'Proposed Signature:'})\n",
- " score = Field(annotation=float required=True json_schema_extra={'desc': \"The expected score for the new signature. Don't write anything after this number.\", '__dspy_field_type': 'output', 'prefix': 'Score:'})\n",
- ")"
- ]
- },
- "execution_count": 17,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "GenerateSignature"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": []
- },
{
"cell_type": "code",
"execution_count": null,