DSPy: The framework for programming—not prompting—language models
Python
WIP
Isaac Miller8f31aea0995d
diff --git a/dsp/modules/lm.py b/dsp/modules/lm.pyindex 7e2626fb5..7f7d064ea 100644--- a/dsp/modules/lm.py+++ b/dsp/modules/lm.py@@ -1,12 +1,16 @@from abc import ABC, abstractmethod+from dataclasses import dataclass, fieldfrom typing import Any, Optional-+@dataclass#(frozen=True) technically should be frozenclass LM(ABC):"""Abstract class for language models."""+ kwargs: dict[str, Any] = field(init=False)+ provider: str = field(init=False)+ history: list[dict[str, Any]] = field(init=False)def __init__(self, model):- self.kwargs = {+ object.__setattr__(self, 'kwargs', {"model": model,"temperature": 0.0,"max_tokens": 150,@@ -14,10 +18,9 @@ class LM(ABC):"frequency_penalty": 0,"presence_penalty": 0,"n": 1,- }- self.provider = "default"-- self.history = []+ })+ object.__setattr__(self, 'provider', "default")+ object.__setattr__(self, 'history', [])@abstractmethoddef basic_request(self, prompt, **kwargs):diff --git a/dsp/modules/openai.py b/dsp/modules/openai.pyindex c55dd331f..721d76773 100644--- a/dsp/modules/openai.py+++ b/dsp/modules/openai.py@@ -9,8 +9,6 @@ import tiktokenfrom collections import defaultdict# These utility functions come from: https://cookbook.openai.com/examples/chat_finetuning_data_prep--def openai_data_validation(dataset: dict[str, Any]) -> Union[dict[str, Any], None]:"""Validate OpenAI data before sending it to the model.diff --git a/examples/finetuning/openai_finetuning_hpqa.ipynb b/examples/finetuning/openai_finetuning_hpqa.ipynbnew file mode 100644index 000000000..679ddd84d--- /dev/null+++ b/examples/finetuning/openai_finetuning_hpqa.ipynb@@ -0,0 +1,707 @@+{+ "cells": [+ {+ "cell_type": "code",+ "execution_count": 1,+ "metadata": {},+ "outputs": [],+ "source": [+ "%load_ext autoreload\n",+ "%autoreload 2"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 2,+ "metadata": {},+ "outputs": [+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "/Users/isaac.miller/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/urllib3/__init__.py:35: NotOpenSSLWarning: urllib3 v2 only supports OpenSSL 1.1.1+, currently the 'ssl' module is compiled with 'LibreSSL 2.8.3'. See: https://github.com/urllib3/urllib3/issues/3020\n",+ " warnings.warn(\n",+ "/Users/isaac.miller/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",+ " from .autonotebook import tqdm as notebook_tqdm\n"+ ]+ }+ ],+ "source": [+ "import sys\n",+ "import os\n",+ "\n",+ "import zipfile\n",+ "import os\n",+ "\n",+ "import dotenv\n",+ "\n",+ "dotenv.load_dotenv()\n",+ "import dspy\n",+ "\n",+ "from dspy.datasets import HotPotQA\n",+ "import re\n",+ "from dspy.evaluate import Evaluate\n",+ "\n",+ "from dsp.utils import EM\n",+ "from dsp.utils.utils import deduplicate\n",+ "import collections"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 3,+ "metadata": {},+ "outputs": [],+ "source": [+ "import openai\n",+ "\n",+ "openai.api_key = os.environ.get('OPENAI_API_KEY')\n",+ "\n",+ "mini = \"gpt-4o-mini-2024-07-18\"\n",+ "colbert_v2_endpoint = \"http://20.102.90.50:2017/wiki17_abstracts\"\n",+ "base_temp = 0.9\n",+ "\n",+ "prompt_model = dspy.OpenAIModel(model=mini, max_tokens=500, temperature=base_temp)\n",+ "\n",+ "colbertv2 = dspy.ColBERTv2(url=colbert_v2_endpoint)\n",+ "\n",+ "dspy.settings.configure(rm=colbertv2) # "+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 4,+ "metadata": {},+ "outputs": [],+ "source": [+ "import dspy.evaluate\n",+ "\n",+ "\n",+ "class AnswerCorrectnessSignature(dspy.Signature):\n",+ " \"\"\"Verify that the predicted answer matches the gold answer.\"\"\"\n",+ "\n",+ " question = dspy.InputField()\n",+ " gold_answer = dspy.InputField(desc=\"correct answer for question\")\n",+ " predicted_answer = dspy.InputField(desc=\"predicted answer for question\")\n",+ " is_correct = dspy.OutputField(desc='True or False')\n",+ "\n",+ "class AnswerCorrectness(dspy.Module):\n",+ " def __init__(self):\n",+ " super().__init__()\n",+ " self.evaluate_correctness = dspy.ChainOfThought(AnswerCorrectnessSignature)\n",+ " \n",+ " def forward(self, example, predicted_answer):\n",+ " question, gold_answer = example.question, example.answer\n",+ " with dspy.context(lm=dspy.OpenAI(model=mini, max_tokens=100, temperature=0.1)):\n",+ " return self.evaluate_correctness(question=question, gold_answer=gold_answer, predicted_answer=predicted_answer)\n",+ " \n",+ "evaluator = AnswerCorrectness()\n",+ "\n",+ "def metric(example, pred, trace=None):\n",+ " result = evaluator(example=example, predicted_answer=pred.answer).is_correct.lower()\n",+ " return \"true\" in result and not \"false\" in result\n",+ "\n",+ "metric = dspy.evaluate.answer_exact_match"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 5,+ "metadata": {},+ "outputs": [],+ "source": [+ "# Load and configure the datasets.\n",+ "TRAIN_SIZE = 500\n",+ "EVAL_SIZE = 500\n",+ "\n",+ "hotpot_dataset = HotPotQA(train_seed=1, eval_seed=2023, test_size=0, keep_details=\"type\")\n",+ "trainset = [x.with_inputs('question') for x in hotpot_dataset.train][:EVAL_SIZE]\n",+ "devset = [x.with_inputs('question') for x in hotpot_dataset.dev][:EVAL_SIZE]\n",+ "\n",+ "# Set up metrics\n",+ "NUM_THREADS = 12"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 6,+ "metadata": {},+ "outputs": [],+ "source": [+ "kwargs = dict(num_threads=NUM_THREADS, display_progress=True)\n",+ "evaluate = Evaluate(devset=devset, metric=metric, **kwargs)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 7,+ "metadata": {},+ "outputs": [],+ "source": [+ "class BasicMH(dspy.Module):\n",+ " def __init__(self, passages_per_hop=3):\n",+ " super().__init__()\n",+ " self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",+ " self.generate_query = [dspy.ChainOfThought(\"context, question -> search_query\") for _ in range(2)]\n",+ " self.generate_answer = dspy.ChainOfThought(\"context, question -> answer\")\n",+ "\n",+ " def forward(self, question, return_trace=False):\n",+ " context = []\n",+ " for hop in range(2):\n",+ " search_query = self.generate_query[hop](context=context, question=question).search_query\n",+ " passages = self.retrieve(search_query).passages\n",+ " context = deduplicate(context + passages)\n",+ "\n",+ " x = self.generate_answer(context=context, question=question).copy(context=context)\n",+ " \n",+ " if return_trace:\n",+ " return x, dspy.settings.trace\n",+ " return x"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 8,+ "metadata": {},+ "outputs": [],+ "source": [+ "from dspy.teleprompt.random_search import BootstrapFewShotWithRandomSearch\n",+ "\n",+ "program_params = {\n",+ " \"passages_per_hop\": 3,\n",+ "}\n",+ "\n",+ "COMPILE = False\n",+ "\n",+ "if COMPILE:\n",+ " with dspy.context(lm=dspy.OpenAI(model=mini, max_tokens=500, temperature=0.0)):\n",+ " max_bootstrapped_demos, max_labeled_demos, num_candidate_programs = 3,3,6\n",+ " config = dict(max_bootstrapped_demos=max_bootstrapped_demos, num_candidate_programs=num_candidate_programs, num_threads=NUM_THREADS)\n",+ " teleprompter = BootstrapFewShotWithRandomSearch(metric=metric, **config)\n",+ " basicmh_bs = teleprompter.compile(BasicMH(**program_params), trainset=trainset[:100], valset=devset[:250])\n",+ " basicmh_bs.save(f\"basicmh_{max_bootstrapped_demos}_{max_labeled_demos}_{num_candidate_programs}.json\")\n",+ "\n",+ " baseline_eval = evaluate(BasicMH(**program_params), devset=devset[:300])\n",+ " bs_eval = evaluate(basicmh_bs, devset=devset[:300])\n",+ "else:\n",+ " basicmh_bs = BasicMH(**program_params)\n",+ " basicmh_bs.load(\"basicmh_3_3_6.json\")"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 9,+ "metadata": {},+ "outputs": [],+ "source": [+ "from collections import Counter\n",+ "from dspy.teleprompt.finetune_teleprompter import bootstrap_multiple_prompt_completion_data\n",+ "from typing import Callable\n",+ "\n",+ "samples = 500\n",+ "max_attempts = 1\n",+ "\n",+ "\n",+ "class DataCollectionCallback:\n",+ " def __init__(self):\n",+ " self.num_attempts = 0\n",+ " self.num_correct = 2\n",+ " self.max_attempts = max_attempts\n",+ "\n",+ " def move_on_callback_correct_with_max(self, dataset_copy, data):\n",+ " correct_counts = Counter(list(map(lambda x: x[\"example\"], data)))\n",+ " examples_still_incorrect = [x for x in dataset_copy if correct_counts.get(x, 0) < self.num_correct]\n",+ " self.num_attempts += 1\n",+ " if self.num_attempts >= self.max_attempts:\n",+ " return []\n",+ " return examples_still_incorrect\n",+ "\n",+ "\n",+ "callback = DataCollectionCallback()\n",+ " # \"dataset\":devset[:samples],\n",+ " # \"metric\":metric,\n",+ "dc_kwargs = {\n",+ " \"include\": None,\n",+ " \"exclude_demos\":True, \n",+ " \"temperature\": base_temp,\n",+ " \"temperature_delta\":0.0001,\n",+ " \"move_on_callback\": callback.move_on_callback_correct_with_max,\n",+ " \"num_threads\": NUM_THREADS,\n",+ "}\n",+ "\n",+ "# results = bootstrap_multiple_prompt_completion_data(program, **kwargs)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 10,+ "metadata": {},+ "outputs": [],+ "source": [+ "from dspy.teleprompt.finetune_teleprompter import BootstrapFinetune\n",+ "\n",+ "teleprompter = BootstrapFinetune(training_kwargs={}, seed=314)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 11,+ "metadata": {},+ "outputs": [],+ "source": [+ "TRAIN_FROM_SCRATCH = False\n",+ "\n",+ "if TRAIN_FROM_SCRATCH:\n",+ " teacher = basicmh_bs\n",+ " lm = dspy.OpenAIModel(model=mini, max_tokens=500, temperature=base_temp)\n",+ " # for predictor in teacher.predictors():\n",+ " # predictor.lm = lm\n",+ " dspy.settings.configure(lm=lm)\n",+ " # TODO: Working here\n",+ " # teleprompter = BootstrapFinetune(training_kwargs={}, seed=314)\n",+ " kwargs = {\"num_threads\": NUM_THREADS, \"combine\": True}\n",+ " # NOTE: LM IS NOT SET PROPERLY\n",+ " basicmh_bs_ft = teleprompter.compile(BasicMH(**program_params), teacher=teacher, dataset=trainset[:samples], metric=metric, data_collection_kwargs=dc_kwargs, **kwargs)\n",+ " basicmh_bs_ft.save('mini_ft_hpqa_100.json')\n",+ "else:\n",+ " basicmh_bs_ft = BasicMH(**program_params)\n",+ " basicmh_bs_ft.load('mini_ft_hpqa_100.json')"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 12,+ "metadata": {},+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "ft:gpt-4o-mini-2024-07-18:anyscale::9tkXPhLI\n"+ ]+ }+ ],+ "source": [+ "print(basicmh_bs_ft.predictors()[0].lm.kwargs[\"model\"])\n",+ "dspy.settings.configure(lm=basicmh_bs_ft.predictors()[0].lm)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 13,+ "metadata": {},+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Going to sample between 1 and 3 traces per predictor.\n",+ "Will attempt to bootstrap 6 candidate sets.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 9 / 10 (90.0): 4%|▎ | 9/250 [00:00<00:01, 136.55it/s]"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 188 / 250 (75.2): 100%|██████████| 250/250 [00:00<00:00, 257.17it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Score: 75.2 for set: [0, 0, 0]\n",+ "New best sscore: 75.2 for seed -3\n",+ "Scores so far: [75.2]\n",+ "Best score: 75.2\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 183 / 250 (73.2): 100%|██████████| 250/250 [00:00<00:00, 533.70it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Score: 73.2 for set: [16, 16, 16]\n",+ "Scores so far: [75.2, 73.2]\n",+ "Best score: 75.2\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 4%|▍ | 4/100 [00:00<00:00, 173.77it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 3 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 184 / 250 (73.6): 100%|██████████| 250/250 [00:00<00:00, 296.67it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Score: 73.6 for set: [16, 16, 16]\n",+ "Scores so far: [75.2, 73.2, 73.6]\n",+ "Best score: 75.2\n",+ "Average of max per entry across top 1 scores: 0.752\n",+ "Average of max per entry across top 2 scores: 0.784\n",+ "Average of max per entry across top 3 scores: 0.8\n",+ "Average of max per entry across top 5 scores: 0.8\n",+ "Average of max per entry across top 8 scores: 0.8\n",+ "Average of max per entry across top 9999 scores: 0.8\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 3%|▎ | 3/100 [00:00<00:00, 240.60it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 2 full traces after 4 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 182 / 250 (72.8): 100%|██████████| 250/250 [00:00<00:00, 260.19it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Score: 72.8 for set: [16, 16, 16]\n",+ "Scores so far: [75.2, 73.2, 73.6, 72.8]\n",+ "Best score: 75.2\n",+ "Average of max per entry across top 1 scores: 0.752\n",+ "Average of max per entry across top 2 scores: 0.784\n",+ "Average of max per entry across top 3 scores: 0.8\n",+ "Average of max per entry across top 5 scores: 0.812\n",+ "Average of max per entry across top 8 scores: 0.812\n",+ "Average of max per entry across top 9999 scores: 0.812\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 1%| | 1/100 [00:00<00:00, 154.99it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 1 full traces after 2 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 178 / 250 (71.2): 100%|██████████| 250/250 [00:00<00:00, 310.60it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Score: 71.2 for set: [16, 16, 16]\n",+ "Scores so far: [75.2, 73.2, 73.6, 72.8, 71.2]\n",+ "Best score: 75.2\n",+ "Average of max per entry across top 1 scores: 0.752\n",+ "Average of max per entry across top 2 scores: 0.784\n",+ "Average of max per entry across top 3 scores: 0.8\n",+ "Average of max per entry across top 5 scores: 0.824\n",+ "Average of max per entry across top 8 scores: 0.824\n",+ "Average of max per entry across top 9999 scores: 0.824\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 1%| | 1/100 [00:00<00:00, 451.92it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 1 full traces after 2 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 182 / 250 (72.8): 100%|██████████| 250/250 [00:00<00:00, 305.13it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Score: 72.8 for set: [16, 16, 16]\n",+ "Scores so far: [75.2, 73.2, 73.6, 72.8, 71.2, 72.8]\n",+ "Best score: 75.2\n",+ "Average of max per entry across top 1 scores: 0.752\n",+ "Average of max per entry across top 2 scores: 0.784\n",+ "Average of max per entry across top 3 scores: 0.8\n",+ "Average of max per entry across top 5 scores: 0.824\n",+ "Average of max per entry across top 8 scores: 0.832\n",+ "Average of max per entry across top 9999 scores: 0.832\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 2%|▏ | 2/100 [00:00<00:00, 148.98it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 1 full traces after 3 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 186 / 250 (74.4): 100%|██████████| 250/250 [00:00<00:00, 289.52it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Score: 74.4 for set: [16, 16, 16]\n",+ "Scores so far: [75.2, 73.2, 73.6, 72.8, 71.2, 72.8, 74.4]\n",+ "Best score: 75.2\n",+ "Average of max per entry across top 1 scores: 0.752\n",+ "Average of max per entry across top 2 scores: 0.796\n",+ "Average of max per entry across top 3 scores: 0.812\n",+ "Average of max per entry across top 5 scores: 0.828\n",+ "Average of max per entry across top 8 scores: 0.84\n",+ "Average of max per entry across top 9999 scores: 0.84\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 1%| | 1/100 [00:00<00:00, 258.21it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 1 full traces after 2 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 185 / 250 (74.0): 100%|██████████| 250/250 [00:00<00:00, 346.57it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Score: 74.0 for set: [16, 16, 16]\n",+ "Scores so far: [75.2, 73.2, 73.6, 72.8, 71.2, 72.8, 74.4, 74.0]\n",+ "Best score: 75.2\n",+ "Average of max per entry across top 1 scores: 0.752\n",+ "Average of max per entry across top 2 scores: 0.796\n",+ "Average of max per entry across top 3 scores: 0.816\n",+ "Average of max per entry across top 5 scores: 0.836\n",+ "Average of max per entry across top 8 scores: 0.848\n",+ "Average of max per entry across top 9999 scores: 0.848\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 3%|▎ | 3/100 [00:00<00:00, 258.47it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 3 full traces after 4 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 186 / 250 (74.4): 100%|██████████| 250/250 [00:02<00:00, 113.88it/s]"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Score: 74.4 for set: [16, 16, 16]\n",+ "Scores so far: [75.2, 73.2, 73.6, 72.8, 71.2, 72.8, 74.4, 74.0, 74.4]\n",+ "Best score: 75.2\n",+ "Average of max per entry across top 1 scores: 0.752\n",+ "Average of max per entry across top 2 scores: 0.796\n",+ "Average of max per entry across top 3 scores: 0.812\n",+ "Average of max per entry across top 5 scores: 0.832\n",+ "Average of max per entry across top 8 scores: 0.844\n",+ "Average of max per entry across top 9999 scores: 0.848\n",+ "9 candidate programs found.\n",+ "[('retrieve', <dspy.retrieve.retrieve.Retrieve object at 0x124d86610>), ('generate_query[0]', Predict(StringSignature(context, question -> rationale, search_query\n",+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the search_query}. We ...', '__dspy_field_type': 'output'})\n",+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",+ "))), ('generate_query[1]', Predict(StringSignature(context, question -> rationale, search_query\n",+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the search_query}. We ...', '__dspy_field_type': 'output'})\n",+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",+ "))), ('generate_answer', Predict(StringSignature(context, question -> rationale, answer\n",+ " instructions='Given the fields `context`, `question`, produce the fields `answer`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the answer}. We ...', '__dspy_field_type': 'output'})\n",+ " answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Answer:', 'desc': '${answer}'})\n",+ ")))]\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "\n"+ ]+ }+ ],+ "source": [+ "RECOMPILE_FT_MODEL = True\n",+ "\n",+ "if RECOMPILE_FT_MODEL:\n",+ " max_bootstrapped_demos, max_labeled_demos, num_candidate_programs = 3,3,6\n",+ " config = dict(max_bootstrapped_demos=max_bootstrapped_demos, num_candidate_programs=num_candidate_programs, num_threads=NUM_THREADS)\n",+ " bsfsrs_teleprompter = BootstrapFewShotWithRandomSearch(metric=metric, **config)\n",+ " basicmh_bs_ft_bs = bsfsrs_teleprompter.compile(student=basicmh_bs_ft, trainset=trainset[:100], valset=devset[:250])\n",+ " basicmh_bs_ft_bs.save('mini_bs_ft_bs_hpqa_100.json')\n",+ "else:\n",+ " basicmh_bs_ft_bs = BasicMH(**program_params)\n",+ " basicmh_bs_ft_bs.load('mini_bs_ft_bs_hpqa_100.json')"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 16,+ "metadata": {},+ "outputs": [+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 219 / 300 (73.0): 100%|██████████| 300/300 [00:00<00:00, 4214.14it/s]\n",+ "Average Metric: 227 / 300 (75.7): 100%|██████████| 300/300 [00:00<00:00, 2704.22it/s]\n",+ "Average Metric: 216 / 300 (72.0): 100%|██████████| 300/300 [00:00<00:00, 2398.68it/s]\n",+ "Average Metric: 222 / 300 (74.0): 100%|██████████| 300/300 [00:00<00:00, 2448.68it/s]"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Results for HotPotQA finetuning gpt-4o-mini with rejection sampling N=500 and up to 1 attempts for each example with one model for all predictors. Tested on first 300 of devset.\n",+ "Non-finetuned model: 73.0\n",+ "Non-finetuned bootstrapped model: 75.67\n",+ "Finetuned model: 72.0\n",+ "Finetuned model with bootstrapping: 74.0\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "\n"+ ]+ }+ ],+ "source": [+ "TEST_SIZE = 300\n",+ "with dspy.context(lm=dspy.OpenAI(model=mini, max_tokens=500, temperature=0.0)):\n",+ " baseline_eval = evaluate(BasicMH(**program_params), devset=devset[:TEST_SIZE])\n",+ " bs_eval = evaluate(basicmh_bs, devset=devset[:TEST_SIZE])\n",+ "bs_ft_eval = evaluate(basicmh_bs_ft, devset=devset[:TEST_SIZE])\n",+ "bs_ft_bs_eval = evaluate(basicmh_bs_ft_bs, devset=devset[:TEST_SIZE])\n",+ "\n",+ "print(f\"Results for HotPotQA finetuning gpt-4o-mini with rejection sampling N={samples} and up to {max_attempts} attempts for each example with one model for all predictors. Tested on first {TEST_SIZE} of devset.\")\n",+ "print(f\"Non-finetuned model: {baseline_eval}\")\n",+ "print(f\"Non-finetuned bootstrapped model: {bs_eval}\")\n",+ "print(f\"Finetuned model: {bs_ft_eval}\")\n",+ "print(f\"Finetuned model with bootstrapping: {bs_ft_bs_eval}\")"+ ]+ }+ ],+ "metadata": {+ "kernelspec": {+ "display_name": ".venv",+ "language": "python",+ "name": "python3"+ },+ "language_info": {+ "codemirror_mode": {+ "name": "ipython",+ "version": 3+ },+ "file_extension": ".py",+ "mimetype": "text/x-python",+ "name": "python",+ "nbconvert_exporter": "python",+ "pygments_lexer": "ipython3",+ "version": "3.9.6"+ }+ },+ "nbformat": 4,+ "nbformat_minor": 2+}