Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

WIP

Isaac Miller8f31aea0995d
diff --git a/dsp/modules/lm.py b/dsp/modules/lm.py
index 7e2626fb5..7f7d064ea 100644
--- a/dsp/modules/lm.py
+++ b/dsp/modules/lm.py
@@ -1,12 +1,16 @@
from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
from typing import Any, Optional
-
+@dataclass#(frozen=True) technically should be frozen
class LM(ABC):
"""Abstract class for language models."""
+ kwargs: dict[str, Any] = field(init=False)
+ provider: str = field(init=False)
+ history: list[dict[str, Any]] = field(init=False)
def __init__(self, model):
- self.kwargs = {
+ object.__setattr__(self, 'kwargs', {
"model": model,
"temperature": 0.0,
"max_tokens": 150,
@@ -14,10 +18,9 @@ class LM(ABC):
"frequency_penalty": 0,
"presence_penalty": 0,
"n": 1,
- }
- self.provider = "default"
-
- self.history = []
+ })
+ object.__setattr__(self, 'provider', "default")
+ object.__setattr__(self, 'history', [])
@abstractmethod
def basic_request(self, prompt, **kwargs):
diff --git a/dsp/modules/openai.py b/dsp/modules/openai.py
index c55dd331f..721d76773 100644
--- a/dsp/modules/openai.py
+++ b/dsp/modules/openai.py
@@ -9,8 +9,6 @@ import tiktoken
from collections import defaultdict
# These utility functions come from: https://cookbook.openai.com/examples/chat_finetuning_data_prep
-
-
def openai_data_validation(dataset: dict[str, Any]) -> Union[dict[str, Any], None]:
"""Validate OpenAI data before sending it to the model.
diff --git a/examples/finetuning/openai_finetuning_hpqa.ipynb b/examples/finetuning/openai_finetuning_hpqa.ipynb
new file mode 100644
index 000000000..679ddd84d
--- /dev/null
+++ b/examples/finetuning/openai_finetuning_hpqa.ipynb
@@ -0,0 +1,707 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%load_ext autoreload\n",
+ "%autoreload 2"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/Users/isaac.miller/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/urllib3/__init__.py:35: NotOpenSSLWarning: urllib3 v2 only supports OpenSSL 1.1.1+, currently the 'ssl' module is compiled with 'LibreSSL 2.8.3'. See: https://github.com/urllib3/urllib3/issues/3020\n",
+ " warnings.warn(\n",
+ "/Users/isaac.miller/projects/dspy-finetuning/.venv/lib/python3.9/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+ " from .autonotebook import tqdm as notebook_tqdm\n"
+ ]
+ }
+ ],
+ "source": [
+ "import sys\n",
+ "import os\n",
+ "\n",
+ "import zipfile\n",
+ "import os\n",
+ "\n",
+ "import dotenv\n",
+ "\n",
+ "dotenv.load_dotenv()\n",
+ "import dspy\n",
+ "\n",
+ "from dspy.datasets import HotPotQA\n",
+ "import re\n",
+ "from dspy.evaluate import Evaluate\n",
+ "\n",
+ "from dsp.utils import EM\n",
+ "from dsp.utils.utils import deduplicate\n",
+ "import collections"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import openai\n",
+ "\n",
+ "openai.api_key = os.environ.get('OPENAI_API_KEY')\n",
+ "\n",
+ "mini = \"gpt-4o-mini-2024-07-18\"\n",
+ "colbert_v2_endpoint = \"http://20.102.90.50:2017/wiki17_abstracts\"\n",
+ "base_temp = 0.9\n",
+ "\n",
+ "prompt_model = dspy.OpenAIModel(model=mini, max_tokens=500, temperature=base_temp)\n",
+ "\n",
+ "colbertv2 = dspy.ColBERTv2(url=colbert_v2_endpoint)\n",
+ "\n",
+ "dspy.settings.configure(rm=colbertv2) # "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import dspy.evaluate\n",
+ "\n",
+ "\n",
+ "class AnswerCorrectnessSignature(dspy.Signature):\n",
+ " \"\"\"Verify that the predicted answer matches the gold answer.\"\"\"\n",
+ "\n",
+ " question = dspy.InputField()\n",
+ " gold_answer = dspy.InputField(desc=\"correct answer for question\")\n",
+ " predicted_answer = dspy.InputField(desc=\"predicted answer for question\")\n",
+ " is_correct = dspy.OutputField(desc='True or False')\n",
+ "\n",
+ "class AnswerCorrectness(dspy.Module):\n",
+ " def __init__(self):\n",
+ " super().__init__()\n",
+ " self.evaluate_correctness = dspy.ChainOfThought(AnswerCorrectnessSignature)\n",
+ " \n",
+ " def forward(self, example, predicted_answer):\n",
+ " question, gold_answer = example.question, example.answer\n",
+ " with dspy.context(lm=dspy.OpenAI(model=mini, max_tokens=100, temperature=0.1)):\n",
+ " return self.evaluate_correctness(question=question, gold_answer=gold_answer, predicted_answer=predicted_answer)\n",
+ " \n",
+ "evaluator = AnswerCorrectness()\n",
+ "\n",
+ "def metric(example, pred, trace=None):\n",
+ " result = evaluator(example=example, predicted_answer=pred.answer).is_correct.lower()\n",
+ " return \"true\" in result and not \"false\" in result\n",
+ "\n",
+ "metric = dspy.evaluate.answer_exact_match"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Load and configure the datasets.\n",
+ "TRAIN_SIZE = 500\n",
+ "EVAL_SIZE = 500\n",
+ "\n",
+ "hotpot_dataset = HotPotQA(train_seed=1, eval_seed=2023, test_size=0, keep_details=\"type\")\n",
+ "trainset = [x.with_inputs('question') for x in hotpot_dataset.train][:EVAL_SIZE]\n",
+ "devset = [x.with_inputs('question') for x in hotpot_dataset.dev][:EVAL_SIZE]\n",
+ "\n",
+ "# Set up metrics\n",
+ "NUM_THREADS = 12"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "kwargs = dict(num_threads=NUM_THREADS, display_progress=True)\n",
+ "evaluate = Evaluate(devset=devset, metric=metric, **kwargs)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "class BasicMH(dspy.Module):\n",
+ " def __init__(self, passages_per_hop=3):\n",
+ " super().__init__()\n",
+ " self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",
+ " self.generate_query = [dspy.ChainOfThought(\"context, question -> search_query\") for _ in range(2)]\n",
+ " self.generate_answer = dspy.ChainOfThought(\"context, question -> answer\")\n",
+ "\n",
+ " def forward(self, question, return_trace=False):\n",
+ " context = []\n",
+ " for hop in range(2):\n",
+ " search_query = self.generate_query[hop](context=context, question=question).search_query\n",
+ " passages = self.retrieve(search_query).passages\n",
+ " context = deduplicate(context + passages)\n",
+ "\n",
+ " x = self.generate_answer(context=context, question=question).copy(context=context)\n",
+ " \n",
+ " if return_trace:\n",
+ " return x, dspy.settings.trace\n",
+ " return x"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from dspy.teleprompt.random_search import BootstrapFewShotWithRandomSearch\n",
+ "\n",
+ "program_params = {\n",
+ " \"passages_per_hop\": 3,\n",
+ "}\n",
+ "\n",
+ "COMPILE = False\n",
+ "\n",
+ "if COMPILE:\n",
+ " with dspy.context(lm=dspy.OpenAI(model=mini, max_tokens=500, temperature=0.0)):\n",
+ " max_bootstrapped_demos, max_labeled_demos, num_candidate_programs = 3,3,6\n",
+ " config = dict(max_bootstrapped_demos=max_bootstrapped_demos, num_candidate_programs=num_candidate_programs, num_threads=NUM_THREADS)\n",
+ " teleprompter = BootstrapFewShotWithRandomSearch(metric=metric, **config)\n",
+ " basicmh_bs = teleprompter.compile(BasicMH(**program_params), trainset=trainset[:100], valset=devset[:250])\n",
+ " basicmh_bs.save(f\"basicmh_{max_bootstrapped_demos}_{max_labeled_demos}_{num_candidate_programs}.json\")\n",
+ "\n",
+ " baseline_eval = evaluate(BasicMH(**program_params), devset=devset[:300])\n",
+ " bs_eval = evaluate(basicmh_bs, devset=devset[:300])\n",
+ "else:\n",
+ " basicmh_bs = BasicMH(**program_params)\n",
+ " basicmh_bs.load(\"basicmh_3_3_6.json\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from collections import Counter\n",
+ "from dspy.teleprompt.finetune_teleprompter import bootstrap_multiple_prompt_completion_data\n",
+ "from typing import Callable\n",
+ "\n",
+ "samples = 500\n",
+ "max_attempts = 1\n",
+ "\n",
+ "\n",
+ "class DataCollectionCallback:\n",
+ " def __init__(self):\n",
+ " self.num_attempts = 0\n",
+ " self.num_correct = 2\n",
+ " self.max_attempts = max_attempts\n",
+ "\n",
+ " def move_on_callback_correct_with_max(self, dataset_copy, data):\n",
+ " correct_counts = Counter(list(map(lambda x: x[\"example\"], data)))\n",
+ " examples_still_incorrect = [x for x in dataset_copy if correct_counts.get(x, 0) < self.num_correct]\n",
+ " self.num_attempts += 1\n",
+ " if self.num_attempts >= self.max_attempts:\n",
+ " return []\n",
+ " return examples_still_incorrect\n",
+ "\n",
+ "\n",
+ "callback = DataCollectionCallback()\n",
+ " # \"dataset\":devset[:samples],\n",
+ " # \"metric\":metric,\n",
+ "dc_kwargs = {\n",
+ " \"include\": None,\n",
+ " \"exclude_demos\":True, \n",
+ " \"temperature\": base_temp,\n",
+ " \"temperature_delta\":0.0001,\n",
+ " \"move_on_callback\": callback.move_on_callback_correct_with_max,\n",
+ " \"num_threads\": NUM_THREADS,\n",
+ "}\n",
+ "\n",
+ "# results = bootstrap_multiple_prompt_completion_data(program, **kwargs)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from dspy.teleprompt.finetune_teleprompter import BootstrapFinetune\n",
+ "\n",
+ "teleprompter = BootstrapFinetune(training_kwargs={}, seed=314)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "TRAIN_FROM_SCRATCH = False\n",
+ "\n",
+ "if TRAIN_FROM_SCRATCH:\n",
+ " teacher = basicmh_bs\n",
+ " lm = dspy.OpenAIModel(model=mini, max_tokens=500, temperature=base_temp)\n",
+ " # for predictor in teacher.predictors():\n",
+ " # predictor.lm = lm\n",
+ " dspy.settings.configure(lm=lm)\n",
+ " # TODO: Working here\n",
+ " # teleprompter = BootstrapFinetune(training_kwargs={}, seed=314)\n",
+ " kwargs = {\"num_threads\": NUM_THREADS, \"combine\": True}\n",
+ " # NOTE: LM IS NOT SET PROPERLY\n",
+ " basicmh_bs_ft = teleprompter.compile(BasicMH(**program_params), teacher=teacher, dataset=trainset[:samples], metric=metric, data_collection_kwargs=dc_kwargs, **kwargs)\n",
+ " basicmh_bs_ft.save('mini_ft_hpqa_100.json')\n",
+ "else:\n",
+ " basicmh_bs_ft = BasicMH(**program_params)\n",
+ " basicmh_bs_ft.load('mini_ft_hpqa_100.json')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "ft:gpt-4o-mini-2024-07-18:anyscale::9tkXPhLI\n"
+ ]
+ }
+ ],
+ "source": [
+ "print(basicmh_bs_ft.predictors()[0].lm.kwargs[\"model\"])\n",
+ "dspy.settings.configure(lm=basicmh_bs_ft.predictors()[0].lm)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Going to sample between 1 and 3 traces per predictor.\n",
+ "Will attempt to bootstrap 6 candidate sets.\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 9 / 10 (90.0): 4%|▎ | 9/250 [00:00<00:01, 136.55it/s]"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 188 / 250 (75.2): 100%|██████████| 250/250 [00:00<00:00, 257.17it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Score: 75.2 for set: [0, 0, 0]\n",
+ "New best sscore: 75.2 for seed -3\n",
+ "Scores so far: [75.2]\n",
+ "Best score: 75.2\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 183 / 250 (73.2): 100%|██████████| 250/250 [00:00<00:00, 533.70it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Score: 73.2 for set: [16, 16, 16]\n",
+ "Scores so far: [75.2, 73.2]\n",
+ "Best score: 75.2\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ " 4%|▍ | 4/100 [00:00<00:00, 173.77it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Bootstrapped 3 full traces after 5 examples in round 0.\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 184 / 250 (73.6): 100%|██████████| 250/250 [00:00<00:00, 296.67it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Score: 73.6 for set: [16, 16, 16]\n",
+ "Scores so far: [75.2, 73.2, 73.6]\n",
+ "Best score: 75.2\n",
+ "Average of max per entry across top 1 scores: 0.752\n",
+ "Average of max per entry across top 2 scores: 0.784\n",
+ "Average of max per entry across top 3 scores: 0.8\n",
+ "Average of max per entry across top 5 scores: 0.8\n",
+ "Average of max per entry across top 8 scores: 0.8\n",
+ "Average of max per entry across top 9999 scores: 0.8\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ " 3%|▎ | 3/100 [00:00<00:00, 240.60it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Bootstrapped 2 full traces after 4 examples in round 0.\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 182 / 250 (72.8): 100%|██████████| 250/250 [00:00<00:00, 260.19it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Score: 72.8 for set: [16, 16, 16]\n",
+ "Scores so far: [75.2, 73.2, 73.6, 72.8]\n",
+ "Best score: 75.2\n",
+ "Average of max per entry across top 1 scores: 0.752\n",
+ "Average of max per entry across top 2 scores: 0.784\n",
+ "Average of max per entry across top 3 scores: 0.8\n",
+ "Average of max per entry across top 5 scores: 0.812\n",
+ "Average of max per entry across top 8 scores: 0.812\n",
+ "Average of max per entry across top 9999 scores: 0.812\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ " 1%| | 1/100 [00:00<00:00, 154.99it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Bootstrapped 1 full traces after 2 examples in round 0.\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 178 / 250 (71.2): 100%|██████████| 250/250 [00:00<00:00, 310.60it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Score: 71.2 for set: [16, 16, 16]\n",
+ "Scores so far: [75.2, 73.2, 73.6, 72.8, 71.2]\n",
+ "Best score: 75.2\n",
+ "Average of max per entry across top 1 scores: 0.752\n",
+ "Average of max per entry across top 2 scores: 0.784\n",
+ "Average of max per entry across top 3 scores: 0.8\n",
+ "Average of max per entry across top 5 scores: 0.824\n",
+ "Average of max per entry across top 8 scores: 0.824\n",
+ "Average of max per entry across top 9999 scores: 0.824\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ " 1%| | 1/100 [00:00<00:00, 451.92it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Bootstrapped 1 full traces after 2 examples in round 0.\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 182 / 250 (72.8): 100%|██████████| 250/250 [00:00<00:00, 305.13it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Score: 72.8 for set: [16, 16, 16]\n",
+ "Scores so far: [75.2, 73.2, 73.6, 72.8, 71.2, 72.8]\n",
+ "Best score: 75.2\n",
+ "Average of max per entry across top 1 scores: 0.752\n",
+ "Average of max per entry across top 2 scores: 0.784\n",
+ "Average of max per entry across top 3 scores: 0.8\n",
+ "Average of max per entry across top 5 scores: 0.824\n",
+ "Average of max per entry across top 8 scores: 0.832\n",
+ "Average of max per entry across top 9999 scores: 0.832\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ " 2%|▏ | 2/100 [00:00<00:00, 148.98it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Bootstrapped 1 full traces after 3 examples in round 0.\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 186 / 250 (74.4): 100%|██████████| 250/250 [00:00<00:00, 289.52it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Score: 74.4 for set: [16, 16, 16]\n",
+ "Scores so far: [75.2, 73.2, 73.6, 72.8, 71.2, 72.8, 74.4]\n",
+ "Best score: 75.2\n",
+ "Average of max per entry across top 1 scores: 0.752\n",
+ "Average of max per entry across top 2 scores: 0.796\n",
+ "Average of max per entry across top 3 scores: 0.812\n",
+ "Average of max per entry across top 5 scores: 0.828\n",
+ "Average of max per entry across top 8 scores: 0.84\n",
+ "Average of max per entry across top 9999 scores: 0.84\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ " 1%| | 1/100 [00:00<00:00, 258.21it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Bootstrapped 1 full traces after 2 examples in round 0.\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 185 / 250 (74.0): 100%|██████████| 250/250 [00:00<00:00, 346.57it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Score: 74.0 for set: [16, 16, 16]\n",
+ "Scores so far: [75.2, 73.2, 73.6, 72.8, 71.2, 72.8, 74.4, 74.0]\n",
+ "Best score: 75.2\n",
+ "Average of max per entry across top 1 scores: 0.752\n",
+ "Average of max per entry across top 2 scores: 0.796\n",
+ "Average of max per entry across top 3 scores: 0.816\n",
+ "Average of max per entry across top 5 scores: 0.836\n",
+ "Average of max per entry across top 8 scores: 0.848\n",
+ "Average of max per entry across top 9999 scores: 0.848\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ " 3%|▎ | 3/100 [00:00<00:00, 258.47it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Bootstrapped 3 full traces after 4 examples in round 0.\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 186 / 250 (74.4): 100%|██████████| 250/250 [00:02<00:00, 113.88it/s]"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Score: 74.4 for set: [16, 16, 16]\n",
+ "Scores so far: [75.2, 73.2, 73.6, 72.8, 71.2, 72.8, 74.4, 74.0, 74.4]\n",
+ "Best score: 75.2\n",
+ "Average of max per entry across top 1 scores: 0.752\n",
+ "Average of max per entry across top 2 scores: 0.796\n",
+ "Average of max per entry across top 3 scores: 0.812\n",
+ "Average of max per entry across top 5 scores: 0.832\n",
+ "Average of max per entry across top 8 scores: 0.844\n",
+ "Average of max per entry across top 9999 scores: 0.848\n",
+ "9 candidate programs found.\n",
+ "[('retrieve', <dspy.retrieve.retrieve.Retrieve object at 0x124d86610>), ('generate_query[0]', Predict(StringSignature(context, question -> rationale, search_query\n",
+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",
+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",
+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the search_query}. We ...', '__dspy_field_type': 'output'})\n",
+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",
+ "))), ('generate_query[1]', Predict(StringSignature(context, question -> rationale, search_query\n",
+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",
+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",
+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the search_query}. We ...', '__dspy_field_type': 'output'})\n",
+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",
+ "))), ('generate_answer', Predict(StringSignature(context, question -> rationale, answer\n",
+ " instructions='Given the fields `context`, `question`, produce the fields `answer`.'\n",
+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",
+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the answer}. We ...', '__dspy_field_type': 'output'})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Answer:', 'desc': '${answer}'})\n",
+ ")))]\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "RECOMPILE_FT_MODEL = True\n",
+ "\n",
+ "if RECOMPILE_FT_MODEL:\n",
+ " max_bootstrapped_demos, max_labeled_demos, num_candidate_programs = 3,3,6\n",
+ " config = dict(max_bootstrapped_demos=max_bootstrapped_demos, num_candidate_programs=num_candidate_programs, num_threads=NUM_THREADS)\n",
+ " bsfsrs_teleprompter = BootstrapFewShotWithRandomSearch(metric=metric, **config)\n",
+ " basicmh_bs_ft_bs = bsfsrs_teleprompter.compile(student=basicmh_bs_ft, trainset=trainset[:100], valset=devset[:250])\n",
+ " basicmh_bs_ft_bs.save('mini_bs_ft_bs_hpqa_100.json')\n",
+ "else:\n",
+ " basicmh_bs_ft_bs = BasicMH(**program_params)\n",
+ " basicmh_bs_ft_bs.load('mini_bs_ft_bs_hpqa_100.json')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 219 / 300 (73.0): 100%|██████████| 300/300 [00:00<00:00, 4214.14it/s]\n",
+ "Average Metric: 227 / 300 (75.7): 100%|██████████| 300/300 [00:00<00:00, 2704.22it/s]\n",
+ "Average Metric: 216 / 300 (72.0): 100%|██████████| 300/300 [00:00<00:00, 2398.68it/s]\n",
+ "Average Metric: 222 / 300 (74.0): 100%|██████████| 300/300 [00:00<00:00, 2448.68it/s]"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Results for HotPotQA finetuning gpt-4o-mini with rejection sampling N=500 and up to 1 attempts for each example with one model for all predictors. Tested on first 300 of devset.\n",
+ "Non-finetuned model: 73.0\n",
+ "Non-finetuned bootstrapped model: 75.67\n",
+ "Finetuned model: 72.0\n",
+ "Finetuned model with bootstrapping: 74.0\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "TEST_SIZE = 300\n",
+ "with dspy.context(lm=dspy.OpenAI(model=mini, max_tokens=500, temperature=0.0)):\n",
+ " baseline_eval = evaluate(BasicMH(**program_params), devset=devset[:TEST_SIZE])\n",
+ " bs_eval = evaluate(basicmh_bs, devset=devset[:TEST_SIZE])\n",
+ "bs_ft_eval = evaluate(basicmh_bs_ft, devset=devset[:TEST_SIZE])\n",
+ "bs_ft_bs_eval = evaluate(basicmh_bs_ft_bs, devset=devset[:TEST_SIZE])\n",
+ "\n",
+ "print(f\"Results for HotPotQA finetuning gpt-4o-mini with rejection sampling N={samples} and up to {max_attempts} attempts for each example with one model for all predictors. Tested on first {TEST_SIZE} of devset.\")\n",
+ "print(f\"Non-finetuned model: {baseline_eval}\")\n",
+ "print(f\"Non-finetuned bootstrapped model: {bs_eval}\")\n",
+ "print(f\"Finetuned model: {bs_ft_eval}\")\n",
+ "print(f\"Finetuned model with bootstrapping: {bs_ft_bs_eval}\")"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": ".venv",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.9.6"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}