Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Added example HumanEval notebook with types

Thomas D Ahlefec7d2e52d3c
diff --git a/dspy/functional/functional.py b/dspy/functional/functional.py
index bc10def22..5bb7d8eff 100644
--- a/dspy/functional/functional.py
+++ b/dspy/functional/functional.py
@@ -28,16 +28,31 @@ class FunctionalModule(dspy.Module):
class TypedPredictor(dspy.Module):
- def __init__(self, signature, chain_of_thought=False, simple_output=False):
+ def __init__(
+ self, signature, chain_of_thought=False, simple_output=False, make_example=False
+ ):
super().__init__()
self.signature = signature
self.predictor = dspy.Predict(signature)
self.chain_of_thought = chain_of_thought
self.simple_output = simple_output
+ self.make_example = make_example
def copy(self):
return TypedPredictor(self.signature, self.chain_of_thought, self.simple_output)
+ @staticmethod
+ def _make_example(type_):
+ return dspy.Predict(
+ dspy.Signature(
+ "json_schema -> json_object",
+ "Make a very succinct json object that validates with the following schema",
+ )
+ )(json_schema=json.dumps(type_.model_json_schema())).json_object
+ # TODO: Another fun idea is to only (but automatically) do this if the output fails.
+ # We could also have a more general "suggest solution" prompt that tries to fix the output
+ # More directly.
+
def _prepare_signature(self):
"""Add formats and parsers to the signature fields, based on the type
annotations of the fields."""
@@ -68,10 +83,14 @@ class TypedPredictor(dspy.Module):
name,
desc=field.json_schema_extra.get("desc", "")
+ (
+ # f". Put the answer in the following JSON structure "
f". Respond with a single JSON object using the schema "
+ json.dumps(type_.model_json_schema())
+ + (". For example: " + self._make_example(type_) if self.make_example else "")
+ ),
+ format=lambda x: (
+ x if isinstance(x, str) else x.model_dump_json()
),
- format=lambda x: x if isinstance(x, str) else x.model_dump_json(),
parser=lambda x: unwrap(
type_.model_validate_json(_unwrap_json(x))
),
diff --git a/examples/functional/functional.ipynb b/examples/functional/functional.ipynb
new file mode 100644
index 000000000..8d3e380c7
--- /dev/null
+++ b/examples/functional/functional.ipynb
@@ -0,0 +1,850 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 192,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "[autoreload of dspy.signatures.signature failed: Traceback (most recent call last):\n",
+ " File \"/opt/homebrew/lib/python3.11/site-packages/IPython/extensions/autoreload.py\", line 276, in check\n",
+ " superreload(m, reload, self.old_objects)\n",
+ " File \"/opt/homebrew/lib/python3.11/site-packages/IPython/extensions/autoreload.py\", line 500, in superreload\n",
+ " update_generic(old_obj, new_obj)\n",
+ " File \"/opt/homebrew/lib/python3.11/site-packages/IPython/extensions/autoreload.py\", line 397, in update_generic\n",
+ " update(a, b)\n",
+ " File \"/opt/homebrew/lib/python3.11/site-packages/IPython/extensions/autoreload.py\", line 365, in update_class\n",
+ " update_instances(old, new)\n",
+ " File \"/opt/homebrew/lib/python3.11/site-packages/IPython/extensions/autoreload.py\", line 323, in update_instances\n",
+ " object.__setattr__(ref, \"__class__\", new)\n",
+ "TypeError: can't apply this __setattr__ to SignatureMeta object\n",
+ "]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "The autoreload extension is already loaded. To reload it, use:\n",
+ " %reload_ext autoreload\n",
+ "Requirement already satisfied: datasets in /opt/homebrew/lib/python3.11/site-packages (2.14.7)\n",
+ "Requirement already satisfied: numpy>=1.17 in /opt/homebrew/lib/python3.11/site-packages (from datasets) (1.26.2)\n",
+ "Requirement already satisfied: pyarrow>=8.0.0 in /opt/homebrew/lib/python3.11/site-packages (from datasets) (12.0.0)\n",
+ "Requirement already satisfied: pyarrow-hotfix in /opt/homebrew/lib/python3.11/site-packages (from datasets) (0.6)\n",
+ "Requirement already satisfied: dill<0.3.8,>=0.3.0 in /opt/homebrew/lib/python3.11/site-packages (from datasets) (0.3.6)\n",
+ "Requirement already satisfied: pandas in /opt/homebrew/lib/python3.11/site-packages (from datasets) (2.1.4)\n",
+ "Requirement already satisfied: requests>=2.19.0 in /opt/homebrew/lib/python3.11/site-packages (from datasets) (2.31.0)\n",
+ "Requirement already satisfied: tqdm>=4.62.1 in /opt/homebrew/lib/python3.11/site-packages (from datasets) (4.66.1)\n",
+ "Requirement already satisfied: xxhash in /opt/homebrew/lib/python3.11/site-packages (from datasets) (3.2.0)\n",
+ "Requirement already satisfied: multiprocess in /opt/homebrew/lib/python3.11/site-packages (from datasets) (0.70.14)\n",
+ "Requirement already satisfied: fsspec<=2023.10.0,>=2023.1.0 in /opt/homebrew/lib/python3.11/site-packages (from fsspec[http]<=2023.10.0,>=2023.1.0->datasets) (2023.10.0)\n",
+ "Requirement already satisfied: aiohttp in /opt/homebrew/lib/python3.11/site-packages (from datasets) (3.9.1)\n",
+ "Requirement already satisfied: huggingface-hub<1.0.0,>=0.14.0 in /opt/homebrew/lib/python3.11/site-packages (from datasets) (0.19.4)\n",
+ "Requirement already satisfied: packaging in /opt/homebrew/lib/python3.11/site-packages (from datasets) (23.2)\n",
+ "Requirement already satisfied: pyyaml>=5.1 in /opt/homebrew/lib/python3.11/site-packages (from datasets) (6.0.1)\n",
+ "Requirement already satisfied: attrs>=17.3.0 in /opt/homebrew/lib/python3.11/site-packages (from aiohttp->datasets) (23.1.0)\n",
+ "Requirement already satisfied: multidict<7.0,>=4.5 in /opt/homebrew/lib/python3.11/site-packages (from aiohttp->datasets) (6.0.4)\n",
+ "Requirement already satisfied: yarl<2.0,>=1.0 in /opt/homebrew/lib/python3.11/site-packages (from aiohttp->datasets) (1.9.3)\n",
+ "Requirement already satisfied: frozenlist>=1.1.1 in /opt/homebrew/lib/python3.11/site-packages (from aiohttp->datasets) (1.4.0)\n",
+ "Requirement already satisfied: aiosignal>=1.1.2 in /opt/homebrew/lib/python3.11/site-packages (from aiohttp->datasets) (1.3.1)\n",
+ "Requirement already satisfied: filelock in /opt/homebrew/lib/python3.11/site-packages (from huggingface-hub<1.0.0,>=0.14.0->datasets) (3.13.1)\n",
+ "Requirement already satisfied: typing-extensions>=3.7.4.3 in /opt/homebrew/lib/python3.11/site-packages (from huggingface-hub<1.0.0,>=0.14.0->datasets) (4.8.0)\n",
+ "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/homebrew/lib/python3.11/site-packages (from requests>=2.19.0->datasets) (3.3.2)\n",
+ "Requirement already satisfied: idna<4,>=2.5 in /opt/homebrew/lib/python3.11/site-packages (from requests>=2.19.0->datasets) (3.6)\n",
+ "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/homebrew/lib/python3.11/site-packages (from requests>=2.19.0->datasets) (2.1.0)\n",
+ "Requirement already satisfied: certifi>=2017.4.17 in /opt/homebrew/lib/python3.11/site-packages (from requests>=2.19.0->datasets) (2023.11.17)\n",
+ "Requirement already satisfied: python-dateutil>=2.8.2 in /opt/homebrew/lib/python3.11/site-packages (from pandas->datasets) (2.8.2)\n",
+ "Requirement already satisfied: pytz>=2020.1 in /opt/homebrew/lib/python3.11/site-packages (from pandas->datasets) (2023.3)\n",
+ "Requirement already satisfied: tzdata>=2022.1 in /opt/homebrew/lib/python3.11/site-packages (from pandas->datasets) (2023.3)\n",
+ "Requirement already satisfied: six>=1.5 in /opt/homebrew/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.16.0)\n",
+ "\n",
+ "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.3.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.0\u001b[0m\n",
+ "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3.11 -m pip install --upgrade pip\u001b[0m\n",
+ "Note: you may need to restart the kernel to use updated packages.\n"
+ ]
+ }
+ ],
+ "source": [
+ "%load_ext autoreload\n",
+ "%autoreload 2\n",
+ "%pip install datasets"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Let's get the dataset and see what it looks like."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 193,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "{'task_id': 'HumanEval/0',\n",
+ " 'prompt': 'from typing import List\\n\\n\\ndef has_close_elements(numbers: List[float], threshold: float) -> bool:\\n \"\"\" Check if in given list of numbers, are any two numbers closer to each other than\\n given threshold.\\n >>> has_close_elements([1.0, 2.0, 3.0], 0.5)\\n False\\n >>> has_close_elements([1.0, 2.8, 3.0, 4.0, 5.0, 2.0], 0.3)\\n True\\n \"\"\"\\n',\n",
+ " 'canonical_solution': ' for idx, elem in enumerate(numbers):\\n for idx2, elem2 in enumerate(numbers):\\n if idx != idx2:\\n distance = abs(elem - elem2)\\n if distance < threshold:\\n return True\\n\\n return False\\n',\n",
+ " 'test': \"\\n\\nMETADATA = {\\n 'author': 'jt',\\n 'dataset': 'test'\\n}\\n\\n\\ndef check(candidate):\\n assert candidate([1.0, 2.0, 3.9, 4.0, 5.0, 2.2], 0.3) == True\\n assert candidate([1.0, 2.0, 3.9, 4.0, 5.0, 2.2], 0.05) == False\\n assert candidate([1.0, 2.0, 5.9, 4.0, 5.0], 0.95) == True\\n assert candidate([1.0, 2.0, 5.9, 4.0, 5.0], 0.8) == False\\n assert candidate([1.0, 2.0, 3.0, 4.0, 5.0, 2.0], 0.1) == True\\n assert candidate([1.1, 2.2, 3.1, 4.1, 5.1], 1.0) == True\\n assert candidate([1.1, 2.2, 3.1, 4.1, 5.1], 0.5) == False\\n\\n\",\n",
+ " 'entry_point': 'has_close_elements'}"
+ ]
+ },
+ "execution_count": 193,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "import datasets\n",
+ "ds = datasets.load_dataset(\"openai_humaneval\")\n",
+ "ds['test'][0]\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Before we try to solve the problem, let's just load a language model and make sure everything works."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 194,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Prediction(\n",
+ " answer='Paris'\n",
+ ")\n"
+ ]
+ }
+ ],
+ "source": [
+ "import dspy, dotenv, os\n",
+ "dotenv.load_dotenv(os.path.expanduser(\"~/.env\")) # load OpenAI API key from .env file\n",
+ "lm = dspy.OpenAI(model=\"gpt-3.5-turbo\", max_tokens=4000)\n",
+ "dspy.settings.configure(lm=lm)\n",
+ "\n",
+ "predictor = dspy.Predict(\"question -> answer\")\n",
+ "print(predictor(question=\"What is the capital of France?\"))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Next let's write a program that actually outputs code."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 195,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Prediction(\n",
+ " reasoning='produce the solution. We need to ensure that the JSON object we provide is valid and follows the specified schema. The error message indicates that there are trailing characters in the JSON object, which is causing it to be invalid. To avoid this error in the future, we need to make sure that the JSON object is correctly formatted and does not contain any additional characters outside of the specified structure.',\n",
+ " solution=PythonCode(code='def has_close_elements(numbers: List[float], threshold: float) -> bool:\\n for i in range(len(numbers)):\\n for j in range(i+1, len(numbers)):\\n if abs(numbers[i] - numbers[j]) < threshold:\\n return True\\n return False')\n",
+ ")\n"
+ ]
+ }
+ ],
+ "source": [
+ "from dspy import InputField, OutputField, Signature\n",
+ "from dspy.functional import TypedPredictor\n",
+ "import pydantic\n",
+ "\n",
+ "# We define a pydantic type that automatically checks if it's argument is valid python code.\n",
+ "class PythonCode(pydantic.BaseModel):\n",
+ " code: str\n",
+ "\n",
+ " @pydantic.field_validator('code')\n",
+ " def check_syntax(cls, v):\n",
+ " try:\n",
+ " # Attempt to compile the code snippet\n",
+ " compile(v, \"<string>\", \"exec\")\n",
+ " except SyntaxError as e:\n",
+ " # If a SyntaxError is raised, the code is not syntactically valid\n",
+ " raise ValueError(f\"Code is not syntactically valid: {e}\")\n",
+ " \n",
+ " return v\n",
+ "\n",
+ "# The signature is the main DSpy object. Note that we have types for the input and output fields,\n",
+ "# which was not possible beofore.\n",
+ "class CodeSignature(Signature):\n",
+ " prompt: str = InputField()\n",
+ " test: PythonCode = InputField()\n",
+ " entry_point: str = InputField()\n",
+ " solution: PythonCode = OutputField()\n",
+ "\n",
+ "predictor = TypedPredictor(CodeSignature, chain_of_thought=True, make_example=True)\n",
+ "prediction = predictor(\n",
+ " prompt=PythonCode(code=ds['test'][0]['prompt']),\n",
+ " test=PythonCode(code=ds['test'][0]['test']),\n",
+ " entry_point=ds['test'][0]['entry_point']\n",
+ ")\n",
+ "\n",
+ "print(prediction)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Let's see what's happening under the hood"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 196,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "Make a very succinct json object that validates with the following schema\n",
+ "\n",
+ "---\n",
+ "\n",
+ "Follow the following format.\n",
+ "\n",
+ "Json Schema: ${json_schema}\n",
+ "Json Object: ${json_object}\n",
+ "\n",
+ "---\n",
+ "\n",
+ "Json Schema: {\"properties\": {\"code\": {\"title\": \"Code\", \"type\": \"string\"}}, \"required\": [\"code\"], \"title\": \"PythonCode\", \"type\": \"object\"}\n",
+ "Json Object:\u001b[32m {\"code\": \"print('Hello, World!')\"}\u001b[0m\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "Given the fields `prompt`, `test`, `entry_point`, produce the fields `solution`.\n",
+ "\n",
+ "---\n",
+ "\n",
+ "Follow the following format.\n",
+ "\n",
+ "Prompt: ${prompt}\n",
+ "\n",
+ "Test: ${test}\n",
+ "\n",
+ "Entry Point: ${entry_point}\n",
+ "\n",
+ "Reasoning: Let's think step by step in order to ${produce the solution}. We ...\n",
+ "\n",
+ "Solution: ${solution}. Respond with a single JSON object using the schema {\"properties\": {\"code\": {\"title\": \"Code\", \"type\": \"string\"}}, \"required\": [\"code\"], \"title\": \"PythonCode\", \"type\": \"object\"}. For example: {\"code\": \"print('Hello, World!')\"}\n",
+ "\n",
+ "---\n",
+ "\n",
+ "Prompt: code='from typing import List\\n\\n\\ndef has_close_elements(numbers: List[float], threshold: float) -> bool:\\n \"\"\" Check if in given list of numbers, are any two numbers closer to each other than\\n given threshold.\\n >>> has_close_elements([1.0, 2.0, 3.0], 0.5)\\n False\\n >>> has_close_elements([1.0, 2.8, 3.0, 4.0, 5.0, 2.0], 0.3)\\n True\\n \"\"\"\\n'\n",
+ "\n",
+ "Test: {\"code\":\"\\n\\nMETADATA = {\\n 'author': 'jt',\\n 'dataset': 'test'\\n}\\n\\n\\ndef check(candidate):\\n assert candidate([1.0, 2.0, 3.9, 4.0, 5.0, 2.2], 0.3) == True\\n assert candidate([1.0, 2.0, 3.9, 4.0, 5.0, 2.2], 0.05) == False\\n assert candidate([1.0, 2.0, 5.9, 4.0, 5.0], 0.95) == True\\n assert candidate([1.0, 2.0, 5.9, 4.0, 5.0], 0.8) == False\\n assert candidate([1.0, 2.0, 3.0, 4.0, 5.0, 2.0], 0.1) == True\\n assert candidate([1.1, 2.2, 3.1, 4.1, 5.1], 1.0) == True\\n assert candidate([1.1, 2.2, 3.1, 4.1, 5.1], 0.5) == False\\n\\n\"}\n",
+ "\n",
+ "Entry Point: has_close_elements\n",
+ "\n",
+ "Reasoning: Let's think step by step in order to\u001b[32m produce the solution. We need to iterate through the list of numbers and compare each pair of numbers to see if their absolute difference is less than the threshold.\n",
+ "\n",
+ "Solution: {\"properties\": {\"code\": {\"title\": \"Code\", \"type\": \"string\"}}, \"required\": [\"code\"], \"title\": \"PythonCode\", \"type\": \"object\"}. For example: {\"code\": \"from typing import List\\n\\ndef has_close_elements(numbers: List[float], threshold: float) -> bool:\\n for i in range(len(numbers)):\\n for j in range(i+1, len(numbers)):\\n if abs(numbers[i] - numbers[j]) < threshold:\\n return True\\n return False\"}\u001b[0m\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "Given the fields `prompt`, `test`, `entry_point`, produce the fields `solution`.\n",
+ "\n",
+ "---\n",
+ "\n",
+ "Follow the following format.\n",
+ "\n",
+ "Prompt: ${prompt}\n",
+ "\n",
+ "Test: ${test}\n",
+ "\n",
+ "Entry Point: ${entry_point}\n",
+ "\n",
+ "Past Error (solution): An error to avoid in the future\n",
+ "\n",
+ "Reasoning: Let's think step by step in order to ${produce the solution}. We ...\n",
+ "\n",
+ "Solution: ${solution}. Respond with a single JSON object using the schema {\"properties\": {\"code\": {\"title\": \"Code\", \"type\": \"string\"}}, \"required\": [\"code\"], \"title\": \"PythonCode\", \"type\": \"object\"}. For example: {\"code\": \"print('Hello, World!')\"}\n",
+ "\n",
+ "---\n",
+ "\n",
+ "Prompt: code='from typing import List\\n\\n\\ndef has_close_elements(numbers: List[float], threshold: float) -> bool:\\n \"\"\" Check if in given list of numbers, are any two numbers closer to each other than\\n given threshold.\\n >>> has_close_elements([1.0, 2.0, 3.0], 0.5)\\n False\\n >>> has_close_elements([1.0, 2.8, 3.0, 4.0, 5.0, 2.0], 0.3)\\n True\\n \"\"\"\\n'\n",
+ "\n",
+ "Test: {\"code\":\"\\n\\nMETADATA = {\\n 'author': 'jt',\\n 'dataset': 'test'\\n}\\n\\n\\ndef check(candidate):\\n assert candidate([1.0, 2.0, 3.9, 4.0, 5.0, 2.2], 0.3) == True\\n assert candidate([1.0, 2.0, 3.9, 4.0, 5.0, 2.2], 0.05) == False\\n assert candidate([1.0, 2.0, 5.9, 4.0, 5.0], 0.95) == True\\n assert candidate([1.0, 2.0, 5.9, 4.0, 5.0], 0.8) == False\\n assert candidate([1.0, 2.0, 3.0, 4.0, 5.0, 2.0], 0.1) == True\\n assert candidate([1.1, 2.2, 3.1, 4.1, 5.1], 1.0) == True\\n assert candidate([1.1, 2.2, 3.1, 4.1, 5.1], 0.5) == False\\n\\n\"}\n",
+ "\n",
+ "Entry Point: has_close_elements\n",
+ "\n",
+ "Past Error (solution): 1 validation error for PythonCode Invalid JSON: trailing characters at line 1 column 125 [type=json_invalid, input_value='{\"properties\": {\"code\": ...ue\\\\n return False\"}', input_type=str] For further information visit https://errors.pydantic.dev/2.5/v/json_invalid\n",
+ "\n",
+ "Reasoning: Let's think step by step in order to\u001b[32m produce the solution. We need to ensure that the JSON object we provide is valid and follows the specified schema. The error message indicates that there are trailing characters in the JSON object, which is causing it to be invalid. To avoid this error in the future, we need to make sure that the JSON object is correctly formatted and does not contain any additional characters outside of the specified structure.\n",
+ "\n",
+ "Solution: {\"code\": \"def has_close_elements(numbers: List[float], threshold: float) -> bool:\\n for i in range(len(numbers)):\\n for j in range(i+1, len(numbers)):\\n if abs(numbers[i] - numbers[j]) < threshold:\\n return True\\n return False\"}\u001b[0m\n",
+ "\n",
+ "\n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "lm.inspect_history(n=3)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "We can see `functional` first created an example value {\"code\": \"print('Hello, World!')\"}, which can be useful to boostrap the json generation.\n",
+ "After that it still failed to generate valid json.\n",
+ "It apparently decided to first repeat the schema, and then give the actual code \"as an example\"\n",
+ "The validator caught the error, and gave it as a \"Past Error\", which made the model finally output a valid output."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "We need a way to run code. This is actually super tricky to do right in python (see https://stackoverflow.com/questions/3068139/how-can-i-sandbox-python-in-pure-python), so we'll just YOLO and call \"exec\" with globals={}."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 197,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def execute_code(code, globals={}, locals=None):\n",
+ " try:\n",
+ " output = exec(code, globals, locals)\n",
+ " return output, None\n",
+ " except Exception as e:\n",
+ " return None, e"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Let's run the evaluator on all the \"canonical solutions\" from HumanEval to check that everything is working."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 198,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Score with the original model:\n",
+ "100.0\n"
+ ]
+ }
+ ],
+ "source": [
+ "from dspy import Example\n",
+ "\n",
+ "devset = [Example(\n",
+ " prompt=PythonCode(code=test['prompt']),\n",
+ " test=PythonCode(code=test['test']),\n",
+ " entry_point=test['entry_point'],\n",
+ " solution=PythonCode(code=test['prompt']+test['canonical_solution']),\n",
+ ").with_inputs('prompt', 'test', 'entry_point') for test in ds['test']]\n",
+ "\n",
+ "trainset = devset[:40]\n",
+ "testset = devset[40:]\n",
+ "\n",
+ "def metric(example, pred, trace=None):\n",
+ " if pred.solution.code is None:\n",
+ " return 0\n",
+ " _output, error = execute_code(\n",
+ " \"from typing import List\\n\"\n",
+ " + f\"{pred.solution.code}\\n\"\n",
+ " + f\"{example.test.code}\\n\"\n",
+ " + f\"check({example.entry_point})\"\n",
+ " )\n",
+ " return int(error is None)\n",
+ "\n",
+ "print(\"Score with the original model:\")\n",
+ "print(100 * sum(metric(example, example) for example in testset) / len(testset))\n",
+ "\n",
+ "for example in devset:\n",
+ " if not metric(example, example):\n",
+ " print(\"Bad example:\")\n",
+ " code = (\n",
+ " \"from typing import List\\n\"\n",
+ " + f\"{example.solution.code}\\n\"\n",
+ " + f\"{example.test.code}\\n\"\n",
+ " + f\"check({example.entry_point})\"\n",
+ " )\n",
+ " print(code)\n",
+ " output, error = execute_code(code)\n",
+ " print(f\"{output=}\")\n",
+ " print(f\"{error=}\")\n",
+ " break\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Now test our program."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 199,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Error for example in dev set: \t\t Too many retries\n",
+ "Error for example in dev set: \t\t Too many retries\n",
+ "Error for example in dev set: \t\t Too many retries\n",
+ "Error for example in dev set: \t\t Too many retries\n",
+ "Error for example in dev set: \t\t Too many retries\n",
+ "Error for example in dev set: \t\t Too many retries\n",
+ "Error for example in dev set: \t\t Too many retries\n",
+ "Error for example in dev set: \t\t Too many retries\n",
+ "Error for example in dev set: \t\t Too many retries\n",
+ "Error for example in dev set: \t\t Too many retries\n",
+ "Error for example in dev set: \t\t Too many retries\n",
+ "Error for example in dev set: \t\t Too many retries\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 70.0 / 124 (56.5): 100%|██████████| 124/124 [00:00<00:00, 1781.00it/s]"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 70.0 / 124 (56.5%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "\n",
+ "/Users/ahle/repos/dspy/dspy/evaluate/evaluate.py:143: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n",
+ " \n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<style type=\"text/css\">\n",
+ "#T_2a332 th {\n",
+ " text-align: left;\n",
+ "}\n",
+ "#T_2a332 td {\n",
+ " text-align: left;\n",
+ "}\n",
+ "#T_2a332_row0_col0, #T_2a332_row0_col1, #T_2a332_row0_col2, #T_2a332_row0_col3, #T_2a332_row0_col4, #T_2a332_row0_col5, #T_2a332_row0_col6, #T_2a332_row0_col7, #T_2a332_row1_col0, #T_2a332_row1_col1, #T_2a332_row1_col2, #T_2a332_row1_col3, #T_2a332_row1_col4, #T_2a332_row1_col5, #T_2a332_row1_col6, #T_2a332_row1_col7, #T_2a332_row2_col0, #T_2a332_row2_col1, #T_2a332_row2_col2, #T_2a332_row2_col3, #T_2a332_row2_col4, #T_2a332_row2_col5, #T_2a332_row2_col6, #T_2a332_row2_col7, #T_2a332_row3_col0, #T_2a332_row3_col1, #T_2a332_row3_col2, #T_2a332_row3_col3, #T_2a332_row3_col4, #T_2a332_row3_col5, #T_2a332_row3_col6, #T_2a332_row3_col7, #T_2a332_row4_col0, #T_2a332_row4_col1, #T_2a332_row4_col2, #T_2a332_row4_col3, #T_2a332_row4_col4, #T_2a332_row4_col5, #T_2a332_row4_col6, #T_2a332_row4_col7 {\n",
+ " text-align: left;\n",
+ " white-space: pre-wrap;\n",
+ " word-wrap: break-word;\n",
+ " max-width: 400px;\n",
+ "}\n",
+ "</style>\n",
+ "<table id=\"T_2a332\">\n",
+ " <thead>\n",
+ " <tr>\n",
+ " <th class=\"blank level0\" >&nbsp;</th>\n",
+ " <th id=\"T_2a332_level0_col0\" class=\"col_heading level0 col0\" >prompt</th>\n",
+ " <th id=\"T_2a332_level0_col1\" class=\"col_heading level0 col1\" >test</th>\n",
+ " <th id=\"T_2a332_level0_col2\" class=\"col_heading level0 col2\" >entry_point</th>\n",
+ " <th id=\"T_2a332_level0_col3\" class=\"col_heading level0 col3\" >example_solution</th>\n",
+ " <th id=\"T_2a332_level0_col4\" class=\"col_heading level0 col4\" >reasoning</th>\n",
+ " <th id=\"T_2a332_level0_col5\" class=\"col_heading level0 col5\" >pred_solution</th>\n",
+ " <th id=\"T_2a332_level0_col6\" class=\"col_heading level0 col6\" >metric</th>\n",
+ " <th id=\"T_2a332_level0_col7\" class=\"col_heading level0 col7\" >solution</th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th id=\"T_2a332_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+ " <td id=\"T_2a332_row0_col0\" class=\"data row0 col0\" >code='\\n\\ndef triples_sum_to_zero(l: list):\\n \"\"\"\\n triples_sum_to_zero takes a list of integers as an input.\\n it returns True if there are three distinct elements in the list...</td>\n",
+ " <td id=\"T_2a332_row0_col1\" class=\"data row0 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate([1, 3, 5, 0]) == False\\n assert candidate([1, 3, 5, -1]) == False\\n assert candidate([1, 3, -2, 1]) == True\\n...</td>\n",
+ " <td id=\"T_2a332_row0_col2\" class=\"data row0 col2\" >triples_sum_to_zero</td>\n",
+ " <td id=\"T_2a332_row0_col3\" class=\"data row0 col3\" >code='\\n\\ndef triples_sum_to_zero(l: list):\\n \"\"\"\\n triples_sum_to_zero takes a list of integers as an input.\\n it returns True if there are three distinct elements in the list...</td>\n",
+ " <td id=\"T_2a332_row0_col4\" class=\"data row0 col4\" >produce the solution. We need to ensure that the JSON object we provide is valid and follows the specified schema. The error message indicates that...</td>\n",
+ " <td id=\"T_2a332_row0_col5\" class=\"data row0 col5\" >code='def triples_sum_to_zero(l: list):\\n for i in range(len(l)):\\n for j in range(i+1, len(l)):\\n for k in range(j+1, len(l)):\\n if l[i] + l[j] + l[k] == 0:\\n...</td>\n",
+ " <td id=\"T_2a332_row0_col6\" class=\"data row0 col6\" >1.0</td>\n",
+ " <td id=\"T_2a332_row0_col7\" class=\"data row0 col7\" >nan</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th id=\"T_2a332_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
+ " <td id=\"T_2a332_row1_col0\" class=\"data row1 col0\" >code='\\n\\ndef car_race_collision(n: int):\\n \"\"\"\\n Imagine a road that\\'s a perfectly straight infinitely long line.\\n n cars are driving left to right; simultaneously, a different set...</td>\n",
+ " <td id=\"T_2a332_row1_col1\" class=\"data row1 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate(2) == 4\\n assert candidate(3) == 9\\n assert candidate(4) == 16\\n assert candidate(8) == 64\\n assert candidate(10) == 100\\n\\n'</td>\n",
+ " <td id=\"T_2a332_row1_col2\" class=\"data row1 col2\" >car_race_collision</td>\n",
+ " <td id=\"T_2a332_row1_col3\" class=\"data row1 col3\" >code='\\n\\ndef car_race_collision(n: int):\\n \"\"\"\\n Imagine a road that\\'s a perfectly straight infinitely long line.\\n n cars are driving left to right; simultaneously, a different set...</td>\n",
+ " <td id=\"T_2a332_row1_col4\" class=\"data row1 col4\" >produce the solution. We need to ensure that the code provided is valid JSON format. The error message indicates that there are trailing characters at...</td>\n",
+ " <td id=\"T_2a332_row1_col5\" class=\"data row1 col5\" >code='def car_race_collision(n: int):\\n return n ** 2'</td>\n",
+ " <td id=\"T_2a332_row1_col6\" class=\"data row1 col6\" >1.0</td>\n",
+ " <td id=\"T_2a332_row1_col7\" class=\"data row1 col7\" >nan</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th id=\"T_2a332_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
+ " <td id=\"T_2a332_row2_col0\" class=\"data row2 col0\" >code='\\n\\ndef incr_list(l: list):\\n \"\"\"Return list with elements incremented by 1.\\n >>> incr_list([1, 2, 3])\\n [2, 3, 4]\\n >>> incr_list([5, 3, 5, 2, 3, 3, 9,...</td>\n",
+ " <td id=\"T_2a332_row2_col1\" class=\"data row2 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate([]) == []\\n assert candidate([3, 2, 1]) == [4, 3, 2]\\n assert candidate([5, 2, 5, 2, 3, 3, 9, 0,...</td>\n",
+ " <td id=\"T_2a332_row2_col2\" class=\"data row2 col2\" >incr_list</td>\n",
+ " <td id=\"T_2a332_row2_col3\" class=\"data row2 col3\" >code='\\n\\ndef incr_list(l: list):\\n \"\"\"Return list with elements incremented by 1.\\n >>> incr_list([1, 2, 3])\\n [2, 3, 4]\\n >>> incr_list([5, 3, 5, 2, 3, 3, 9,...</td>\n",
+ " <td id=\"T_2a332_row2_col4\" class=\"data row2 col4\" >produce the solution. We need to ensure that the JSON object we provide follows the correct format and does not contain any trailing characters that...</td>\n",
+ " <td id=\"T_2a332_row2_col5\" class=\"data row2 col5\" >code='def incr_list(l: list):\\n return [x + 1 for x in l]'</td>\n",
+ " <td id=\"T_2a332_row2_col6\" class=\"data row2 col6\" >1.0</td>\n",
+ " <td id=\"T_2a332_row2_col7\" class=\"data row2 col7\" >nan</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th id=\"T_2a332_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
+ " <td id=\"T_2a332_row3_col0\" class=\"data row3 col0\" >code='\\n\\ndef pairs_sum_to_zero(l):\\n \"\"\"\\n pairs_sum_to_zero takes a list of integers as an input.\\n it returns True if there are two distinct elements in the list that\\n...</td>\n",
+ " <td id=\"T_2a332_row3_col1\" class=\"data row3 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate([1, 3, 5, 0]) == False\\n assert candidate([1, 3, -2, 1]) == False\\n assert candidate([1, 2, 3, 7]) == False\\n...</td>\n",
+ " <td id=\"T_2a332_row3_col2\" class=\"data row3 col2\" >pairs_sum_to_zero</td>\n",
+ " <td id=\"T_2a332_row3_col3\" class=\"data row3 col3\" >code='\\n\\ndef pairs_sum_to_zero(l):\\n \"\"\"\\n pairs_sum_to_zero takes a list of integers as an input.\\n it returns True if there are two distinct elements in the list that\\n...</td>\n",
+ " <td id=\"T_2a332_row3_col4\" class=\"data row3 col4\" >solve this issue. The error message indicates that there are trailing characters in the JSON object provided. To fix this, we need to ensure that...</td>\n",
+ " <td id=\"T_2a332_row3_col5\" class=\"data row3 col5\" >code='def pairs_sum_to_zero(l):\\n for i in range(len(l)):\\n for j in range(i+1, len(l)):\\n if l[i] + l[j] == 0:\\n return True\\n return False'</td>\n",
+ " <td id=\"T_2a332_row3_col6\" class=\"data row3 col6\" >1.0</td>\n",
+ " <td id=\"T_2a332_row3_col7\" class=\"data row3 col7\" >nan</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th id=\"T_2a332_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",
+ " <td id=\"T_2a332_row4_col0\" class=\"data row4 col0\" >code='\\n\\ndef change_base(x: int, base: int):\\n \"\"\"Change numerical base of input number x to base.\\n return string representation after the conversion.\\n base numbers are less than...</td>\n",
+ " <td id=\"T_2a332_row4_col1\" class=\"data row4 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate(8, 3) == \"22\"\\n assert candidate(9, 3) == \"100\"\\n assert candidate(234, 2) == \"11101010\"\\n assert candidate(16, 2) == \"10000\"\\n assert...</td>\n",
+ " <td id=\"T_2a332_row4_col2\" class=\"data row4 col2\" >change_base</td>\n",
+ " <td id=\"T_2a332_row4_col3\" class=\"data row4 col3\" >code='\\n\\ndef change_base(x: int, base: int):\\n \"\"\"Change numerical base of input number x to base.\\n return string representation after the conversion.\\n base numbers are less than...</td>\n",
+ " <td id=\"T_2a332_row4_col4\" class=\"data row4 col4\" >produce the solution. We need to ensure that the JSON object provided follows the correct format specified in the schema. The error message indicates that...</td>\n",
+ " <td id=\"T_2a332_row4_col5\" class=\"data row4 col5\" >code=\"def change_base(x: int, base: int):\\n result = ''\\n while x > 0:\\n result = str(x % base) + result\\n x //= base\\n return result\"</td>\n",
+ " <td id=\"T_2a332_row4_col6\" class=\"data row4 col6\" >1.0</td>\n",
+ " <td id=\"T_2a332_row4_col7\" class=\"data row4 col7\" >nan</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n"
+ ],
+ "text/plain": [
+ "<pandas.io.formats.style.Styler at 0x15f344710>"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ " <div style='\n",
+ " text-align: center; \n",
+ " font-size: 16px; \n",
+ " font-weight: bold; \n",
+ " color: #555; \n",
+ " margin: 10px 0;'>\n",
+ " ... 119 more rows not displayed ...\n",
+ " </div>\n",
+ " "
+ ],
+ "text/plain": [
+ "<IPython.core.display.HTML object>"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "56.45\n"
+ ]
+ }
+ ],
+ "source": [
+ "from dspy.evaluate.evaluate import Evaluate\n",
+ "evaluator = Evaluate(\n",
+ " devset=testset, num_threads=30, display_progress=True, display_table=5, max_errors=100,\n",
+ ")\n",
+ "res = evaluator(predictor, metric)\n",
+ "print(res)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Let's try to optimize it a bit"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 200,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Compiling...\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ " 12%|█▎ | 5/40 [00:00<00:00, 237.80it/s]"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Bootstrapped 4 full traces after 6 examples in round 0.\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "from dspy.teleprompt.bootstrap import BootstrapFewShot\n",
+ "\n",
+ "print(\"Compiling...\")\n",
+ "compiled = BootstrapFewShot(\n",
+ " metric=metric\n",
+ ").compile(\n",
+ " predictor,\n",
+ " trainset=trainset,\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Finally evaluate the trained model"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 201,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Evaluating...\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 84 / 124 (67.7): 100%|██████████| 124/124 [00:00<00:00, 2265.09it/s]"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 84 / 124 (67.7%)\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<style type=\"text/css\">\n",
+ "#T_729fd th {\n",
+ " text-align: left;\n",
+ "}\n",
+ "#T_729fd td {\n",
+ " text-align: left;\n",
+ "}\n",
+ "#T_729fd_row0_col0, #T_729fd_row0_col1, #T_729fd_row0_col2, #T_729fd_row0_col3, #T_729fd_row0_col4, #T_729fd_row0_col5, #T_729fd_row0_col6, #T_729fd_row1_col0, #T_729fd_row1_col1, #T_729fd_row1_col2, #T_729fd_row1_col3, #T_729fd_row1_col4, #T_729fd_row1_col5, #T_729fd_row1_col6, #T_729fd_row2_col0, #T_729fd_row2_col1, #T_729fd_row2_col2, #T_729fd_row2_col3, #T_729fd_row2_col4, #T_729fd_row2_col5, #T_729fd_row2_col6, #T_729fd_row3_col0, #T_729fd_row3_col1, #T_729fd_row3_col2, #T_729fd_row3_col3, #T_729fd_row3_col4, #T_729fd_row3_col5, #T_729fd_row3_col6, #T_729fd_row4_col0, #T_729fd_row4_col1, #T_729fd_row4_col2, #T_729fd_row4_col3, #T_729fd_row4_col4, #T_729fd_row4_col5, #T_729fd_row4_col6 {\n",
+ " text-align: left;\n",
+ " white-space: pre-wrap;\n",
+ " word-wrap: break-word;\n",
+ " max-width: 400px;\n",
+ "}\n",
+ "</style>\n",
+ "<table id=\"T_729fd\">\n",
+ " <thead>\n",
+ " <tr>\n",
+ " <th class=\"blank level0\" >&nbsp;</th>\n",
+ " <th id=\"T_729fd_level0_col0\" class=\"col_heading level0 col0\" >prompt</th>\n",
+ " <th id=\"T_729fd_level0_col1\" class=\"col_heading level0 col1\" >test</th>\n",
+ " <th id=\"T_729fd_level0_col2\" class=\"col_heading level0 col2\" >entry_point</th>\n",
+ " <th id=\"T_729fd_level0_col3\" class=\"col_heading level0 col3\" >example_solution</th>\n",
+ " <th id=\"T_729fd_level0_col4\" class=\"col_heading level0 col4\" >reasoning</th>\n",
+ " <th id=\"T_729fd_level0_col5\" class=\"col_heading level0 col5\" >pred_solution</th>\n",
+ " <th id=\"T_729fd_level0_col6\" class=\"col_heading level0 col6\" >metric</th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th id=\"T_729fd_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+ " <td id=\"T_729fd_row0_col0\" class=\"data row0 col0\" >code='\\n\\ndef triples_sum_to_zero(l: list):\\n \"\"\"\\n triples_sum_to_zero takes a list of integers as an input.\\n it returns True if there are three distinct elements in the list...</td>\n",
+ " <td id=\"T_729fd_row0_col1\" class=\"data row0 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate([1, 3, 5, 0]) == False\\n assert candidate([1, 3, 5, -1]) == False\\n assert candidate([1, 3, -2, 1]) == True\\n...</td>\n",
+ " <td id=\"T_729fd_row0_col2\" class=\"data row0 col2\" >triples_sum_to_zero</td>\n",
+ " <td id=\"T_729fd_row0_col3\" class=\"data row0 col3\" >code='\\n\\ndef triples_sum_to_zero(l: list):\\n \"\"\"\\n triples_sum_to_zero takes a list of integers as an input.\\n it returns True if there are three distinct elements in the list...</td>\n",
+ " <td id=\"T_729fd_row0_col4\" class=\"data row0 col4\" >produce the solution. We need to find three distinct elements in the list that sum to zero. One way to approach this is to use...</td>\n",
+ " <td id=\"T_729fd_row0_col5\" class=\"data row0 col5\" >code='\\n\\ndef triples_sum_to_zero(l: list):\\n \"\"\"\\n triples_sum_to_zero takes a list of integers as an input.\\n it returns True if there are three distinct elements in the list...</td>\n",
+ " <td id=\"T_729fd_row0_col6\" class=\"data row0 col6\" >1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th id=\"T_729fd_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
+ " <td id=\"T_729fd_row1_col0\" class=\"data row1 col0\" >code='\\n\\ndef car_race_collision(n: int):\\n \"\"\"\\n Imagine a road that\\'s a perfectly straight infinitely long line.\\n n cars are driving left to right; simultaneously, a different set...</td>\n",
+ " <td id=\"T_729fd_row1_col1\" class=\"data row1 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate(2) == 4\\n assert candidate(3) == 9\\n assert candidate(4) == 16\\n assert candidate(8) == 64\\n assert candidate(10) == 100\\n\\n'</td>\n",
+ " <td id=\"T_729fd_row1_col2\" class=\"data row1 col2\" >car_race_collision</td>\n",
+ " <td id=\"T_729fd_row1_col3\" class=\"data row1 col3\" >code='\\n\\ndef car_race_collision(n: int):\\n \"\"\"\\n Imagine a road that\\'s a perfectly straight infinitely long line.\\n n cars are driving left to right; simultaneously, a different set...</td>\n",
+ " <td id=\"T_729fd_row1_col4\" class=\"data row1 col4\" >produce the solution. We can visualize the cars moving towards each other on the road and calculate the number of collisions that would occur when...</td>\n",
+ " <td id=\"T_729fd_row1_col5\" class=\"data row1 col5\" >code='\\n\\ndef car_race_collision(n: int):\\n \"\"\"\\n Imagine a road that\\'s a perfectly straight infinitely long line.\\n n cars are driving left to right; simultaneously, a different set...</td>\n",
+ " <td id=\"T_729fd_row1_col6\" class=\"data row1 col6\" >1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th id=\"T_729fd_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
+ " <td id=\"T_729fd_row2_col0\" class=\"data row2 col0\" >code='\\n\\ndef incr_list(l: list):\\n \"\"\"Return list with elements incremented by 1.\\n >>> incr_list([1, 2, 3])\\n [2, 3, 4]\\n >>> incr_list([5, 3, 5, 2, 3, 3, 9,...</td>\n",
+ " <td id=\"T_729fd_row2_col1\" class=\"data row2 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate([]) == []\\n assert candidate([3, 2, 1]) == [4, 3, 2]\\n assert candidate([5, 2, 5, 2, 3, 3, 9, 0,...</td>\n",
+ " <td id=\"T_729fd_row2_col2\" class=\"data row2 col2\" >incr_list</td>\n",
+ " <td id=\"T_729fd_row2_col3\" class=\"data row2 col3\" >code='\\n\\ndef incr_list(l: list):\\n \"\"\"Return list with elements incremented by 1.\\n >>> incr_list([1, 2, 3])\\n [2, 3, 4]\\n >>> incr_list([5, 3, 5, 2, 3, 3, 9,...</td>\n",
+ " <td id=\"T_729fd_row2_col4\" class=\"data row2 col4\" >produce the solution. We need to iterate through the list and increment each element by 1.</td>\n",
+ " <td id=\"T_729fd_row2_col5\" class=\"data row2 col5\" >code='\\n\\ndef incr_list(l: list):\\n \"\"\"Return list with elements incremented by 1.\\n >>> incr_list([1, 2, 3])\\n [2, 3, 4]\\n >>> incr_list([5, 3, 5, 2, 3, 3, 9,...</td>\n",
+ " <td id=\"T_729fd_row2_col6\" class=\"data row2 col6\" >1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th id=\"T_729fd_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
+ " <td id=\"T_729fd_row3_col0\" class=\"data row3 col0\" >code='\\n\\ndef pairs_sum_to_zero(l):\\n \"\"\"\\n pairs_sum_to_zero takes a list of integers as an input.\\n it returns True if there are two distinct elements in the list that\\n...</td>\n",
+ " <td id=\"T_729fd_row3_col1\" class=\"data row3 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate([1, 3, 5, 0]) == False\\n assert candidate([1, 3, -2, 1]) == False\\n assert candidate([1, 2, 3, 7]) == False\\n...</td>\n",
+ " <td id=\"T_729fd_row3_col2\" class=\"data row3 col2\" >pairs_sum_to_zero</td>\n",
+ " <td id=\"T_729fd_row3_col3\" class=\"data row3 col3\" >code='\\n\\ndef pairs_sum_to_zero(l):\\n \"\"\"\\n pairs_sum_to_zero takes a list of integers as an input.\\n it returns True if there are two distinct elements in the list that\\n...</td>\n",
+ " <td id=\"T_729fd_row3_col4\" class=\"data row3 col4\" >produce the solution. We need to iterate through the list and check if there are any two distinct elements that sum up to zero.</td>\n",
+ " <td id=\"T_729fd_row3_col5\" class=\"data row3 col5\" >code='\\n\\ndef pairs_sum_to_zero(l):\\n \"\"\"\\n pairs_sum_to_zero takes a list of integers as an input.\\n it returns True if there are two distinct elements in the list that\\n...</td>\n",
+ " <td id=\"T_729fd_row3_col6\" class=\"data row3 col6\" >1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th id=\"T_729fd_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",
+ " <td id=\"T_729fd_row4_col0\" class=\"data row4 col0\" >code='\\n\\ndef change_base(x: int, base: int):\\n \"\"\"Change numerical base of input number x to base.\\n return string representation after the conversion.\\n base numbers are less than...</td>\n",
+ " <td id=\"T_729fd_row4_col1\" class=\"data row4 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate(8, 3) == \"22\"\\n assert candidate(9, 3) == \"100\"\\n assert candidate(234, 2) == \"11101010\"\\n assert candidate(16, 2) == \"10000\"\\n assert...</td>\n",
+ " <td id=\"T_729fd_row4_col2\" class=\"data row4 col2\" >change_base</td>\n",
+ " <td id=\"T_729fd_row4_col3\" class=\"data row4 col3\" >code='\\n\\ndef change_base(x: int, base: int):\\n \"\"\"Change numerical base of input number x to base.\\n return string representation after the conversion.\\n base numbers are less than...</td>\n",
+ " <td id=\"T_729fd_row4_col4\" class=\"data row4 col4\" >produce the solution. We need to convert the input number `x` to the desired base `base`. To do this, we can repeatedly divide `x` by...</td>\n",
+ " <td id=\"T_729fd_row4_col5\" class=\"data row4 col5\" >code='\\n\\ndef change_base(x: int, base: int):\\n \"\"\"Change numerical base of input number x to base.\\n return string representation after the conversion.\\n base numbers are less than...</td>\n",
+ " <td id=\"T_729fd_row4_col6\" class=\"data row4 col6\" >1</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n"
+ ],
+ "text/plain": [
+ "<pandas.io.formats.style.Styler at 0x29b8fc2d0>"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ " <div style='\n",
+ " text-align: center; \n",
+ " font-size: 16px; \n",
+ " font-weight: bold; \n",
+ " color: #555; \n",
+ " margin: 10px 0;'>\n",
+ " ... 119 more rows not displayed ...\n",
+ " </div>\n",
+ " "
+ ],
+ "text/plain": [
+ "<IPython.core.display.HTML object>"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Compiled HumanEval score: 67.74\n"
+ ]
+ }
+ ],
+ "source": [
+ "print(\"Evaluating...\")\n",
+ "print(\n",
+ " \"Compiled HumanEval score:\",\n",
+ " evaluator(compiled, metric=metric),\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.11.7"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}