Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Use timeout for code evaluation

Thomas D Ahlee9204f45b460
diff --git a/dspy/teleprompt/random_search.py b/dspy/teleprompt/random_search.py
index 8605cd427..f5b8eecdd 100644
--- a/dspy/teleprompt/random_search.py
+++ b/dspy/teleprompt/random_search.py
@@ -93,7 +93,7 @@ class BootstrapFewShotWithRandomSearch(Teleprompter):
program2 = teleprompter.compile(student, teacher=teacher, trainset=trainset2)
evaluate = Evaluate(devset=self.valset, metric=self.metric, num_threads=self.num_threads,
- display_table=False, display_progress=True)
+ display_table=False, display_progress=True, max_errors=100)
score, subscores = evaluate(program2, return_all_scores=True)
diff --git a/examples/functional/functional.ipynb b/examples/functional/functional.ipynb
index 01d2c3088..611cc15c4 100644
--- a/examples/functional/functional.ipynb
+++ b/examples/functional/functional.ipynb
@@ -2,52 +2,9 @@
"cells": [
{
"cell_type": "code",
- "execution_count": 202,
+ "execution_count": null,
"metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "The autoreload extension is already loaded. To reload it, use:\n",
- " %reload_ext autoreload\n",
- "Requirement already satisfied: datasets in /opt/homebrew/lib/python3.11/site-packages (2.14.7)\n",
- "Requirement already satisfied: numpy>=1.17 in /opt/homebrew/lib/python3.11/site-packages (from datasets) (1.26.2)\n",
- "Requirement already satisfied: pyarrow>=8.0.0 in /opt/homebrew/lib/python3.11/site-packages (from datasets) (12.0.0)\n",
- "Requirement already satisfied: pyarrow-hotfix in /opt/homebrew/lib/python3.11/site-packages (from datasets) (0.6)\n",
- "Requirement already satisfied: dill<0.3.8,>=0.3.0 in /opt/homebrew/lib/python3.11/site-packages (from datasets) (0.3.6)\n",
- "Requirement already satisfied: pandas in /opt/homebrew/lib/python3.11/site-packages (from datasets) (2.1.4)\n",
- "Requirement already satisfied: requests>=2.19.0 in /opt/homebrew/lib/python3.11/site-packages (from datasets) (2.31.0)\n",
- "Requirement already satisfied: tqdm>=4.62.1 in /opt/homebrew/lib/python3.11/site-packages (from datasets) (4.66.1)\n",
- "Requirement already satisfied: xxhash in /opt/homebrew/lib/python3.11/site-packages (from datasets) (3.2.0)\n",
- "Requirement already satisfied: multiprocess in /opt/homebrew/lib/python3.11/site-packages (from datasets) (0.70.14)\n",
- "Requirement already satisfied: fsspec<=2023.10.0,>=2023.1.0 in /opt/homebrew/lib/python3.11/site-packages (from fsspec[http]<=2023.10.0,>=2023.1.0->datasets) (2023.10.0)\n",
- "Requirement already satisfied: aiohttp in /opt/homebrew/lib/python3.11/site-packages (from datasets) (3.9.1)\n",
- "Requirement already satisfied: huggingface-hub<1.0.0,>=0.14.0 in /opt/homebrew/lib/python3.11/site-packages (from datasets) (0.19.4)\n",
- "Requirement already satisfied: packaging in /opt/homebrew/lib/python3.11/site-packages (from datasets) (23.2)\n",
- "Requirement already satisfied: pyyaml>=5.1 in /opt/homebrew/lib/python3.11/site-packages (from datasets) (6.0.1)\n",
- "Requirement already satisfied: attrs>=17.3.0 in /opt/homebrew/lib/python3.11/site-packages (from aiohttp->datasets) (23.1.0)\n",
- "Requirement already satisfied: multidict<7.0,>=4.5 in /opt/homebrew/lib/python3.11/site-packages (from aiohttp->datasets) (6.0.4)\n",
- "Requirement already satisfied: yarl<2.0,>=1.0 in /opt/homebrew/lib/python3.11/site-packages (from aiohttp->datasets) (1.9.3)\n",
- "Requirement already satisfied: frozenlist>=1.1.1 in /opt/homebrew/lib/python3.11/site-packages (from aiohttp->datasets) (1.4.0)\n",
- "Requirement already satisfied: aiosignal>=1.1.2 in /opt/homebrew/lib/python3.11/site-packages (from aiohttp->datasets) (1.3.1)\n",
- "Requirement already satisfied: filelock in /opt/homebrew/lib/python3.11/site-packages (from huggingface-hub<1.0.0,>=0.14.0->datasets) (3.13.1)\n",
- "Requirement already satisfied: typing-extensions>=3.7.4.3 in /opt/homebrew/lib/python3.11/site-packages (from huggingface-hub<1.0.0,>=0.14.0->datasets) (4.8.0)\n",
- "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/homebrew/lib/python3.11/site-packages (from requests>=2.19.0->datasets) (3.3.2)\n",
- "Requirement already satisfied: idna<4,>=2.5 in /opt/homebrew/lib/python3.11/site-packages (from requests>=2.19.0->datasets) (3.6)\n",
- "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/homebrew/lib/python3.11/site-packages (from requests>=2.19.0->datasets) (2.1.0)\n",
- "Requirement already satisfied: certifi>=2017.4.17 in /opt/homebrew/lib/python3.11/site-packages (from requests>=2.19.0->datasets) (2023.11.17)\n",
- "Requirement already satisfied: python-dateutil>=2.8.2 in /opt/homebrew/lib/python3.11/site-packages (from pandas->datasets) (2.8.2)\n",
- "Requirement already satisfied: pytz>=2020.1 in /opt/homebrew/lib/python3.11/site-packages (from pandas->datasets) (2023.3)\n",
- "Requirement already satisfied: tzdata>=2022.1 in /opt/homebrew/lib/python3.11/site-packages (from pandas->datasets) (2023.3)\n",
- "Requirement already satisfied: six>=1.5 in /opt/homebrew/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.16.0)\n",
- "\n",
- "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.3.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.0\u001b[0m\n",
- "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3.11 -m pip install --upgrade pip\u001b[0m\n",
- "Note: you may need to restart the kernel to use updated packages.\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"%load_ext autoreload\n",
"%autoreload 2\n",
@@ -63,9 +20,17 @@
},
{
"cell_type": "code",
- "execution_count": 193,
+ "execution_count": 2,
"metadata": {},
"outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/opt/homebrew/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+ " from .autonotebook import tqdm as notebook_tqdm\n"
+ ]
+ },
{
"data": {
"text/plain": [
@@ -76,7 +41,7 @@
" 'entry_point': 'has_close_elements'}"
]
},
- "execution_count": 193,
+ "execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
@@ -96,7 +61,7 @@
},
{
"cell_type": "code",
- "execution_count": 194,
+ "execution_count": 3,
"metadata": {},
"outputs": [
{
@@ -128,7 +93,7 @@
},
{
"cell_type": "code",
- "execution_count": 195,
+ "execution_count": 4,
"metadata": {},
"outputs": [
{
@@ -189,7 +154,7 @@
},
{
"cell_type": "code",
- "execution_count": 196,
+ "execution_count": 5,
"metadata": {},
"outputs": [
{
@@ -314,16 +279,23 @@
},
{
"cell_type": "code",
- "execution_count": 197,
+ "execution_count": 6,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "3\n",
+ "None\n",
+ "AssertionError()\n"
+ ]
+ }
+ ],
"source": [
- "def execute_code(code, globals={}, locals=None):\n",
- " try:\n",
- " output = exec(code, globals, locals)\n",
- " return output, None\n",
- " except Exception as e:\n",
- " return None, e"
+ "from repl import execute_code\n",
+ "print(execute_code(\"print(3)\"))\n",
+ "print(execute_code(\"assert False\"))"
]
},
{
@@ -335,7 +307,7 @@
},
{
"cell_type": "code",
- "execution_count": 198,
+ "execution_count": 7,
"metadata": {},
"outputs": [
{
@@ -363,7 +335,7 @@
"def metric(example, pred, trace=None):\n",
" if pred.solution.code is None:\n",
" return 0\n",
- " _output, error = execute_code(\n",
+ " error = execute_code(\n",
" \"from typing import List\\n\"\n",
" + f\"{pred.solution.code}\\n\"\n",
" + f\"{example.test.code}\\n\"\n",
@@ -384,8 +356,7 @@
" + f\"check({example.entry_point})\"\n",
" )\n",
" print(code)\n",
- " output, error = execute_code(code)\n",
- " print(f\"{output=}\")\n",
+ " error = execute_code(code)\n",
" print(f\"{error=}\")\n",
" break\n"
]
@@ -399,23 +370,111 @@
},
{
"cell_type": "code",
- "execution_count": 199,
+ "execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
- "Error for example in dev set: \t\t Too many retries\n",
- "Error for example in dev set: \t\t Too many retries\n",
- "Error for example in dev set: \t\t Too many retries\n",
- "Error for example in dev set: \t\t Too many retries\n",
- "Error for example in dev set: \t\t Too many retries\n",
- "Error for example in dev set: \t\t Too many retries\n",
- "Error for example in dev set: \t\t Too many retries\n",
- "Error for example in dev set: \t\t Too many retries\n",
- "Error for example in dev set: \t\t Too many retries\n",
- "Error for example in dev set: \t\t Too many retries\n",
+ "Error for example in dev set: \t\t Too many retries\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 0.0 / 1 (0.0): 0%| | 0/124 [00:00<?, ?it/s]"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Error for example in dev set: \t\t Too many retries\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 0.0 / 2 (0.0): 1%| | 1/124 [00:00<00:01, 66.59it/s]"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Error for example in dev set: \t\t Too many retries\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 0.0 / 3 (0.0): 2%|▏ | 2/124 [00:00<00:01, 81.27it/s]"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Error for example in dev set: \t\t Too many retries\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 7.0 / 14 (50.0): 10%|█ | 13/124 [00:00<00:02, 49.54it/s]"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Error for example in dev set: \t\t Too many retries\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 14.0 / 21 (66.7): 16%|█▌ | 20/124 [00:00<00:02, 49.54it/s]"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Error for example in dev set: \t\t Too many retries\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 17.0 / 24 (70.8): 19%|█▊ | 23/124 [00:00<00:02, 49.54it/s]"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Error for example in dev set: \t\t Too many retries\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 29.0 / 41 (70.7): 32%|███▏ | 40/124 [00:00<00:01, 69.00it/s]"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
"Error for example in dev set: \t\t Too many retries\n",
"Error for example in dev set: \t\t Too many retries\n"
]
@@ -424,117 +483,166 @@
"name": "stderr",
"output_type": "stream",
"text": [
- "Average Metric: 70.0 / 124 (56.5): 100%|██████████| 124/124 [00:00<00:00, 1781.00it/s]"
+ "Average Metric: 34.0 / 55 (61.8): 44%|████▎ | 54/124 [00:00<00:01, 68.27it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
- "Average Metric: 70.0 / 124 (56.5%)\n"
+ "Error for example in dev set: \t\t Too many retries\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
- "\n",
+ "Average Metric: 38.0 / 67 (56.7): 54%|█████▍ | 67/124 [00:01<00:00, 60.79it/s]"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Error for example in dev set: \t\t Too many retries"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 40.0 / 70 (57.1): 56%|█████▌ | 69/124 [00:01<00:00, 60.79it/s]"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 41.0 / 73 (56.2): 58%|█████▊ | 72/124 [00:01<00:00, 60.79it/s]"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Error for example in dev set: \t\t Too many retries\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 62.0 / 124 (50.0): 100%|██████████| 124/124 [00:01<00:00, 92.27it/s]\n",
"/Users/ahle/repos/dspy/dspy/evaluate/evaluate.py:143: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n",
- " \n"
+ " df = df.applymap(truncate_cell)\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 62.0 / 124 (50.0%)\n"
]
},
{
"data": {
"text/html": [
"<style type=\"text/css\">\n",
- "#T_2a332 th {\n",
+ "#T_3b532 th {\n",
" text-align: left;\n",
"}\n",
- "#T_2a332 td {\n",
+ "#T_3b532 td {\n",
" text-align: left;\n",
"}\n",
- "#T_2a332_row0_col0, #T_2a332_row0_col1, #T_2a332_row0_col2, #T_2a332_row0_col3, #T_2a332_row0_col4, #T_2a332_row0_col5, #T_2a332_row0_col6, #T_2a332_row0_col7, #T_2a332_row1_col0, #T_2a332_row1_col1, #T_2a332_row1_col2, #T_2a332_row1_col3, #T_2a332_row1_col4, #T_2a332_row1_col5, #T_2a332_row1_col6, #T_2a332_row1_col7, #T_2a332_row2_col0, #T_2a332_row2_col1, #T_2a332_row2_col2, #T_2a332_row2_col3, #T_2a332_row2_col4, #T_2a332_row2_col5, #T_2a332_row2_col6, #T_2a332_row2_col7, #T_2a332_row3_col0, #T_2a332_row3_col1, #T_2a332_row3_col2, #T_2a332_row3_col3, #T_2a332_row3_col4, #T_2a332_row3_col5, #T_2a332_row3_col6, #T_2a332_row3_col7, #T_2a332_row4_col0, #T_2a332_row4_col1, #T_2a332_row4_col2, #T_2a332_row4_col3, #T_2a332_row4_col4, #T_2a332_row4_col5, #T_2a332_row4_col6, #T_2a332_row4_col7 {\n",
+ "#T_3b532_row0_col0, #T_3b532_row0_col1, #T_3b532_row0_col2, #T_3b532_row0_col3, #T_3b532_row0_col4, #T_3b532_row0_col5, #T_3b532_row0_col6, #T_3b532_row0_col7, #T_3b532_row1_col0, #T_3b532_row1_col1, #T_3b532_row1_col2, #T_3b532_row1_col3, #T_3b532_row1_col4, #T_3b532_row1_col5, #T_3b532_row1_col6, #T_3b532_row1_col7, #T_3b532_row2_col0, #T_3b532_row2_col1, #T_3b532_row2_col2, #T_3b532_row2_col3, #T_3b532_row2_col4, #T_3b532_row2_col5, #T_3b532_row2_col6, #T_3b532_row2_col7, #T_3b532_row3_col0, #T_3b532_row3_col1, #T_3b532_row3_col2, #T_3b532_row3_col3, #T_3b532_row3_col4, #T_3b532_row3_col5, #T_3b532_row3_col6, #T_3b532_row3_col7, #T_3b532_row4_col0, #T_3b532_row4_col1, #T_3b532_row4_col2, #T_3b532_row4_col3, #T_3b532_row4_col4, #T_3b532_row4_col5, #T_3b532_row4_col6, #T_3b532_row4_col7 {\n",
" text-align: left;\n",
" white-space: pre-wrap;\n",
" word-wrap: break-word;\n",
" max-width: 400px;\n",
"}\n",
"</style>\n",
- "<table id=\"T_2a332\">\n",
+ "<table id=\"T_3b532\">\n",
" <thead>\n",
" <tr>\n",
" <th class=\"blank level0\" >&nbsp;</th>\n",
- " <th id=\"T_2a332_level0_col0\" class=\"col_heading level0 col0\" >prompt</th>\n",
- " <th id=\"T_2a332_level0_col1\" class=\"col_heading level0 col1\" >test</th>\n",
- " <th id=\"T_2a332_level0_col2\" class=\"col_heading level0 col2\" >entry_point</th>\n",
- " <th id=\"T_2a332_level0_col3\" class=\"col_heading level0 col3\" >example_solution</th>\n",
- " <th id=\"T_2a332_level0_col4\" class=\"col_heading level0 col4\" >reasoning</th>\n",
- " <th id=\"T_2a332_level0_col5\" class=\"col_heading level0 col5\" >pred_solution</th>\n",
- " <th id=\"T_2a332_level0_col6\" class=\"col_heading level0 col6\" >metric</th>\n",
- " <th id=\"T_2a332_level0_col7\" class=\"col_heading level0 col7\" >solution</th>\n",
+ " <th id=\"T_3b532_level0_col0\" class=\"col_heading level0 col0\" >prompt</th>\n",
+ " <th id=\"T_3b532_level0_col1\" class=\"col_heading level0 col1\" >test</th>\n",
+ " <th id=\"T_3b532_level0_col2\" class=\"col_heading level0 col2\" >entry_point</th>\n",
+ " <th id=\"T_3b532_level0_col3\" class=\"col_heading level0 col3\" >example_solution</th>\n",
+ " <th id=\"T_3b532_level0_col4\" class=\"col_heading level0 col4\" >reasoning</th>\n",
+ " <th id=\"T_3b532_level0_col5\" class=\"col_heading level0 col5\" >pred_solution</th>\n",
+ " <th id=\"T_3b532_level0_col6\" class=\"col_heading level0 col6\" >metric</th>\n",
+ " <th id=\"T_3b532_level0_col7\" class=\"col_heading level0 col7\" >solution</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
- " <th id=\"T_2a332_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
- " <td id=\"T_2a332_row0_col0\" class=\"data row0 col0\" >code='\\n\\ndef triples_sum_to_zero(l: list):\\n \"\"\"\\n triples_sum_to_zero takes a list of integers as an input.\\n it returns True if there are three distinct elements in the list...</td>\n",
- " <td id=\"T_2a332_row0_col1\" class=\"data row0 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate([1, 3, 5, 0]) == False\\n assert candidate([1, 3, 5, -1]) == False\\n assert candidate([1, 3, -2, 1]) == True\\n...</td>\n",
- " <td id=\"T_2a332_row0_col2\" class=\"data row0 col2\" >triples_sum_to_zero</td>\n",
- " <td id=\"T_2a332_row0_col3\" class=\"data row0 col3\" >code='\\n\\ndef triples_sum_to_zero(l: list):\\n \"\"\"\\n triples_sum_to_zero takes a list of integers as an input.\\n it returns True if there are three distinct elements in the list...</td>\n",
- " <td id=\"T_2a332_row0_col4\" class=\"data row0 col4\" >produce the solution. We need to ensure that the JSON object we provide is valid and follows the specified schema. The error message indicates that...</td>\n",
- " <td id=\"T_2a332_row0_col5\" class=\"data row0 col5\" >code='def triples_sum_to_zero(l: list):\\n for i in range(len(l)):\\n for j in range(i+1, len(l)):\\n for k in range(j+1, len(l)):\\n if l[i] + l[j] + l[k] == 0:\\n...</td>\n",
- " <td id=\"T_2a332_row0_col6\" class=\"data row0 col6\" >1.0</td>\n",
- " <td id=\"T_2a332_row0_col7\" class=\"data row0 col7\" >nan</td>\n",
+ " <th id=\"T_3b532_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+ " <td id=\"T_3b532_row0_col0\" class=\"data row0 col0\" >code='\\n\\ndef triples_sum_to_zero(l: list):\\n \"\"\"\\n triples_sum_to_zero takes a list of integers as an input.\\n it returns True if there are three distinct elements in the list...</td>\n",
+ " <td id=\"T_3b532_row0_col1\" class=\"data row0 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate([1, 3, 5, 0]) == False\\n assert candidate([1, 3, 5, -1]) == False\\n assert candidate([1, 3, -2, 1]) == True\\n...</td>\n",
+ " <td id=\"T_3b532_row0_col2\" class=\"data row0 col2\" >triples_sum_to_zero</td>\n",
+ " <td id=\"T_3b532_row0_col3\" class=\"data row0 col3\" >code='\\n\\ndef triples_sum_to_zero(l: list):\\n \"\"\"\\n triples_sum_to_zero takes a list of integers as an input.\\n it returns True if there are three distinct elements in the list...</td>\n",
+ " <td id=\"T_3b532_row0_col4\" class=\"data row0 col4\" >produce the solution. We need to ensure that the JSON object we provide is valid and follows the specified schema. The error message indicates that...</td>\n",
+ " <td id=\"T_3b532_row0_col5\" class=\"data row0 col5\" >code='def triples_sum_to_zero(l: list):\\n for i in range(len(l)):\\n for j in range(i+1, len(l)):\\n for k in range(j+1, len(l)):\\n if l[i] + l[j] + l[k] == 0:\\n...</td>\n",
+ " <td id=\"T_3b532_row0_col6\" class=\"data row0 col6\" >0.0</td>\n",
+ " <td id=\"T_3b532_row0_col7\" class=\"data row0 col7\" >nan</td>\n",
" </tr>\n",
" <tr>\n",
- " <th id=\"T_2a332_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
- " <td id=\"T_2a332_row1_col0\" class=\"data row1 col0\" >code='\\n\\ndef car_race_collision(n: int):\\n \"\"\"\\n Imagine a road that\\'s a perfectly straight infinitely long line.\\n n cars are driving left to right; simultaneously, a different set...</td>\n",
- " <td id=\"T_2a332_row1_col1\" class=\"data row1 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate(2) == 4\\n assert candidate(3) == 9\\n assert candidate(4) == 16\\n assert candidate(8) == 64\\n assert candidate(10) == 100\\n\\n'</td>\n",
- " <td id=\"T_2a332_row1_col2\" class=\"data row1 col2\" >car_race_collision</td>\n",
- " <td id=\"T_2a332_row1_col3\" class=\"data row1 col3\" >code='\\n\\ndef car_race_collision(n: int):\\n \"\"\"\\n Imagine a road that\\'s a perfectly straight infinitely long line.\\n n cars are driving left to right; simultaneously, a different set...</td>\n",
- " <td id=\"T_2a332_row1_col4\" class=\"data row1 col4\" >produce the solution. We need to ensure that the code provided is valid JSON format. The error message indicates that there are trailing characters at...</td>\n",
- " <td id=\"T_2a332_row1_col5\" class=\"data row1 col5\" >code='def car_race_collision(n: int):\\n return n ** 2'</td>\n",
- " <td id=\"T_2a332_row1_col6\" class=\"data row1 col6\" >1.0</td>\n",
- " <td id=\"T_2a332_row1_col7\" class=\"data row1 col7\" >nan</td>\n",
+ " <th id=\"T_3b532_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
+ " <td id=\"T_3b532_row1_col0\" class=\"data row1 col0\" >code='\\n\\ndef car_race_collision(n: int):\\n \"\"\"\\n Imagine a road that\\'s a perfectly straight infinitely long line.\\n n cars are driving left to right; simultaneously, a different set...</td>\n",
+ " <td id=\"T_3b532_row1_col1\" class=\"data row1 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate(2) == 4\\n assert candidate(3) == 9\\n assert candidate(4) == 16\\n assert candidate(8) == 64\\n assert candidate(10) == 100\\n\\n'</td>\n",
+ " <td id=\"T_3b532_row1_col2\" class=\"data row1 col2\" >car_race_collision</td>\n",
+ " <td id=\"T_3b532_row1_col3\" class=\"data row1 col3\" >code='\\n\\ndef car_race_collision(n: int):\\n \"\"\"\\n Imagine a road that\\'s a perfectly straight infinitely long line.\\n n cars are driving left to right; simultaneously, a different set...</td>\n",
+ " <td id=\"T_3b532_row1_col4\" class=\"data row1 col4\" >produce the solution. We need to ensure that the code provided is valid JSON format. The error message indicates that there are trailing characters at...</td>\n",
+ " <td id=\"T_3b532_row1_col5\" class=\"data row1 col5\" >code='def car_race_collision(n: int):\\n return n ** 2'</td>\n",
+ " <td id=\"T_3b532_row1_col6\" class=\"data row1 col6\" >1.0</td>\n",
+ " <td id=\"T_3b532_row1_col7\" class=\"data row1 col7\" >nan</td>\n",
" </tr>\n",
" <tr>\n",
- " <th id=\"T_2a332_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
- " <td id=\"T_2a332_row2_col0\" class=\"data row2 col0\" >code='\\n\\ndef incr_list(l: list):\\n \"\"\"Return list with elements incremented by 1.\\n >>> incr_list([1, 2, 3])\\n [2, 3, 4]\\n >>> incr_list([5, 3, 5, 2, 3, 3, 9,...</td>\n",
- " <td id=\"T_2a332_row2_col1\" class=\"data row2 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate([]) == []\\n assert candidate([3, 2, 1]) == [4, 3, 2]\\n assert candidate([5, 2, 5, 2, 3, 3, 9, 0,...</td>\n",
- " <td id=\"T_2a332_row2_col2\" class=\"data row2 col2\" >incr_list</td>\n",
- " <td id=\"T_2a332_row2_col3\" class=\"data row2 col3\" >code='\\n\\ndef incr_list(l: list):\\n \"\"\"Return list with elements incremented by 1.\\n >>> incr_list([1, 2, 3])\\n [2, 3, 4]\\n >>> incr_list([5, 3, 5, 2, 3, 3, 9,...</td>\n",
- " <td id=\"T_2a332_row2_col4\" class=\"data row2 col4\" >produce the solution. We need to ensure that the JSON object we provide follows the correct format and does not contain any trailing characters that...</td>\n",
- " <td id=\"T_2a332_row2_col5\" class=\"data row2 col5\" >code='def incr_list(l: list):\\n return [x + 1 for x in l]'</td>\n",
- " <td id=\"T_2a332_row2_col6\" class=\"data row2 col6\" >1.0</td>\n",
- " <td id=\"T_2a332_row2_col7\" class=\"data row2 col7\" >nan</td>\n",
+ " <th id=\"T_3b532_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
+ " <td id=\"T_3b532_row2_col0\" class=\"data row2 col0\" >code='\\n\\ndef incr_list(l: list):\\n \"\"\"Return list with elements incremented by 1.\\n >>> incr_list([1, 2, 3])\\n [2, 3, 4]\\n >>> incr_list([5, 3, 5, 2, 3, 3, 9,...</td>\n",
+ " <td id=\"T_3b532_row2_col1\" class=\"data row2 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate([]) == []\\n assert candidate([3, 2, 1]) == [4, 3, 2]\\n assert candidate([5, 2, 5, 2, 3, 3, 9, 0,...</td>\n",
+ " <td id=\"T_3b532_row2_col2\" class=\"data row2 col2\" >incr_list</td>\n",
+ " <td id=\"T_3b532_row2_col3\" class=\"data row2 col3\" >code='\\n\\ndef incr_list(l: list):\\n \"\"\"Return list with elements incremented by 1.\\n >>> incr_list([1, 2, 3])\\n [2, 3, 4]\\n >>> incr_list([5, 3, 5, 2, 3, 3, 9,...</td>\n",
+ " <td id=\"T_3b532_row2_col4\" class=\"data row2 col4\" >produce the solution. We need to ensure that the JSON object we provide follows the correct format and does not contain any trailing characters that...</td>\n",
+ " <td id=\"T_3b532_row2_col5\" class=\"data row2 col5\" >code='def incr_list(l: list):\\n return [x + 1 for x in l]'</td>\n",
+ " <td id=\"T_3b532_row2_col6\" class=\"data row2 col6\" >1.0</td>\n",
+ " <td id=\"T_3b532_row2_col7\" class=\"data row2 col7\" >nan</td>\n",
" </tr>\n",
" <tr>\n",
- " <th id=\"T_2a332_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
- " <td id=\"T_2a332_row3_col0\" class=\"data row3 col0\" >code='\\n\\ndef pairs_sum_to_zero(l):\\n \"\"\"\\n pairs_sum_to_zero takes a list of integers as an input.\\n it returns True if there are two distinct elements in the list that\\n...</td>\n",
- " <td id=\"T_2a332_row3_col1\" class=\"data row3 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate([1, 3, 5, 0]) == False\\n assert candidate([1, 3, -2, 1]) == False\\n assert candidate([1, 2, 3, 7]) == False\\n...</td>\n",
- " <td id=\"T_2a332_row3_col2\" class=\"data row3 col2\" >pairs_sum_to_zero</td>\n",
- " <td id=\"T_2a332_row3_col3\" class=\"data row3 col3\" >code='\\n\\ndef pairs_sum_to_zero(l):\\n \"\"\"\\n pairs_sum_to_zero takes a list of integers as an input.\\n it returns True if there are two distinct elements in the list that\\n...</td>\n",
- " <td id=\"T_2a332_row3_col4\" class=\"data row3 col4\" >solve this issue. The error message indicates that there are trailing characters in the JSON object provided. To fix this, we need to ensure that...</td>\n",
- " <td id=\"T_2a332_row3_col5\" class=\"data row3 col5\" >code='def pairs_sum_to_zero(l):\\n for i in range(len(l)):\\n for j in range(i+1, len(l)):\\n if l[i] + l[j] == 0:\\n return True\\n return False'</td>\n",
- " <td id=\"T_2a332_row3_col6\" class=\"data row3 col6\" >1.0</td>\n",
- " <td id=\"T_2a332_row3_col7\" class=\"data row3 col7\" >nan</td>\n",
+ " <th id=\"T_3b532_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
+ " <td id=\"T_3b532_row3_col0\" class=\"data row3 col0\" >code='\\n\\ndef pairs_sum_to_zero(l):\\n \"\"\"\\n pairs_sum_to_zero takes a list of integers as an input.\\n it returns True if there are two distinct elements in the list that\\n...</td>\n",
+ " <td id=\"T_3b532_row3_col1\" class=\"data row3 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate([1, 3, 5, 0]) == False\\n assert candidate([1, 3, -2, 1]) == False\\n assert candidate([1, 2, 3, 7]) == False\\n...</td>\n",
+ " <td id=\"T_3b532_row3_col2\" class=\"data row3 col2\" >pairs_sum_to_zero</td>\n",
+ " <td id=\"T_3b532_row3_col3\" class=\"data row3 col3\" >code='\\n\\ndef pairs_sum_to_zero(l):\\n \"\"\"\\n pairs_sum_to_zero takes a list of integers as an input.\\n it returns True if there are two distinct elements in the list that\\n...</td>\n",
+ " <td id=\"T_3b532_row3_col4\" class=\"data row3 col4\" >solve this issue. The error message indicates that there are trailing characters in the JSON object provided. To fix this, we need to ensure that...</td>\n",
+ " <td id=\"T_3b532_row3_col5\" class=\"data row3 col5\" >code='def pairs_sum_to_zero(l):\\n for i in range(len(l)):\\n for j in range(i+1, len(l)):\\n if l[i] + l[j] == 0:\\n return True\\n return False'</td>\n",
+ " <td id=\"T_3b532_row3_col6\" class=\"data row3 col6\" >1.0</td>\n",
+ " <td id=\"T_3b532_row3_col7\" class=\"data row3 col7\" >nan</td>\n",
" </tr>\n",
" <tr>\n",
- " <th id=\"T_2a332_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",
- " <td id=\"T_2a332_row4_col0\" class=\"data row4 col0\" >code='\\n\\ndef change_base(x: int, base: int):\\n \"\"\"Change numerical base of input number x to base.\\n return string representation after the conversion.\\n base numbers are less than...</td>\n",
- " <td id=\"T_2a332_row4_col1\" class=\"data row4 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate(8, 3) == \"22\"\\n assert candidate(9, 3) == \"100\"\\n assert candidate(234, 2) == \"11101010\"\\n assert candidate(16, 2) == \"10000\"\\n assert...</td>\n",
- " <td id=\"T_2a332_row4_col2\" class=\"data row4 col2\" >change_base</td>\n",
- " <td id=\"T_2a332_row4_col3\" class=\"data row4 col3\" >code='\\n\\ndef change_base(x: int, base: int):\\n \"\"\"Change numerical base of input number x to base.\\n return string representation after the conversion.\\n base numbers are less than...</td>\n",
- " <td id=\"T_2a332_row4_col4\" class=\"data row4 col4\" >produce the solution. We need to ensure that the JSON object provided follows the correct format specified in the schema. The error message indicates that...</td>\n",
- " <td id=\"T_2a332_row4_col5\" class=\"data row4 col5\" >code=\"def change_base(x: int, base: int):\\n result = ''\\n while x > 0:\\n result = str(x % base) + result\\n x //= base\\n return result\"</td>\n",
- " <td id=\"T_2a332_row4_col6\" class=\"data row4 col6\" >1.0</td>\n",
- " <td id=\"T_2a332_row4_col7\" class=\"data row4 col7\" >nan</td>\n",
+ " <th id=\"T_3b532_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",
+ " <td id=\"T_3b532_row4_col0\" class=\"data row4 col0\" >code='\\n\\ndef change_base(x: int, base: int):\\n \"\"\"Change numerical base of input number x to base.\\n return string representation after the conversion.\\n base numbers are less than...</td>\n",
+ " <td id=\"T_3b532_row4_col1\" class=\"data row4 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate(8, 3) == \"22\"\\n assert candidate(9, 3) == \"100\"\\n assert candidate(234, 2) == \"11101010\"\\n assert candidate(16, 2) == \"10000\"\\n assert...</td>\n",
+ " <td id=\"T_3b532_row4_col2\" class=\"data row4 col2\" >change_base</td>\n",
+ " <td id=\"T_3b532_row4_col3\" class=\"data row4 col3\" >code='\\n\\ndef change_base(x: int, base: int):\\n \"\"\"Change numerical base of input number x to base.\\n return string representation after the conversion.\\n base numbers are less than...</td>\n",
+ " <td id=\"T_3b532_row4_col4\" class=\"data row4 col4\" >produce the solution. We need to ensure that the JSON object provided follows the correct format specified in the schema. The error message indicates that...</td>\n",
+ " <td id=\"T_3b532_row4_col5\" class=\"data row4 col5\" >code=\"def change_base(x: int, base: int):\\n result = ''\\n while x > 0:\\n result = str(x % base) + result\\n x //= base\\n return result\"</td>\n",
+ " <td id=\"T_3b532_row4_col6\" class=\"data row4 col6\" >1.0</td>\n",
+ " <td id=\"T_3b532_row4_col7\" class=\"data row4 col7\" >nan</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n"
],
"text/plain": [
- "<pandas.io.formats.style.Styler at 0x15f344710>"
+ "<pandas.io.formats.style.Styler at 0x1777bb350>"
]
},
"metadata": {},
@@ -565,14 +673,17 @@
"name": "stdout",
"output_type": "stream",
"text": [
- "56.45\n"
+ "50.0\n"
]
}
],
"source": [
"from dspy.evaluate.evaluate import Evaluate\n",
"evaluator = Evaluate(\n",
- " devset=testset, num_threads=30, display_progress=True, display_table=5, max_errors=100,\n",
+ " devset=testset, num_threads=30,\n",
+ " display_progress=True,\n",
+ " display_table=5,\n",
+ " max_errors=100,\n",
")\n",
"res = evaluator(predictor, metric)\n",
"print(res)"
@@ -587,44 +698,20 @@
},
{
"cell_type": "code",
- "execution_count": 200,
+ "execution_count": null,
"metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Compiling...\n"
- ]
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- " 12%|█▎ | 5/40 [00:00<00:00, 237.80it/s]"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Bootstrapped 4 full traces after 6 examples in round 0.\n"
- ]
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"from dspy.teleprompt.bootstrap import BootstrapFewShot\n",
+ "from dspy.teleprompt.random_search import BootstrapFewShotWithRandomSearch\n",
"\n",
"print(\"Compiling...\")\n",
- "compiled = BootstrapFewShot(\n",
- " metric=metric\n",
+ "compiled = BootstrapFewShotWithRandomSearch(\n",
+ "#compiled = BootstrapFewShot(\n",
+ " metric=metric,\n",
+ " num_threads=30,\n",
+ " num_candidate_programs=5,\n",
+ " max_labeled_demos=8,\n",
").compile(\n",
" predictor,\n",
" trainset=trainset,\n",
@@ -640,7 +727,7 @@
},
{
"cell_type": "code",
- "execution_count": 201,
+ "execution_count": 10,
"metadata": {},
"outputs": [
{
@@ -654,14 +741,14 @@
"name": "stderr",
"output_type": "stream",
"text": [
- "Average Metric: 84 / 124 (67.7): 100%|██████████| 124/124 [00:00<00:00, 2265.09it/s]"
+ "Average Metric: 87 / 124 (70.2): 100%|██████████| 124/124 [00:29<00:00, 4.25it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
- "Average Metric: 84 / 124 (67.7%)\n"
+ "Average Metric: 87 / 124 (70.2%)\n"
]
},
{
@@ -675,88 +762,88 @@
"data": {
"text/html": [
"<style type=\"text/css\">\n",
- "#T_729fd th {\n",
+ "#T_bfcaa th {\n",
" text-align: left;\n",
"}\n",
- "#T_729fd td {\n",
+ "#T_bfcaa td {\n",
" text-align: left;\n",
"}\n",
- "#T_729fd_row0_col0, #T_729fd_row0_col1, #T_729fd_row0_col2, #T_729fd_row0_col3, #T_729fd_row0_col4, #T_729fd_row0_col5, #T_729fd_row0_col6, #T_729fd_row1_col0, #T_729fd_row1_col1, #T_729fd_row1_col2, #T_729fd_row1_col3, #T_729fd_row1_col4, #T_729fd_row1_col5, #T_729fd_row1_col6, #T_729fd_row2_col0, #T_729fd_row2_col1, #T_729fd_row2_col2, #T_729fd_row2_col3, #T_729fd_row2_col4, #T_729fd_row2_col5, #T_729fd_row2_col6, #T_729fd_row3_col0, #T_729fd_row3_col1, #T_729fd_row3_col2, #T_729fd_row3_col3, #T_729fd_row3_col4, #T_729fd_row3_col5, #T_729fd_row3_col6, #T_729fd_row4_col0, #T_729fd_row4_col1, #T_729fd_row4_col2, #T_729fd_row4_col3, #T_729fd_row4_col4, #T_729fd_row4_col5, #T_729fd_row4_col6 {\n",
+ "#T_bfcaa_row0_col0, #T_bfcaa_row0_col1, #T_bfcaa_row0_col2, #T_bfcaa_row0_col3, #T_bfcaa_row0_col4, #T_bfcaa_row0_col5, #T_bfcaa_row0_col6, #T_bfcaa_row1_col0, #T_bfcaa_row1_col1, #T_bfcaa_row1_col2, #T_bfcaa_row1_col3, #T_bfcaa_row1_col4, #T_bfcaa_row1_col5, #T_bfcaa_row1_col6, #T_bfcaa_row2_col0, #T_bfcaa_row2_col1, #T_bfcaa_row2_col2, #T_bfcaa_row2_col3, #T_bfcaa_row2_col4, #T_bfcaa_row2_col5, #T_bfcaa_row2_col6, #T_bfcaa_row3_col0, #T_bfcaa_row3_col1, #T_bfcaa_row3_col2, #T_bfcaa_row3_col3, #T_bfcaa_row3_col4, #T_bfcaa_row3_col5, #T_bfcaa_row3_col6, #T_bfcaa_row4_col0, #T_bfcaa_row4_col1, #T_bfcaa_row4_col2, #T_bfcaa_row4_col3, #T_bfcaa_row4_col4, #T_bfcaa_row4_col5, #T_bfcaa_row4_col6 {\n",
" text-align: left;\n",
" white-space: pre-wrap;\n",
" word-wrap: break-word;\n",
" max-width: 400px;\n",
"}\n",
"</style>\n",
- "<table id=\"T_729fd\">\n",
+ "<table id=\"T_bfcaa\">\n",
" <thead>\n",
" <tr>\n",
" <th class=\"blank level0\" >&nbsp;</th>\n",
- " <th id=\"T_729fd_level0_col0\" class=\"col_heading level0 col0\" >prompt</th>\n",
- " <th id=\"T_729fd_level0_col1\" class=\"col_heading level0 col1\" >test</th>\n",
- " <th id=\"T_729fd_level0_col2\" class=\"col_heading level0 col2\" >entry_point</th>\n",
- " <th id=\"T_729fd_level0_col3\" class=\"col_heading level0 col3\" >example_solution</th>\n",
- " <th id=\"T_729fd_level0_col4\" class=\"col_heading level0 col4\" >reasoning</th>\n",
- " <th id=\"T_729fd_level0_col5\" class=\"col_heading level0 col5\" >pred_solution</th>\n",
- " <th id=\"T_729fd_level0_col6\" class=\"col_heading level0 col6\" >metric</th>\n",
+ " <th id=\"T_bfcaa_level0_col0\" class=\"col_heading level0 col0\" >prompt</th>\n",
+ " <th id=\"T_bfcaa_level0_col1\" class=\"col_heading level0 col1\" >test</th>\n",
+ " <th id=\"T_bfcaa_level0_col2\" class=\"col_heading level0 col2\" >entry_point</th>\n",
+ " <th id=\"T_bfcaa_level0_col3\" class=\"col_heading level0 col3\" >example_solution</th>\n",
+ " <th id=\"T_bfcaa_level0_col4\" class=\"col_heading level0 col4\" >reasoning</th>\n",
+ " <th id=\"T_bfcaa_level0_col5\" class=\"col_heading level0 col5\" >pred_solution</th>\n",
+ " <th id=\"T_bfcaa_level0_col6\" class=\"col_heading level0 col6\" >metric</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
- " <th id=\"T_729fd_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
- " <td id=\"T_729fd_row0_col0\" class=\"data row0 col0\" >code='\\n\\ndef triples_sum_to_zero(l: list):\\n \"\"\"\\n triples_sum_to_zero takes a list of integers as an input.\\n it returns True if there are three distinct elements in the list...</td>\n",
- " <td id=\"T_729fd_row0_col1\" class=\"data row0 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate([1, 3, 5, 0]) == False\\n assert candidate([1, 3, 5, -1]) == False\\n assert candidate([1, 3, -2, 1]) == True\\n...</td>\n",
- " <td id=\"T_729fd_row0_col2\" class=\"data row0 col2\" >triples_sum_to_zero</td>\n",
- " <td id=\"T_729fd_row0_col3\" class=\"data row0 col3\" >code='\\n\\ndef triples_sum_to_zero(l: list):\\n \"\"\"\\n triples_sum_to_zero takes a list of integers as an input.\\n it returns True if there are three distinct elements in the list...</td>\n",
- " <td id=\"T_729fd_row0_col4\" class=\"data row0 col4\" >produce the solution. We need to find three distinct elements in the list that sum to zero. One way to approach this is to use...</td>\n",
- " <td id=\"T_729fd_row0_col5\" class=\"data row0 col5\" >code='\\n\\ndef triples_sum_to_zero(l: list):\\n \"\"\"\\n triples_sum_to_zero takes a list of integers as an input.\\n it returns True if there are three distinct elements in the list...</td>\n",
- " <td id=\"T_729fd_row0_col6\" class=\"data row0 col6\" >1</td>\n",
+ " <th id=\"T_bfcaa_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+ " <td id=\"T_bfcaa_row0_col0\" class=\"data row0 col0\" >code='\\n\\ndef triples_sum_to_zero(l: list):\\n \"\"\"\\n triples_sum_to_zero takes a list of integers as an input.\\n it returns True if there are three distinct elements in the list...</td>\n",
+ " <td id=\"T_bfcaa_row0_col1\" class=\"data row0 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate([1, 3, 5, 0]) == False\\n assert candidate([1, 3, 5, -1]) == False\\n assert candidate([1, 3, -2, 1]) == True\\n...</td>\n",
+ " <td id=\"T_bfcaa_row0_col2\" class=\"data row0 col2\" >triples_sum_to_zero</td>\n",
+ " <td id=\"T_bfcaa_row0_col3\" class=\"data row0 col3\" >code='\\n\\ndef triples_sum_to_zero(l: list):\\n \"\"\"\\n triples_sum_to_zero takes a list of integers as an input.\\n it returns True if there are three distinct elements in the list...</td>\n",
+ " <td id=\"T_bfcaa_row0_col4\" class=\"data row0 col4\" >produce the solution. We need to find three distinct elements in the list that sum to zero. One way to approach this is to iterate...</td>\n",
+ " <td id=\"T_bfcaa_row0_col5\" class=\"data row0 col5\" >code='\\n\\ndef triples_sum_to_zero(l: list):\\n \"\"\"\\n triples_sum_to_zero takes a list of integers as an input.\\n it returns True if there are three distinct elements in the list...</td>\n",
+ " <td id=\"T_bfcaa_row0_col6\" class=\"data row0 col6\" >1</td>\n",
" </tr>\n",
" <tr>\n",
- " <th id=\"T_729fd_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
- " <td id=\"T_729fd_row1_col0\" class=\"data row1 col0\" >code='\\n\\ndef car_race_collision(n: int):\\n \"\"\"\\n Imagine a road that\\'s a perfectly straight infinitely long line.\\n n cars are driving left to right; simultaneously, a different set...</td>\n",
- " <td id=\"T_729fd_row1_col1\" class=\"data row1 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate(2) == 4\\n assert candidate(3) == 9\\n assert candidate(4) == 16\\n assert candidate(8) == 64\\n assert candidate(10) == 100\\n\\n'</td>\n",
- " <td id=\"T_729fd_row1_col2\" class=\"data row1 col2\" >car_race_collision</td>\n",
- " <td id=\"T_729fd_row1_col3\" class=\"data row1 col3\" >code='\\n\\ndef car_race_collision(n: int):\\n \"\"\"\\n Imagine a road that\\'s a perfectly straight infinitely long line.\\n n cars are driving left to right; simultaneously, a different set...</td>\n",
- " <td id=\"T_729fd_row1_col4\" class=\"data row1 col4\" >produce the solution. We can visualize the cars moving towards each other on the road and calculate the number of collisions that would occur when...</td>\n",
- " <td id=\"T_729fd_row1_col5\" class=\"data row1 col5\" >code='\\n\\ndef car_race_collision(n: int):\\n \"\"\"\\n Imagine a road that\\'s a perfectly straight infinitely long line.\\n n cars are driving left to right; simultaneously, a different set...</td>\n",
- " <td id=\"T_729fd_row1_col6\" class=\"data row1 col6\" >1</td>\n",
+ " <th id=\"T_bfcaa_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
+ " <td id=\"T_bfcaa_row1_col0\" class=\"data row1 col0\" >code='\\n\\ndef car_race_collision(n: int):\\n \"\"\"\\n Imagine a road that\\'s a perfectly straight infinitely long line.\\n n cars are driving left to right; simultaneously, a different set...</td>\n",
+ " <td id=\"T_bfcaa_row1_col1\" class=\"data row1 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate(2) == 4\\n assert candidate(3) == 9\\n assert candidate(4) == 16\\n assert candidate(8) == 64\\n assert candidate(10) == 100\\n\\n'</td>\n",
+ " <td id=\"T_bfcaa_row1_col2\" class=\"data row1 col2\" >car_race_collision</td>\n",
+ " <td id=\"T_bfcaa_row1_col3\" class=\"data row1 col3\" >code='\\n\\ndef car_race_collision(n: int):\\n \"\"\"\\n Imagine a road that\\'s a perfectly straight infinitely long line.\\n n cars are driving left to right; simultaneously, a different set...</td>\n",
+ " <td id=\"T_bfcaa_row1_col4\" class=\"data row1 col4\" >produce the solution. We can visualize the cars moving towards each other on an infinitely long road and calculate the number of collisions that would...</td>\n",
+ " <td id=\"T_bfcaa_row1_col5\" class=\"data row1 col5\" >code='\\n\\ndef car_race_collision(n: int):\\n \"\"\"\\n Imagine a road that\\'s a perfectly straight infinitely long line.\\n n cars are driving left to right; simultaneously, a different set...</td>\n",
+ " <td id=\"T_bfcaa_row1_col6\" class=\"data row1 col6\" >1</td>\n",
" </tr>\n",
" <tr>\n",
- " <th id=\"T_729fd_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
- " <td id=\"T_729fd_row2_col0\" class=\"data row2 col0\" >code='\\n\\ndef incr_list(l: list):\\n \"\"\"Return list with elements incremented by 1.\\n >>> incr_list([1, 2, 3])\\n [2, 3, 4]\\n >>> incr_list([5, 3, 5, 2, 3, 3, 9,...</td>\n",
- " <td id=\"T_729fd_row2_col1\" class=\"data row2 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate([]) == []\\n assert candidate([3, 2, 1]) == [4, 3, 2]\\n assert candidate([5, 2, 5, 2, 3, 3, 9, 0,...</td>\n",
- " <td id=\"T_729fd_row2_col2\" class=\"data row2 col2\" >incr_list</td>\n",
- " <td id=\"T_729fd_row2_col3\" class=\"data row2 col3\" >code='\\n\\ndef incr_list(l: list):\\n \"\"\"Return list with elements incremented by 1.\\n >>> incr_list([1, 2, 3])\\n [2, 3, 4]\\n >>> incr_list([5, 3, 5, 2, 3, 3, 9,...</td>\n",
- " <td id=\"T_729fd_row2_col4\" class=\"data row2 col4\" >produce the solution. We need to iterate through the list and increment each element by 1.</td>\n",
- " <td id=\"T_729fd_row2_col5\" class=\"data row2 col5\" >code='\\n\\ndef incr_list(l: list):\\n \"\"\"Return list with elements incremented by 1.\\n >>> incr_list([1, 2, 3])\\n [2, 3, 4]\\n >>> incr_list([5, 3, 5, 2, 3, 3, 9,...</td>\n",
- " <td id=\"T_729fd_row2_col6\" class=\"data row2 col6\" >1</td>\n",
+ " <th id=\"T_bfcaa_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
+ " <td id=\"T_bfcaa_row2_col0\" class=\"data row2 col0\" >code='\\n\\ndef incr_list(l: list):\\n \"\"\"Return list with elements incremented by 1.\\n >>> incr_list([1, 2, 3])\\n [2, 3, 4]\\n >>> incr_list([5, 3, 5, 2, 3, 3, 9,...</td>\n",
+ " <td id=\"T_bfcaa_row2_col1\" class=\"data row2 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate([]) == []\\n assert candidate([3, 2, 1]) == [4, 3, 2]\\n assert candidate([5, 2, 5, 2, 3, 3, 9, 0,...</td>\n",
+ " <td id=\"T_bfcaa_row2_col2\" class=\"data row2 col2\" >incr_list</td>\n",
+ " <td id=\"T_bfcaa_row2_col3\" class=\"data row2 col3\" >code='\\n\\ndef incr_list(l: list):\\n \"\"\"Return list with elements incremented by 1.\\n >>> incr_list([1, 2, 3])\\n [2, 3, 4]\\n >>> incr_list([5, 3, 5, 2, 3, 3, 9,...</td>\n",
+ " <td id=\"T_bfcaa_row2_col4\" class=\"data row2 col4\" >produce the solution. We need to iterate through each element in the list and increment it by 1.</td>\n",
+ " <td id=\"T_bfcaa_row2_col5\" class=\"data row2 col5\" >code='\\n\\ndef incr_list(l: list):\\n \"\"\"Return list with elements incremented by 1.\\n >>> incr_list([1, 2, 3])\\n [2, 3, 4]\\n >>> incr_list([5, 3, 5, 2, 3, 3, 9,...</td>\n",
+ " <td id=\"T_bfcaa_row2_col6\" class=\"data row2 col6\" >1</td>\n",
" </tr>\n",
" <tr>\n",
- " <th id=\"T_729fd_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
- " <td id=\"T_729fd_row3_col0\" class=\"data row3 col0\" >code='\\n\\ndef pairs_sum_to_zero(l):\\n \"\"\"\\n pairs_sum_to_zero takes a list of integers as an input.\\n it returns True if there are two distinct elements in the list that\\n...</td>\n",
- " <td id=\"T_729fd_row3_col1\" class=\"data row3 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate([1, 3, 5, 0]) == False\\n assert candidate([1, 3, -2, 1]) == False\\n assert candidate([1, 2, 3, 7]) == False\\n...</td>\n",
- " <td id=\"T_729fd_row3_col2\" class=\"data row3 col2\" >pairs_sum_to_zero</td>\n",
- " <td id=\"T_729fd_row3_col3\" class=\"data row3 col3\" >code='\\n\\ndef pairs_sum_to_zero(l):\\n \"\"\"\\n pairs_sum_to_zero takes a list of integers as an input.\\n it returns True if there are two distinct elements in the list that\\n...</td>\n",
- " <td id=\"T_729fd_row3_col4\" class=\"data row3 col4\" >produce the solution. We need to iterate through the list and check if there are any two distinct elements that sum up to zero.</td>\n",
- " <td id=\"T_729fd_row3_col5\" class=\"data row3 col5\" >code='\\n\\ndef pairs_sum_to_zero(l):\\n \"\"\"\\n pairs_sum_to_zero takes a list of integers as an input.\\n it returns True if there are two distinct elements in the list that\\n...</td>\n",
- " <td id=\"T_729fd_row3_col6\" class=\"data row3 col6\" >1</td>\n",
+ " <th id=\"T_bfcaa_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
+ " <td id=\"T_bfcaa_row3_col0\" class=\"data row3 col0\" >code='\\n\\ndef pairs_sum_to_zero(l):\\n \"\"\"\\n pairs_sum_to_zero takes a list of integers as an input.\\n it returns True if there are two distinct elements in the list that\\n...</td>\n",
+ " <td id=\"T_bfcaa_row3_col1\" class=\"data row3 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate([1, 3, 5, 0]) == False\\n assert candidate([1, 3, -2, 1]) == False\\n assert candidate([1, 2, 3, 7]) == False\\n...</td>\n",
+ " <td id=\"T_bfcaa_row3_col2\" class=\"data row3 col2\" >pairs_sum_to_zero</td>\n",
+ " <td id=\"T_bfcaa_row3_col3\" class=\"data row3 col3\" >code='\\n\\ndef pairs_sum_to_zero(l):\\n \"\"\"\\n pairs_sum_to_zero takes a list of integers as an input.\\n it returns True if there are two distinct elements in the list that\\n...</td>\n",
+ " <td id=\"T_bfcaa_row3_col4\" class=\"data row3 col4\" >produce the solution. We need to iterate through the list and check if there are two distinct elements that sum up to zero. To do...</td>\n",
+ " <td id=\"T_bfcaa_row3_col5\" class=\"data row3 col5\" >code='\\n\\ndef pairs_sum_to_zero(l):\\n \"\"\"\\n pairs_sum_to_zero takes a list of integers as an input.\\n it returns True if there are two distinct elements in the list that\\n...</td>\n",
+ " <td id=\"T_bfcaa_row3_col6\" class=\"data row3 col6\" >1</td>\n",
" </tr>\n",
" <tr>\n",
- " <th id=\"T_729fd_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",
- " <td id=\"T_729fd_row4_col0\" class=\"data row4 col0\" >code='\\n\\ndef change_base(x: int, base: int):\\n \"\"\"Change numerical base of input number x to base.\\n return string representation after the conversion.\\n base numbers are less than...</td>\n",
- " <td id=\"T_729fd_row4_col1\" class=\"data row4 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate(8, 3) == \"22\"\\n assert candidate(9, 3) == \"100\"\\n assert candidate(234, 2) == \"11101010\"\\n assert candidate(16, 2) == \"10000\"\\n assert...</td>\n",
- " <td id=\"T_729fd_row4_col2\" class=\"data row4 col2\" >change_base</td>\n",
- " <td id=\"T_729fd_row4_col3\" class=\"data row4 col3\" >code='\\n\\ndef change_base(x: int, base: int):\\n \"\"\"Change numerical base of input number x to base.\\n return string representation after the conversion.\\n base numbers are less than...</td>\n",
- " <td id=\"T_729fd_row4_col4\" class=\"data row4 col4\" >produce the solution. We need to convert the input number `x` to the desired base `base`. To do this, we can repeatedly divide `x` by...</td>\n",
- " <td id=\"T_729fd_row4_col5\" class=\"data row4 col5\" >code='\\n\\ndef change_base(x: int, base: int):\\n \"\"\"Change numerical base of input number x to base.\\n return string representation after the conversion.\\n base numbers are less than...</td>\n",
- " <td id=\"T_729fd_row4_col6\" class=\"data row4 col6\" >1</td>\n",
+ " <th id=\"T_bfcaa_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",
+ " <td id=\"T_bfcaa_row4_col0\" class=\"data row4 col0\" >code='\\n\\ndef change_base(x: int, base: int):\\n \"\"\"Change numerical base of input number x to base.\\n return string representation after the conversion.\\n base numbers are less than...</td>\n",
+ " <td id=\"T_bfcaa_row4_col1\" class=\"data row4 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate(8, 3) == \"22\"\\n assert candidate(9, 3) == \"100\"\\n assert candidate(234, 2) == \"11101010\"\\n assert candidate(16, 2) == \"10000\"\\n assert...</td>\n",
+ " <td id=\"T_bfcaa_row4_col2\" class=\"data row4 col2\" >change_base</td>\n",
+ " <td id=\"T_bfcaa_row4_col3\" class=\"data row4 col3\" >code='\\n\\ndef change_base(x: int, base: int):\\n \"\"\"Change numerical base of input number x to base.\\n return string representation after the conversion.\\n base numbers are less than...</td>\n",
+ " <td id=\"T_bfcaa_row4_col4\" class=\"data row4 col4\" >produce the solution. We need to convert the number `x` from its current base to the desired `base`. To do this, we can repeatedly divide...</td>\n",
+ " <td id=\"T_bfcaa_row4_col5\" class=\"data row4 col5\" >code='\\n\\ndef change_base(x: int, base: int):\\n \"\"\"Change numerical base of input number x to base.\\n return string representation after the conversion.\\n base numbers are less than...</td>\n",
+ " <td id=\"T_bfcaa_row4_col6\" class=\"data row4 col6\" >1</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n"
],
"text/plain": [
- "<pandas.io.formats.style.Styler at 0x29b8fc2d0>"
+ "<pandas.io.formats.style.Styler at 0x29730c990>"
]
},
"metadata": {},
@@ -787,7 +874,7 @@
"name": "stdout",
"output_type": "stream",
"text": [
- "Compiled HumanEval score: 67.74\n"
+ "Compiled HumanEval score: 70.16\n"
]
}
],
@@ -798,13 +885,6 @@
" evaluator(compiled, metric=metric),\n",
")"
]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": []
}
],
"metadata": {
diff --git a/examples/functional/repl.py b/examples/functional/repl.py
new file mode 100644
index 000000000..f0fc1801f
--- /dev/null
+++ b/examples/functional/repl.py
@@ -0,0 +1,29 @@
+import multiprocessing
+from typing import Optional
+
+
+class PythonREPL:
+ @classmethod
+ def worker(cls, code: str, globals, locals, queue):
+ try:
+ exec(code, globals, locals)
+ queue.put(None)
+ except Exception as e:
+ queue.put(repr(e))
+
+ def run(self, command: str, timeout=5, globals={}, locals=None) -> Optional[str]:
+ queue: multiprocessing.Queue = multiprocessing.Queue()
+ p = multiprocessing.Process(
+ target=self.worker, args=(command, globals, locals, queue)
+ )
+ p.start()
+ p.join(timeout)
+ if p.is_alive():
+ p.terminate()
+ return "TimeoutError"
+ return queue.get_nowait()
+
+
+# We had to move the code above to a class for pickle reasons
+def execute_code(command: str, timeout=5, globals={}, locals=None) -> Optional[str]:
+ return PythonREPL().run(command, timeout, globals, locals)