DSPy: The framework for programming—not prompting—language models
Python
Use timeout for code evaluation
Thomas D Ahlee9204f45b460
diff --git a/dspy/teleprompt/random_search.py b/dspy/teleprompt/random_search.pyindex 8605cd427..f5b8eecdd 100644--- a/dspy/teleprompt/random_search.py+++ b/dspy/teleprompt/random_search.py@@ -93,7 +93,7 @@ class BootstrapFewShotWithRandomSearch(Teleprompter):program2 = teleprompter.compile(student, teacher=teacher, trainset=trainset2)evaluate = Evaluate(devset=self.valset, metric=self.metric, num_threads=self.num_threads,- display_table=False, display_progress=True)+ display_table=False, display_progress=True, max_errors=100)score, subscores = evaluate(program2, return_all_scores=True)diff --git a/examples/functional/functional.ipynb b/examples/functional/functional.ipynbindex 01d2c3088..611cc15c4 100644--- a/examples/functional/functional.ipynb+++ b/examples/functional/functional.ipynb@@ -2,52 +2,9 @@"cells": [{"cell_type": "code",- "execution_count": 202,+ "execution_count": null,"metadata": {},- "outputs": [- {- "name": "stdout",- "output_type": "stream",- "text": [- "The autoreload extension is already loaded. To reload it, use:\n",- " %reload_ext autoreload\n",- "Requirement already satisfied: datasets in /opt/homebrew/lib/python3.11/site-packages (2.14.7)\n",- "Requirement already satisfied: numpy>=1.17 in /opt/homebrew/lib/python3.11/site-packages (from datasets) (1.26.2)\n",- "Requirement already satisfied: pyarrow>=8.0.0 in /opt/homebrew/lib/python3.11/site-packages (from datasets) (12.0.0)\n",- "Requirement already satisfied: pyarrow-hotfix in /opt/homebrew/lib/python3.11/site-packages (from datasets) (0.6)\n",- "Requirement already satisfied: dill<0.3.8,>=0.3.0 in /opt/homebrew/lib/python3.11/site-packages (from datasets) (0.3.6)\n",- "Requirement already satisfied: pandas in /opt/homebrew/lib/python3.11/site-packages (from datasets) (2.1.4)\n",- "Requirement already satisfied: requests>=2.19.0 in /opt/homebrew/lib/python3.11/site-packages (from datasets) (2.31.0)\n",- "Requirement already satisfied: tqdm>=4.62.1 in /opt/homebrew/lib/python3.11/site-packages (from datasets) (4.66.1)\n",- "Requirement already satisfied: xxhash in /opt/homebrew/lib/python3.11/site-packages (from datasets) (3.2.0)\n",- "Requirement already satisfied: multiprocess in /opt/homebrew/lib/python3.11/site-packages (from datasets) (0.70.14)\n",- "Requirement already satisfied: fsspec<=2023.10.0,>=2023.1.0 in /opt/homebrew/lib/python3.11/site-packages (from fsspec[http]<=2023.10.0,>=2023.1.0->datasets) (2023.10.0)\n",- "Requirement already satisfied: aiohttp in /opt/homebrew/lib/python3.11/site-packages (from datasets) (3.9.1)\n",- "Requirement already satisfied: huggingface-hub<1.0.0,>=0.14.0 in /opt/homebrew/lib/python3.11/site-packages (from datasets) (0.19.4)\n",- "Requirement already satisfied: packaging in /opt/homebrew/lib/python3.11/site-packages (from datasets) (23.2)\n",- "Requirement already satisfied: pyyaml>=5.1 in /opt/homebrew/lib/python3.11/site-packages (from datasets) (6.0.1)\n",- "Requirement already satisfied: attrs>=17.3.0 in /opt/homebrew/lib/python3.11/site-packages (from aiohttp->datasets) (23.1.0)\n",- "Requirement already satisfied: multidict<7.0,>=4.5 in /opt/homebrew/lib/python3.11/site-packages (from aiohttp->datasets) (6.0.4)\n",- "Requirement already satisfied: yarl<2.0,>=1.0 in /opt/homebrew/lib/python3.11/site-packages (from aiohttp->datasets) (1.9.3)\n",- "Requirement already satisfied: frozenlist>=1.1.1 in /opt/homebrew/lib/python3.11/site-packages (from aiohttp->datasets) (1.4.0)\n",- "Requirement already satisfied: aiosignal>=1.1.2 in /opt/homebrew/lib/python3.11/site-packages (from aiohttp->datasets) (1.3.1)\n",- "Requirement already satisfied: filelock in /opt/homebrew/lib/python3.11/site-packages (from huggingface-hub<1.0.0,>=0.14.0->datasets) (3.13.1)\n",- "Requirement already satisfied: typing-extensions>=3.7.4.3 in /opt/homebrew/lib/python3.11/site-packages (from huggingface-hub<1.0.0,>=0.14.0->datasets) (4.8.0)\n",- "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/homebrew/lib/python3.11/site-packages (from requests>=2.19.0->datasets) (3.3.2)\n",- "Requirement already satisfied: idna<4,>=2.5 in /opt/homebrew/lib/python3.11/site-packages (from requests>=2.19.0->datasets) (3.6)\n",- "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/homebrew/lib/python3.11/site-packages (from requests>=2.19.0->datasets) (2.1.0)\n",- "Requirement already satisfied: certifi>=2017.4.17 in /opt/homebrew/lib/python3.11/site-packages (from requests>=2.19.0->datasets) (2023.11.17)\n",- "Requirement already satisfied: python-dateutil>=2.8.2 in /opt/homebrew/lib/python3.11/site-packages (from pandas->datasets) (2.8.2)\n",- "Requirement already satisfied: pytz>=2020.1 in /opt/homebrew/lib/python3.11/site-packages (from pandas->datasets) (2023.3)\n",- "Requirement already satisfied: tzdata>=2022.1 in /opt/homebrew/lib/python3.11/site-packages (from pandas->datasets) (2023.3)\n",- "Requirement already satisfied: six>=1.5 in /opt/homebrew/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.16.0)\n",- "\n",- "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.3.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.0\u001b[0m\n",- "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3.11 -m pip install --upgrade pip\u001b[0m\n",- "Note: you may need to restart the kernel to use updated packages.\n"- ]- }- ],+ "outputs": [],"source": ["%load_ext autoreload\n","%autoreload 2\n",@@ -63,9 +20,17 @@},{"cell_type": "code",- "execution_count": 193,+ "execution_count": 2,"metadata": {},"outputs": [+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "/opt/homebrew/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",+ " from .autonotebook import tqdm as notebook_tqdm\n"+ ]+ },{"data": {"text/plain": [@@ -76,7 +41,7 @@" 'entry_point': 'has_close_elements'}"]},- "execution_count": 193,+ "execution_count": 2,"metadata": {},"output_type": "execute_result"}@@ -96,7 +61,7 @@},{"cell_type": "code",- "execution_count": 194,+ "execution_count": 3,"metadata": {},"outputs": [{@@ -128,7 +93,7 @@},{"cell_type": "code",- "execution_count": 195,+ "execution_count": 4,"metadata": {},"outputs": [{@@ -189,7 +154,7 @@},{"cell_type": "code",- "execution_count": 196,+ "execution_count": 5,"metadata": {},"outputs": [{@@ -314,16 +279,23 @@},{"cell_type": "code",- "execution_count": 197,+ "execution_count": 6,"metadata": {},- "outputs": [],+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "3\n",+ "None\n",+ "AssertionError()\n"+ ]+ }+ ],"source": [- "def execute_code(code, globals={}, locals=None):\n",- " try:\n",- " output = exec(code, globals, locals)\n",- " return output, None\n",- " except Exception as e:\n",- " return None, e"+ "from repl import execute_code\n",+ "print(execute_code(\"print(3)\"))\n",+ "print(execute_code(\"assert False\"))"]},{@@ -335,7 +307,7 @@},{"cell_type": "code",- "execution_count": 198,+ "execution_count": 7,"metadata": {},"outputs": [{@@ -363,7 +335,7 @@"def metric(example, pred, trace=None):\n"," if pred.solution.code is None:\n"," return 0\n",- " _output, error = execute_code(\n",+ " error = execute_code(\n"," \"from typing import List\\n\"\n"," + f\"{pred.solution.code}\\n\"\n"," + f\"{example.test.code}\\n\"\n",@@ -384,8 +356,7 @@" + f\"check({example.entry_point})\"\n"," )\n"," print(code)\n",- " output, error = execute_code(code)\n",- " print(f\"{output=}\")\n",+ " error = execute_code(code)\n"," print(f\"{error=}\")\n"," break\n"]@@ -399,23 +370,111 @@},{"cell_type": "code",- "execution_count": 199,+ "execution_count": 8,"metadata": {},"outputs": [{"name": "stdout","output_type": "stream","text": [- "Error for example in dev set: \t\t Too many retries\n",- "Error for example in dev set: \t\t Too many retries\n",- "Error for example in dev set: \t\t Too many retries\n",- "Error for example in dev set: \t\t Too many retries\n",- "Error for example in dev set: \t\t Too many retries\n",- "Error for example in dev set: \t\t Too many retries\n",- "Error for example in dev set: \t\t Too many retries\n",- "Error for example in dev set: \t\t Too many retries\n",- "Error for example in dev set: \t\t Too many retries\n",- "Error for example in dev set: \t\t Too many retries\n",+ "Error for example in dev set: \t\t Too many retries\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 0.0 / 1 (0.0): 0%| | 0/124 [00:00<?, ?it/s]"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Error for example in dev set: \t\t Too many retries\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 0.0 / 2 (0.0): 1%| | 1/124 [00:00<00:01, 66.59it/s]"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Error for example in dev set: \t\t Too many retries\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 0.0 / 3 (0.0): 2%|▏ | 2/124 [00:00<00:01, 81.27it/s]"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Error for example in dev set: \t\t Too many retries\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 7.0 / 14 (50.0): 10%|█ | 13/124 [00:00<00:02, 49.54it/s]"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Error for example in dev set: \t\t Too many retries\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 14.0 / 21 (66.7): 16%|█▌ | 20/124 [00:00<00:02, 49.54it/s]"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Error for example in dev set: \t\t Too many retries\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 17.0 / 24 (70.8): 19%|█▊ | 23/124 [00:00<00:02, 49.54it/s]"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Error for example in dev set: \t\t Too many retries\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 29.0 / 41 (70.7): 32%|███▏ | 40/124 [00:00<00:01, 69.00it/s]"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": ["Error for example in dev set: \t\t Too many retries\n","Error for example in dev set: \t\t Too many retries\n"]@@ -424,117 +483,166 @@"name": "stderr","output_type": "stream","text": [- "Average Metric: 70.0 / 124 (56.5): 100%|██████████| 124/124 [00:00<00:00, 1781.00it/s]"+ "Average Metric: 34.0 / 55 (61.8): 44%|████▎ | 54/124 [00:00<00:01, 68.27it/s]"]},{"name": "stdout","output_type": "stream","text": [- "Average Metric: 70.0 / 124 (56.5%)\n"+ "Error for example in dev set: \t\t Too many retries\n"]},{"name": "stderr","output_type": "stream","text": [- "\n",+ "Average Metric: 38.0 / 67 (56.7): 54%|█████▍ | 67/124 [00:01<00:00, 60.79it/s]"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Error for example in dev set: \t\t Too many retries"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 40.0 / 70 (57.1): 56%|█████▌ | 69/124 [00:01<00:00, 60.79it/s]"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 41.0 / 73 (56.2): 58%|█████▊ | 72/124 [00:01<00:00, 60.79it/s]"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Error for example in dev set: \t\t Too many retries\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 62.0 / 124 (50.0): 100%|██████████| 124/124 [00:01<00:00, 92.27it/s]\n","/Users/ahle/repos/dspy/dspy/evaluate/evaluate.py:143: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n",- " \n"+ " df = df.applymap(truncate_cell)\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Average Metric: 62.0 / 124 (50.0%)\n"]},{"data": {"text/html": ["<style type=\"text/css\">\n",- "#T_2a332 th {\n",+ "#T_3b532 th {\n"," text-align: left;\n","}\n",- "#T_2a332 td {\n",+ "#T_3b532 td {\n"," text-align: left;\n","}\n",- "#T_2a332_row0_col0, #T_2a332_row0_col1, #T_2a332_row0_col2, #T_2a332_row0_col3, #T_2a332_row0_col4, #T_2a332_row0_col5, #T_2a332_row0_col6, #T_2a332_row0_col7, #T_2a332_row1_col0, #T_2a332_row1_col1, #T_2a332_row1_col2, #T_2a332_row1_col3, #T_2a332_row1_col4, #T_2a332_row1_col5, #T_2a332_row1_col6, #T_2a332_row1_col7, #T_2a332_row2_col0, #T_2a332_row2_col1, #T_2a332_row2_col2, #T_2a332_row2_col3, #T_2a332_row2_col4, #T_2a332_row2_col5, #T_2a332_row2_col6, #T_2a332_row2_col7, #T_2a332_row3_col0, #T_2a332_row3_col1, #T_2a332_row3_col2, #T_2a332_row3_col3, #T_2a332_row3_col4, #T_2a332_row3_col5, #T_2a332_row3_col6, #T_2a332_row3_col7, #T_2a332_row4_col0, #T_2a332_row4_col1, #T_2a332_row4_col2, #T_2a332_row4_col3, #T_2a332_row4_col4, #T_2a332_row4_col5, #T_2a332_row4_col6, #T_2a332_row4_col7 {\n",+ "#T_3b532_row0_col0, #T_3b532_row0_col1, #T_3b532_row0_col2, #T_3b532_row0_col3, #T_3b532_row0_col4, #T_3b532_row0_col5, #T_3b532_row0_col6, #T_3b532_row0_col7, #T_3b532_row1_col0, #T_3b532_row1_col1, #T_3b532_row1_col2, #T_3b532_row1_col3, #T_3b532_row1_col4, #T_3b532_row1_col5, #T_3b532_row1_col6, #T_3b532_row1_col7, #T_3b532_row2_col0, #T_3b532_row2_col1, #T_3b532_row2_col2, #T_3b532_row2_col3, #T_3b532_row2_col4, #T_3b532_row2_col5, #T_3b532_row2_col6, #T_3b532_row2_col7, #T_3b532_row3_col0, #T_3b532_row3_col1, #T_3b532_row3_col2, #T_3b532_row3_col3, #T_3b532_row3_col4, #T_3b532_row3_col5, #T_3b532_row3_col6, #T_3b532_row3_col7, #T_3b532_row4_col0, #T_3b532_row4_col1, #T_3b532_row4_col2, #T_3b532_row4_col3, #T_3b532_row4_col4, #T_3b532_row4_col5, #T_3b532_row4_col6, #T_3b532_row4_col7 {\n"," text-align: left;\n"," white-space: pre-wrap;\n"," word-wrap: break-word;\n"," max-width: 400px;\n","}\n","</style>\n",- "<table id=\"T_2a332\">\n",+ "<table id=\"T_3b532\">\n"," <thead>\n"," <tr>\n"," <th class=\"blank level0\" > </th>\n",- " <th id=\"T_2a332_level0_col0\" class=\"col_heading level0 col0\" >prompt</th>\n",- " <th id=\"T_2a332_level0_col1\" class=\"col_heading level0 col1\" >test</th>\n",- " <th id=\"T_2a332_level0_col2\" class=\"col_heading level0 col2\" >entry_point</th>\n",- " <th id=\"T_2a332_level0_col3\" class=\"col_heading level0 col3\" >example_solution</th>\n",- " <th id=\"T_2a332_level0_col4\" class=\"col_heading level0 col4\" >reasoning</th>\n",- " <th id=\"T_2a332_level0_col5\" class=\"col_heading level0 col5\" >pred_solution</th>\n",- " <th id=\"T_2a332_level0_col6\" class=\"col_heading level0 col6\" >metric</th>\n",- " <th id=\"T_2a332_level0_col7\" class=\"col_heading level0 col7\" >solution</th>\n",+ " <th id=\"T_3b532_level0_col0\" class=\"col_heading level0 col0\" >prompt</th>\n",+ " <th id=\"T_3b532_level0_col1\" class=\"col_heading level0 col1\" >test</th>\n",+ " <th id=\"T_3b532_level0_col2\" class=\"col_heading level0 col2\" >entry_point</th>\n",+ " <th id=\"T_3b532_level0_col3\" class=\"col_heading level0 col3\" >example_solution</th>\n",+ " <th id=\"T_3b532_level0_col4\" class=\"col_heading level0 col4\" >reasoning</th>\n",+ " <th id=\"T_3b532_level0_col5\" class=\"col_heading level0 col5\" >pred_solution</th>\n",+ " <th id=\"T_3b532_level0_col6\" class=\"col_heading level0 col6\" >metric</th>\n",+ " <th id=\"T_3b532_level0_col7\" class=\"col_heading level0 col7\" >solution</th>\n"," </tr>\n"," </thead>\n"," <tbody>\n"," <tr>\n",- " <th id=\"T_2a332_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",- " <td id=\"T_2a332_row0_col0\" class=\"data row0 col0\" >code='\\n\\ndef triples_sum_to_zero(l: list):\\n \"\"\"\\n triples_sum_to_zero takes a list of integers as an input.\\n it returns True if there are three distinct elements in the list...</td>\n",- " <td id=\"T_2a332_row0_col1\" class=\"data row0 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate([1, 3, 5, 0]) == False\\n assert candidate([1, 3, 5, -1]) == False\\n assert candidate([1, 3, -2, 1]) == True\\n...</td>\n",- " <td id=\"T_2a332_row0_col2\" class=\"data row0 col2\" >triples_sum_to_zero</td>\n",- " <td id=\"T_2a332_row0_col3\" class=\"data row0 col3\" >code='\\n\\ndef triples_sum_to_zero(l: list):\\n \"\"\"\\n triples_sum_to_zero takes a list of integers as an input.\\n it returns True if there are three distinct elements in the list...</td>\n",- " <td id=\"T_2a332_row0_col4\" class=\"data row0 col4\" >produce the solution. We need to ensure that the JSON object we provide is valid and follows the specified schema. The error message indicates that...</td>\n",- " <td id=\"T_2a332_row0_col5\" class=\"data row0 col5\" >code='def triples_sum_to_zero(l: list):\\n for i in range(len(l)):\\n for j in range(i+1, len(l)):\\n for k in range(j+1, len(l)):\\n if l[i] + l[j] + l[k] == 0:\\n...</td>\n",- " <td id=\"T_2a332_row0_col6\" class=\"data row0 col6\" >1.0</td>\n",- " <td id=\"T_2a332_row0_col7\" class=\"data row0 col7\" >nan</td>\n",+ " <th id=\"T_3b532_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",+ " <td id=\"T_3b532_row0_col0\" class=\"data row0 col0\" >code='\\n\\ndef triples_sum_to_zero(l: list):\\n \"\"\"\\n triples_sum_to_zero takes a list of integers as an input.\\n it returns True if there are three distinct elements in the list...</td>\n",+ " <td id=\"T_3b532_row0_col1\" class=\"data row0 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate([1, 3, 5, 0]) == False\\n assert candidate([1, 3, 5, -1]) == False\\n assert candidate([1, 3, -2, 1]) == True\\n...</td>\n",+ " <td id=\"T_3b532_row0_col2\" class=\"data row0 col2\" >triples_sum_to_zero</td>\n",+ " <td id=\"T_3b532_row0_col3\" class=\"data row0 col3\" >code='\\n\\ndef triples_sum_to_zero(l: list):\\n \"\"\"\\n triples_sum_to_zero takes a list of integers as an input.\\n it returns True if there are three distinct elements in the list...</td>\n",+ " <td id=\"T_3b532_row0_col4\" class=\"data row0 col4\" >produce the solution. We need to ensure that the JSON object we provide is valid and follows the specified schema. The error message indicates that...</td>\n",+ " <td id=\"T_3b532_row0_col5\" class=\"data row0 col5\" >code='def triples_sum_to_zero(l: list):\\n for i in range(len(l)):\\n for j in range(i+1, len(l)):\\n for k in range(j+1, len(l)):\\n if l[i] + l[j] + l[k] == 0:\\n...</td>\n",+ " <td id=\"T_3b532_row0_col6\" class=\"data row0 col6\" >0.0</td>\n",+ " <td id=\"T_3b532_row0_col7\" class=\"data row0 col7\" >nan</td>\n"," </tr>\n"," <tr>\n",- " <th id=\"T_2a332_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",- " <td id=\"T_2a332_row1_col0\" class=\"data row1 col0\" >code='\\n\\ndef car_race_collision(n: int):\\n \"\"\"\\n Imagine a road that\\'s a perfectly straight infinitely long line.\\n n cars are driving left to right; simultaneously, a different set...</td>\n",- " <td id=\"T_2a332_row1_col1\" class=\"data row1 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate(2) == 4\\n assert candidate(3) == 9\\n assert candidate(4) == 16\\n assert candidate(8) == 64\\n assert candidate(10) == 100\\n\\n'</td>\n",- " <td id=\"T_2a332_row1_col2\" class=\"data row1 col2\" >car_race_collision</td>\n",- " <td id=\"T_2a332_row1_col3\" class=\"data row1 col3\" >code='\\n\\ndef car_race_collision(n: int):\\n \"\"\"\\n Imagine a road that\\'s a perfectly straight infinitely long line.\\n n cars are driving left to right; simultaneously, a different set...</td>\n",- " <td id=\"T_2a332_row1_col4\" class=\"data row1 col4\" >produce the solution. We need to ensure that the code provided is valid JSON format. The error message indicates that there are trailing characters at...</td>\n",- " <td id=\"T_2a332_row1_col5\" class=\"data row1 col5\" >code='def car_race_collision(n: int):\\n return n ** 2'</td>\n",- " <td id=\"T_2a332_row1_col6\" class=\"data row1 col6\" >1.0</td>\n",- " <td id=\"T_2a332_row1_col7\" class=\"data row1 col7\" >nan</td>\n",+ " <th id=\"T_3b532_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",+ " <td id=\"T_3b532_row1_col0\" class=\"data row1 col0\" >code='\\n\\ndef car_race_collision(n: int):\\n \"\"\"\\n Imagine a road that\\'s a perfectly straight infinitely long line.\\n n cars are driving left to right; simultaneously, a different set...</td>\n",+ " <td id=\"T_3b532_row1_col1\" class=\"data row1 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate(2) == 4\\n assert candidate(3) == 9\\n assert candidate(4) == 16\\n assert candidate(8) == 64\\n assert candidate(10) == 100\\n\\n'</td>\n",+ " <td id=\"T_3b532_row1_col2\" class=\"data row1 col2\" >car_race_collision</td>\n",+ " <td id=\"T_3b532_row1_col3\" class=\"data row1 col3\" >code='\\n\\ndef car_race_collision(n: int):\\n \"\"\"\\n Imagine a road that\\'s a perfectly straight infinitely long line.\\n n cars are driving left to right; simultaneously, a different set...</td>\n",+ " <td id=\"T_3b532_row1_col4\" class=\"data row1 col4\" >produce the solution. We need to ensure that the code provided is valid JSON format. The error message indicates that there are trailing characters at...</td>\n",+ " <td id=\"T_3b532_row1_col5\" class=\"data row1 col5\" >code='def car_race_collision(n: int):\\n return n ** 2'</td>\n",+ " <td id=\"T_3b532_row1_col6\" class=\"data row1 col6\" >1.0</td>\n",+ " <td id=\"T_3b532_row1_col7\" class=\"data row1 col7\" >nan</td>\n"," </tr>\n"," <tr>\n",- " <th id=\"T_2a332_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",- " <td id=\"T_2a332_row2_col0\" class=\"data row2 col0\" >code='\\n\\ndef incr_list(l: list):\\n \"\"\"Return list with elements incremented by 1.\\n >>> incr_list([1, 2, 3])\\n [2, 3, 4]\\n >>> incr_list([5, 3, 5, 2, 3, 3, 9,...</td>\n",- " <td id=\"T_2a332_row2_col1\" class=\"data row2 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate([]) == []\\n assert candidate([3, 2, 1]) == [4, 3, 2]\\n assert candidate([5, 2, 5, 2, 3, 3, 9, 0,...</td>\n",- " <td id=\"T_2a332_row2_col2\" class=\"data row2 col2\" >incr_list</td>\n",- " <td id=\"T_2a332_row2_col3\" class=\"data row2 col3\" >code='\\n\\ndef incr_list(l: list):\\n \"\"\"Return list with elements incremented by 1.\\n >>> incr_list([1, 2, 3])\\n [2, 3, 4]\\n >>> incr_list([5, 3, 5, 2, 3, 3, 9,...</td>\n",- " <td id=\"T_2a332_row2_col4\" class=\"data row2 col4\" >produce the solution. We need to ensure that the JSON object we provide follows the correct format and does not contain any trailing characters that...</td>\n",- " <td id=\"T_2a332_row2_col5\" class=\"data row2 col5\" >code='def incr_list(l: list):\\n return [x + 1 for x in l]'</td>\n",- " <td id=\"T_2a332_row2_col6\" class=\"data row2 col6\" >1.0</td>\n",- " <td id=\"T_2a332_row2_col7\" class=\"data row2 col7\" >nan</td>\n",+ " <th id=\"T_3b532_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",+ " <td id=\"T_3b532_row2_col0\" class=\"data row2 col0\" >code='\\n\\ndef incr_list(l: list):\\n \"\"\"Return list with elements incremented by 1.\\n >>> incr_list([1, 2, 3])\\n [2, 3, 4]\\n >>> incr_list([5, 3, 5, 2, 3, 3, 9,...</td>\n",+ " <td id=\"T_3b532_row2_col1\" class=\"data row2 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate([]) == []\\n assert candidate([3, 2, 1]) == [4, 3, 2]\\n assert candidate([5, 2, 5, 2, 3, 3, 9, 0,...</td>\n",+ " <td id=\"T_3b532_row2_col2\" class=\"data row2 col2\" >incr_list</td>\n",+ " <td id=\"T_3b532_row2_col3\" class=\"data row2 col3\" >code='\\n\\ndef incr_list(l: list):\\n \"\"\"Return list with elements incremented by 1.\\n >>> incr_list([1, 2, 3])\\n [2, 3, 4]\\n >>> incr_list([5, 3, 5, 2, 3, 3, 9,...</td>\n",+ " <td id=\"T_3b532_row2_col4\" class=\"data row2 col4\" >produce the solution. We need to ensure that the JSON object we provide follows the correct format and does not contain any trailing characters that...</td>\n",+ " <td id=\"T_3b532_row2_col5\" class=\"data row2 col5\" >code='def incr_list(l: list):\\n return [x + 1 for x in l]'</td>\n",+ " <td id=\"T_3b532_row2_col6\" class=\"data row2 col6\" >1.0</td>\n",+ " <td id=\"T_3b532_row2_col7\" class=\"data row2 col7\" >nan</td>\n"," </tr>\n"," <tr>\n",- " <th id=\"T_2a332_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",- " <td id=\"T_2a332_row3_col0\" class=\"data row3 col0\" >code='\\n\\ndef pairs_sum_to_zero(l):\\n \"\"\"\\n pairs_sum_to_zero takes a list of integers as an input.\\n it returns True if there are two distinct elements in the list that\\n...</td>\n",- " <td id=\"T_2a332_row3_col1\" class=\"data row3 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate([1, 3, 5, 0]) == False\\n assert candidate([1, 3, -2, 1]) == False\\n assert candidate([1, 2, 3, 7]) == False\\n...</td>\n",- " <td id=\"T_2a332_row3_col2\" class=\"data row3 col2\" >pairs_sum_to_zero</td>\n",- " <td id=\"T_2a332_row3_col3\" class=\"data row3 col3\" >code='\\n\\ndef pairs_sum_to_zero(l):\\n \"\"\"\\n pairs_sum_to_zero takes a list of integers as an input.\\n it returns True if there are two distinct elements in the list that\\n...</td>\n",- " <td id=\"T_2a332_row3_col4\" class=\"data row3 col4\" >solve this issue. The error message indicates that there are trailing characters in the JSON object provided. To fix this, we need to ensure that...</td>\n",- " <td id=\"T_2a332_row3_col5\" class=\"data row3 col5\" >code='def pairs_sum_to_zero(l):\\n for i in range(len(l)):\\n for j in range(i+1, len(l)):\\n if l[i] + l[j] == 0:\\n return True\\n return False'</td>\n",- " <td id=\"T_2a332_row3_col6\" class=\"data row3 col6\" >1.0</td>\n",- " <td id=\"T_2a332_row3_col7\" class=\"data row3 col7\" >nan</td>\n",+ " <th id=\"T_3b532_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",+ " <td id=\"T_3b532_row3_col0\" class=\"data row3 col0\" >code='\\n\\ndef pairs_sum_to_zero(l):\\n \"\"\"\\n pairs_sum_to_zero takes a list of integers as an input.\\n it returns True if there are two distinct elements in the list that\\n...</td>\n",+ " <td id=\"T_3b532_row3_col1\" class=\"data row3 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate([1, 3, 5, 0]) == False\\n assert candidate([1, 3, -2, 1]) == False\\n assert candidate([1, 2, 3, 7]) == False\\n...</td>\n",+ " <td id=\"T_3b532_row3_col2\" class=\"data row3 col2\" >pairs_sum_to_zero</td>\n",+ " <td id=\"T_3b532_row3_col3\" class=\"data row3 col3\" >code='\\n\\ndef pairs_sum_to_zero(l):\\n \"\"\"\\n pairs_sum_to_zero takes a list of integers as an input.\\n it returns True if there are two distinct elements in the list that\\n...</td>\n",+ " <td id=\"T_3b532_row3_col4\" class=\"data row3 col4\" >solve this issue. The error message indicates that there are trailing characters in the JSON object provided. To fix this, we need to ensure that...</td>\n",+ " <td id=\"T_3b532_row3_col5\" class=\"data row3 col5\" >code='def pairs_sum_to_zero(l):\\n for i in range(len(l)):\\n for j in range(i+1, len(l)):\\n if l[i] + l[j] == 0:\\n return True\\n return False'</td>\n",+ " <td id=\"T_3b532_row3_col6\" class=\"data row3 col6\" >1.0</td>\n",+ " <td id=\"T_3b532_row3_col7\" class=\"data row3 col7\" >nan</td>\n"," </tr>\n"," <tr>\n",- " <th id=\"T_2a332_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",- " <td id=\"T_2a332_row4_col0\" class=\"data row4 col0\" >code='\\n\\ndef change_base(x: int, base: int):\\n \"\"\"Change numerical base of input number x to base.\\n return string representation after the conversion.\\n base numbers are less than...</td>\n",- " <td id=\"T_2a332_row4_col1\" class=\"data row4 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate(8, 3) == \"22\"\\n assert candidate(9, 3) == \"100\"\\n assert candidate(234, 2) == \"11101010\"\\n assert candidate(16, 2) == \"10000\"\\n assert...</td>\n",- " <td id=\"T_2a332_row4_col2\" class=\"data row4 col2\" >change_base</td>\n",- " <td id=\"T_2a332_row4_col3\" class=\"data row4 col3\" >code='\\n\\ndef change_base(x: int, base: int):\\n \"\"\"Change numerical base of input number x to base.\\n return string representation after the conversion.\\n base numbers are less than...</td>\n",- " <td id=\"T_2a332_row4_col4\" class=\"data row4 col4\" >produce the solution. We need to ensure that the JSON object provided follows the correct format specified in the schema. The error message indicates that...</td>\n",- " <td id=\"T_2a332_row4_col5\" class=\"data row4 col5\" >code=\"def change_base(x: int, base: int):\\n result = ''\\n while x > 0:\\n result = str(x % base) + result\\n x //= base\\n return result\"</td>\n",- " <td id=\"T_2a332_row4_col6\" class=\"data row4 col6\" >1.0</td>\n",- " <td id=\"T_2a332_row4_col7\" class=\"data row4 col7\" >nan</td>\n",+ " <th id=\"T_3b532_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",+ " <td id=\"T_3b532_row4_col0\" class=\"data row4 col0\" >code='\\n\\ndef change_base(x: int, base: int):\\n \"\"\"Change numerical base of input number x to base.\\n return string representation after the conversion.\\n base numbers are less than...</td>\n",+ " <td id=\"T_3b532_row4_col1\" class=\"data row4 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate(8, 3) == \"22\"\\n assert candidate(9, 3) == \"100\"\\n assert candidate(234, 2) == \"11101010\"\\n assert candidate(16, 2) == \"10000\"\\n assert...</td>\n",+ " <td id=\"T_3b532_row4_col2\" class=\"data row4 col2\" >change_base</td>\n",+ " <td id=\"T_3b532_row4_col3\" class=\"data row4 col3\" >code='\\n\\ndef change_base(x: int, base: int):\\n \"\"\"Change numerical base of input number x to base.\\n return string representation after the conversion.\\n base numbers are less than...</td>\n",+ " <td id=\"T_3b532_row4_col4\" class=\"data row4 col4\" >produce the solution. We need to ensure that the JSON object provided follows the correct format specified in the schema. The error message indicates that...</td>\n",+ " <td id=\"T_3b532_row4_col5\" class=\"data row4 col5\" >code=\"def change_base(x: int, base: int):\\n result = ''\\n while x > 0:\\n result = str(x % base) + result\\n x //= base\\n return result\"</td>\n",+ " <td id=\"T_3b532_row4_col6\" class=\"data row4 col6\" >1.0</td>\n",+ " <td id=\"T_3b532_row4_col7\" class=\"data row4 col7\" >nan</td>\n"," </tr>\n"," </tbody>\n","</table>\n"],"text/plain": [- "<pandas.io.formats.style.Styler at 0x15f344710>"+ "<pandas.io.formats.style.Styler at 0x1777bb350>"]},"metadata": {},@@ -565,14 +673,17 @@"name": "stdout","output_type": "stream","text": [- "56.45\n"+ "50.0\n"]}],"source": ["from dspy.evaluate.evaluate import Evaluate\n","evaluator = Evaluate(\n",- " devset=testset, num_threads=30, display_progress=True, display_table=5, max_errors=100,\n",+ " devset=testset, num_threads=30,\n",+ " display_progress=True,\n",+ " display_table=5,\n",+ " max_errors=100,\n",")\n","res = evaluator(predictor, metric)\n","print(res)"@@ -587,44 +698,20 @@},{"cell_type": "code",- "execution_count": 200,+ "execution_count": null,"metadata": {},- "outputs": [- {- "name": "stdout",- "output_type": "stream",- "text": [- "Compiling...\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- " 12%|█▎ | 5/40 [00:00<00:00, 237.80it/s]"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Bootstrapped 4 full traces after 6 examples in round 0.\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "\n"- ]- }- ],+ "outputs": [],"source": ["from dspy.teleprompt.bootstrap import BootstrapFewShot\n",+ "from dspy.teleprompt.random_search import BootstrapFewShotWithRandomSearch\n","\n","print(\"Compiling...\")\n",- "compiled = BootstrapFewShot(\n",- " metric=metric\n",+ "compiled = BootstrapFewShotWithRandomSearch(\n",+ "#compiled = BootstrapFewShot(\n",+ " metric=metric,\n",+ " num_threads=30,\n",+ " num_candidate_programs=5,\n",+ " max_labeled_demos=8,\n",").compile(\n"," predictor,\n"," trainset=trainset,\n",@@ -640,7 +727,7 @@},{"cell_type": "code",- "execution_count": 201,+ "execution_count": 10,"metadata": {},"outputs": [{@@ -654,14 +741,14 @@"name": "stderr","output_type": "stream","text": [- "Average Metric: 84 / 124 (67.7): 100%|██████████| 124/124 [00:00<00:00, 2265.09it/s]"+ "Average Metric: 87 / 124 (70.2): 100%|██████████| 124/124 [00:29<00:00, 4.25it/s]"]},{"name": "stdout","output_type": "stream","text": [- "Average Metric: 84 / 124 (67.7%)\n"+ "Average Metric: 87 / 124 (70.2%)\n"]},{@@ -675,88 +762,88 @@"data": {"text/html": ["<style type=\"text/css\">\n",- "#T_729fd th {\n",+ "#T_bfcaa th {\n"," text-align: left;\n","}\n",- "#T_729fd td {\n",+ "#T_bfcaa td {\n"," text-align: left;\n","}\n",- "#T_729fd_row0_col0, #T_729fd_row0_col1, #T_729fd_row0_col2, #T_729fd_row0_col3, #T_729fd_row0_col4, #T_729fd_row0_col5, #T_729fd_row0_col6, #T_729fd_row1_col0, #T_729fd_row1_col1, #T_729fd_row1_col2, #T_729fd_row1_col3, #T_729fd_row1_col4, #T_729fd_row1_col5, #T_729fd_row1_col6, #T_729fd_row2_col0, #T_729fd_row2_col1, #T_729fd_row2_col2, #T_729fd_row2_col3, #T_729fd_row2_col4, #T_729fd_row2_col5, #T_729fd_row2_col6, #T_729fd_row3_col0, #T_729fd_row3_col1, #T_729fd_row3_col2, #T_729fd_row3_col3, #T_729fd_row3_col4, #T_729fd_row3_col5, #T_729fd_row3_col6, #T_729fd_row4_col0, #T_729fd_row4_col1, #T_729fd_row4_col2, #T_729fd_row4_col3, #T_729fd_row4_col4, #T_729fd_row4_col5, #T_729fd_row4_col6 {\n",+ "#T_bfcaa_row0_col0, #T_bfcaa_row0_col1, #T_bfcaa_row0_col2, #T_bfcaa_row0_col3, #T_bfcaa_row0_col4, #T_bfcaa_row0_col5, #T_bfcaa_row0_col6, #T_bfcaa_row1_col0, #T_bfcaa_row1_col1, #T_bfcaa_row1_col2, #T_bfcaa_row1_col3, #T_bfcaa_row1_col4, #T_bfcaa_row1_col5, #T_bfcaa_row1_col6, #T_bfcaa_row2_col0, #T_bfcaa_row2_col1, #T_bfcaa_row2_col2, #T_bfcaa_row2_col3, #T_bfcaa_row2_col4, #T_bfcaa_row2_col5, #T_bfcaa_row2_col6, #T_bfcaa_row3_col0, #T_bfcaa_row3_col1, #T_bfcaa_row3_col2, #T_bfcaa_row3_col3, #T_bfcaa_row3_col4, #T_bfcaa_row3_col5, #T_bfcaa_row3_col6, #T_bfcaa_row4_col0, #T_bfcaa_row4_col1, #T_bfcaa_row4_col2, #T_bfcaa_row4_col3, #T_bfcaa_row4_col4, #T_bfcaa_row4_col5, #T_bfcaa_row4_col6 {\n"," text-align: left;\n"," white-space: pre-wrap;\n"," word-wrap: break-word;\n"," max-width: 400px;\n","}\n","</style>\n",- "<table id=\"T_729fd\">\n",+ "<table id=\"T_bfcaa\">\n"," <thead>\n"," <tr>\n"," <th class=\"blank level0\" > </th>\n",- " <th id=\"T_729fd_level0_col0\" class=\"col_heading level0 col0\" >prompt</th>\n",- " <th id=\"T_729fd_level0_col1\" class=\"col_heading level0 col1\" >test</th>\n",- " <th id=\"T_729fd_level0_col2\" class=\"col_heading level0 col2\" >entry_point</th>\n",- " <th id=\"T_729fd_level0_col3\" class=\"col_heading level0 col3\" >example_solution</th>\n",- " <th id=\"T_729fd_level0_col4\" class=\"col_heading level0 col4\" >reasoning</th>\n",- " <th id=\"T_729fd_level0_col5\" class=\"col_heading level0 col5\" >pred_solution</th>\n",- " <th id=\"T_729fd_level0_col6\" class=\"col_heading level0 col6\" >metric</th>\n",+ " <th id=\"T_bfcaa_level0_col0\" class=\"col_heading level0 col0\" >prompt</th>\n",+ " <th id=\"T_bfcaa_level0_col1\" class=\"col_heading level0 col1\" >test</th>\n",+ " <th id=\"T_bfcaa_level0_col2\" class=\"col_heading level0 col2\" >entry_point</th>\n",+ " <th id=\"T_bfcaa_level0_col3\" class=\"col_heading level0 col3\" >example_solution</th>\n",+ " <th id=\"T_bfcaa_level0_col4\" class=\"col_heading level0 col4\" >reasoning</th>\n",+ " <th id=\"T_bfcaa_level0_col5\" class=\"col_heading level0 col5\" >pred_solution</th>\n",+ " <th id=\"T_bfcaa_level0_col6\" class=\"col_heading level0 col6\" >metric</th>\n"," </tr>\n"," </thead>\n"," <tbody>\n"," <tr>\n",- " <th id=\"T_729fd_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",- " <td id=\"T_729fd_row0_col0\" class=\"data row0 col0\" >code='\\n\\ndef triples_sum_to_zero(l: list):\\n \"\"\"\\n triples_sum_to_zero takes a list of integers as an input.\\n it returns True if there are three distinct elements in the list...</td>\n",- " <td id=\"T_729fd_row0_col1\" class=\"data row0 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate([1, 3, 5, 0]) == False\\n assert candidate([1, 3, 5, -1]) == False\\n assert candidate([1, 3, -2, 1]) == True\\n...</td>\n",- " <td id=\"T_729fd_row0_col2\" class=\"data row0 col2\" >triples_sum_to_zero</td>\n",- " <td id=\"T_729fd_row0_col3\" class=\"data row0 col3\" >code='\\n\\ndef triples_sum_to_zero(l: list):\\n \"\"\"\\n triples_sum_to_zero takes a list of integers as an input.\\n it returns True if there are three distinct elements in the list...</td>\n",- " <td id=\"T_729fd_row0_col4\" class=\"data row0 col4\" >produce the solution. We need to find three distinct elements in the list that sum to zero. One way to approach this is to use...</td>\n",- " <td id=\"T_729fd_row0_col5\" class=\"data row0 col5\" >code='\\n\\ndef triples_sum_to_zero(l: list):\\n \"\"\"\\n triples_sum_to_zero takes a list of integers as an input.\\n it returns True if there are three distinct elements in the list...</td>\n",- " <td id=\"T_729fd_row0_col6\" class=\"data row0 col6\" >1</td>\n",+ " <th id=\"T_bfcaa_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",+ " <td id=\"T_bfcaa_row0_col0\" class=\"data row0 col0\" >code='\\n\\ndef triples_sum_to_zero(l: list):\\n \"\"\"\\n triples_sum_to_zero takes a list of integers as an input.\\n it returns True if there are three distinct elements in the list...</td>\n",+ " <td id=\"T_bfcaa_row0_col1\" class=\"data row0 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate([1, 3, 5, 0]) == False\\n assert candidate([1, 3, 5, -1]) == False\\n assert candidate([1, 3, -2, 1]) == True\\n...</td>\n",+ " <td id=\"T_bfcaa_row0_col2\" class=\"data row0 col2\" >triples_sum_to_zero</td>\n",+ " <td id=\"T_bfcaa_row0_col3\" class=\"data row0 col3\" >code='\\n\\ndef triples_sum_to_zero(l: list):\\n \"\"\"\\n triples_sum_to_zero takes a list of integers as an input.\\n it returns True if there are three distinct elements in the list...</td>\n",+ " <td id=\"T_bfcaa_row0_col4\" class=\"data row0 col4\" >produce the solution. We need to find three distinct elements in the list that sum to zero. One way to approach this is to iterate...</td>\n",+ " <td id=\"T_bfcaa_row0_col5\" class=\"data row0 col5\" >code='\\n\\ndef triples_sum_to_zero(l: list):\\n \"\"\"\\n triples_sum_to_zero takes a list of integers as an input.\\n it returns True if there are three distinct elements in the list...</td>\n",+ " <td id=\"T_bfcaa_row0_col6\" class=\"data row0 col6\" >1</td>\n"," </tr>\n"," <tr>\n",- " <th id=\"T_729fd_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",- " <td id=\"T_729fd_row1_col0\" class=\"data row1 col0\" >code='\\n\\ndef car_race_collision(n: int):\\n \"\"\"\\n Imagine a road that\\'s a perfectly straight infinitely long line.\\n n cars are driving left to right; simultaneously, a different set...</td>\n",- " <td id=\"T_729fd_row1_col1\" class=\"data row1 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate(2) == 4\\n assert candidate(3) == 9\\n assert candidate(4) == 16\\n assert candidate(8) == 64\\n assert candidate(10) == 100\\n\\n'</td>\n",- " <td id=\"T_729fd_row1_col2\" class=\"data row1 col2\" >car_race_collision</td>\n",- " <td id=\"T_729fd_row1_col3\" class=\"data row1 col3\" >code='\\n\\ndef car_race_collision(n: int):\\n \"\"\"\\n Imagine a road that\\'s a perfectly straight infinitely long line.\\n n cars are driving left to right; simultaneously, a different set...</td>\n",- " <td id=\"T_729fd_row1_col4\" class=\"data row1 col4\" >produce the solution. We can visualize the cars moving towards each other on the road and calculate the number of collisions that would occur when...</td>\n",- " <td id=\"T_729fd_row1_col5\" class=\"data row1 col5\" >code='\\n\\ndef car_race_collision(n: int):\\n \"\"\"\\n Imagine a road that\\'s a perfectly straight infinitely long line.\\n n cars are driving left to right; simultaneously, a different set...</td>\n",- " <td id=\"T_729fd_row1_col6\" class=\"data row1 col6\" >1</td>\n",+ " <th id=\"T_bfcaa_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",+ " <td id=\"T_bfcaa_row1_col0\" class=\"data row1 col0\" >code='\\n\\ndef car_race_collision(n: int):\\n \"\"\"\\n Imagine a road that\\'s a perfectly straight infinitely long line.\\n n cars are driving left to right; simultaneously, a different set...</td>\n",+ " <td id=\"T_bfcaa_row1_col1\" class=\"data row1 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate(2) == 4\\n assert candidate(3) == 9\\n assert candidate(4) == 16\\n assert candidate(8) == 64\\n assert candidate(10) == 100\\n\\n'</td>\n",+ " <td id=\"T_bfcaa_row1_col2\" class=\"data row1 col2\" >car_race_collision</td>\n",+ " <td id=\"T_bfcaa_row1_col3\" class=\"data row1 col3\" >code='\\n\\ndef car_race_collision(n: int):\\n \"\"\"\\n Imagine a road that\\'s a perfectly straight infinitely long line.\\n n cars are driving left to right; simultaneously, a different set...</td>\n",+ " <td id=\"T_bfcaa_row1_col4\" class=\"data row1 col4\" >produce the solution. We can visualize the cars moving towards each other on an infinitely long road and calculate the number of collisions that would...</td>\n",+ " <td id=\"T_bfcaa_row1_col5\" class=\"data row1 col5\" >code='\\n\\ndef car_race_collision(n: int):\\n \"\"\"\\n Imagine a road that\\'s a perfectly straight infinitely long line.\\n n cars are driving left to right; simultaneously, a different set...</td>\n",+ " <td id=\"T_bfcaa_row1_col6\" class=\"data row1 col6\" >1</td>\n"," </tr>\n"," <tr>\n",- " <th id=\"T_729fd_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",- " <td id=\"T_729fd_row2_col0\" class=\"data row2 col0\" >code='\\n\\ndef incr_list(l: list):\\n \"\"\"Return list with elements incremented by 1.\\n >>> incr_list([1, 2, 3])\\n [2, 3, 4]\\n >>> incr_list([5, 3, 5, 2, 3, 3, 9,...</td>\n",- " <td id=\"T_729fd_row2_col1\" class=\"data row2 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate([]) == []\\n assert candidate([3, 2, 1]) == [4, 3, 2]\\n assert candidate([5, 2, 5, 2, 3, 3, 9, 0,...</td>\n",- " <td id=\"T_729fd_row2_col2\" class=\"data row2 col2\" >incr_list</td>\n",- " <td id=\"T_729fd_row2_col3\" class=\"data row2 col3\" >code='\\n\\ndef incr_list(l: list):\\n \"\"\"Return list with elements incremented by 1.\\n >>> incr_list([1, 2, 3])\\n [2, 3, 4]\\n >>> incr_list([5, 3, 5, 2, 3, 3, 9,...</td>\n",- " <td id=\"T_729fd_row2_col4\" class=\"data row2 col4\" >produce the solution. We need to iterate through the list and increment each element by 1.</td>\n",- " <td id=\"T_729fd_row2_col5\" class=\"data row2 col5\" >code='\\n\\ndef incr_list(l: list):\\n \"\"\"Return list with elements incremented by 1.\\n >>> incr_list([1, 2, 3])\\n [2, 3, 4]\\n >>> incr_list([5, 3, 5, 2, 3, 3, 9,...</td>\n",- " <td id=\"T_729fd_row2_col6\" class=\"data row2 col6\" >1</td>\n",+ " <th id=\"T_bfcaa_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",+ " <td id=\"T_bfcaa_row2_col0\" class=\"data row2 col0\" >code='\\n\\ndef incr_list(l: list):\\n \"\"\"Return list with elements incremented by 1.\\n >>> incr_list([1, 2, 3])\\n [2, 3, 4]\\n >>> incr_list([5, 3, 5, 2, 3, 3, 9,...</td>\n",+ " <td id=\"T_bfcaa_row2_col1\" class=\"data row2 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate([]) == []\\n assert candidate([3, 2, 1]) == [4, 3, 2]\\n assert candidate([5, 2, 5, 2, 3, 3, 9, 0,...</td>\n",+ " <td id=\"T_bfcaa_row2_col2\" class=\"data row2 col2\" >incr_list</td>\n",+ " <td id=\"T_bfcaa_row2_col3\" class=\"data row2 col3\" >code='\\n\\ndef incr_list(l: list):\\n \"\"\"Return list with elements incremented by 1.\\n >>> incr_list([1, 2, 3])\\n [2, 3, 4]\\n >>> incr_list([5, 3, 5, 2, 3, 3, 9,...</td>\n",+ " <td id=\"T_bfcaa_row2_col4\" class=\"data row2 col4\" >produce the solution. We need to iterate through each element in the list and increment it by 1.</td>\n",+ " <td id=\"T_bfcaa_row2_col5\" class=\"data row2 col5\" >code='\\n\\ndef incr_list(l: list):\\n \"\"\"Return list with elements incremented by 1.\\n >>> incr_list([1, 2, 3])\\n [2, 3, 4]\\n >>> incr_list([5, 3, 5, 2, 3, 3, 9,...</td>\n",+ " <td id=\"T_bfcaa_row2_col6\" class=\"data row2 col6\" >1</td>\n"," </tr>\n"," <tr>\n",- " <th id=\"T_729fd_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",- " <td id=\"T_729fd_row3_col0\" class=\"data row3 col0\" >code='\\n\\ndef pairs_sum_to_zero(l):\\n \"\"\"\\n pairs_sum_to_zero takes a list of integers as an input.\\n it returns True if there are two distinct elements in the list that\\n...</td>\n",- " <td id=\"T_729fd_row3_col1\" class=\"data row3 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate([1, 3, 5, 0]) == False\\n assert candidate([1, 3, -2, 1]) == False\\n assert candidate([1, 2, 3, 7]) == False\\n...</td>\n",- " <td id=\"T_729fd_row3_col2\" class=\"data row3 col2\" >pairs_sum_to_zero</td>\n",- " <td id=\"T_729fd_row3_col3\" class=\"data row3 col3\" >code='\\n\\ndef pairs_sum_to_zero(l):\\n \"\"\"\\n pairs_sum_to_zero takes a list of integers as an input.\\n it returns True if there are two distinct elements in the list that\\n...</td>\n",- " <td id=\"T_729fd_row3_col4\" class=\"data row3 col4\" >produce the solution. We need to iterate through the list and check if there are any two distinct elements that sum up to zero.</td>\n",- " <td id=\"T_729fd_row3_col5\" class=\"data row3 col5\" >code='\\n\\ndef pairs_sum_to_zero(l):\\n \"\"\"\\n pairs_sum_to_zero takes a list of integers as an input.\\n it returns True if there are two distinct elements in the list that\\n...</td>\n",- " <td id=\"T_729fd_row3_col6\" class=\"data row3 col6\" >1</td>\n",+ " <th id=\"T_bfcaa_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",+ " <td id=\"T_bfcaa_row3_col0\" class=\"data row3 col0\" >code='\\n\\ndef pairs_sum_to_zero(l):\\n \"\"\"\\n pairs_sum_to_zero takes a list of integers as an input.\\n it returns True if there are two distinct elements in the list that\\n...</td>\n",+ " <td id=\"T_bfcaa_row3_col1\" class=\"data row3 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate([1, 3, 5, 0]) == False\\n assert candidate([1, 3, -2, 1]) == False\\n assert candidate([1, 2, 3, 7]) == False\\n...</td>\n",+ " <td id=\"T_bfcaa_row3_col2\" class=\"data row3 col2\" >pairs_sum_to_zero</td>\n",+ " <td id=\"T_bfcaa_row3_col3\" class=\"data row3 col3\" >code='\\n\\ndef pairs_sum_to_zero(l):\\n \"\"\"\\n pairs_sum_to_zero takes a list of integers as an input.\\n it returns True if there are two distinct elements in the list that\\n...</td>\n",+ " <td id=\"T_bfcaa_row3_col4\" class=\"data row3 col4\" >produce the solution. We need to iterate through the list and check if there are two distinct elements that sum up to zero. To do...</td>\n",+ " <td id=\"T_bfcaa_row3_col5\" class=\"data row3 col5\" >code='\\n\\ndef pairs_sum_to_zero(l):\\n \"\"\"\\n pairs_sum_to_zero takes a list of integers as an input.\\n it returns True if there are two distinct elements in the list that\\n...</td>\n",+ " <td id=\"T_bfcaa_row3_col6\" class=\"data row3 col6\" >1</td>\n"," </tr>\n"," <tr>\n",- " <th id=\"T_729fd_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",- " <td id=\"T_729fd_row4_col0\" class=\"data row4 col0\" >code='\\n\\ndef change_base(x: int, base: int):\\n \"\"\"Change numerical base of input number x to base.\\n return string representation after the conversion.\\n base numbers are less than...</td>\n",- " <td id=\"T_729fd_row4_col1\" class=\"data row4 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate(8, 3) == \"22\"\\n assert candidate(9, 3) == \"100\"\\n assert candidate(234, 2) == \"11101010\"\\n assert candidate(16, 2) == \"10000\"\\n assert...</td>\n",- " <td id=\"T_729fd_row4_col2\" class=\"data row4 col2\" >change_base</td>\n",- " <td id=\"T_729fd_row4_col3\" class=\"data row4 col3\" >code='\\n\\ndef change_base(x: int, base: int):\\n \"\"\"Change numerical base of input number x to base.\\n return string representation after the conversion.\\n base numbers are less than...</td>\n",- " <td id=\"T_729fd_row4_col4\" class=\"data row4 col4\" >produce the solution. We need to convert the input number `x` to the desired base `base`. To do this, we can repeatedly divide `x` by...</td>\n",- " <td id=\"T_729fd_row4_col5\" class=\"data row4 col5\" >code='\\n\\ndef change_base(x: int, base: int):\\n \"\"\"Change numerical base of input number x to base.\\n return string representation after the conversion.\\n base numbers are less than...</td>\n",- " <td id=\"T_729fd_row4_col6\" class=\"data row4 col6\" >1</td>\n",+ " <th id=\"T_bfcaa_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",+ " <td id=\"T_bfcaa_row4_col0\" class=\"data row4 col0\" >code='\\n\\ndef change_base(x: int, base: int):\\n \"\"\"Change numerical base of input number x to base.\\n return string representation after the conversion.\\n base numbers are less than...</td>\n",+ " <td id=\"T_bfcaa_row4_col1\" class=\"data row4 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate(8, 3) == \"22\"\\n assert candidate(9, 3) == \"100\"\\n assert candidate(234, 2) == \"11101010\"\\n assert candidate(16, 2) == \"10000\"\\n assert...</td>\n",+ " <td id=\"T_bfcaa_row4_col2\" class=\"data row4 col2\" >change_base</td>\n",+ " <td id=\"T_bfcaa_row4_col3\" class=\"data row4 col3\" >code='\\n\\ndef change_base(x: int, base: int):\\n \"\"\"Change numerical base of input number x to base.\\n return string representation after the conversion.\\n base numbers are less than...</td>\n",+ " <td id=\"T_bfcaa_row4_col4\" class=\"data row4 col4\" >produce the solution. We need to convert the number `x` from its current base to the desired `base`. To do this, we can repeatedly divide...</td>\n",+ " <td id=\"T_bfcaa_row4_col5\" class=\"data row4 col5\" >code='\\n\\ndef change_base(x: int, base: int):\\n \"\"\"Change numerical base of input number x to base.\\n return string representation after the conversion.\\n base numbers are less than...</td>\n",+ " <td id=\"T_bfcaa_row4_col6\" class=\"data row4 col6\" >1</td>\n"," </tr>\n"," </tbody>\n","</table>\n"],"text/plain": [- "<pandas.io.formats.style.Styler at 0x29b8fc2d0>"+ "<pandas.io.formats.style.Styler at 0x29730c990>"]},"metadata": {},@@ -787,7 +874,7 @@"name": "stdout","output_type": "stream","text": [- "Compiled HumanEval score: 67.74\n"+ "Compiled HumanEval score: 70.16\n"]}],@@ -798,13 +885,6 @@" evaluator(compiled, metric=metric),\n",")"]- },- {- "cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],- "source": []}],"metadata": {diff --git a/examples/functional/repl.py b/examples/functional/repl.pynew file mode 100644index 000000000..f0fc1801f--- /dev/null+++ b/examples/functional/repl.py@@ -0,0 +1,29 @@+import multiprocessing+from typing import Optional+++class PythonREPL:+ @classmethod+ def worker(cls, code: str, globals, locals, queue):+ try:+ exec(code, globals, locals)+ queue.put(None)+ except Exception as e:+ queue.put(repr(e))++ def run(self, command: str, timeout=5, globals={}, locals=None) -> Optional[str]:+ queue: multiprocessing.Queue = multiprocessing.Queue()+ p = multiprocessing.Process(+ target=self.worker, args=(command, globals, locals, queue)+ )+ p.start()+ p.join(timeout)+ if p.is_alive():+ p.terminate()+ return "TimeoutError"+ return queue.get_nowait()+++# We had to move the code above to a class for pickle reasons+def execute_code(command: str, timeout=5, globals={}, locals=None) -> Optional[str]:+ return PythonREPL().run(command, timeout, globals, locals)