Skip to content

A programming framework for agentic AI

response filter (#1039)

Chi Wange463146cb815
diff --git a/.github/workflows/openai.yml b/.github/workflows/openai.yml
index 3ec06f847..f074f2ffd 100644
--- a/.github/workflows/openai.yml
+++ b/.github/workflows/openai.yml
@@ -7,10 +7,10 @@ on:
pull_request:
branches: ['main']
paths:
- - 'flaml/integrations/oai/**'
- - 'test/openai/**'
- - 'notebook/integrate_openai.ipynb'
- - 'notebook/integrate_chatgpt_math.ipynb'
+ - 'flaml/autogen/**'
+ - 'test/autogen/**'
+ - 'notebook/autogen_openai_completion.ipynb'
+ - 'notebook/autogen_chatgpt_gpt4.ipynb'
- '.github/workflows/openai.yml'
jobs:
@@ -18,7 +18,7 @@ jobs:
strategy:
matrix:
os: [ubuntu-latest]
- python-version: [3.9]
+ python-version: ["3.9", "3.10", "3.11"]
runs-on: ${{ matrix.os }}
environment: openai
steps:
@@ -33,14 +33,27 @@ jobs:
python -m pip install --upgrade pip wheel
pip install -e .[autogen,blendsearch]
python -c "import flaml"
+ pip install coverage pytest datasets
- name: Coverage
+ if: matrix.python-version == '3.9'
env:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+ AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
+ AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }}
run: |
- pip install coverage pytest datasets nbconvert nbformat ipykernel
- coverage run -a -m pytest test/openai
+ coverage run -a -m pytest test/autogen
coverage xml
- cat "$(pwd)/test/openai/executed_openai_notebook_output.txt"
+ - name: Coverage and check notebook outputs
+ if: matrix.python-version != '3.9'
+ env:
+ OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+ AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
+ AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }}
+ run: |
+ pip install nbconvert nbformat ipykernel
+ coverage run -a -m pytest test/autogen/oai/test_notebook.py
+ coverage xml
+ cat "$(pwd)/test/autogen/oai/executed_openai_notebook_output.txt"
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v3
with:
diff --git a/flaml/autogen/agent/agent.py b/flaml/autogen/agent/agent.py
index ec4165515..bbac5b0f0 100644
--- a/flaml/autogen/agent/agent.py
+++ b/flaml/autogen/agent/agent.py
@@ -37,7 +37,7 @@ class Agent:
def _receive(self, message, sender):
"""Receive a message from another agent."""
- print("****", self.name, "received message from", sender.name, "****")
+ print("\n****", self.name, "received message from", sender.name, "****\n")
print(message)
self._conversations[sender.name].append({"content": message, "role": "user"})
diff --git a/flaml/autogen/agent/human_proxy_agent.py b/flaml/autogen/agent/user_proxy_agent.py
similarity index 89%
rename from flaml/autogen/agent/human_proxy_agent.py
rename to flaml/autogen/agent/user_proxy_agent.py
index 70cd285e4..14e663ceb 100644
--- a/flaml/autogen/agent/human_proxy_agent.py
+++ b/flaml/autogen/agent/user_proxy_agent.py
@@ -3,8 +3,8 @@ from flaml.autogen.code_utils import extract_code, execute_code
from collections import defaultdict
-class HumanProxyAgent(Agent):
- """(Experimental) A proxy agent for human, that can execute code and provide feedback to the other agents."""
+class UserProxyAgent(Agent):
+ """(Experimental) A proxy agent for the user, that can execute code and provide feedback to the other agents."""
MAX_CONSECUTIVE_AUTO_REPLY = 100 # maximum number of consecutive auto replies (subject to future change)
@@ -16,6 +16,7 @@ class HumanProxyAgent(Agent):
human_input_mode="ALWAYS",
max_consecutive_auto_reply=None,
is_termination_msg=None,
+ use_docker=True,
**config,
):
"""
@@ -51,22 +52,26 @@ class HumanProxyAgent(Agent):
max_consecutive_auto_reply if max_consecutive_auto_reply is not None else self.MAX_CONSECUTIVE_AUTO_REPLY
)
self._consecutive_auto_reply_counter = defaultdict(int)
+ self._use_docker = use_docker
def _execute_code(self, code, lang):
"""Execute the code and return the result."""
- if lang == "bash":
- assert code.startswith("python "), code
+ if lang in ["bash", "shell"]:
+ if not code.startswith("python "):
+ return 1, f"please do not suggest bash or shell commands like {code}"
file_name = code[len("python ") :]
- exitcode, logs = execute_code(filename=file_name, work_dir=self._work_dir)
+ exitcode, logs = execute_code(filename=file_name, work_dir=self._work_dir, use_docker=self._use_docker)
+ logs = logs.decode("utf-8")
elif lang == "python":
if code.startswith("# filename: "):
filename = code[11 : code.find("\n")].strip()
else:
filename = None
- exitcode, logs = execute_code(code, work_dir=self._work_dir, filename=filename)
+ exitcode, logs = execute_code(code, work_dir=self._work_dir, filename=filename, use_docker=self._use_docker)
+ logs = logs.decode("utf-8")
else:
# TODO: could this happen?
- exitcode, logs = 1, "unknown language"
+ exitcode, logs = 1, f"unknown language {lang}"
# raise NotImplementedError
return exitcode, logs
@@ -80,7 +85,7 @@ class HumanProxyAgent(Agent):
# try to execute the code
exitcode, logs = self._execute_code(code, lang)
exitcode2str = "execution succeeded" if exitcode == 0 else "execution failed"
- self._send(f"exitcode: {exitcode} ({exitcode2str})\nCode output: {logs.decode('utf-8')}", sender)
+ self._send(f"exitcode: {exitcode} ({exitcode2str})\nCode output: {logs}", sender)
def receive(self, message, sender):
"""Receive a message from the sender agent.
diff --git a/flaml/autogen/code_utils.py b/flaml/autogen/code_utils.py
index bbfac38b6..8264ef2ed 100644
--- a/flaml/autogen/code_utils.py
+++ b/flaml/autogen/code_utils.py
@@ -37,8 +37,7 @@ def generate_code(pattern: str = CODE_BLOCK_PATTERN, **config) -> Tuple[str, flo
float: The cost of the generation.
"""
response = oai.Completion.create(**config)
- cost = oai.Completion.cost(response)
- return extract_code(oai.Completion.extract_text(response)[0], pattern), cost
+ return extract_code(oai.Completion.extract_text(response)[0], pattern), response["cost"]
_IMPROVE_FUNCTION_CONFIG = {
@@ -59,8 +58,7 @@ def improve_function(file_name, func_name, objective, **config):
response = oai.Completion.create(
{"func_name": func_name, "objective": objective, "file_string": file_string}, **params
)
- cost = oai.Completion.cost(response)
- return oai.Completion.extract_text(response)[0], cost
+ return oai.Completion.extract_text(response)[0], response["cost"]
_IMPROVE_CODE_CONFIG = {
@@ -97,8 +95,7 @@ def improve_code(files, objective, suggest_only=True, **config):
params = {**_IMPROVE_CODE_CONFIG, **config}
followup = "" if suggest_only else " followed by the improved code"
response = oai.Completion.create({"objective": objective, "code": code, "followup": followup}, **params)
- cost = oai.Completion.cost(response)
- return oai.Completion.extract_text(response)[0], cost
+ return oai.Completion.extract_text(response)[0], response["cost"]
def timeout_handler(signum, frame):
@@ -281,9 +278,8 @@ def generate_assertions(definition: str, **config) -> Tuple[str, float]:
{"definition": definition},
**params,
)
- cost = oai.Completion.cost(response)
assertions = oai.Completion.extract_text(response)[0]
- return assertions, cost
+ return assertions, response["cost"]
def _remove_check(response):
@@ -387,6 +383,23 @@ _IMPLEMENT_CONFIGS = [
]
+class PassAssertionFilter:
+ def __init__(self, assertions):
+ self._assertions = assertions
+ self.cost = 0
+ self.metrics = self.responses = None
+
+ def pass_assertions(self, context, response, **_):
+ """Check if the response passes the assertions."""
+ responses = oai.Completion.extract_text(response)
+ metrics = eval_function_completions(responses, context["definition"], assertions=self._assertions)
+ self._assertions = metrics["assertions"]
+ self.cost += metrics["gen_cost"]
+ self.metrics = metrics
+ self.responses = responses
+ return metrics["succeed_assertions"]
+
+
def implement(
definition: str,
configs: Optional[List[Dict]] = None,
@@ -408,12 +421,19 @@ def implement(
configs = configs or _IMPLEMENT_CONFIGS
if len(configs) > 1 and callable(assertions):
assertions, cost = assertions(definition)
- for i, config in enumerate(configs):
- response = oai.Completion.create({"definition": definition}, **config)
- cost += oai.Completion.cost(response)
- responses = oai.Completion.extract_text(response)
- metrics = eval_function_completions(responses, definition, assertions=assertions)
- assertions = metrics["assertions"]
- cost += metrics["gen_cost"]
- if metrics["succeed_assertions"] or i == len(configs) - 1:
- return responses[metrics["index_selected"]], cost, i
+ assertion_filter = PassAssertionFilter(assertions)
+ response = oai.Completion.create(
+ {"definition": definition}, config_list=configs, filter_func=assertion_filter.pass_assertions
+ )
+ cost += assertion_filter.cost + response["cost"]
+ return assertion_filter.responses[assertion_filter.metrics["index_selected"]], cost, response["config_id"]
+
+ # for i, config in enumerate(configs):
+ # response = oai.Completion.create({"definition": definition}, **config)
+ # cost += oai.Completion.cost(response)
+ # responses = oai.Completion.extract_text(response)
+ # metrics = eval_function_completions(responses, definition, assertions=assertions)
+ # assertions = metrics["assertions"]
+ # cost += metrics["gen_cost"]
+ # if metrics["succeed_assertions"] or i == len(configs) - 1:
+ # return responses[metrics["index_selected"]], cost, i
diff --git a/flaml/autogen/math_utils.py b/flaml/autogen/math_utils.py
index fdf1cfbe8..304f68be9 100644
--- a/flaml/autogen/math_utils.py
+++ b/flaml/autogen/math_utils.py
@@ -20,9 +20,8 @@ def solve_problem(problem: str, **config) -> str:
"""
params = {**_MATH_CONFIG, **config}
response = oai.Completion.create({"problem": problem}, **params)
- cost = oai.Completion.cost(response)
results = eval_math_responses(oai.Completion.extract_text(response))
- return results.get("voted_answer"), cost
+ return results.get("voted_answer"), response["cost"]
def remove_boxed(string: str) -> Optional[str]:
diff --git a/flaml/autogen/oai/__init__.py b/flaml/autogen/oai/__init__.py
index c845911b4..5457351b1 100644
--- a/flaml/autogen/oai/__init__.py
+++ b/flaml/autogen/oai/__init__.py
@@ -1,3 +1,4 @@
from flaml.autogen.oai.completion import Completion, ChatCompletion
+from flaml.autogen.oai.openai_utils import get_config_list, config_list_gpt4_gpt35, config_list_openai_aoai
-__all__ = ["Completion", "ChatCompletion"]
+__all__ = ["Completion", "ChatCompletion", "get_config_list", "config_list_gpt4_gpt35", "config_list_openai_aoai"]
diff --git a/flaml/autogen/oai/completion.py b/flaml/autogen/oai/completion.py
index cc95803f1..f98ccd8a8 100644
--- a/flaml/autogen/oai/completion.py
+++ b/flaml/autogen/oai/completion.py
@@ -2,11 +2,13 @@ from time import sleep
import logging
import numpy as np
import time
-from typing import List, Optional, Dict
+from typing import List, Optional, Dict, Callable, Any
import sys
-import json
+import shutil
from flaml import tune, BlendSearch
+from flaml.tune.space import is_constant
from flaml.automl.logger import logger_formatter
+from .openai_utils import get_key
try:
import openai
@@ -34,23 +36,6 @@ if not logger.handlers:
logger.addHandler(_ch)
-def get_key(config):
- """Get a unique identifier of a configuration.
-
- Args:
- config (dict or list): A configuration.
-
- Returns:
- tuple: A unique identifier which can be used as a key for a dict.
- """
- # if isinstance(config, dict):
- # return tuple(get_key(x) for x in sorted(config.items()))
- # if isinstance(config, list):
- # return tuple(get_key(x) for x in config)
- # return config
- return json.dumps(config, sort_keys=True)
-
-
class Completion(openai_Completion):
"""A class for OpenAI completion API.
@@ -123,7 +108,7 @@ class Completion(openai_Completion):
_history_dict = _count_create = None
@classmethod
- def set_cache(cls, seed=41, cache_path=".cache"):
+ def set_cache(cls, seed: Optional[int] = 41, cache_path_root: Optional[str] = ".cache"):
"""Set cache path.
Args:
@@ -133,11 +118,29 @@ class Completion(openai_Completion):
The complete cache path will be {cache_path}/{seed}.
"""
cls.seed = seed
- cls.cache_path = f"{cache_path}/{seed}"
+ cls.cache_path = f"{cache_path_root}/{seed}"
+
+ @classmethod
+ def clear_cache(cls, seed: Optional[int] = None, cache_path_root: Optional[str] = ".cache"):
+ """Clear cache.
+
+ Args:
+ seed (int, Optional): The integer identifier for the pseudo seed.
+ If omitted, all caches under cache_path_root will be cleared.
+ cache_path (str, Optional): The root path for the cache.
+ The complete cache path will be {cache_path}/{seed}.
+ """
+ if seed is None:
+ shutil.rmtree(cache_path_root, ignore_errors=True)
+ return
+ with diskcache.Cache(f"{cache_path_root}/{seed}") as cache:
+ cache.clear()
@classmethod
def _book_keeping(cls, config: Dict, response):
"""Book keeping for the created completions."""
+ if response != -1 and "cost" not in response:
+ response["cost"] = cls.cost(response)
if cls._history_dict is None:
return
if cls._history_compact:
@@ -154,7 +157,7 @@ class Completion(openai_Completion):
else:
key = get_key([config["prompt"]] + [choice.get("text") for choice in response["choices"]])
value["created_at"].append(cls._count_create)
- value["cost"].append(cls.cost(response))
+ value["cost"].append(response["cost"])
cls._history_dict[key] = value
cls._count_create += 1
return
@@ -165,7 +168,7 @@ class Completion(openai_Completion):
cls._count_create += 1
@classmethod
- def _get_response(cls, config: Dict, eval_only=False, use_cache=True):
+ def _get_response(cls, config: Dict, raise_error=False, use_cache=True):
"""Get the response from the openai api call.
Try cache first. If not found, call the openai api. If the api call fails, retry after retry_time.
@@ -175,7 +178,7 @@ class Completion(openai_Completion):
key = get_key(config)
if use_cache:
response = cls._cache.get(key, None)
- if response is not None and (response != -1 or not eval_only):
+ if response is not None and (response != -1 or not raise_error):
# print("using cached response")
cls._book_keeping(config, response)
return response
@@ -197,7 +200,7 @@ class Completion(openai_Completion):
APIConnectionError,
):
# transient error
- logger.warning(f"retrying in {cls.retry_time} seconds...", exc_info=1)
+ logger.info(f"retrying in {cls.retry_time} seconds...", exc_info=1)
sleep(cls.retry_time)
except APIError as err:
error_code = err and err.json_body and err.json_body.get("error")
@@ -205,7 +208,7 @@ class Completion(openai_Completion):
if error_code == "content_filter":
raise
# transient error
- logger.warning(f"retrying in {cls.retry_time} seconds...", exc_info=1)
+ logger.info(f"retrying in {cls.retry_time} seconds...", exc_info=1)
sleep(cls.retry_time)
except (RateLimitError, Timeout) as err:
time_left = cls.retry_timeout - (time.time() - start_time + cls.retry_time)
@@ -216,10 +219,16 @@ class Completion(openai_Completion):
and isinstance(err, Timeout)
):
logger.info(f"retrying in {cls.retry_time} seconds...", exc_info=1)
- elif eval_only:
+ elif raise_error:
raise
else:
- break
+ response = -1
+ if use_cache and isinstance(err, Timeout):
+ cls._cache.set(key, response)
+ logger.warning(
+ f"Failed to get response from openai api due to getting RateLimitError or Timeout for {cls.retry_timeout} seconds."
+ )
+ return response
if isinstance(err, Timeout):
if "request_timeout" in config:
raise
@@ -237,13 +246,6 @@ class Completion(openai_Completion):
cls._cache.set(key, response)
cls._book_keeping(config, response)
return response
- logger.warning(
- f"Failed to get response from openai api due to getting RateLimitError or Timeout for {cls.retry_timeout} seconds."
- )
- response = -1
- if use_cache:
- cls._cache.set(key, response)
- return response
@classmethod
def _get_max_valid_n(cls, key, max_tokens):
@@ -266,6 +268,7 @@ class Completion(openai_Completion):
@classmethod
def _get_region_key(cls, config):
# get a key for the valid/invalid region corresponding to the given config
+ config = cls._pop_subspace(config, always_copy=False)
return (
config["model"],
config.get("prompt", config.get("messages")),
@@ -282,31 +285,28 @@ class Completion(openai_Completion):
invalid_n[max_tokens] = min(num_completions, invalid_n.get(max_tokens, np.inf))
@classmethod
- def _pop_subspace(cls, config):
+ def _pop_subspace(cls, config, always_copy=True):
if "subspace" in config:
config = config.copy()
config.update(config.pop("subspace"))
- return config
+ return config.copy() if always_copy else config
@classmethod
- def _get_prompt_messages_from_config(cls, model, config):
- prompt, messages = None, None
- if model in cls.chat_models or issubclass(cls, ChatCompletion):
- # either "prompt" should be in config (for being compatible with non-chat models)
- # or "messages" should be in config (for tuning chat models only)
- prompt = config.get("prompt")
- messages = config.get("messages")
- # either prompt or messages should be in config, but not both
- assert (prompt is None) != (
- messages is None
- ), "Either prompt or messages should be in config for chat models."
- if prompt is None:
- messages = cls._messages[messages]
- else:
- prompt = cls._prompts[prompt]
+ def _get_params_for_create(cls, config: Dict) -> Dict:
+ """Get the params for the openai api call from a config in the search space."""
+ params = cls._pop_subspace(config)
+ if cls._prompts:
+ params["prompt"] = cls._prompts[config["prompt"]]
else:
- prompt = cls._prompts[config["prompt"]]
- return prompt, messages
+ params["messages"] = cls._messages[config["messages"]]
+ if "stop" in params:
+ params["stop"] = cls._stops and cls._stops[params["stop"]]
+ temperature_or_top_p = params.pop("temperature_or_top_p", None)
+ if temperature_or_top_p:
+ params.update(temperature_or_top_p)
+ if cls._config_list and "config_list" not in params:
+ params["config_list"] = cls._config_list
+ return params
@classmethod
def _eval(cls, config: dict, prune=True, eval_only=False):
@@ -315,7 +315,8 @@ class Completion(openai_Completion):
Args:
config (dict): Hyperparameter setting for the openai api call.
prune (bool, optional): Whether to enable pruning. Defaults to True.
- eval_only (bool, optional): Whether to evaluate only (ignore the inference budget and no timeout).
+ eval_only (bool, optional): Whether to evaluate only
+ (ignore the inference budget and do not rasie error when a request fails).
Defaults to False.
Returns:
@@ -323,18 +324,18 @@ class Completion(openai_Completion):
"""
cost = 0
data = cls.data
- config = cls._pop_subspace(config)
- model = config["model"]
+ params = cls._get_params_for_create(config)
+ model = params["model"]
data_length = len(data)
price = cls.price1K.get(model)
price_input, price_output = price if isinstance(price, tuple) else (price, price)
inference_budget = getattr(cls, "inference_budget", None)
prune_hp = getattr(cls, "_prune_hp", "n")
metric = cls._metric
- config_n = config.get(prune_hp, 1) # default value in OpenAI is 1
- max_tokens = config.get("max_tokens", np.inf if model in cls.chat_models else 16)
- prompt, messages = cls._get_prompt_messages_from_config(model, config)
- stop = cls._stops and cls._stops[config["stop"]]
+ config_n = params.get(prune_hp, 1) # default value in OpenAI is 1
+ max_tokens = params.get(
+ "max_tokens", np.inf if model in cls.chat_models or issubclass(cls, ChatCompletion) else 16
+ )
target_output_tokens = None
if not cls.avg_input_tokens:
input_tokens = [None] * data_length
@@ -365,12 +366,6 @@ class Completion(openai_Completion):
else:
start_n = config_n
region_key = None
- params = config.copy()
- if "stop" in config:
- params["stop"] = stop
- temperature_or_top_p = params.pop("temperature_or_top_p", None)
- if temperature_or_top_p:
- params.update(temperature_or_top_p)
num_completions, previous_num_completions = start_n, 0
n_tokens_list, result, responses_list = [], {}, []
while True: # n <= config_n
@@ -383,9 +378,9 @@ class Completion(openai_Completion):
for i in range(prev_data_limit, data_limit):
logger.debug(f"num_completions={num_completions}, data instance={i}")
data_i = data[i]
- params = cls._construct_params(data_i, params, prompt, messages)
- response = cls._get_response(params, eval_only)
- if response == -1: # rate limit error, treat as invalid
+ # params = cls._construct_params(data_i, params, prompt, messages)
+ response = cls.create(data_i, raise_error=eval_only, **params)
+ if response == -1: # rate limit/timeout error, treat as invalid
cls._update_invalid_n(prune, region_key, max_tokens, num_completions)
result[metric] = 0
result["cost"] = cost
@@ -398,7 +393,7 @@ class Completion(openai_Completion):
if not cls.avg_input_tokens and not input_tokens[i]:
# store the # input tokens
input_tokens[i] = n_input_tokens
- query_cost = (price_input * n_input_tokens + price_output * n_output_tokens) / 1000
+ query_cost = response["cost"]
cls._total_cost += query_cost
cost += query_cost
if cls.optimization_budget and cls._total_cost >= cls.optimization_budget and not eval_only:
@@ -489,15 +484,15 @@ class Completion(openai_Completion):
@classmethod
def tune(
cls,
- data,
- metric,
- mode,
- eval_func,
- log_file_name=None,
- inference_budget=None,
- optimization_budget=None,
- num_samples=1,
- logging_level=logging.WARNING,
+ data: List[Dict],
+ metric: str,
+ mode: str,
+ eval_func: Callable,
+ log_file_name: Optional[str] = None,
+ inference_budget: Optional[float] = None,
+ optimization_budget: Optional[float] = None,
+ num_samples: Optional[int] = 1,
+ logging_level: Optional[int] = logging.WARNING,
**config,
):
"""Tune the parameters for the OpenAI API call.
@@ -597,6 +592,11 @@ class Completion(openai_Completion):
if not (isinstance(cls._stops, list) and isinstance(cls._stops[0], list)):
cls._stops = [cls._stops]
space["stop"] = tune.choice(list(range(len(cls._stops))))
+ cls._config_list = space.get("config_list")
+ if cls._config_list is not None:
+ is_const = is_constant(cls._config_list)
+ if is_const:
+ space.pop("config_list")
cls._metric, cls._mode = metric, mode
cls._total_cost = 0 # total optimization cost
cls._eval_func = eval_func
@@ -663,16 +663,9 @@ class Completion(openai_Completion):
verbose=3,
)
config = analysis.best_config
- params = cls._pop_subspace(config)
- if cls._prompts:
- params["prompt"] = cls._prompts[config["prompt"]]
- else:
- params["messages"] = cls._messages[config["messages"]]
- stop = cls._stops and cls._stops[config["stop"]]
- params["stop"] = stop
- temperature_or_top_p = params.pop("temperature_or_top_p", None)
- if temperature_or_top_p:
- params.update(temperature_or_top_p)
+ params = cls._get_params_for_create(config)
+ if cls._config_list is not None and is_const:
+ params.pop("config_list")
logger.setLevel(old_level)
return params, analysis
@@ -681,14 +674,16 @@ class Completion(openai_Completion):
cls,
context: Optional[Dict] = None,
use_cache: Optional[bool] = True,
- config_list: Optional[List] = None,
+ config_list: Optional[List[Dict]] = None,
+ filter_func: Optional[Callable[[Dict, Dict, Dict], bool]] = None,
+ raise_error: Optional[bool] = True,
**config,
):
"""Make a completion for a given context.
Args:
context (Dict, Optional): The context to instantiate the prompt.
- It needs to contain keys that are used by the prompt template.
+ It needs to contain keys that are used by the prompt template or the filter function.
E.g., `prompt="Complete the following sentence: {prefix}, context={"prefix": "Today I feel"}`.
The actual prompt will be:
"Complete the following sentence: Today I feel".
@@ -714,19 +709,28 @@ class Completion(openai_Completion):
"api_key": os.environ.get("OPENAI_API_KEY"),
"api_type": "open_ai",
"api_base": "https://api.openai.com/v1",
- "api_version": None,
},
{
"model": "llama-7B",
"api_base": "http://127.0.0.1:8080",
"api_type": "open_ai",
- "api_version": None,
}
],
prompt="Hi",
)
```
+ filter_func (Callable, Optional): A function that takes in the context, the config and the response and returns a boolean to indicate whether the response is valid. E.g.,
+
+ ```python
+ def yes_or_no_filter(context, config, response):
+ return context.get("yes_or_no_choice", False) is False or any(
+ text in ["Yes.", "No."] for text in oai.Completion.extract_text(response)
+ )
+ ```
+
+ raise_error (bool, Optional): Whether to raise error when all configs fail.
+ When set to False, -1 will be returned when all configs fail.
**config: Configuration for the completion.
Besides the parameters for the openai API call, it can also contain a seed (int) for the cache.
This is useful when implementing "controlled randomness" for the completion.
@@ -739,28 +743,39 @@ class Completion(openai_Completion):
raise ERROR
if config_list:
retry_timeout = cls.retry_timeout
+ last = len(config_list) - 1
+ cost = 0
for i, each_config in enumerate(config_list):
base_config = config.copy()
base_config.update(each_config)
try:
- cls.retry_timeout = 0 if i < len(config_list) - 1 else retry_timeout
- # retry_timeout = 0 to avoid retrying
- return cls.create(context, use_cache, **base_config)
+ cls.retry_timeout = 0 if i < last and filter_func is None else retry_timeout
+ # retry_timeout = 0 to avoid retrying when no filter is given
+ response = cls.create(context, use_cache, **base_config)
+ pass_filter = filter_func is None or filter_func(
+ context=context, base_config=config, response=response
+ )
+ if pass_filter or i == last:
+ response["cost"] = cost + response["cost"]
+ response["config_id"] = i
+ response["pass_filter"] = pass_filter
+ return response
+ cost += response["cost"]
except (AuthenticationError, RateLimitError, Timeout):
- logger.info(f"failed with config {i}", exc_info=1)
- if i == len(config_list) - 1:
+ logger.debug(f"failed with config {i}", exc_info=1)
+ if i == last:
raise
finally:
cls.retry_timeout = retry_timeout
params = cls._construct_params(context, config)
if not use_cache:
- return cls._get_response(params, eval_only=True, use_cache=False)
+ return cls._get_response(params, raise_error=raise_error, use_cache=False)
seed = cls.seed
if "seed" in params:
cls.set_cache(params.pop("seed"))
with diskcache.Cache(cls.cache_path) as cls._cache:
cls.set_cache(seed)
- return cls._get_response(params, eval_only=True)
+ return cls._get_response(params, raise_error=raise_error)
@classmethod
def _instantiate(cls, template: str, context: Optional[Dict] = None):
@@ -810,18 +825,17 @@ class Completion(openai_Completion):
def test(
cls,
data,
- config,
eval_func=None,
use_cache=True,
agg_method="avg",
return_responses_and_per_instance_result=False,
logging_level=logging.WARNING,
+ **config,
):
"""Evaluate the responses created with the config for the OpenAI API call.
Args:
data (list): The list of test data points.
- config (dict): Hyperparameter setting for the openai api call.
eval_func (Callable): The evaluation function for responses per data instance.
The function should take a list of responses and a data point as input,
and return a dict of metrics. You need to either provide a valid callable
@@ -867,6 +881,7 @@ class Completion(openai_Completion):
return_responses_and_per_instance_result (bool): Whether to also return responses
and per instance results in addition to the aggregated results.
logging_level (optional): logging level. Defaults to logging.WARNING.
+ **config (dict): parametes passed to the openai api call `create()`.
Returns:
None when no valid eval_func is provided in either test or tune;
@@ -881,7 +896,7 @@ class Completion(openai_Completion):
for i, data_i in enumerate(data):
logger.info(f"evaluating data instance {i}")
response = cls.create(data_i, use_cache, **config)
- cost += cls.cost(response)
+ cost += response["cost"]
# evaluate the quality of the responses
responses = cls.extract_text(response)
if eval_func is not None:
@@ -947,11 +962,12 @@ class Completion(openai_Completion):
response (dict): The response from OpenAI API.
Returns:
- The cost in USD.
+ The cost in USD. 0 if the model is not supported.
"""
model = response["model"]
if model not in cls.price1K:
- raise ValueError(f"Unknown model: {model}")
+ return 0
+ # raise ValueError(f"Unknown model: {model}")
usage = response["usage"]
n_input_tokens = usage["prompt_tokens"]
n_output_tokens = usage.get("completion_tokens", 0)
diff --git a/flaml/autogen/oai/openai_utils.py b/flaml/autogen/oai/openai_utils.py
new file mode 100644
index 000000000..44bc0ad61
--- /dev/null
+++ b/flaml/autogen/oai/openai_utils.py
@@ -0,0 +1,142 @@
+import os
+import json
+from typing import List, Optional, Dict
+import logging
+
+NON_CACHE_KEY = ["api_key", "api_base", "api_type", "api_version"]
+
+
+def get_key(config):
+ """Get a unique identifier of a configuration.
+
+ Args:
+ config (dict or list): A configuration.
+
+ Returns:
+ tuple: A unique identifier which can be used as a key for a dict.
+ """
+ copied = False
+ for key in NON_CACHE_KEY:
+ if key in config:
+ config, copied = config.copy() if not copied else config, True
+ config.pop(key)
+ # if isinstance(config, dict):
+ # return tuple(get_key(x) for x in sorted(config.items()))
+ # if isinstance(config, list):
+ # return tuple(get_key(x) for x in config)
+ # return config
+ return json.dumps(config, sort_keys=True)
+
+
+def get_config_list(
+ api_keys: List, api_bases: Optional[List] = None, api_type: Optional[str] = None, api_version: Optional[str] = None
+) -> List[Dict]:
+ """Get a list of configs for openai api calls.
+
+ Args:
+ api_keys (list): The api keys for openai api calls.
+ api_bases (list, optional): The api bases for openai api calls.
+ api_type (str, optional): The api type for openai api calls.
+ api_version (str, optional): The api version for openai api calls.
+ """
+ config_list = []
+ for i, api_key in enumerate(api_keys):
+ if not api_key.strip():
+ continue
+ config = {"api_key": api_key}
+ if api_bases:
+ config["api_base"] = api_bases[i]
+ if api_type:
+ config["api_type"] = api_type
+ if api_version:
+ config["api_version"] = api_version
+ config_list.append(config)
+ return config_list
+
+
+def config_list_openai_aoai(
+ key_file_path: Optional[str] = ".",
+ openai_api_key_file: Optional[str] = "key_openai.txt",
+ aoai_api_key_file: Optional[str] = "key_aoai.txt",
+ aoai_api_base_file: Optional[str] = "base_aoai.txt",
+) -> List[Dict]:
+ """Get a list of configs for openai + azure openai api calls.
+
+ Args:
+ key_file_path (str, optional): The path to the key files.
+ openai_api_key_file (str, optional): The file name of the openai api key.
+ aoai_api_key_file (str, optional): The file name of the azure openai api key.
+ aoai_api_base_file (str, optional): The file name of the azure openai api base.
+
+ Returns:
+ list: A list of configs for openai api calls.
+ """
+ if "OPENAI_API_KEY" not in os.environ:
+ try:
+ os.environ["OPENAI_API_KEY"] = open(f"{key_file_path}/{openai_api_key_file}").read().strip()
+ except FileNotFoundError:
+ logging.info(
+ "To use OpenAI API, please set OPENAI_API_KEY in os.environ "
+ "or create key_openai.txt in the specified path, or specify the api_key in config_list."
+ )
+ if "AZURE_OPENAI_API_KEY" not in os.environ:
+ try:
+ os.environ["AZURE_OPENAI_API_KEY"] = open(f"{key_file_path}/{aoai_api_key_file}").read().strip()
+ except FileNotFoundError:
+ logging.info(
+ "To use Azure OpenAI API, please set AZURE_OPENAI_API_KEY in os.environ "
+ "or create key_aoai.txt in the specified path, or specify the api_key in config_list."
+ )
+ if "AZURE_OPENAI_API_BASE" not in os.environ:
+ try:
+ os.environ["AZURE_OPENAI_API_BASE"] = open(f"{key_file_path}/{aoai_api_base_file}").read().strip()
+ except FileNotFoundError:
+ logging.info(
+ "To use Azure OpenAI API, please set AZURE_OPENAI_API_BASE in os.environ "
+ "or create base_aoai.txt in the specified path, or specify the api_base in config_list."
+ )
+ aoai_config = get_config_list(
+ # Assuming Azure OpenAI api keys in os.environ["AZURE_OPENAI_API_KEY"], in separated lines
+ api_keys=os.environ.get("AZURE_OPENAI_API_KEY", "").split("\n"),
+ # Assuming Azure OpenAI api bases in os.environ["AZURE_OPENAI_API_BASE"], in separated lines
+ api_bases=os.environ.get("AZURE_OPENAI_API_BASE", "").split("\n"),
+ api_type="azure",
+ api_version="2023-03-15-preview", # change if necessary
+ )
+ openai_config = get_config_list(
+ # Assuming OpenAI API_KEY in os.environ["OPENAI_API_KEY"]
+ api_keys=os.environ.get("OPENAI_API_KEY", "").split("\n"),
+ # "api_type": "open_ai",
+ # "api_base": "https://api.openai.com/v1",
+ )
+ config_list = openai_config + aoai_config
+ return config_list
+
+
+def config_list_gpt4_gpt35(
+ key_file_path: Optional[str] = ".",
+ openai_api_key_file: Optional[str] = "key_openai.txt",
+ aoai_api_key_file: Optional[str] = "key_aoai.txt",
+ aoai_api_base_file: Optional[str] = "base_aoai.txt",
+) -> List[Dict]:
+ """Get a list of configs for gpt-4 followed by gpt-3.5 api calls.
+
+ Args:
+ key_file_path (str, optional): The path to the key files.
+ openai_api_key_file (str, optional): The file name of the openai api key.
+ aoai_api_key_file (str, optional): The file name of the azure openai api key.
+ aoai_api_base_file (str, optional): The file name of the azure openai api base.
+
+ Returns:
+ list: A list of configs for openai api calls.
+ """
+
+ config_list = config_list_openai_aoai(
+ key_file_path,
+ openai_api_key_file,
+ aoai_api_key_file,
+ aoai_api_base_file,
+ )
+ return [{**config, "model": "gpt-4"} for config in config_list] + [
+ {**config, "model": "gpt-3.5-turbo"} for config in config_list
+ ]
diff --git a/flaml/tune/space.py b/flaml/tune/space.py
index 72a08cf1c..a46738df6 100644
--- a/flaml/tune/space.py
+++ b/flaml/tune/space.py
@@ -11,7 +11,7 @@ try:
except (ImportError, AssertionError):
from . import sample
from .searcher.variant_generator import generate_variants
-from typing import Dict, Optional, Any, Tuple, Generator
+from typing import Dict, Optional, Any, Tuple, Generator, List, Union
import numpy as np
import logging
@@ -27,6 +27,29 @@ def generate_variants_compatible(
return generate_variants(unresolved_spec, constant_grid_search)
+def is_constant(space: Union[Dict, List]) -> bool:
+ """Whether the search space is all constant.
+
+ Returns:
+ A bool of whether the search space is all constant.
+ """
+ if isinstance(space, dict):
+ for domain in space.values():
+ if isinstance(domain, (dict, list)):
+ if not is_constant(domain):
+ return False
+ continue
+ if isinstance(domain, sample.Domain):
+ return False
+ return True
+ elif isinstance(space, list):
+ for item in space:
+ if not is_constant(item):
+ return False
+ return True
+ return not isinstance(space, sample.Domain)
+
+
def define_by_run_func(trial, space: Dict, path: str = "") -> Optional[Dict[str, Any]]:
"""Define-by-run function to create the search space.
diff --git a/notebook/autogen_chatgpt.ipynb b/notebook/autogen_chatgpt.ipynb
deleted file mode 100644
index 672038b9c..000000000
--- a/notebook/autogen_chatgpt.ipynb
+++ /dev/null
@@ -1,1525 +0,0 @@
-{
- "cells": [
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {
- "slideshow": {
- "slide_type": "slide"
- }
- },
- "source": [
- "Copyright (c) Microsoft Corporation. All rights reserved. \n",
- "\n",
- "Licensed under the MIT License.\n",
- "\n",
- "# Use FLAML to Tune ChatGPT\n",
- "\n",
- "FLAML offers a cost-effective hyperparameter optimization technique [EcoOptiGen](https://arxiv.org/abs/2303.04673) for tuning Large Language Models. Our study finds that tuning hyperparameters can significantly improve the utility of LLMs.\n",
- "\n",
- "In this notebook, we tune OpenAI ChatGPT (both GPT-3.5 and GPT-4) models for math problem solving. We use [the MATH benchmark](https://crfm.stanford.edu/helm/latest/?group=math_chain_of_thought) for measuring mathematical problem solving on competition math problems with chain-of-thoughts style reasoning. \n",
- "\n",
- "## Requirements\n",
- "\n",
- "FLAML requires `Python>=3.7`. To run this notebook example, please install flaml with the [openai,blendsearch] option:\n",
- "```bash\n",
- "pip install flaml[openai,blendsearch]==1.2.2\n",
- "```"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 1,
- "metadata": {
- "execution": {
- "iopub.execute_input": "2023-02-13T23:40:52.317406Z",
- "iopub.status.busy": "2023-02-13T23:40:52.316561Z",
- "iopub.status.idle": "2023-02-13T23:40:52.321193Z",
- "shell.execute_reply": "2023-02-13T23:40:52.320628Z"
- }
- },
- "outputs": [],
- "source": [
- "# %pip install flaml[openai,blendsearch]==1.2.2 datasets"
- ]
- },
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Set your OpenAI key:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 2,
- "metadata": {
- "execution": {
- "iopub.execute_input": "2023-02-13T23:40:52.324240Z",
- "iopub.status.busy": "2023-02-13T23:40:52.323783Z",
- "iopub.status.idle": "2023-02-13T23:40:52.330570Z",
- "shell.execute_reply": "2023-02-13T23:40:52.329750Z"
- }
- },
- "outputs": [],
- "source": [
- "import os\n",
- "\n",
- "if \"OPENAI_API_KEY\" not in os.environ:\n",
- " os.environ[\"OPENAI_API_KEY\"] = \"<your OpenAI API key here>\""
- ]
- },
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Uncomment the following to use Azure OpenAI:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 3,
- "metadata": {
- "execution": {
- "iopub.execute_input": "2023-02-13T23:40:52.333547Z",
- "iopub.status.busy": "2023-02-13T23:40:52.333249Z",
- "iopub.status.idle": "2023-02-13T23:40:52.336508Z",
- "shell.execute_reply": "2023-02-13T23:40:52.335858Z"
- }
- },
- "outputs": [],
- "source": [
- "# import openai\n",
- "# openai.api_type = \"azure\"\n",
- "# openai.api_base = \"https://<your_endpoint>.openai.azure.com/\"\n",
- "# openai.api_version = \"2023-03-15-preview\""
- ]
- },
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Load dataset\n",
- "\n",
- "First, we load the competition_math dataset. The dataset contains 201 \"Level 2\" Algebra examples. We use a random sample of 20 examples for tuning the generation hyperparameters and the remaining for evaluation."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 4,
- "metadata": {
- "execution": {
- "iopub.execute_input": "2023-02-13T23:40:52.339977Z",
- "iopub.status.busy": "2023-02-13T23:40:52.339556Z",
- "iopub.status.idle": "2023-02-13T23:40:54.603349Z",
- "shell.execute_reply": "2023-02-13T23:40:54.602630Z"
- }
- },
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "Using custom data configuration default\n",
- "Found cached dataset competition_math (/home/vscode/.cache/huggingface/datasets/competition_math/default/1.0.0/2a2a2995c2847186883ecd64f69be7d602b8a6f6b51950624d4dc2263f93333b)\n"
- ]
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "c23bfd043e284ea29f8a6b4de2974637",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- " 0%| | 0/2 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "Loading cached shuffled indices for dataset at /home/vscode/.cache/huggingface/datasets/competition_math/default/1.0.0/2a2a2995c2847186883ecd64f69be7d602b8a6f6b51950624d4dc2263f93333b/cache-f1cfe8228271b121.arrow\n",
- "Loading cached shuffled indices for dataset at /home/vscode/.cache/huggingface/datasets/competition_math/default/1.0.0/2a2a2995c2847186883ecd64f69be7d602b8a6f6b51950624d4dc2263f93333b/cache-d155a2d38c23bd53.arrow\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "20 201\n"
- ]
- }
- ],
- "source": [
- "import datasets\n",
- "\n",
- "seed = 41\n",
- "data = datasets.load_dataset(\"competition_math\")\n",
- "train_data = data[\"train\"].shuffle(seed=seed)\n",
- "test_data = data[\"test\"].shuffle(seed=seed)\n",
- "n_tune_data = 20\n",
- "tune_data = [\n",
- " {\n",
- " \"problem\": train_data[x][\"problem\"],\n",
- " \"solution\": train_data[x][\"solution\"],\n",
- " }\n",
- " for x in range(len(train_data)) if train_data[x][\"level\"] == \"Level 2\" and train_data[x][\"type\"] == \"Algebra\"\n",
- "][:n_tune_data]\n",
- "test_data = [\n",
- " {\n",
- " \"problem\": test_data[x][\"problem\"],\n",
- " \"solution\": test_data[x][\"solution\"],\n",
- " }\n",
- " for x in range(len(test_data)) if test_data[x][\"level\"] == \"Level 2\" and test_data[x][\"type\"] == \"Algebra\"\n",
- "]\n",
- "print(len(tune_data), len(test_data))\n"
- ]
- },
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {
- "slideshow": {
- "slide_type": "slide"
- }
- },
- "source": [
- "Check a tuning example:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 5,
- "metadata": {
- "execution": {
- "iopub.execute_input": "2023-02-13T23:40:54.607152Z",
- "iopub.status.busy": "2023-02-13T23:40:54.606441Z",
- "iopub.status.idle": "2023-02-13T23:40:54.610504Z",
- "shell.execute_reply": "2023-02-13T23:40:54.609759Z"
- },
- "slideshow": {
- "slide_type": "subslide"
- },
- "tags": []
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Rationalize the denominator of $\\displaystyle\\frac{21}{\\sqrt{21}}$.\n"
- ]
- }
- ],
- "source": [
- "print(tune_data[1][\"problem\"])"
- ]
- },
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Here is one example of the canonical solution:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 6,
- "metadata": {
- "execution": {
- "iopub.execute_input": "2023-02-13T23:40:54.613590Z",
- "iopub.status.busy": "2023-02-13T23:40:54.613168Z",
- "iopub.status.idle": "2023-02-13T23:40:54.616873Z",
- "shell.execute_reply": "2023-02-13T23:40:54.616193Z"
- }
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "$\\dfrac{21}{\\sqrt{21}} = \\dfrac{21}{\\sqrt{21}} \\cdot \\dfrac{\\sqrt{21}}{\\sqrt{21}} = \\dfrac{21\\sqrt{21}}{21} = \\boxed{\\!\\sqrt{21}}$.\n"
- ]
- }
- ],
- "source": [
- "print(tune_data[1][\"solution\"])"
- ]
- },
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Define Success Metric\n",
- "\n",
- "Before we start tuning, we need to define the success metric we want to optimize. For each math task, we use voting to select a response with the most common answers out of all the generated responses. If it has an equivalent answer to the canonical solution, we consider the task as successfully solved. Then we can optimize the mean success rate of a collection of tasks."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 7,
- "metadata": {
- "execution": {
- "iopub.execute_input": "2023-02-13T23:40:54.626998Z",
- "iopub.status.busy": "2023-02-13T23:40:54.626593Z",
- "iopub.status.idle": "2023-02-13T23:40:54.631383Z",
- "shell.execute_reply": "2023-02-13T23:40:54.630770Z"
- }
- },
- "outputs": [],
- "source": [
- "from flaml.autogen.math_utils import eval_math_responses"
- ]
- },
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {
- "slideshow": {
- "slide_type": "slide"
- }
- },
- "source": [
- "## Use the tuning data to find a good configuration\n",
- "\n",
- "### Import the oai and tune subpackages from flaml.\n",
- "\n",
- "FLAML has provided an API for hyperparameter optimization of OpenAI ChatGPT models: `oai.ChatCompletion.tune` and to make a request with the tuned config: `oai.ChatCompletion.create`. First, we import oai from flaml:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 8,
- "metadata": {
- "execution": {
- "iopub.execute_input": "2023-02-13T23:40:54.634335Z",
- "iopub.status.busy": "2023-02-13T23:40:54.633929Z",
- "iopub.status.idle": "2023-02-13T23:40:56.105700Z",
- "shell.execute_reply": "2023-02-13T23:40:56.105085Z"
- },
- "slideshow": {
- "slide_type": "slide"
- }
- },
- "outputs": [],
- "source": [
- "from flaml import oai"
- ]
- },
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "For (local) reproducibility and cost efficiency, we cache responses from OpenAI."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 9,
- "metadata": {
- "execution": {
- "iopub.execute_input": "2023-02-13T23:40:56.109177Z",
- "iopub.status.busy": "2023-02-13T23:40:56.108624Z",
- "iopub.status.idle": "2023-02-13T23:40:56.112651Z",
- "shell.execute_reply": "2023-02-13T23:40:56.112076Z"
- },
- "slideshow": {
- "slide_type": "slide"
- }
- },
- "outputs": [],
- "source": [
- "oai.ChatCompletion.set_cache(seed)"
- ]
- },
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "This will create a disk cache in \".cache/{seed}\". You can change `cache_path` in `set_cache()`. The cache for different seeds are stored separately.\n",
- "\n",
- "### Perform tuning\n",
- "\n",
- "The tuning will take a while to finish, depending on the optimization budget. The tuning will be performed under the specified optimization budgets.\n",
- "\n",
- "* `inference_budget` is the target average inference budget per instance in the benchmark. For example, 0.004 means the target inference budget is 0.004 dollars, which translates to 2000 tokens (input + output combined) if the gpt-3.5-turbo model is used.\n",
- "* `optimization_budget` is the total budget allowed to perform the tuning. For example, 1 means 1 dollars are allowed in total, which translates to 500K tokens for the gpt-3.5-turbo model.\n",
- "* `num_sumples` is the number of different hyperparameter configurations which is allowed to try. The tuning will stop after either num_samples trials or after optimization_budget dollars spent, whichever happens first. -1 means no hard restriction in the number of trials and the actual number is decided by `optimization_budget`.\n",
- "\n",
- "Users can specify tuning data, optimization metric, optimization mode, evaluation function, search spaces etc.. The default search space is:\n",
- "\n",
- "```python\n",
- "default_search_space = {\n",
- " \"model\": tune.choice([\n",
- " \"gpt-3.5-turbo\",\n",
- " \"gpt-4\",\n",
- " ]),\n",
- " \"temperature_or_top_p\": tune.choice(\n",
- " [\n",
- " {\"temperature\": tune.uniform(0, 1)},\n",
- " {\"top_p\": tune.uniform(0, 1)},\n",
- " ]\n",
- " ),\n",
- " \"max_tokens\": tune.lograndint(50, 1000),\n",
- " \"n\": tune.randint(1, 100),\n",
- " \"prompt\": \"{prompt}\",\n",
- "}\n",
- "```\n",
- "\n",
- "The default search space can be overridden by users' input.\n",
- "For example, the following code specifies a fixed prompt template. For hyperparameters which don't appear in users' input, the default search space will be used."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 10,
- "metadata": {
- "execution": {
- "iopub.execute_input": "2023-02-13T23:40:56.115383Z",
- "iopub.status.busy": "2023-02-13T23:40:56.114975Z",
- "iopub.status.idle": "2023-02-13T23:41:55.045654Z",
- "shell.execute_reply": "2023-02-13T23:41:55.044973Z"
- }
- },
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "\u001b[32m[I 2023-04-02 21:11:48,741]\u001b[0m A new study created in memory with name: optuna\u001b[0m\n",
- "\u001b[32m[I 2023-04-02 21:11:48,744]\u001b[0m A new study created in memory with name: optuna\u001b[0m\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "[flaml.tune.tune: 04-02 21:11:48] {832} INFO - trial 1 config: {'prompt': 0, 'subspace': {'model': 'gpt-4', 'max_tokens': 347, 'temperature_or_top_p': {'temperature': 0.36865945026811975}, 'n': 1}}\n",
- "[flaml.tune.tune: 04-02 21:11:48] {215} INFO - result: {'expected_success': 0.8, 'success': 0.8, 'success_vote': 0.8, 'voted_answer': 'We use the distance formula to find the distance between the points $(0,4)$ and $(3,0)$: $$\\\\sqrt{(3-0)^2+(0-4)^2}=\\\\sqrt{3^2+(-4)^2}=\\\\sqrt{9+16}=\\\\sqrt{25}=\\\\boxed{5}.$$', 'votes': 1.0, 'total_cost': 0.14595, 'cost': 0.14595, 'inference_cost': 0.007297499999999999, 'training_iteration': 0, 'config': {'prompt': 0, 'subspace': {'model': 'gpt-4', 'max_tokens': 347, 'temperature_or_top_p': {'temperature': 0.36865945026811975}, 'n': 1}}, 'config/prompt': 0, 'config/subspace': {'model': 'gpt-4', 'max_tokens': 347, 'temperature_or_top_p': {'temperature': 0.36865945026811975}, 'n': 1}, 'experiment_tag': 'exp', 'time_total_s': 0.005024433135986328}\n",
- "[flaml.tune.tune: 04-02 21:11:48] {832} INFO - trial 2 config: {'prompt': 0, 'subspace': {'model': 'gpt-3.5-turbo', 'max_tokens': 347, 'temperature_or_top_p': {'temperature': 0.36865945026811975}, 'n': 1}}\n",
- "[flaml.tune.tune: 04-02 21:11:48] {215} INFO - result: {'expected_success': 0.75, 'success': 0.75, 'success_vote': 0.75, 'voted_answer': 'Using the distance formula, we have:\\n\\n$$\\\\sqrt{(3-0)^2+(0-4)^2}=\\\\sqrt{9+16}=\\\\sqrt{25}=5$$\\n\\nTherefore, the distance between the points (0,4) and (3,0) is $\\\\boxed{5}$.', 'votes': 1.0, 'total_cost': 0.153144, 'cost': 0.007194, 'inference_cost': 0.0003577, 'training_iteration': 0, 'config': {'prompt': 0, 'subspace': {'model': 'gpt-3.5-turbo', 'max_tokens': 347, 'temperature_or_top_p': {'temperature': 0.36865945026811975}, 'n': 1}}, 'config/prompt': 0, 'config/subspace': {'model': 'gpt-3.5-turbo', 'max_tokens': 347, 'temperature_or_top_p': {'temperature': 0.36865945026811975}, 'n': 1}, 'experiment_tag': 'exp', 'time_total_s': 0.004560708999633789}\n",
- "[flaml.tune.tune: 04-02 21:11:48] {832} INFO - trial 3 config: {'prompt': 0, 'subspace': {'max_tokens': 332, 'temperature_or_top_p': {'temperature': 0.22479664553084766}, 'n': 20, 'model': 'gpt-3.5-turbo'}}\n",
- "[flaml.tune.tune: 04-02 21:14:17] {215} INFO - result: {'expected_success': 0.9135658788605563, 'success': 0.95, 'success_vote': 0.8, 'voted_answer': 'We use the distance formula: $$\\\\sqrt{(x_2-x_1)^2+(y_2-y_1)^2}.$$ Plugging in the coordinates of the two points, we get: $$\\\\sqrt{(3-0)^2+(0-4)^2}.$$ Simplifying, we have: $$\\\\sqrt{9+16}.$$ This equals $\\\\sqrt{25}=5$. Therefore, the distance between the two points is $\\\\boxed{5}$.', 'votes': 15.45, 'total_cost': 0.261568, 'cost': 0.108424, 'inference_cost': 0.0054192, 'training_iteration': 0, 'config': {'prompt': 0, 'subspace': {'max_tokens': 332, 'temperature_or_top_p': {'temperature': 0.22479664553084766}, 'n': 20, 'model': 'gpt-3.5-turbo'}}, 'config/prompt': 0, 'config/subspace': {'max_tokens': 332, 'temperature_or_top_p': {'temperature': 0.22479664553084766}, 'n': 20, 'model': 'gpt-3.5-turbo'}, 'experiment_tag': 'exp', 'time_total_s': 148.96061491966248}\n",
- "[flaml.tune.tune: 04-02 21:14:17] {832} INFO - trial 4 config: {'prompt': 0, 'subspace': {'max_tokens': 65, 'temperature_or_top_p': {'top_p': 0.003948266327914451}, 'n': 51, 'model': 'gpt-3.5-turbo'}}\n",
- "[flaml.tune.tune: 04-02 21:15:13] {215} INFO - result: {'expected_success': 0.25, 'success': 0.25, 'success_vote': 0.25, 'voted_answer': 'We use the distance formula: $$\\\\sqrt{(x_2-x_1)^2+(y_2-y_1)^2}$$ Letting $(x_1,y_1)=(0,4)$ and $(x_2,y_2)=(3,0)$, we have: $$\\\\sqrt{(3', 'votes': 15.3, 'total_cost': 0.3915420000000002, 'cost': 0.12997399999999998, 'inference_cost': 0.006496699999999999, 'training_iteration': 0, 'config': {'prompt': 0, 'subspace': {'max_tokens': 65, 'temperature_or_top_p': {'top_p': 0.003948266327914451}, 'n': 51, 'model': 'gpt-3.5-turbo'}}, 'config/prompt': 0, 'config/subspace': {'max_tokens': 65, 'temperature_or_top_p': {'top_p': 0.003948266327914451}, 'n': 51, 'model': 'gpt-3.5-turbo'}, 'experiment_tag': 'exp', 'time_total_s': 56.23015236854553}\n",
- "[flaml.tune.tune: 04-02 21:15:13] {832} INFO - trial 5 config: {'prompt': 0, 'subspace': {'max_tokens': 433, 'temperature_or_top_p': {'top_p': 0.7145757833976906}, 'n': 54, 'model': 'gpt-3.5-turbo'}}\n",
- "[flaml.tune.tune: 04-02 21:18:45] {215} INFO - result: {'expected_success': 0.9999912490789106, 'success': 1.0, 'success_vote': 0.9, 'voted_answer': 'We use the distance formula: $$\\\\sqrt{(x_2-x_1)^2+(y_2-y_1)^2}.$$ Letting $(x_1,y_1)=(0,4)$ and $(x_2,y_2)=(3,0)$, we have: \\\\begin{align*}\\n\\\\sqrt{(3-0)^2+(0-4)^2} &= \\\\sqrt{9+16} \\\\\\\\\\n&= \\\\sqrt{25} \\\\\\\\\\n&= \\\\boxed{5}.\\n\\\\end{align*}', 'votes': 41.65, 'total_cost': 0.6993100000000001, 'cost': 0.30776799999999993, 'inference_cost': 0.014812200000000001, 'training_iteration': 0, 'config': {'prompt': 0, 'subspace': {'max_tokens': 433, 'temperature_or_top_p': {'top_p': 0.7145757833976906}, 'n': 54, 'model': 'gpt-3.5-turbo'}}, 'config/prompt': 0, 'config/subspace': {'max_tokens': 433, 'temperature_or_top_p': {'top_p': 0.7145757833976906}, 'n': 54, 'model': 'gpt-3.5-turbo'}, 'experiment_tag': 'exp', 'time_total_s': 211.06462264060974}\n",
- "[flaml.tune.tune: 04-02 21:18:45] {832} INFO - trial 6 config: {'prompt': 0, 'subspace': {'max_tokens': 375, 'temperature_or_top_p': {'temperature': 0.6177669784693172}, 'n': 51, 'model': 'gpt-4'}}\n",
- "[flaml.tune.tune: 04-02 21:19:09] {215} INFO - result: {'success_vote': 0, 'total_cost': 0.75685, 'cost': 0.057539999999999994, 'training_iteration': 0, 'config': {'prompt': 0, 'subspace': {'max_tokens': 375, 'temperature_or_top_p': {'temperature': 0.6177669784693172}, 'n': 51, 'model': 'gpt-4'}}, 'config/prompt': 0, 'config/subspace': {'max_tokens': 375, 'temperature_or_top_p': {'temperature': 0.6177669784693172}, 'n': 51, 'model': 'gpt-4'}, 'experiment_tag': 'exp', 'time_total_s': 24.581016063690186}\n",
- "[flaml.tune.tune: 04-02 21:19:09] {832} INFO - trial 7 config: {'prompt': 0, 'subspace': {'max_tokens': 557, 'temperature_or_top_p': {'top_p': 0.3192360889885453}, 'n': 9, 'model': 'gpt-3.5-turbo'}}\n",
- "[flaml.tune.tune: 04-02 21:20:54] {215} INFO - result: {'expected_success': 0.7499999998709412, 'success': 0.75, 'success_vote': 0.75, 'voted_answer': 'We use the distance formula: $$\\\\sqrt{(3-0)^2+(0-4)^2}=\\\\sqrt{9+16}=\\\\sqrt{25}=5.$$ Therefore, the distance between the points (0,4) and (3,0) is $\\\\boxed{5}$.', 'votes': 8.45, 'total_cost': 0.8058679999999998, 'cost': 0.049018, 'inference_cost': 0.0024489, 'training_iteration': 0, 'config': {'prompt': 0, 'subspace': {'max_tokens': 557, 'temperature_or_top_p': {'top_p': 0.3192360889885453}, 'n': 9, 'model': 'gpt-3.5-turbo'}}, 'config/prompt': 0, 'config/subspace': {'max_tokens': 557, 'temperature_or_top_p': {'top_p': 0.3192360889885453}, 'n': 9, 'model': 'gpt-3.5-turbo'}, 'experiment_tag': 'exp', 'time_total_s': 104.52026915550232}\n",
- "[flaml.tune.tune: 04-02 21:20:54] {832} INFO - trial 8 config: {'prompt': 0, 'subspace': {'max_tokens': 597, 'temperature_or_top_p': {'top_p': 0.5475861559192435}, 'n': 82, 'model': 'gpt-3.5-turbo'}}\n",
- "[flaml.tune.tune: 04-02 21:22:29] {215} INFO - result: {'success_vote': 0, 'total_cost': 0.9975759999999998, 'cost': 0.19170800000000002, 'training_iteration': 0, 'config': {'prompt': 0, 'subspace': {'max_tokens': 597, 'temperature_or_top_p': {'top_p': 0.5475861559192435}, 'n': 82, 'model': 'gpt-3.5-turbo'}}, 'config/prompt': 0, 'config/subspace': {'max_tokens': 597, 'temperature_or_top_p': {'top_p': 0.5475861559192435}, 'n': 82, 'model': 'gpt-3.5-turbo'}, 'experiment_tag': 'exp', 'time_total_s': 94.87062358856201}\n",
- "[flaml.tune.tune: 04-02 21:22:29] {832} INFO - trial 9 config: {'prompt': 0, 'subspace': {'max_tokens': 142, 'temperature_or_top_p': {'temperature': 0.8839364795611863}, 'n': 33, 'model': 'gpt-4'}}\n",
- "[flaml.tune.tune: 04-02 21:22:38] {215} INFO - result: {'success_vote': 0, 'total_cost': 1.0206759999999997, 'cost': 0.0231, 'training_iteration': 0, 'config': {'prompt': 0, 'subspace': {'max_tokens': 142, 'temperature_or_top_p': {'temperature': 0.8839364795611863}, 'n': 33, 'model': 'gpt-4'}}, 'config/prompt': 0, 'config/subspace': {'max_tokens': 142, 'temperature_or_top_p': {'temperature': 0.8839364795611863}, 'n': 33, 'model': 'gpt-4'}, 'experiment_tag': 'exp', 'time_total_s': 9.344996690750122}\n",
- "[flaml.tune.tune: 04-02 21:22:38] {855} WARNING - fail to sample a trial for 100 times in a row, stopping.\n"
- ]
- }
- ],
- "source": [
- "import logging\n",
- "\n",
- "prompts = [\"{problem} Solve the problem carefully. Simplify your answer as much as possible. Put the final answer in \\\\boxed{{}}.\"]\n",
- "config, analysis = oai.ChatCompletion.tune(\n",
- " data=tune_data, # the data for tuning\n",
- " metric=\"success_vote\", # the metric to optimize\n",
- " mode=\"max\", # the optimization mode\n",
- " eval_func=eval_math_responses, # the evaluation function to return the success metrics\n",
- " # log_file_name=\"logs/math.log\", # the log file name\n",
- " inference_budget=0.02, # the inference budget (dollar per instance)\n",
- " optimization_budget=1, # the optimization budget (dollar in total)\n",
- " # num_samples can further limit the number of trials for different hyperparameter configurations;\n",
- " # -1 means decided by the optimization budget only\n",
- " num_samples=-1,\n",
- " # model=\"gpt-3.5-turbo\", # uncomment if you don't have access to gpt-4\n",
- " prompt=prompts, # the prompt templates to choose from\n",
- " # stop=\"###\", # the stop sequence\n",
- " logging_level=logging.INFO, # the logging level\n",
- ")\n"
- ]
- },
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### Output tuning results\n",
- "\n",
- "After the tuning, we can print out the config and the result found by FLAML:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 11,
- "metadata": {
- "execution": {
- "iopub.execute_input": "2023-02-13T23:41:55.049204Z",
- "iopub.status.busy": "2023-02-13T23:41:55.048871Z",
- "iopub.status.idle": "2023-02-13T23:41:55.053284Z",
- "shell.execute_reply": "2023-02-13T23:41:55.052574Z"
- }
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "optimized config {'prompt': '{problem} Solve the problem carefully. Simplify your answer as much as possible. Put the final answer in \\\\boxed{{}}.', 'max_tokens': 433, 'n': 54, 'model': 'gpt-3.5-turbo', 'stop': None, 'top_p': 0.7145757833976906}\n",
- "best result on tuning data {'expected_success': 0.9999912490789106, 'success': 1.0, 'success_vote': 0.9, 'voted_answer': 'We use the distance formula: $$\\\\sqrt{(x_2-x_1)^2+(y_2-y_1)^2}.$$ Letting $(x_1,y_1)=(0,4)$ and $(x_2,y_2)=(3,0)$, we have: \\\\begin{align*}\\n\\\\sqrt{(3-0)^2+(0-4)^2} &= \\\\sqrt{9+16} \\\\\\\\\\n&= \\\\sqrt{25} \\\\\\\\\\n&= \\\\boxed{5}.\\n\\\\end{align*}', 'votes': 41.65, 'total_cost': 0.6993100000000001, 'cost': 0.30776799999999993, 'inference_cost': 0.014812200000000001, 'training_iteration': 0, 'config': {'prompt': 0, 'subspace': {'max_tokens': 433, 'temperature_or_top_p': {'top_p': 0.7145757833976906}, 'n': 54, 'model': 'gpt-3.5-turbo'}}, 'config/prompt': 0, 'config/subspace': {'max_tokens': 433, 'temperature_or_top_p': {'top_p': 0.7145757833976906}, 'n': 54, 'model': 'gpt-3.5-turbo'}, 'experiment_tag': 'exp', 'time_total_s': 211.06462264060974}\n"
- ]
- }
- ],
- "source": [
- "print(\"optimized config\", config)\n",
- "print(\"best result on tuning data\", analysis.best_result)"
- ]
- },
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {
- "slideshow": {
- "slide_type": "slide"
- }
- },
- "source": [
- "### Make a request with the tuned config\n",
- "\n",
- "We can apply the tuned config on the request for an example task:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 12,
- "metadata": {
- "execution": {
- "iopub.execute_input": "2023-02-13T23:41:55.056205Z",
- "iopub.status.busy": "2023-02-13T23:41:55.055631Z",
- "iopub.status.idle": "2023-02-13T23:41:56.039259Z",
- "shell.execute_reply": "2023-02-13T23:41:56.038427Z"
- },
- "slideshow": {
- "slide_type": "subslide"
- },
- "tags": []
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "response on an example data instance: {\n",
- " \"choices\": [\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 0,\n",
- " \"message\": {\n",
- " \"content\": \"We want to eliminate the radical in the denominator. We can do this by multiplying the numerator and denominator by $\\\\sqrt{21}$: $$\\\\frac{21}{\\\\sqrt{21}}=\\\\frac{21\\\\cdot\\\\sqrt{21}}{\\\\sqrt{21}\\\\cdot\\\\sqrt{21}}=\\\\frac{21\\\\sqrt{21}}{21}=\\\\sqrt{21}.$$Therefore, the simplified expression is $\\\\boxed{\\\\sqrt{21}}$.\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 1,\n",
- " \"message\": {\n",
- " \"content\": \"We have $\\\\displaystyle\\\\frac{21}{\\\\sqrt{21}}=\\\\frac{21}{\\\\sqrt{21}}\\\\cdot\\\\frac{\\\\sqrt{21}}{\\\\sqrt{21}}=\\\\frac{21\\\\sqrt{21}}{21}=\\\\boxed{\\\\sqrt{21}}$.\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 2,\n",
- " \"message\": {\n",
- " \"content\": \"We can simplify the expression by multiplying both the numerator and the denominator by $\\\\sqrt{21}$: $$\\\\frac{21}{\\\\sqrt{21}}=\\\\frac{21\\\\cdot\\\\sqrt{21}}{\\\\sqrt{21}\\\\cdot\\\\sqrt{21}}=\\\\frac{21\\\\sqrt{21}}{21}=\\\\sqrt{21}.$$Therefore, $\\\\displaystyle\\\\frac{21}{\\\\sqrt{21}}=\\\\boxed{\\\\sqrt{21}}$.\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 3,\n",
- " \"message\": {\n",
- " \"content\": \"We can simplify $\\\\sqrt{21}$ by factoring it as $\\\\sqrt{21}=\\\\sqrt{3\\\\cdot7}=\\\\sqrt{3}\\\\cdot\\\\sqrt{7}$. Therefore, we have \\\\begin{align*}\\n\\\\frac{21}{\\\\sqrt{21}}&=\\\\frac{21}{\\\\sqrt{3}\\\\cdot\\\\sqrt{7}}\\\\\\\\\\n&=\\\\frac{21}{\\\\sqrt{3}\\\\cdot\\\\sqrt{7}}\\\\cdot\\\\frac{\\\\sqrt{3}}{\\\\sqrt{3}}\\\\\\\\\\n&=\\\\frac{21\\\\sqrt{3}}{3\\\\sqrt{7}}\\\\\\\\\\n&=\\\\frac{7\\\\sqrt{3}}{\\\\sqrt{7}}\\\\\\\\\\n&=\\\\frac{7\\\\sqrt{3}\\\\cdot\\\\sqrt{7}}{7}\\\\\\\\\\n&=\\\\boxed{\\\\sqrt{21}}.\\n\\\\end{align*}\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 4,\n",
- " \"message\": {\n",
- " \"content\": \"We begin by multiplying the numerator and denominator of the fraction by $\\\\sqrt{21}$: $$\\\\frac{21}{\\\\sqrt{21}}=\\\\frac{21\\\\cdot\\\\sqrt{21}}{\\\\sqrt{21}\\\\cdot\\\\sqrt{21}}=\\\\frac{21\\\\sqrt{21}}{21}.$$ Simplifying, we get: $$\\\\frac{21\\\\sqrt{21}}{21}=\\\\frac{\\\\cancelto{1}{21}\\\\cdot\\\\sqrt{21}}{\\\\cancelto{1}{21}}=\\\\boxed{\\\\sqrt{21}}.$$\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 5,\n",
- " \"message\": {\n",
- " \"content\": \"We can start by simplifying the fraction by canceling a factor of $\\\\sqrt{21}$ from the numerator and denominator: $$\\\\frac{21}{\\\\sqrt{21}} = \\\\frac{21}{\\\\sqrt{21}} \\\\cdot \\\\frac{\\\\sqrt{21}}{\\\\sqrt{21}} = \\\\frac{21\\\\sqrt{21}}{21} = \\\\sqrt{21}.$$ Therefore, the denominator is already rationalized, and the answer is $\\\\boxed{\\\\sqrt{21}}$.\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 6,\n",
- " \"message\": {\n",
- " \"content\": \"We have $$\\\\frac{21}{\\\\sqrt{21}} = \\\\frac{21}{\\\\sqrt{21}} \\\\cdot \\\\frac{\\\\sqrt{21}}{\\\\sqrt{21}} = \\\\frac{21\\\\sqrt{21}}{21} = \\\\boxed{\\\\sqrt{21}}.$$\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 7,\n",
- " \"message\": {\n",
- " \"content\": \"We want to eliminate the radical in the denominator. To do this, we multiply both the numerator and denominator by $\\\\sqrt{21}$: $$\\\\frac{21}{\\\\sqrt{21}} \\\\cdot \\\\frac{\\\\sqrt{21}}{\\\\sqrt{21}} = \\\\frac{21\\\\sqrt{21}}{21}.$$ Simplifying, we get $\\\\boxed{\\\\sqrt{21}}$.\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 8,\n",
- " \"message\": {\n",
- " \"content\": \"We can simplify $\\\\sqrt{21}$ by factoring it as $\\\\sqrt{21} = \\\\sqrt{3\\\\cdot7} = \\\\sqrt{3}\\\\cdot\\\\sqrt{7}$. Therefore, we have \\\\begin{align*}\\n\\\\frac{21}{\\\\sqrt{21}} &= \\\\frac{21}{\\\\sqrt{3}\\\\cdot\\\\sqrt{7}}\\\\\\\\\\n&= \\\\frac{21}{\\\\sqrt{3}}\\\\cdot\\\\frac{1}{\\\\sqrt{7}}\\\\\\\\\\n&= \\\\frac{21}{\\\\sqrt{3}}\\\\cdot\\\\frac{\\\\sqrt{7}}{\\\\sqrt{7}\\\\cdot\\\\sqrt{7}}\\\\\\\\\\n&= \\\\frac{21\\\\sqrt{7}}{\\\\sqrt{3\\\\cdot7}}\\\\\\\\\\n&= \\\\frac{21\\\\sqrt{7}}{\\\\sqrt{21}}.\\n\\\\end{align*}Thus, the denominator is rationalized, and the simplified answer is $\\\\boxed{\\\\frac{21\\\\sqrt{7}}{21}}$, which can be further simplified to $\\\\boxed{\\\\sqrt{7}}$.\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 9,\n",
- " \"message\": {\n",
- " \"content\": \"We can simplify $\\\\displaystyle\\\\frac{21}{\\\\sqrt{21}}$ by multiplying the numerator and denominator by $\\\\sqrt{21}$. This gives us: $$\\\\frac{21}{\\\\sqrt{21}}\\\\cdot\\\\frac{\\\\sqrt{21}}{\\\\sqrt{21}}=\\\\frac{21\\\\sqrt{21}}{21}=\\\\sqrt{21}.$$ Therefore, $\\\\displaystyle\\\\frac{21}{\\\\sqrt{21}}=\\\\boxed{\\\\sqrt{21}}$.\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 10,\n",
- " \"message\": {\n",
- " \"content\": \"We can simplify $\\\\sqrt{21}$ by factoring it as $\\\\sqrt{21} = \\\\sqrt{3 \\\\cdot 7} = \\\\sqrt{3} \\\\cdot \\\\sqrt{7}$. Therefore, we have \\\\begin{align*}\\n\\\\frac{21}{\\\\sqrt{21}} &= \\\\frac{21}{\\\\sqrt{3} \\\\cdot \\\\sqrt{7}} \\\\\\\\\\n&= \\\\frac{21}{\\\\sqrt{3}} \\\\cdot \\\\frac{1}{\\\\sqrt{7}} \\\\\\\\\\n&= \\\\frac{21}{\\\\sqrt{3}} \\\\cdot \\\\frac{\\\\sqrt{7}}{\\\\sqrt{7} \\\\cdot \\\\sqrt{7}} \\\\\\\\\\n&= \\\\frac{21\\\\sqrt{7}}{\\\\sqrt{3 \\\\cdot 7}} \\\\\\\\\\n&= \\\\frac{21\\\\sqrt{7}}{\\\\sqrt{21}} \\\\\\\\\\n&= \\\\frac{21\\\\sqrt{7}}{\\\\sqrt{3} \\\\cdot \\\\sqrt{7}} \\\\\\\\\\n&= \\\\frac{21\\\\sqrt{7}}{3\\\\sqrt{7}} \\\\\\\\\\n&= \\\\frac{21}{3} \\\\\\\\\\n&= \\\\boxed{7}.\\n\\\\end{align*}\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 11,\n",
- " \"message\": {\n",
- " \"content\": \"We can simplify $\\\\frac{21}{\\\\sqrt{21}}$ by multiplying both the numerator and denominator by $\\\\sqrt{21}$: $$\\\\frac{21}{\\\\sqrt{21}}=\\\\frac{21\\\\cdot\\\\sqrt{21}}{\\\\sqrt{21}\\\\cdot\\\\sqrt{21}}=\\\\frac{21\\\\sqrt{21}}{21}=\\\\sqrt{21}.$$Therefore, the rationalized form of the denominator is $\\\\boxed{\\\\sqrt{21}}$.\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 12,\n",
- " \"message\": {\n",
- " \"content\": \"We want to get rid of the radical in the denominator, so we multiply both the numerator and denominator by $\\\\sqrt{21}$: $$\\\\frac{21}{\\\\sqrt{21}} \\\\cdot \\\\frac{\\\\sqrt{21}}{\\\\sqrt{21}} = \\\\frac{21\\\\sqrt{21}}{21}.$$ Simplifying the fraction, we get $\\\\frac{\\\\cancel{21}\\\\sqrt{21}}{\\\\cancel{21}} = \\\\sqrt{21}$. Thus, $\\\\displaystyle\\\\frac{21}{\\\\sqrt{21}} = \\\\boxed{\\\\sqrt{21}}$.\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 13,\n",
- " \"message\": {\n",
- " \"content\": \"We want to eliminate the radical from the denominator. To do this, we multiply both the numerator and denominator by $\\\\sqrt{21}$: \\\\begin{align*}\\n\\\\frac{21}{\\\\sqrt{21}} &= \\\\frac{21}{\\\\sqrt{21}} \\\\cdot \\\\frac{\\\\sqrt{21}}{\\\\sqrt{21}} \\\\\\\\\\n&= \\\\frac{21\\\\sqrt{21}}{21} \\\\\\\\\\n&= \\\\boxed{\\\\sqrt{21}}.\\n\\\\end{align*}\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 14,\n",
- " \"message\": {\n",
- " \"content\": \"We have $$\\\\frac{21}{\\\\sqrt{21}}=\\\\frac{21}{\\\\sqrt{21}}\\\\cdot\\\\frac{\\\\sqrt{21}}{\\\\sqrt{21}}=\\\\frac{21\\\\sqrt{21}}{21}=\\\\sqrt{21}.$$Therefore, the answer is $\\\\boxed{\\\\sqrt{21}}$.\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 15,\n",
- " \"message\": {\n",
- " \"content\": \"We begin by multiplying the numerator and denominator of the fraction by $\\\\sqrt{21}$: $$\\\\frac{21}{\\\\sqrt{21}} = \\\\frac{21\\\\cdot\\\\sqrt{21}}{\\\\sqrt{21}\\\\cdot\\\\sqrt{21}} = \\\\frac{21\\\\sqrt{21}}{21}.$$Simplifying, we have $\\\\frac{21\\\\sqrt{21}}{21} = \\\\sqrt{21}$. Thus, the answer is $\\\\boxed{\\\\sqrt{21}}$.\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 16,\n",
- " \"message\": {\n",
- " \"content\": \"We can simplify $\\\\frac{21}{\\\\sqrt{21}}$ by multiplying the numerator and denominator by $\\\\sqrt{21}$: $$\\\\frac{21}{\\\\sqrt{21}} = \\\\frac{21\\\\cdot \\\\sqrt{21}}{\\\\sqrt{21}\\\\cdot \\\\sqrt{21}} = \\\\frac{21\\\\cdot \\\\sqrt{21}}{21} = \\\\sqrt{21}.$$ Therefore, the rationalized form of the denominator is $\\\\boxed{\\\\sqrt{21}}$.\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 17,\n",
- " \"message\": {\n",
- " \"content\": \"We begin by multiplying the numerator and denominator by $\\\\sqrt{21}$: $$\\\\frac{21}{\\\\sqrt{21}}=\\\\frac{21\\\\cdot\\\\sqrt{21}}{\\\\sqrt{21}\\\\cdot\\\\sqrt{21}}=\\\\frac{21\\\\sqrt{21}}{21}.$$ Simplifying, we get $\\\\boxed{\\\\sqrt{21}}$.\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 18,\n",
- " \"message\": {\n",
- " \"content\": \"We want to get rid of the square root in the denominator, so we multiply the numerator and denominator by $\\\\sqrt{21}$: \\\\begin{align*}\\n\\\\frac{21}{\\\\sqrt{21}} &= \\\\frac{21\\\\cdot\\\\sqrt{21}}{\\\\sqrt{21}\\\\cdot\\\\sqrt{21}} \\\\\\\\\\n&= \\\\frac{21\\\\cdot\\\\sqrt{21}}{21} \\\\\\\\\\n&= \\\\boxed{\\\\sqrt{21}}.\\n\\\\end{align*}\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 19,\n",
- " \"message\": {\n",
- " \"content\": \"We can start by simplifying the fraction $\\\\frac{21}{\\\\sqrt{21}}$ by dividing the numerator and denominator by the greatest perfect square that divides both of them, which is 3. This gives us:\\n\\n$$\\\\frac{21}{\\\\sqrt{21}}=\\\\frac{7\\\\cdot 3}{\\\\sqrt{7\\\\cdot 3}}=\\\\frac{7\\\\cdot 3}{\\\\sqrt{7}\\\\cdot \\\\sqrt{3}}$$\\n\\nNext, we can rationalize the denominator by multiplying the numerator and denominator by $\\\\sqrt{3}$:\\n\\n$$\\\\frac{7\\\\cdot 3}{\\\\sqrt{7}\\\\cdot \\\\sqrt{3}}=\\\\frac{7\\\\cdot 3\\\\cdot \\\\sqrt{3}}{\\\\sqrt{7}\\\\cdot \\\\sqrt{3}\\\\cdot \\\\sqrt{3}}=\\\\frac{21\\\\sqrt{3}}{3}$$\\n\\nFinally, we can simplify the fraction by dividing both the numerator and denominator by 3:\\n\\n$$\\\\frac{21\\\\sqrt{3}}{3}=7\\\\sqrt{3}$$\\n\\nTherefore, the final answer is $\\\\boxed{7\\\\sqrt{3}}$.\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 20,\n",
- " \"message\": {\n",
- " \"content\": \"We begin by multiplying both the numerator and denominator of the fraction by $\\\\sqrt{21}$: $$\\\\frac{21}{\\\\sqrt{21}}=\\\\frac{21\\\\cdot\\\\sqrt{21}}{\\\\sqrt{21}\\\\cdot\\\\sqrt{21}}=\\\\frac{21\\\\cdot\\\\sqrt{21}}{21}=\\\\boxed{\\\\sqrt{21}}.$$\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 21,\n",
- " \"message\": {\n",
- " \"content\": \"We can start by simplifying the denominator using the property $\\\\sqrt{a^2}=|a|$. Since $21$ is positive, $\\\\sqrt{21}=|21|=21$. Therefore, we have: $$\\\\frac{21}{\\\\sqrt{21}}=\\\\frac{21}{21}=\\\\boxed{1}$$\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 22,\n",
- " \"message\": {\n",
- " \"content\": \"We can simplify $\\\\frac{21}{\\\\sqrt{21}}$ by multiplying the numerator and denominator by $\\\\sqrt{21}$: $$\\\\frac{21}{\\\\sqrt{21}} = \\\\frac{21\\\\cdot \\\\sqrt{21}}{\\\\sqrt{21}\\\\cdot \\\\sqrt{21}} = \\\\frac{21\\\\cdot \\\\sqrt{21}}{21}.$$ Simplifying, we get $\\\\boxed{\\\\sqrt{21}}$.\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 23,\n",
- " \"message\": {\n",
- " \"content\": \"We can simplify $\\\\sqrt{21}$ by finding its prime factorization: $$\\\\sqrt{21}=\\\\sqrt{3\\\\cdot7}=\\\\sqrt{3}\\\\cdot\\\\sqrt{7}$$ So, we have $$\\\\frac{21}{\\\\sqrt{21}}=\\\\frac{21}{\\\\sqrt{3}\\\\cdot\\\\sqrt{7}}$$ To rationalize the denominator, we need to get rid of the square root in the denominator by multiplying both the numerator and denominator by $\\\\sqrt{3}\\\\cdot\\\\sqrt{7}$: $$\\\\frac{21}{\\\\sqrt{3}\\\\cdot\\\\sqrt{7}}\\\\cdot\\\\frac{\\\\sqrt{3}\\\\cdot\\\\sqrt{7}}{\\\\sqrt{3}\\\\cdot\\\\sqrt{7}}=\\\\frac{21\\\\sqrt{3}\\\\cdot\\\\sqrt{7}}{3\\\\cdot7}=\\\\frac{3\\\\cdot7\\\\cdot\\\\sqrt{3}\\\\cdot\\\\sqrt{7}}{3\\\\cdot7}=\\\\sqrt{3}\\\\cdot\\\\sqrt{7}$$ Therefore, $\\\\displaystyle\\\\frac{21}{\\\\sqrt{21}}=\\\\boxed{\\\\sqrt{3}\\\\cdot\\\\sqrt{7}}$.\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 24,\n",
- " \"message\": {\n",
- " \"content\": \"We begin by multiplying the numerator and denominator by $\\\\sqrt{21}$: $$\\\\frac{21}{\\\\sqrt{21}} = \\\\frac{21 \\\\cdot \\\\sqrt{21}}{\\\\sqrt{21} \\\\cdot \\\\sqrt{21}} = \\\\frac{21\\\\sqrt{21}}{21}.$$ Simplifying, we get $$\\\\frac{21\\\\sqrt{21}}{21} = \\\\boxed{\\\\sqrt{21}}.$$\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 25,\n",
- " \"message\": {\n",
- " \"content\": \"We can simplify $\\\\frac{21}{\\\\sqrt{21}}$ by multiplying both the numerator and denominator by $\\\\sqrt{21}$: $$\\\\frac{21}{\\\\sqrt{21}} = \\\\frac{21\\\\cdot\\\\sqrt{21}}{\\\\sqrt{21}\\\\cdot\\\\sqrt{21}} = \\\\frac{21\\\\sqrt{21}}{21}.$$ Simplifying the fraction, we have $$\\\\frac{21\\\\sqrt{21}}{21} = \\\\boxed{\\\\sqrt{21}}.$$\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 26,\n",
- " \"message\": {\n",
- " \"content\": \"We begin by multiplying the numerator and denominator by $\\\\sqrt{21}$: $$\\\\frac{21}{\\\\sqrt{21}}=\\\\frac{21\\\\cdot\\\\sqrt{21}}{\\\\sqrt{21}\\\\cdot\\\\sqrt{21}}=\\\\frac{21\\\\sqrt{21}}{21}.$$ Simplifying the numerator and denominator, we have: $$\\\\frac{21\\\\sqrt{21}}{21}=\\\\frac{3\\\\cdot7\\\\cdot\\\\sqrt{3\\\\cdot7}}{3\\\\cdot7}=\\\\frac{\\\\sqrt{3\\\\cdot7}}{1}=\\\\sqrt{21}.$$ Therefore, $\\\\displaystyle\\\\frac{21}{\\\\sqrt{21}}=\\\\boxed{\\\\sqrt{21}}$.\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 27,\n",
- " \"message\": {\n",
- " \"content\": \"We want to get rid of the square root in the denominator. One way to do this is to multiply the numerator and denominator by $\\\\sqrt{21}$: $$\\\\frac{21}{\\\\sqrt{21}}=\\\\frac{21\\\\cdot\\\\sqrt{21}}{\\\\sqrt{21}\\\\cdot\\\\sqrt{21}}=\\\\frac{21\\\\sqrt{21}}{21}.$$ Now we can simplify by canceling the common factor of 21: $$\\\\frac{21\\\\sqrt{21}}{21}=\\\\frac{\\\\cancel{21}\\\\sqrt{21}}{\\\\cancel{21}}=\\\\sqrt{21}.$$ Therefore, $\\\\displaystyle\\\\frac{21}{\\\\sqrt{21}}=\\\\boxed{\\\\sqrt{21}}$.\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 28,\n",
- " \"message\": {\n",
- " \"content\": \"We can simplify $\\\\displaystyle\\\\frac{21}{\\\\sqrt{21}}$ by multiplying the numerator and denominator by $\\\\sqrt{21}$: $$\\\\frac{21}{\\\\sqrt{21}}=\\\\frac{21\\\\cdot\\\\sqrt{21}}{\\\\sqrt{21}\\\\cdot\\\\sqrt{21}}=\\\\frac{21\\\\sqrt{21}}{21}=\\\\sqrt{21}.$$ Therefore, the simplified expression is $\\\\boxed{\\\\sqrt{21}}$.\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 29,\n",
- " \"message\": {\n",
- " \"content\": \"We have $$\\\\frac{21}{\\\\sqrt{21}}=\\\\frac{21}{\\\\sqrt{21}}\\\\cdot\\\\frac{\\\\sqrt{21}}{\\\\sqrt{21}}=\\\\frac{21\\\\sqrt{21}}{21}=\\\\boxed{\\\\sqrt{21}}.$$\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 30,\n",
- " \"message\": {\n",
- " \"content\": \"We have $$\\\\frac{21}{\\\\sqrt{21}}=\\\\frac{21}{\\\\sqrt{21}}\\\\cdot\\\\frac{\\\\sqrt{21}}{\\\\sqrt{21}}=\\\\frac{21\\\\sqrt{21}}{21}=\\\\sqrt{21}.$$ Therefore, the answer is $\\\\boxed{\\\\sqrt{21}}$.\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 31,\n",
- " \"message\": {\n",
- " \"content\": \"We want to get rid of the radical in the denominator. To do this, we multiply the numerator and denominator by $\\\\sqrt{21}$: $$\\\\frac{21}{\\\\sqrt{21}}\\\\cdot\\\\frac{\\\\sqrt{21}}{\\\\sqrt{21}}=\\\\frac{21\\\\sqrt{21}}{21}.$$ Simplifying, we have $$\\\\frac{21\\\\sqrt{21}}{21}=\\\\boxed{\\\\sqrt{21}}.$$\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 32,\n",
- " \"message\": {\n",
- " \"content\": \"We have $\\\\dfrac{21}{\\\\sqrt{21}} = \\\\dfrac{21}{\\\\sqrt{21}}\\\\cdot\\\\dfrac{\\\\sqrt{21}}{\\\\sqrt{21}} = \\\\dfrac{21\\\\sqrt{21}}{21} = \\\\boxed{\\\\sqrt{21}}$.\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 33,\n",
- " \"message\": {\n",
- " \"content\": \"We can start by simplifying the square root in the denominator: $$\\\\frac{21}{\\\\sqrt{21}}=\\\\frac{21}{\\\\sqrt{3\\\\cdot7}}=\\\\frac{21}{\\\\sqrt{3}\\\\cdot\\\\sqrt{7}}$$ To rationalize the denominator, we need to get rid of the square root in the denominator. We can do this by multiplying the numerator and denominator by $\\\\sqrt{7}$: $$\\\\frac{21}{\\\\sqrt{3}\\\\cdot\\\\sqrt{7}}\\\\cdot\\\\frac{\\\\sqrt{7}}{\\\\sqrt{7}}=\\\\frac{21\\\\sqrt{7}}{\\\\sqrt{3}\\\\cdot\\\\sqrt{7}\\\\cdot\\\\sqrt{7}}=\\\\frac{21\\\\sqrt{7}}{\\\\sqrt{3\\\\cdot7}}$$ Simplifying the denominator gives us: $$\\\\frac{21\\\\sqrt{7}}{\\\\sqrt{3\\\\cdot7}}=\\\\frac{21\\\\sqrt{7}}{\\\\sqrt{21}}$$ Therefore, the simplified form of $\\\\displaystyle\\\\frac{21}{\\\\sqrt{21}}$ is $\\\\boxed{\\\\frac{21\\\\sqrt{7}}{21}}$, or simply $\\\\boxed{\\\\sqrt{7}}$.\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 34,\n",
- " \"message\": {\n",
- " \"content\": \"We begin by multiplying the numerator and denominator of the fraction by $\\\\sqrt{21}$: $$\\\\frac{21}{\\\\sqrt{21}}=\\\\frac{21\\\\cdot\\\\sqrt{21}}{\\\\sqrt{21}\\\\cdot\\\\sqrt{21}}=\\\\frac{21\\\\sqrt{21}}{21}.$$ Simplifying the numerator, we have $\\\\frac{21\\\\sqrt{21}}{21}=\\\\sqrt{21}$. Therefore, $\\\\frac{21}{\\\\sqrt{21}}=\\\\boxed{\\\\sqrt{21}}$.\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 35,\n",
- " \"message\": {\n",
- " \"content\": \"We want to get rid of the square root in the denominator. We can do this by multiplying both the numerator and denominator by $\\\\sqrt{21}$: $$\\\\frac{21}{\\\\sqrt{21}}\\\\cdot\\\\frac{\\\\sqrt{21}}{\\\\sqrt{21}}=\\\\frac{21\\\\sqrt{21}}{21}.$$ Simplifying, we get $\\\\boxed{\\\\sqrt{21}}$.\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 36,\n",
- " \"message\": {\n",
- " \"content\": \"We can rationalize the denominator by multiplying the numerator and denominator by $\\\\sqrt{21}$: \\\\begin{align*}\\n\\\\frac{21}{\\\\sqrt{21}} &= \\\\frac{21\\\\cdot\\\\sqrt{21}}{\\\\sqrt{21}\\\\cdot\\\\sqrt{21}} \\\\\\\\\\n&= \\\\frac{21\\\\sqrt{21}}{21} \\\\\\\\\\n&= \\\\sqrt{21}.\\n\\\\end{align*}Therefore, the answer is $\\\\boxed{\\\\sqrt{21}}$.\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 37,\n",
- " \"message\": {\n",
- " \"content\": \"We can rationalize the denominator by multiplying both the numerator and denominator by $\\\\sqrt{21}$: $$\\\\frac{21}{\\\\sqrt{21}}=\\\\frac{21\\\\cdot\\\\sqrt{21}}{\\\\sqrt{21}\\\\cdot\\\\sqrt{21}}=\\\\frac{21\\\\sqrt{21}}{21}=\\\\sqrt{21}.$$ Therefore, the answer is $\\\\boxed{\\\\sqrt{21}}$.\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 38,\n",
- " \"message\": {\n",
- " \"content\": \"We can simplify the denominator by using the fact that $\\\\sqrt{21} = \\\\sqrt{3\\\\cdot7} = \\\\sqrt{3}\\\\cdot\\\\sqrt{7}$. Therefore, $$\\\\frac{21}{\\\\sqrt{21}} = \\\\frac{21}{\\\\sqrt{3}\\\\cdot\\\\sqrt{7}}.$$To rationalize the denominator, we multiply both the numerator and denominator by $\\\\sqrt{3}\\\\cdot\\\\sqrt{7}$: $$\\\\frac{21}{\\\\sqrt{3}\\\\cdot\\\\sqrt{7}} \\\\cdot \\\\frac{\\\\sqrt{3}\\\\cdot\\\\sqrt{7}}{\\\\sqrt{3}\\\\cdot\\\\sqrt{7}} = \\\\frac{21\\\\sqrt{3}\\\\cdot\\\\sqrt{7}}{3\\\\cdot7}.$$Simplifying, we get $$\\\\frac{21\\\\sqrt{3}\\\\cdot\\\\sqrt{7}}{3\\\\cdot7} = \\\\frac{3\\\\cdot7\\\\cdot\\\\sqrt{3}\\\\cdot\\\\sqrt{7}}{3\\\\cdot7} = \\\\boxed{\\\\sqrt{3}\\\\cdot\\\\sqrt{7}}.$$\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 39,\n",
- " \"message\": {\n",
- " \"content\": \"We can rationalize the denominator by multiplying both the numerator and denominator by $\\\\sqrt{21}$: $$\\\\frac{21}{\\\\sqrt{21}}=\\\\frac{21\\\\cdot\\\\sqrt{21}}{\\\\sqrt{21}\\\\cdot\\\\sqrt{21}}=\\\\frac{21\\\\sqrt{21}}{21}=\\\\sqrt{21}.$$Therefore, the answer is $\\\\boxed{\\\\sqrt{21}}$.\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 40,\n",
- " \"message\": {\n",
- " \"content\": \"We want to get rid of the square root in the denominator. One way to do this is to multiply both the numerator and denominator by $\\\\sqrt{21}$. We get $$\\\\frac{21}{\\\\sqrt{21}}=\\\\frac{21\\\\cdot\\\\sqrt{21}}{\\\\sqrt{21}\\\\cdot\\\\sqrt{21}}=\\\\frac{21\\\\sqrt{21}}{21}.$$ Simplifying, we get $\\\\boxed{\\\\sqrt{21}}$.\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 41,\n",
- " \"message\": {\n",
- " \"content\": \"We want to get rid of the radical in the denominator. To do this, we multiply the numerator and denominator by $\\\\sqrt{21}$: \\\\begin{align*}\\n\\\\frac{21}{\\\\sqrt{21}} &= \\\\frac{21\\\\cdot\\\\sqrt{21}}{\\\\sqrt{21}\\\\cdot\\\\sqrt{21}} \\\\\\\\\\n&= \\\\frac{21\\\\cdot\\\\sqrt{21}}{21} \\\\\\\\\\n&= \\\\boxed{\\\\sqrt{21}}.\\n\\\\end{align*} We simplified the expression as much as possible by canceling the factor of 21 in the numerator and denominator.\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 42,\n",
- " \"message\": {\n",
- " \"content\": \"We want to eliminate the radical from the denominator. To do this, we can multiply the numerator and denominator by $\\\\sqrt{21}$: \\\\begin{align*}\\n\\\\frac{21}{\\\\sqrt{21}} &= \\\\frac{21\\\\cdot\\\\sqrt{21}}{\\\\sqrt{21}\\\\cdot\\\\sqrt{21}}\\\\\\\\\\n&= \\\\frac{21\\\\sqrt{21}}{21}\\\\\\\\\\n&= \\\\sqrt{21}\\n\\\\end{align*}Therefore, the simplified expression is $\\\\boxed{\\\\sqrt{21}}$.\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 43,\n",
- " \"message\": {\n",
- " \"content\": \"We want to get rid of the radical in the denominator. One way to do this is to multiply both the numerator and denominator by $\\\\sqrt{21}$: $$\\\\frac{21}{\\\\sqrt{21}} = \\\\frac{21\\\\cdot\\\\sqrt{21}}{\\\\sqrt{21}\\\\cdot\\\\sqrt{21}} = \\\\frac{21\\\\sqrt{21}}{21}.$$ Simplifying, we get $\\\\frac{21\\\\sqrt{21}}{21} = \\\\sqrt{21}$. Therefore, $\\\\displaystyle\\\\frac{21}{\\\\sqrt{21}} = \\\\boxed{\\\\sqrt{21}}$.\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 44,\n",
- " \"message\": {\n",
- " \"content\": \"We want to eliminate the radical from the denominator. To do this, we multiply the numerator and denominator by $\\\\sqrt{21}$: \\\\begin{align*}\\n\\\\frac{21}{\\\\sqrt{21}} &= \\\\frac{21\\\\cdot\\\\sqrt{21}}{\\\\sqrt{21}\\\\cdot\\\\sqrt{21}}\\\\\\\\\\n&= \\\\frac{21\\\\sqrt{21}}{21}\\\\\\\\\\n&= \\\\sqrt{21}.\\n\\\\end{align*}Therefore, $\\\\displaystyle\\\\frac{21}{\\\\sqrt{21}} = \\\\boxed{\\\\sqrt{21}}$.\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 45,\n",
- " \"message\": {\n",
- " \"content\": \"We begin by multiplying both the numerator and denominator of the fraction by $\\\\sqrt{21}$: $$\\\\frac{21}{\\\\sqrt{21}}=\\\\frac{21\\\\cdot\\\\sqrt{21}}{\\\\sqrt{21}\\\\cdot\\\\sqrt{21}}=\\\\frac{21\\\\sqrt{21}}{21}.$$ We can simplify this by canceling a factor of 21: $$\\\\frac{21\\\\sqrt{21}}{21}=\\\\frac{\\\\cancelto{1}{21}\\\\sqrt{21}}{\\\\cancelto{1}{21}}=\\\\boxed{\\\\sqrt{21}}.$$\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 46,\n",
- " \"message\": {\n",
- " \"content\": \"We begin by multiplying both the numerator and the denominator by $\\\\sqrt{21}$ to get rid of the radical in the denominator: \\\\begin{align*}\\n\\\\frac{21}{\\\\sqrt{21}} &= \\\\frac{21}{\\\\sqrt{21}}\\\\cdot\\\\frac{\\\\sqrt{21}}{\\\\sqrt{21}} \\\\\\\\\\n&= \\\\frac{21\\\\cdot\\\\sqrt{21}}{21} \\\\\\\\\\n&= \\\\frac{\\\\cancel{21}\\\\cdot\\\\sqrt{21}}{\\\\cancel{21}} \\\\\\\\\\n&= \\\\sqrt{21}.\\n\\\\end{align*}Therefore, $\\\\displaystyle\\\\frac{21}{\\\\sqrt{21}}=\\\\boxed{\\\\sqrt{21}}$.\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 47,\n",
- " \"message\": {\n",
- " \"content\": \"We can start by simplifying the square root in the denominator: $$\\\\frac{21}{\\\\sqrt{21}}=\\\\frac{21}{\\\\sqrt{3\\\\cdot7}}.$$ Since $\\\\sqrt{3\\\\cdot7}=\\\\sqrt{3}\\\\cdot\\\\sqrt{7}$, we can rewrite the fraction as follows: $$\\\\frac{21}{\\\\sqrt{3\\\\cdot7}}=\\\\frac{21}{\\\\sqrt{3}\\\\cdot\\\\sqrt{7}}.$$ Now, we can multiply the numerator and denominator by $\\\\sqrt{3}$ to get rid of the radical in the denominator: $$\\\\frac{21}{\\\\sqrt{3}\\\\cdot\\\\sqrt{7}}\\\\cdot\\\\frac{\\\\sqrt{3}}{\\\\sqrt{3}}=\\\\frac{21\\\\sqrt{3}}{\\\\sqrt{3}\\\\cdot\\\\sqrt{3\\\\cdot7}}.$$ Simplifying the denominator, we have $\\\\sqrt{3}\\\\cdot\\\\sqrt{3\\\\cdot7}=\\\\sqrt{3^2\\\\cdot7}=3\\\\sqrt{7}$. Therefore, we can simplify the expression as follows: $$\\\\frac{21\\\\sqrt{3}}{\\\\sqrt{3}\\\\cdot\\\\sqrt{3\\\\cdot7}}=\\\\frac{21\\\\sqrt{3}}{3\\\\sqrt{7}}=\\\\frac{7\\\\sqrt{3}}{\\\\sqrt{7}}.$$ To rationalize the denominator, we can multiply the numerator and denominator by $\\\\sqrt{7}$: $$\\\\frac{7\\\\sqrt{3}}{\\\\sqrt{7}}\\\\cdot\\\\frac{\\\\sqrt{7}}{\\\\sqrt{7}}=\\\\frac{7\\\\sqrt{3}\\\\cdot\\\\sqrt{7}}{7}=\\\\sqrt{3}\\\\cdot\\\\sqrt{7}.$$ Therefore, the final answer is $\\\\boxed{\\\\sqrt{3}\\\\cdot\\\\sqrt{7}}$.\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 48,\n",
- " \"message\": {\n",
- " \"content\": \"We want to get rid of the square root in the denominator. We can do this by multiplying both the numerator and denominator by $\\\\sqrt{21}$: $$\\\\frac{21}{\\\\sqrt{21}}=\\\\frac{21\\\\cdot\\\\sqrt{21}}{\\\\sqrt{21}\\\\cdot\\\\sqrt{21}}=\\\\frac{21\\\\sqrt{21}}{21}.$$Simplifying, we get $\\\\frac{21\\\\sqrt{21}}{21}=\\\\boxed{\\\\sqrt{21}}$.\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 49,\n",
- " \"message\": {\n",
- " \"content\": \"We have $$\\\\frac{21}{\\\\sqrt{21}} = \\\\frac{21}{\\\\sqrt{21}} \\\\cdot \\\\frac{\\\\sqrt{21}}{\\\\sqrt{21}} = \\\\frac{21\\\\sqrt{21}}{21} = \\\\boxed{\\\\sqrt{21}}.$$\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 50,\n",
- " \"message\": {\n",
- " \"content\": \"We want to get rid of the square root in the denominator. To do this, we can multiply both the numerator and denominator by $\\\\sqrt{21}$: $$\\\\frac{21}{\\\\sqrt{21}}=\\\\frac{21\\\\cdot\\\\sqrt{21}}{\\\\sqrt{21}\\\\cdot\\\\sqrt{21}}=\\\\frac{21\\\\sqrt{21}}{21}=\\\\sqrt{21}.$$ Therefore, $\\\\displaystyle\\\\frac{21}{\\\\sqrt{21}}=\\\\boxed{\\\\sqrt{21}}$.\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 51,\n",
- " \"message\": {\n",
- " \"content\": \"We begin by multiplying both the numerator and the denominator by $\\\\sqrt{21}$: $$\\\\frac{21}{\\\\sqrt{21}}=\\\\frac{21\\\\cdot\\\\sqrt{21}}{\\\\sqrt{21}\\\\cdot\\\\sqrt{21}}=\\\\frac{21\\\\sqrt{21}}{21}.$$ Simplifying the numerator, we have $\\\\frac{21\\\\sqrt{21}}{21}=\\\\sqrt{21}$. Thus, the rationalized form of $\\\\frac{21}{\\\\sqrt{21}}$ is $\\\\boxed{\\\\sqrt{21}}$.\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 52,\n",
- " \"message\": {\n",
- " \"content\": \"We want to eliminate the radical in the denominator. To do this, we can multiply both the numerator and denominator by $\\\\sqrt{21}$: $$\\\\frac{21}{\\\\sqrt{21}}\\\\cdot\\\\frac{\\\\sqrt{21}}{\\\\sqrt{21}}=\\\\frac{21\\\\sqrt{21}}{21}.$$ Simplifying, we get $$\\\\frac{21\\\\sqrt{21}}{21}=\\\\frac{\\\\cancelto{1}{21}\\\\cdot\\\\sqrt{21}}{\\\\cancelto{1}{21}}=\\\\boxed{\\\\sqrt{21}}.$$\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " },\n",
- " {\n",
- " \"finish_reason\": \"stop\",\n",
- " \"index\": 53,\n",
- " \"message\": {\n",
- " \"content\": \"We can start by simplifying the denominator. Since $\\\\sqrt{21}$ can be written as $\\\\sqrt{3\\\\cdot7}$, we can simplify as follows: $$\\\\frac{21}{\\\\sqrt{21}} = \\\\frac{21}{\\\\sqrt{3\\\\cdot7}} = \\\\frac{21}{\\\\sqrt{3}\\\\cdot\\\\sqrt{7}}.$$ Now we can rationalize the denominator by multiplying the numerator and denominator by $\\\\sqrt{7}$: $$\\\\frac{21}{\\\\sqrt{3}\\\\cdot\\\\sqrt{7}}\\\\cdot\\\\frac{\\\\sqrt{7}}{\\\\sqrt{7}} = \\\\frac{21\\\\sqrt{7}}{\\\\sqrt{3}\\\\cdot\\\\sqrt{7}\\\\cdot\\\\sqrt{7}} = \\\\frac{21\\\\sqrt{7}}{\\\\sqrt{3\\\\cdot7}}.$$ Finally, we can simplify the denominator: $$\\\\frac{21\\\\sqrt{7}}{\\\\sqrt{3\\\\cdot7}} = \\\\frac{21\\\\sqrt{7}}{\\\\sqrt{21}} = \\\\frac{21}{\\\\sqrt{3}}.$$ Thus, the answer is $\\\\boxed{\\\\frac{21}{\\\\sqrt{3}}}$.\",\n",
- " \"role\": \"assistant\"\n",
- " }\n",
- " }\n",
- " ],\n",
- " \"created\": 1680470558,\n",
- " \"id\": \"chatcmpl-70zRmEgMXBlt1xSrJIWXTfYPYGoeU\",\n",
- " \"model\": \"gpt-3.5-turbo-0301\",\n",
- " \"object\": \"chat.completion\",\n",
- " \"usage\": {\n",
- " \"completion_tokens\": 6606,\n",
- " \"prompt_tokens\": 50,\n",
- " \"total_tokens\": 6656\n",
- " }\n",
- "}\n",
- "metric_results on the example data instance: {'expected_success': 1.0, 'success': True, 'success_vote': 1.0, 'voted_answer': 'We want to eliminate the radical in the denominator. We can do this by multiplying the numerator and denominator by $\\\\sqrt{21}$: $$\\\\frac{21}{\\\\sqrt{21}}=\\\\frac{21\\\\cdot\\\\sqrt{21}}{\\\\sqrt{21}\\\\cdot\\\\sqrt{21}}=\\\\frac{21\\\\sqrt{21}}{21}=\\\\sqrt{21}.$$Therefore, the simplified expression is $\\\\boxed{\\\\sqrt{21}}$.', 'votes': 45}\n"
- ]
- }
- ],
- "source": [
- "response = oai.ChatCompletion.create(context=tune_data[1], **config)\n",
- "metric_results = eval_math_responses(oai.ChatCompletion.extract_text(response), **tune_data[1])\n",
- "print(\"response on an example data instance:\", response)\n",
- "print(\"metric_results on the example data instance:\", metric_results)\n"
- ]
- },
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### Evaluate the success rate on the test data\n",
- "\n",
- "You can use flaml's `oai.ChatCompletion.test` to evaluate the performance of an entire dataset with the tuned config. The following code will take a while (30 mins to 1 hour) to evaluate all the test data instances if uncommented and run. It will cost roughly $3. "
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 13,
- "metadata": {
- "execution": {
- "iopub.execute_input": "2023-02-13T23:41:56.042764Z",
- "iopub.status.busy": "2023-02-13T23:41:56.042086Z",
- "iopub.status.idle": "2023-02-13T23:53:05.597643Z",
- "shell.execute_reply": "2023-02-13T23:53:05.596603Z"
- }
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "performance on test data with the tuned config: {'expected_success': 0.9765082554311302, 'success': 0.9800995024875622, 'success_vote': 0.9104477611940298, 'votes': 39.87064676616915, 'cost': 3.0702499999999997, 'inference_cost': 0.015274875621890546}\n"
- ]
- }
- ],
- "source": [
- "# result = oai.Completion.test(test_data, config, eval_math_responses)\n",
- "# print(\"performance on test data with the tuned config:\", result)"
- ]
- },
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "What about the default, untuned gpt-4 config (with the same prompt as the tuned config)? We can evaluate it and compare:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 14,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "performance on test data from gpt-4 with a default config: {'expected_success': 0.6716417910447762, 'success': 0.6716417910447762, 'success_vote': 0.6716417910447762, 'votes': 0.9900497512437811, 'cost': 1.92477, 'inference_cost': 0.009575970149253732}\n"
- ]
- }
- ],
- "source": [
- "# assuming you have access to gpt-4; otherwise use gpt-3.5-turbo\n",
- "# the following code will cost roughly $2 if uncommented and run.\n",
- "\n",
- "# default_config = {\"model\": 'gpt-4', \"prompt\": prompts[0]}\n",
- "# default_result = oai.Completion.test(test_data, default_config, eval_math_responses)\n",
- "# print(\"performance on test data from gpt-4 with a default config:\", default_result)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 15,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "tuned config succeeds in 91.0% test cases\n",
- "untuned config succeeds in 67.2% test cases\n"
- ]
- }
- ],
- "source": [
- "# print(\"tuned config succeeds in {:.1f}% test cases\".format(result[\"success_vote\"] * 100))\n",
- "# print(\"untuned config succeeds in {:.1f}% test cases\".format(default_result[\"success_vote\"] * 100))"
- ]
- },
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "The default use of GPT-4 has a much lower accuracy. Note that the default config has a lower inference cost. What if we heuristically increase the number of responses n?"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 16,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "performance on test data from gpt-4 with a default config and n=2: {'expected_success': 0.7699004975124378, 'success': 0.835820895522388, 'success_vote': 0.7064676616915423, 'votes': 1.5870646766169154, 'cost': 3.267810000000001, 'inference_cost': 0.016257761194029857}\n"
- ]
- }
- ],
- "source": [
- "# The following evaluation costs $3 and longer than one hour if you uncomment it and run it.\n",
- "\n",
- "# config_n2 = {\"model\": 'gpt-4', \"prompt\": prompts[0], \"n\": 2}\n",
- "# result_n2 = oai.ChatCompletion.test(test_data, config_n2, eval_math_responses)\n",
- "# print(\"performance on test data from gpt-4 with a default config and n=2:\", result_n2)\n"
- ]
- },
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "The inference cost is doubled and matches the tuned config. But the success rate doesn't improve much. What if we further increase the number of responses n to 5?"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 17,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "performance on test data from gpt-4 with a default config and n=5: {'expected_success': 0.9197771144278607, 'success': 0.9552238805970149, 'success_vote': 0.8656716417910447, 'votes': 3.691542288557214, 'cost': 7.518809999999999, 'inference_cost': 0.03740701492537313}\n"
- ]
- }
- ],
- "source": [
- "# The following evaluation costs $8 and longer than one hour if you uncomment it and run it.\n",
- "\n",
- "# config_n5 = {\"model\": 'gpt-4', \"prompt\": prompts[0], \"n\": 5}\n",
- "# result_n5 = oai.ChatCompletion.test(test_data, config_n5, eval_math_responses)\n",
- "# print(\"performance on test data from gpt-4 with a default config and n=5:\", result_n5)"
- ]
- },
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "We find that the 'success_vote' metric is increased at the cost of exceeding the inference budget. But the tuned configuration has both higher 'success_vote' (91% vs. 87%) and lower average inference cost ($0.015 vs. $0.037 per instance).\n",
- "\n",
- "A developer could use flaml to tune the configuration to satisfy the target inference budget while maximizing the value out of it."
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 3",
- "language": "python",
- "name": "python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.9.16"
- },
- "vscode": {
- "interpreter": {
- "hash": "949777d72b0d2535278d3dc13498b2535136f6dfe0678499012e853ee9abcab1"
- }
- },
- "widgets": {
- "application/vnd.jupyter.widget-state+json": {
- "state": {
- "2d910cfd2d2a4fc49fc30fbbdc5576a7": {
- "model_module": "@jupyter-widgets/base",
- "model_module_version": "2.0.0",
- "model_name": "LayoutModel",
- "state": {
- "_model_module": "@jupyter-widgets/base",
- "_model_module_version": "2.0.0",
- "_model_name": "LayoutModel",
- "_view_count": null,
- "_view_module": "@jupyter-widgets/base",
- "_view_module_version": "2.0.0",
- "_view_name": "LayoutView",
- "align_content": null,
- "align_items": null,
- "align_self": null,
- "border_bottom": null,
- "border_left": null,
- "border_right": null,
- "border_top": null,
- "bottom": null,
- "display": null,
- "flex": null,
- "flex_flow": null,
- "grid_area": null,
- "grid_auto_columns": null,
- "grid_auto_flow": null,
- "grid_auto_rows": null,
- "grid_column": null,
- "grid_gap": null,
- "grid_row": null,
- "grid_template_areas": null,
- "grid_template_columns": null,
- "grid_template_rows": null,
- "height": null,
- "justify_content": null,
- "justify_items": null,
- "left": null,
- "margin": null,
- "max_height": null,
- "max_width": null,
- "min_height": null,
- "min_width": null,
- "object_fit": null,
- "object_position": null,
- "order": null,
- "overflow": null,
- "padding": null,
- "right": null,
- "top": null,
- "visibility": null,
- "width": null
- }
- },
- "454146d0f7224f038689031002906e6f": {
- "model_module": "@jupyter-widgets/controls",
- "model_module_version": "2.0.0",
- "model_name": "HBoxModel",
- "state": {
- "_dom_classes": [],
- "_model_module": "@jupyter-widgets/controls",
- "_model_module_version": "2.0.0",
- "_model_name": "HBoxModel",
- "_view_count": null,
- "_view_module": "@jupyter-widgets/controls",
- "_view_module_version": "2.0.0",
- "_view_name": "HBoxView",
- "box_style": "",
- "children": [
- "IPY_MODEL_e4ae2b6f5a974fd4bafb6abb9d12ff26",
- "IPY_MODEL_577e1e3cc4db4942b0883577b3b52755",
- "IPY_MODEL_b40bdfb1ac1d4cffb7cefcb870c64d45"
- ],
- "layout": "IPY_MODEL_dc83c7bff2f241309537a8119dfc7555",
- "tabbable": null,
- "tooltip": null
- }
- },
- "577e1e3cc4db4942b0883577b3b52755": {
- "model_module": "@jupyter-widgets/controls",
- "model_module_version": "2.0.0",
- "model_name": "FloatProgressModel",
- "state": {
- "_dom_classes": [],
- "_model_module": "@jupyter-widgets/controls",
- "_model_module_version": "2.0.0",
- "_model_name": "FloatProgressModel",
- "_view_count": null,
- "_view_module": "@jupyter-widgets/controls",
- "_view_module_version": "2.0.0",
- "_view_name": "ProgressView",
- "bar_style": "success",
- "description": "",
- "description_allow_html": false,
- "layout": "IPY_MODEL_2d910cfd2d2a4fc49fc30fbbdc5576a7",
- "max": 1,
- "min": 0,
- "orientation": "horizontal",
- "style": "IPY_MODEL_74a6ba0c3cbc4051be0a83e152fe1e62",
- "tabbable": null,
- "tooltip": null,
- "value": 1
- }
- },
- "6086462a12d54bafa59d3c4566f06cb2": {
- "model_module": "@jupyter-widgets/base",
- "model_module_version": "2.0.0",
- "model_name": "LayoutModel",
- "state": {
- "_model_module": "@jupyter-widgets/base",
- "_model_module_version": "2.0.0",
- "_model_name": "LayoutModel",
- "_view_count": null,
- "_view_module": "@jupyter-widgets/base",
- "_view_module_version": "2.0.0",
- "_view_name": "LayoutView",
- "align_content": null,
- "align_items": null,
- "align_self": null,
- "border_bottom": null,
- "border_left": null,
- "border_right": null,
- "border_top": null,
- "bottom": null,
- "display": null,
- "flex": null,
- "flex_flow": null,
- "grid_area": null,
- "grid_auto_columns": null,
- "grid_auto_flow": null,
- "grid_auto_rows": null,
- "grid_column": null,
- "grid_gap": null,
- "grid_row": null,
- "grid_template_areas": null,
- "grid_template_columns": null,
- "grid_template_rows": null,
- "height": null,
- "justify_content": null,
- "justify_items": null,
- "left": null,
- "margin": null,
- "max_height": null,
- "max_width": null,
- "min_height": null,
- "min_width": null,
- "object_fit": null,
- "object_position": null,
- "order": null,
- "overflow": null,
- "padding": null,
- "right": null,
- "top": null,
- "visibility": null,
- "width": null
- }
- },
- "74a6ba0c3cbc4051be0a83e152fe1e62": {
- "model_module": "@jupyter-widgets/controls",
- "model_module_version": "2.0.0",
- "model_name": "ProgressStyleModel",
- "state": {
- "_model_module": "@jupyter-widgets/controls",
- "_model_module_version": "2.0.0",
- "_model_name": "ProgressStyleModel",
- "_view_count": null,
- "_view_module": "@jupyter-widgets/base",
- "_view_module_version": "2.0.0",
- "_view_name": "StyleView",
- "bar_color": null,
- "description_width": ""
- }
- },
- "7d3f3d9e15894d05a4d188ff4f466554": {
- "model_module": "@jupyter-widgets/controls",
- "model_module_version": "2.0.0",
- "model_name": "HTMLStyleModel",
- "state": {
- "_model_module": "@jupyter-widgets/controls",
- "_model_module_version": "2.0.0",
- "_model_name": "HTMLStyleModel",
- "_view_count": null,
- "_view_module": "@jupyter-widgets/base",
- "_view_module_version": "2.0.0",
- "_view_name": "StyleView",
- "background": null,
- "description_width": "",
- "font_size": null,
- "text_color": null
- }
- },
- "b40bdfb1ac1d4cffb7cefcb870c64d45": {
- "model_module": "@jupyter-widgets/controls",
- "model_module_version": "2.0.0",
- "model_name": "HTMLModel",
- "state": {
- "_dom_classes": [],
- "_model_module": "@jupyter-widgets/controls",
- "_model_module_version": "2.0.0",
- "_model_name": "HTMLModel",
- "_view_count": null,
- "_view_module": "@jupyter-widgets/controls",
- "_view_module_version": "2.0.0",
- "_view_name": "HTMLView",
- "description": "",
- "description_allow_html": false,
- "layout": "IPY_MODEL_f1355871cc6f4dd4b50d9df5af20e5c8",
- "placeholder": "​",
- "style": "IPY_MODEL_ca245376fd9f4354af6b2befe4af4466",
- "tabbable": null,
- "tooltip": null,
- "value": " 1/1 [00:00&lt;00:00, 44.69it/s]"
- }
- },
- "ca245376fd9f4354af6b2befe4af4466": {
- "model_module": "@jupyter-widgets/controls",
- "model_module_version": "2.0.0",
- "model_name": "HTMLStyleModel",
- "state": {
- "_model_module": "@jupyter-widgets/controls",
- "_model_module_version": "2.0.0",
- "_model_name": "HTMLStyleModel",
- "_view_count": null,
- "_view_module": "@jupyter-widgets/base",
- "_view_module_version": "2.0.0",
- "_view_name": "StyleView",
- "background": null,
- "description_width": "",
- "font_size": null,
- "text_color": null
- }
- },
- "dc83c7bff2f241309537a8119dfc7555": {
- "model_module": "@jupyter-widgets/base",
- "model_module_version": "2.0.0",
- "model_name": "LayoutModel",
- "state": {
- "_model_module": "@jupyter-widgets/base",
- "_model_module_version": "2.0.0",
- "_model_name": "LayoutModel",
- "_view_count": null,
- "_view_module": "@jupyter-widgets/base",
- "_view_module_version": "2.0.0",
- "_view_name": "LayoutView",
- "align_content": null,
- "align_items": null,
- "align_self": null,
- "border_bottom": null,
- "border_left": null,
- "border_right": null,
- "border_top": null,
- "bottom": null,
- "display": null,
- "flex": null,
- "flex_flow": null,
- "grid_area": null,
- "grid_auto_columns": null,
- "grid_auto_flow": null,
- "grid_auto_rows": null,
- "grid_column": null,
- "grid_gap": null,
- "grid_row": null,
- "grid_template_areas": null,
- "grid_template_columns": null,
- "grid_template_rows": null,
- "height": null,
- "justify_content": null,
- "justify_items": null,
- "left": null,
- "margin": null,
- "max_height": null,
- "max_width": null,
- "min_height": null,
- "min_width": null,
- "object_fit": null,
- "object_position": null,
- "order": null,
- "overflow": null,
- "padding": null,
- "right": null,
- "top": null,
- "visibility": null,
- "width": null
- }
- },
- "e4ae2b6f5a974fd4bafb6abb9d12ff26": {
- "model_module": "@jupyter-widgets/controls",
- "model_module_version": "2.0.0",
- "model_name": "HTMLModel",
- "state": {
- "_dom_classes": [],
- "_model_module": "@jupyter-widgets/controls",
- "_model_module_version": "2.0.0",
- "_model_name": "HTMLModel",
- "_view_count": null,
- "_view_module": "@jupyter-widgets/controls",
- "_view_module_version": "2.0.0",
- "_view_name": "HTMLView",
- "description": "",
- "description_allow_html": false,
- "layout": "IPY_MODEL_6086462a12d54bafa59d3c4566f06cb2",
- "placeholder": "​",
- "style": "IPY_MODEL_7d3f3d9e15894d05a4d188ff4f466554",
- "tabbable": null,
- "tooltip": null,
- "value": "100%"
- }
- },
- "f1355871cc6f4dd4b50d9df5af20e5c8": {
- "model_module": "@jupyter-widgets/base",
- "model_module_version": "2.0.0",
- "model_name": "LayoutModel",
- "state": {
- "_model_module": "@jupyter-widgets/base",
- "_model_module_version": "2.0.0",
- "_model_name": "LayoutModel",
- "_view_count": null,
- "_view_module": "@jupyter-widgets/base",
- "_view_module_version": "2.0.0",
- "_view_name": "LayoutView",
- "align_content": null,
- "align_items": null,
- "align_self": null,
- "border_bottom": null,
- "border_left": null,
- "border_right": null,
- "border_top": null,
- "bottom": null,
- "display": null,
- "flex": null,
- "flex_flow": null,
- "grid_area": null,
- "grid_auto_columns": null,
- "grid_auto_flow": null,
- "grid_auto_rows": null,
- "grid_column": null,
- "grid_gap": null,
- "grid_row": null,
- "grid_template_areas": null,
- "grid_template_columns": null,
- "grid_template_rows": null,
- "height": null,
- "justify_content": null,
- "justify_items": null,
- "left": null,
- "margin": null,
- "max_height": null,
- "max_width": null,
- "min_height": null,
- "min_width": null,
- "object_fit": null,
- "object_position": null,
- "order": null,
- "overflow": null,
- "padding": null,
- "right": null,
- "top": null,
- "visibility": null,
- "width": null
- }
- }
- },
- "version_major": 2,
- "version_minor": 0
- }
- }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}
diff --git a/notebook/autogen_chatgpt_gpt4.ipynb b/notebook/autogen_chatgpt_gpt4.ipynb
new file mode 100644
index 000000000..9e9db0cec
--- /dev/null
+++ b/notebook/autogen_chatgpt_gpt4.ipynb
@@ -0,0 +1,1682 @@
+{
+ "cells": [
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "slideshow": {
+ "slide_type": "slide"
+ }
+ },
+ "source": [
+ "Copyright (c) Microsoft Corporation. All rights reserved. \n",
+ "\n",
+ "Licensed under the MIT License.\n",
+ "\n",
+ "# Use FLAML to Tune ChatGPT\n",
+ "\n",
+ "FLAML offers a cost-effective hyperparameter optimization technique [EcoOptiGen](https://arxiv.org/abs/2303.04673) for tuning Large Language Models. Our study finds that tuning hyperparameters can significantly improve the utility of LLMs.\n",
+ "\n",
+ "In this notebook, we tune OpenAI ChatGPT (both GPT-3.5 and GPT-4) models for math problem solving. We use [the MATH benchmark](https://crfm.stanford.edu/helm/latest/?group=math_chain_of_thought) for measuring mathematical problem solving on competition math problems with chain-of-thoughts style reasoning. \n",
+ "\n",
+ "## Requirements\n",
+ "\n",
+ "FLAML requires `Python>=3.7`. To run this notebook example, please install flaml with the [openai,blendsearch] option:\n",
+ "```bash\n",
+ "pip install flaml[openai,blendsearch]==1.2.2\n",
+ "```"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {
+ "execution": {
+ "iopub.execute_input": "2023-02-13T23:40:52.317406Z",
+ "iopub.status.busy": "2023-02-13T23:40:52.316561Z",
+ "iopub.status.idle": "2023-02-13T23:40:52.321193Z",
+ "shell.execute_reply": "2023-02-13T23:40:52.320628Z"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "# %pip install flaml[openai,blendsearch]==1.2.2 datasets"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Import the oai and tune subpackages from flaml\n",
+ "\n",
+ "FLAML has provided an API for hyperparameter optimization of OpenAI ChatGPT models: `oai.ChatCompletion.tune` and to make a request with the tuned config: `oai.ChatCompletion.create`. First, we import oai from flaml:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {
+ "execution": {
+ "iopub.execute_input": "2023-02-13T23:40:54.634335Z",
+ "iopub.status.busy": "2023-02-13T23:40:54.633929Z",
+ "iopub.status.idle": "2023-02-13T23:40:56.105700Z",
+ "shell.execute_reply": "2023-02-13T23:40:56.105085Z"
+ },
+ "slideshow": {
+ "slide_type": "slide"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "from flaml import oai"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Set your API Endpoint\n",
+ "\n",
+ "The [`config_list_openai_aoai`](https://microsoft.github.io/FLAML/docs/reference/autogen/oai/openai_utils#config_list_openai_aoai) function tries to create a list of Azure OpenAI endpoints and OpenAI endpoints. It assumes the api keys and api bases are stored in the corresponding environment variables or local txt files:\n",
+ "\n",
+ "- OpenAI API key: os.environ[\"OPENAI_API_KEY\"] or `openai_api_key_file=\"key_openai.txt\"`.\n",
+ "- Azure OpenAI API key: os.environ[\"AZURE_OPENAI_API_KEY\"] or `aoai_api_key_file=\"key_aoai.txt\"`. Multiple keys can be stored, one per line.\n",
+ "- Azure OpenAI API base: os.environ[\"AZURE_OPENAI_API_BASE\"] or `aoai_api_base_file=\"base_aoai.txt\"`. Multiple bases can be stored, one per line.\n",
+ "\n",
+ "It's OK to have only the OpenAI API key, or only the Azure Open API key + base.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {
+ "execution": {
+ "iopub.execute_input": "2023-02-13T23:40:52.324240Z",
+ "iopub.status.busy": "2023-02-13T23:40:52.323783Z",
+ "iopub.status.idle": "2023-02-13T23:40:52.330570Z",
+ "shell.execute_reply": "2023-02-13T23:40:52.329750Z"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "config_list = oai.config_list_openai_aoai()"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "The config list looks like the following:\n",
+ "```python\n",
+ "config_list = [\n",
+ " {'api_key': '<your OpenAI API key here>'}, # only if OpenAI API key is found\n",
+ " {\n",
+ " 'api_key': '<your first Azure OpenAI API key here>',\n",
+ " 'api_base': '<your first Azure OpenAI API base here>',\n",
+ " 'api_type': 'azure',\n",
+ " 'api_version': '2023-03-15-preview',\n",
+ " }, # only if the at least one Azure OpenAI API key is found\n",
+ " {\n",
+ " 'api_key': '<your second Azure OpenAI API key here>',\n",
+ " 'api_base': '<your second Azure OpenAI API base here>',\n",
+ " 'api_type': 'azure',\n",
+ " 'api_version': '2023-03-15-preview',\n",
+ " }, # only if the second Azure OpenAI API key is found\n",
+ "]\n",
+ "```\n",
+ "\n",
+ "You can directly override it if the above function returns an empty list, i.e., it doesn't find the keys in the specified locations."
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Load dataset\n",
+ "\n",
+ "We load the competition_math dataset. The dataset contains 201 \"Level 2\" Algebra examples. We use a random sample of 20 examples for tuning the generation hyperparameters and the remaining for evaluation."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {
+ "execution": {
+ "iopub.execute_input": "2023-02-13T23:40:52.339977Z",
+ "iopub.status.busy": "2023-02-13T23:40:52.339556Z",
+ "iopub.status.idle": "2023-02-13T23:40:54.603349Z",
+ "shell.execute_reply": "2023-02-13T23:40:54.602630Z"
+ }
+ },
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Using custom data configuration default\n",
+ "Found cached dataset competition_math (/home/vscode/.cache/huggingface/datasets/competition_math/default/1.0.0/2a2a2995c2847186883ecd64f69be7d602b8a6f6b51950624d4dc2263f93333b)\n"
+ ]
+ },
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "9e8a0c134dcc447db7395ad52f12f8bd",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ " 0%| | 0/2 [00:00<?, ?it/s]"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Loading cached shuffled indices for dataset at /home/vscode/.cache/huggingface/datasets/competition_math/default/1.0.0/2a2a2995c2847186883ecd64f69be7d602b8a6f6b51950624d4dc2263f93333b/cache-f1cfe8228271b121.arrow\n",
+ "Loading cached shuffled indices for dataset at /home/vscode/.cache/huggingface/datasets/competition_math/default/1.0.0/2a2a2995c2847186883ecd64f69be7d602b8a6f6b51950624d4dc2263f93333b/cache-d155a2d38c23bd53.arrow\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "20 201\n"
+ ]
+ }
+ ],
+ "source": [
+ "import datasets\n",
+ "\n",
+ "seed = 41\n",
+ "data = datasets.load_dataset(\"competition_math\")\n",
+ "train_data = data[\"train\"].shuffle(seed=seed)\n",
+ "test_data = data[\"test\"].shuffle(seed=seed)\n",
+ "n_tune_data = 20\n",
+ "tune_data = [\n",
+ " {\n",
+ " \"problem\": train_data[x][\"problem\"],\n",
+ " \"solution\": train_data[x][\"solution\"],\n",
+ " }\n",
+ " for x in range(len(train_data)) if train_data[x][\"level\"] == \"Level 2\" and train_data[x][\"type\"] == \"Algebra\"\n",
+ "][:n_tune_data]\n",
+ "test_data = [\n",
+ " {\n",
+ " \"problem\": test_data[x][\"problem\"],\n",
+ " \"solution\": test_data[x][\"solution\"],\n",
+ " }\n",
+ " for x in range(len(test_data)) if test_data[x][\"level\"] == \"Level 2\" and test_data[x][\"type\"] == \"Algebra\"\n",
+ "]\n",
+ "print(len(tune_data), len(test_data))\n"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "slideshow": {
+ "slide_type": "slide"
+ }
+ },
+ "source": [
+ "Check a tuning example:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {
+ "execution": {
+ "iopub.execute_input": "2023-02-13T23:40:54.607152Z",
+ "iopub.status.busy": "2023-02-13T23:40:54.606441Z",
+ "iopub.status.idle": "2023-02-13T23:40:54.610504Z",
+ "shell.execute_reply": "2023-02-13T23:40:54.609759Z"
+ },
+ "slideshow": {
+ "slide_type": "subslide"
+ },
+ "tags": []
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "What is the slope of a line parallel to the line $2x - 4y = 9$? Express your answer as a common fraction.\n"
+ ]
+ }
+ ],
+ "source": [
+ "print(tune_data[1][\"problem\"])"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Here is one example of the canonical solution:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {
+ "execution": {
+ "iopub.execute_input": "2023-02-13T23:40:54.613590Z",
+ "iopub.status.busy": "2023-02-13T23:40:54.613168Z",
+ "iopub.status.idle": "2023-02-13T23:40:54.616873Z",
+ "shell.execute_reply": "2023-02-13T23:40:54.616193Z"
+ }
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "The slope-intercept form of the equation of a line is $y = m x + b$ where $m$ is the slope. So if we get $y$ on the opposite side from $x$ and make it have a coefficient of 1, the slope of the line will be the coefficient of $x$. Thus we add $4y$ to both sides and divide everything by 4 which makes the coefficient of $x$ equal to $\\boxed{\\frac{1}{2}}$.\n"
+ ]
+ }
+ ],
+ "source": [
+ "print(tune_data[1][\"solution\"])"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Define Success Metric\n",
+ "\n",
+ "Before we start tuning, we need to define the success metric we want to optimize. For each math task, we use voting to select a response with the most common answers out of all the generated responses. If it has an equivalent answer to the canonical solution, we consider the task as successfully solved. Then we can optimize the mean success rate of a collection of tasks."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {
+ "execution": {
+ "iopub.execute_input": "2023-02-13T23:40:54.626998Z",
+ "iopub.status.busy": "2023-02-13T23:40:54.626593Z",
+ "iopub.status.idle": "2023-02-13T23:40:54.631383Z",
+ "shell.execute_reply": "2023-02-13T23:40:54.630770Z"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "from flaml.autogen.math_utils import eval_math_responses"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "slideshow": {
+ "slide_type": "slide"
+ }
+ },
+ "source": [
+ "## Use the tuning data to find a good configuration\n"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "For (local) reproducibility and cost efficiency, we cache responses from OpenAI."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {
+ "execution": {
+ "iopub.execute_input": "2023-02-13T23:40:56.109177Z",
+ "iopub.status.busy": "2023-02-13T23:40:56.108624Z",
+ "iopub.status.idle": "2023-02-13T23:40:56.112651Z",
+ "shell.execute_reply": "2023-02-13T23:40:56.112076Z"
+ },
+ "slideshow": {
+ "slide_type": "slide"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "oai.ChatCompletion.set_cache(seed)"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "This will create a disk cache in \".cache/{seed}\". You can change `cache_path` in `set_cache()`. The cache for different seeds are stored separately.\n",
+ "\n",
+ "### Perform tuning\n",
+ "\n",
+ "The tuning will take a while to finish, depending on the optimization budget. The tuning will be performed under the specified optimization budgets.\n",
+ "\n",
+ "* `inference_budget` is the target average inference budget per instance in the benchmark. For example, 0.004 means the target inference budget is 0.004 dollars, which translates to 2000 tokens (input + output combined) if the gpt-3.5-turbo model is used.\n",
+ "* `optimization_budget` is the total budget allowed to perform the tuning. For example, 1 means 1 dollars are allowed in total, which translates to 500K tokens for the gpt-3.5-turbo model.\n",
+ "* `num_sumples` is the number of different hyperparameter configurations which is allowed to try. The tuning will stop after either num_samples trials or after optimization_budget dollars spent, whichever happens first. -1 means no hard restriction in the number of trials and the actual number is decided by `optimization_budget`.\n",
+ "\n",
+ "Users can specify tuning data, optimization metric, optimization mode, evaluation function, search spaces etc.. The default search space is:\n",
+ "\n",
+ "```python\n",
+ "default_search_space = {\n",
+ " \"model\": tune.choice([\n",
+ " \"gpt-3.5-turbo\",\n",
+ " \"gpt-4\",\n",
+ " ]),\n",
+ " \"temperature_or_top_p\": tune.choice(\n",
+ " [\n",
+ " {\"temperature\": tune.uniform(0, 2)},\n",
+ " {\"top_p\": tune.uniform(0, 1)},\n",
+ " ]\n",
+ " ),\n",
+ " \"max_tokens\": tune.lograndint(50, 1000),\n",
+ " \"n\": tune.randint(1, 100),\n",
+ " \"prompt\": \"{prompt}\",\n",
+ "}\n",
+ "```\n",
+ "\n",
+ "The default search space can be overridden by users' input.\n",
+ "For example, the following code specifies a fixed prompt template. For hyperparameters which don't appear in users' input, the default search space will be used."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {
+ "execution": {
+ "iopub.execute_input": "2023-02-13T23:40:56.115383Z",
+ "iopub.status.busy": "2023-02-13T23:40:56.114975Z",
+ "iopub.status.idle": "2023-02-13T23:41:55.045654Z",
+ "shell.execute_reply": "2023-02-13T23:41:55.044973Z"
+ }
+ },
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "\u001b[32m[I 2023-05-20 22:12:09,860]\u001b[0m A new study created in memory with name: optuna\u001b[0m\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "[flaml.tune.tune: 05-20 22:12:09] {773} INFO - trial 1 config: {'model': 'gpt-3.5-turbo', 'temperature_or_top_p': {'top_p': 0.36280922847807595}, 'max_tokens': 347, 'n': 10, 'prompt': 0}\n",
+ "[flaml.tune.tune: 05-20 22:12:10] {197} INFO - result: {'expected_success': 0.8043947377449999, 'success': 0.85, 'success_vote': 0.7, 'voted_answer': 'We have $(-27)^{5/3} = \\\\left[(-27)^{1/3}\\\\right]^5$. Since $(-3)^3 = -27$, we have $(-27)^{1/3} = -3$. Therefore, $\\\\left[(-27)^{1/3}\\\\right]^5 = (-3)^5 = \\\\boxed{-243}$.', 'votes': 8.7, 'total_cost': 0.05771399999999999, 'cost': 0.05771399999999999, 'inference_cost': 0.0025029, 'training_iteration': 0, 'config': {'model': 'gpt-3.5-turbo', 'temperature_or_top_p': {'top_p': 0.36280922847807595}, 'max_tokens': 347, 'n': 10, 'prompt': 0}, 'config/model': 'gpt-3.5-turbo', 'config/temperature_or_top_p': {'top_p': 0.36280922847807595}, 'config/max_tokens': 347, 'config/n': 10, 'config/prompt': 0, 'experiment_tag': 'exp', 'time_total_s': 0.22889995574951172}\n",
+ "[flaml.tune.tune: 05-20 22:12:10] {773} INFO - trial 2 config: {'temperature_or_top_p': {'temperature': 1.2672964698525508}, 'max_tokens': 470, 'n': 50, 'prompt': 0, 'model': 'gpt-3.5-turbo'}\n",
+ "[flaml.tune.tune: 05-20 22:13:52] {197} INFO - result: {'expected_success': 0.9999161685004145, 'success': 1.0, 'success_vote': 0.9, 'voted_answer': 'We have \\\\begin{align*}\\n(-27)^{5/3} &= ((-27)^{1/3})^{5}\\\\\\\\\\n&=(\\\\sqrt[3]{-27})^5 &&\\\\mbox{(by definition of cube root)}\\\\\\\\\\n&= (-3)^5 && \\\\mbox{(because $(-a)^{2n+1}= -a^{2n+1}$ for all real numbers $a$ and integers $n \\\\geq 0$)} \\\\\\\\\\n&= -243.\\n\\\\end{align*}Thus, $(-27)^{5/3} = \\\\boxed{-243}$.', 'votes': 33.8, 'total_cost': 0.37417999999999996, 'cost': 0.316466, 'inference_cost': 0.0149563, 'training_iteration': 0, 'config': {'temperature_or_top_p': {'temperature': 1.2672964698525508}, 'max_tokens': 470, 'n': 50, 'prompt': 0, 'model': 'gpt-3.5-turbo'}, 'config/temperature_or_top_p': {'temperature': 1.2672964698525508}, 'config/max_tokens': 470, 'config/n': 50, 'config/prompt': 0, 'config/model': 'gpt-3.5-turbo', 'experiment_tag': 'exp', 'time_total_s': 102.0125892162323}\n",
+ "[flaml.tune.tune: 05-20 22:13:52] {773} INFO - trial 3 config: {'temperature_or_top_p': {'temperature': 1.5210614243979175}, 'max_tokens': 82, 'n': 9, 'prompt': 0, 'model': 'gpt-3.5-turbo'}\n",
+ "[flaml.tune.tune: 05-20 22:14:04] {197} INFO - result: {'expected_success': 0.49315329667554064, 'success': 0.55, 'success_vote': 0.55, 'voted_answer': 'Note that $(-27)^{1/3}=-3$, so \\\\[(-27)^{5/3}=(-27)^{1/3\\\\cdot 5}=\\\\left[(-27)^{1/3}\\\\right]^5=(-3)^5=-243.\\\\]Thus, $(-27)^{5/3} = \\\\boxed{-243}$.', 'votes': 2.05, 'total_cost': 0.403738, 'cost': 0.029557999999999994, 'inference_cost': 0.0014779, 'training_iteration': 0, 'config': {'temperature_or_top_p': {'temperature': 1.5210614243979175}, 'max_tokens': 82, 'n': 9, 'prompt': 0, 'model': 'gpt-3.5-turbo'}, 'config/temperature_or_top_p': {'temperature': 1.5210614243979175}, 'config/max_tokens': 82, 'config/n': 9, 'config/prompt': 0, 'config/model': 'gpt-3.5-turbo', 'experiment_tag': 'exp', 'time_total_s': 12.128517150878906}\n",
+ "[flaml.tune.tune: 05-20 22:14:04] {773} INFO - trial 4 config: {'temperature_or_top_p': {'top_p': 0.003948266327914451}, 'max_tokens': 231, 'n': 81, 'prompt': 0, 'model': 'gpt-3.5-turbo'}\n",
+ "[flaml.tune.tune: 05-20 22:15:10] {197} INFO - result: {'expected_success': 0.75, 'success': 0.75, 'success_vote': 0.75, 'voted_answer': 'We have $(-27)^{5/3} = \\\\left[(-27)^{1/3}\\\\right]^5$. Since $(-3)^3 = -27$, we have $(-27)^{1/3} = -3$. Therefore, $\\\\left[(-27)^{1/3}\\\\right]^5 = (-3)^5 = \\\\boxed{-243}$.', 'votes': 76.15, 'total_cost': 0.7355139999999999, 'cost': 0.331776, 'inference_cost': 0.0161767, 'training_iteration': 0, 'config': {'temperature_or_top_p': {'top_p': 0.003948266327914451}, 'max_tokens': 231, 'n': 81, 'prompt': 0, 'model': 'gpt-3.5-turbo'}, 'config/temperature_or_top_p': {'top_p': 0.003948266327914451}, 'config/max_tokens': 231, 'config/n': 81, 'config/prompt': 0, 'config/model': 'gpt-3.5-turbo', 'experiment_tag': 'exp', 'time_total_s': 66.01478743553162}\n",
+ "[flaml.tune.tune: 05-20 22:15:10] {773} INFO - trial 5 config: {'temperature_or_top_p': {'top_p': 0.29187606817063316}, 'max_tokens': 781, 'n': 71, 'prompt': 0, 'model': 'gpt-3.5-turbo'}\n",
+ "[flaml.tune.tune: 05-20 22:16:50] {197} INFO - result: {'success_vote': 0, 'total_cost': 1.008044, 'cost': 0.27253000000000005, 'training_iteration': 0, 'config': {'temperature_or_top_p': {'top_p': 0.29187606817063316}, 'max_tokens': 781, 'n': 71, 'prompt': 0, 'model': 'gpt-3.5-turbo'}, 'config/temperature_or_top_p': {'top_p': 0.29187606817063316}, 'config/max_tokens': 781, 'config/n': 71, 'config/prompt': 0, 'config/model': 'gpt-3.5-turbo', 'experiment_tag': 'exp', 'time_total_s': 100.23262238502502}\n",
+ "[flaml.tune.tune: 05-20 22:16:50] {796} WARNING - fail to sample a trial for 100 times in a row, stopping.\n"
+ ]
+ }
+ ],
+ "source": [
+ "import logging\n",
+ "\n",
+ "prompts = [\"{problem} Solve the problem carefully. Simplify your answer as much as possible. Put the final answer in \\\\boxed{{}}.\"]\n",
+ "config, analysis = oai.ChatCompletion.tune(\n",
+ " data=tune_data, # the data for tuning\n",
+ " metric=\"success_vote\", # the metric to optimize\n",
+ " mode=\"max\", # the optimization mode\n",
+ " eval_func=eval_math_responses, # the evaluation function to return the success metrics\n",
+ " # log_file_name=\"logs/math.log\", # the log file name\n",
+ " inference_budget=0.02, # the inference budget (dollar per instance)\n",
+ " optimization_budget=1, # the optimization budget (dollar in total)\n",
+ " # num_samples can further limit the number of trials for different hyperparameter configurations;\n",
+ " # -1 means decided by the optimization budget only\n",
+ " num_samples=20,\n",
+ " model=\"gpt-3.5-turbo\", # comment to tune both gpt-3.5-turbo and gpt-4\n",
+ " prompt=prompts, # the prompt templates to choose from\n",
+ " # stop=\"###\", # the stop sequence\n",
+ " config_list=config_list, # the endpoint list\n",
+ " # logging_level=logging.INFO, # the logging level\n",
+ ")\n"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Output tuning results\n",
+ "\n",
+ "After the tuning, we can print out the config and the result found by FLAML:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {
+ "execution": {
+ "iopub.execute_input": "2023-02-13T23:41:55.049204Z",
+ "iopub.status.busy": "2023-02-13T23:41:55.048871Z",
+ "iopub.status.idle": "2023-02-13T23:41:55.053284Z",
+ "shell.execute_reply": "2023-02-13T23:41:55.052574Z"
+ }
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "optimized config {'max_tokens': 470, 'n': 50, 'prompt': '{problem} Solve the problem carefully. Simplify your answer as much as possible. Put the final answer in \\\\boxed{{}}.', 'model': 'gpt-3.5-turbo', 'temperature': 1.2672964698525508}\n",
+ "best result on tuning data {'expected_success': 0.9999161685004145, 'success': 1.0, 'success_vote': 0.9, 'voted_answer': 'We have \\\\begin{align*}\\n(-27)^{5/3} &= ((-27)^{1/3})^{5}\\\\\\\\\\n&=(\\\\sqrt[3]{-27})^5 &&\\\\mbox{(by definition of cube root)}\\\\\\\\\\n&= (-3)^5 && \\\\mbox{(because $(-a)^{2n+1}= -a^{2n+1}$ for all real numbers $a$ and integers $n \\\\geq 0$)} \\\\\\\\\\n&= -243.\\n\\\\end{align*}Thus, $(-27)^{5/3} = \\\\boxed{-243}$.', 'votes': 33.8, 'total_cost': 0.37417999999999996, 'cost': 0.316466, 'inference_cost': 0.0149563, 'training_iteration': 0, 'config': {'temperature_or_top_p': {'temperature': 1.2672964698525508}, 'max_tokens': 470, 'n': 50, 'prompt': 0, 'model': 'gpt-3.5-turbo'}, 'config/temperature_or_top_p': {'temperature': 1.2672964698525508}, 'config/max_tokens': 470, 'config/n': 50, 'config/prompt': 0, 'config/model': 'gpt-3.5-turbo', 'experiment_tag': 'exp', 'time_total_s': 102.0125892162323}\n"
+ ]
+ }
+ ],
+ "source": [
+ "print(\"optimized config\", config)\n",
+ "print(\"best result on tuning data\", analysis.best_result)"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "slideshow": {
+ "slide_type": "slide"
+ }
+ },
+ "source": [
+ "### Make a request with the tuned config\n",
+ "\n",
+ "We can apply the tuned config on the request for an example task:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {
+ "execution": {
+ "iopub.execute_input": "2023-02-13T23:41:55.056205Z",
+ "iopub.status.busy": "2023-02-13T23:41:55.055631Z",
+ "iopub.status.idle": "2023-02-13T23:41:56.039259Z",
+ "shell.execute_reply": "2023-02-13T23:41:56.038427Z"
+ },
+ "slideshow": {
+ "slide_type": "subslide"
+ },
+ "tags": []
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "response on an example data instance: {\n",
+ " \"choices\": [\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 0,\n",
+ " \"message\": {\n",
+ " \"content\": \"Given $2x-4y=9$. Putting this in slope-intercept form yields $-2y = -2x + 9 \\\\Rightarrow y=x-\\\\frac{9}{2}$, so the slope is $\\\\boxed{1}$. Parallel lines have equal slopes.\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 1,\n",
+ " \"message\": {\n",
+ " \"content\": \"To find the slope of a line parallel to $2x-4y=9,$ we need to find the slope of the line $2x-4y=9.$ Solving for $y$ in this first equation, we get $$\\\\begin{aligned} 2x-4y&=9\\n\\\\\\\\\\\\Rightarrow\\\\qquad 4y&=2x-9\\n\\\\\\\\\\\\Rightarrow\\\\qquad y&=\\\\frac{1}{2}x-\\\\frac{9}{4}.\\\\end{aligned}$$So the slope of this line is $\\\\frac{1}{2}.$ Therefore, the slope of any line parallel to this line will also be $\\\\boxed{\\\\frac{1}{2}}.$\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 2,\n",
+ " \"message\": {\n",
+ " \"content\": \"We will first convert the equation $2x-4y=9$ into slope-intercept form, $y=mx+b$. Solving this equation for $y$, we get $$-4y=-2x+9\\\\Rightarrow y=\\\\dfrac12 x -\\\\dfrac94.$$Therefore, the slope of this line is $\\\\dfrac 12$.\\n\\nSince two lines are parallel if and only if they have the same slope, any line that is parallel to $2x-4y=9$ must also have a slope of $\\\\boxed{\\\\dfrac12}$.\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 3,\n",
+ " \"message\": {\n",
+ " \"content\": \"To find the slope of a line parallel to $2x-4y=9$, we first need to write the equation in slope-intercept form, $y=mx+c$, where $m$ is the slope and $c$ is the $y$-intercept.\\n\\nFirst let's move the $-4y$ to the right side of the equation by adding it to the both sides:\\n\\\\begin{align*}\\n2x-4y&=9 \\\\\\\\\\n2x&=4y+9 \\\\\\\\\\ny&=\\\\frac12x-\\\\frac94\\n\\\\end{align*}\\n\\nNow we find the slope by looking at the coefficient of $x$: $m=\\\\frac12$.\\n\\nTherefore, any line that is parallel to $2x-4y=9$ has the same slope of $\\\\frac12$, and our final answer is $\\\\boxed{\\\\frac12}$.\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 4,\n",
+ " \"message\": {\n",
+ " \"content\": \"To figure out the slope, we simply need to put the line $2x - 4y = 9$ into slope-intercept form, $y = mx+b$, solve for $m$, and recognize that a line that is parallel to it will have the same slope. We can get $y$ by moving $2x - 9$ to the other side and dividing by $-4$, to get $y = \\\\frac{1}{2} x - \\\\frac{9}{4}$. Therefore, the slope of the line is $\\\\boxed{\\\\frac{1}{2}}$.\\n\\nAn alternative method is to recognize that parallel lines have the same slope. We first put the given line in the slope-intercept form as above, so $y = \\\\frac{1}{2} x - \\\\frac{9}{4}$. We now know the slope of a parallel line to this line is $\\\\boxed{\\\\frac{1}{2}}$.\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 5,\n",
+ " \"message\": {\n",
+ " \"content\": \"We can determine the slope of the line $2x-4y=9$ by rearranging it to slope-intercept form, $y=\\\\frac{1}{2}x-\\\\frac{9}{4}$, which indicates that the slope is $\\\\frac{1}{2}$.\\nIf a line is parallel to this line, it will have the same slope of $\\\\boxed{\\\\frac12}$.\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 6,\n",
+ " \"message\": {\n",
+ " \"content\": \"In order for two lines to be parallel, they must have the same slope. To find the slope of a line, we can put it into slope-intercept form ($y=mx+b$), where $m$ is the slope.\\n\\nStarting with $2x-4y=9$, we can isolate $y$ to get it into slope-intercept form: $$2x-4y=9$$ $$-4y=-2x+9$$ $$y=\\\\frac12x-\\\\frac94$$\\n\\nTherefore, the slope of the given line is $\\\\frac12$.\\n\\nSince we are looking for a line that is parallel, it will have the same slope. Thus, the slope of the desired line is also $\\\\frac12$.\\n\\nTherefore, the final answer is \\\\boxed{\\\\frac{1}{2}}.\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 7,\n",
+ " \"message\": {\n",
+ " \"content\": \"A line parallel to $2x-4y=9$ will have the same slope. To find the slope, we can solve $2x-4y=9$ for $y$ in terms of $x$. First, we add $2y$ to both sides, to get $2x-9=4y$. Then we divide by $4$ to get $y=\\\\frac{1}{4}x-\\\\frac{9}{4}$. Taking this in slope-intercept form we get that the slope of $2x - 4y = 9$ is $\\\\frac14$, so our final answer is $\\\\boxed{\\\\frac14}$.\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 8,\n",
+ " \"message\": {\n",
+ " \"content\": \"Recall that parallel lines have equal slopes.\\n\\nFirst, put the equation $2x - 4y = 9$ in slope-intercept form $y = mx + b$, where $m$ is the slope.\\n\\n\\\\[2x - 4y = 9 \\\\qquad \\\\Rightarrow \\\\qquad -4y = -2x + 9 \\\\qquad \\\\Rightarrow \\\\qquad y = \\\\frac{1}{2}x - \\\\frac{9}{4}\\\\]\\n\\nTherefore, the slope of the line $2x - 4y = 9$ is $\\\\frac{1}{2}$.\\n\\nSince the slope of any line parallel to this line also has to be $\\\\frac{1}{2}$, our answer is $\\\\boxed{\\\\frac12}$.\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 9,\n",
+ " \"message\": {\n",
+ " \"content\": \"Recall that two lines are parallel if and only if they have the same slope. We need to find the slope of the line $2x-4y=9$ to use this fact. Solving this equation for $y$ yields $y=\\\\frac 12 x - \\\\frac 94$, so the slope of this line is $\\\\frac12$. Therefore, any line parallel to $2x-4y=9$ also has slope $\\\\boxed{\\\\frac{1}{2}}$.\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 10,\n",
+ " \"message\": {\n",
+ " \"content\": \"To find the slope of the line, we must rewrite the equation in slope-intercept form, $y = mx + b$, where the coefficient of $x$, $m$, is the slope of the line. Solving for $y$ in $2x - 4y = 9$, we have \\\\[\\n-4y = -2x + 9\\\\quad \\\\Rightarrow\\\\quad y = \\\\frac12 x - \\\\frac94.\\n\\\\]So, a line parallel to this line will have a slope of $\\\\boxed{\\\\frac12}$.\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 11,\n",
+ " \"message\": {\n",
+ " \"content\": \"To determine the slope of a line parallel to $2x-4y=9$, we need to find the slope of this line first. We can then use the fact that parallel lines have the same slope.\\n\\nRewriting this equation of the line in slope-intercept form, we get:\\n\\n\\\\begin{align*}\\n2x-4y&=9\\\\\\\\\\n-4y&=-2x+9\\\\\\\\\\ny&=\\\\frac{1}{2}x-\\\\frac{9}{4}\\n\\\\end{align*}\\n\\nSo the slope of this line is $\\\\frac{1}{2}$. Therefore, any line parallel to this line will also have a slope of $\\\\boxed{\\\\frac{1}{2}}$.\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 12,\n",
+ " \"message\": {\n",
+ " \"content\": \"To solve the problem at hand, we take note that parallel lines must have the same slope. So, to find the slope of the line in question, we just need to rearrange $2x - 4y = 9$ to be in the slope-intercept form. Adding $4y$ and then dividing by $2$, we get $y = \\\\frac12 x - \\\\frac94$. Therefore, the slope of the line is $\\\\boxed{\\\\frac12}$.\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 13,\n",
+ " \"message\": {\n",
+ " \"content\": \"To find the slope of $2x - 4y = 9$, we can put it into slope-intercept form $y = mx + b$ by solving for $y$, giving $-4y = -2x + 9 \\\\Rightarrow y = \\\\frac{1}{2}x - \\\\frac{9}{4}$. Therefore, the slope of the given line is $\\\\frac12$. \\n\\nParallel lines have the same slope, so any line parallel to this one has a slope of $\\\\boxed{\\\\frac{1}{2}}$.\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 14,\n",
+ " \"message\": {\n",
+ " \"content\": \"We can begin by solving the equation $2x-4y=9$ for $y$ in terms of $x$ to find its slope: $$2x-4y=9\\\\Rightarrow -4y=-2x+9\\\\Rightarrow y=\\\\frac12x-\\\\frac94.$$We recognize $\\\\frac12$ as the slope of this line -- and therefore the slope of any line parallel to it. Thus, our final answer is $\\\\boxed{\\\\frac12}$.\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 15,\n",
+ " \"message\": {\n",
+ " \"content\": \"Note that if two lines in the coordinate plane are parallel, then they have the same slope (indeed, take any two distinct points on a parallel line to get a unique slope, and note that this will hold for all pairs of distinct points on the line due to parallelism).\\n\\nWe now try to put the given line into slope-intercept form ($y = mx + b$). We can do this by solving for $y$:, should get $$2x - 4y = 9\\\\qquad\\\\Longrightarrow\\\\qquad y = \\\\frac{1}{2}x - \\\\frac{9}{4}.$$ Therefore, a line parallel to $2x - 4y = 9$ must have a slope of $\\\\boxed{\\\\frac{1}{2}}$, the same slope as $y = \\\\frac{1}{2}x - \\\\frac{9}{4}.$\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 16,\n",
+ " \"message\": {\n",
+ " \"content\": \"To find the slope of the line $2x - 4y = 9,$ we can rewrite it in slope-intercept form: \\\\begin{align*}\\n2x - 4y &= 9 \\\\\\\\\\n-4y &= -2x + 9 \\\\\\\\\\ny &= \\\\frac{1}{2} x - \\\\frac{9}{4}.\\n\\\\end{align*}Therefore, the slope of this line is $\\\\boxed{\\\\frac{1}{2}}.$\\n\\nRecall that if two lines are parallel, then they have the same slope.\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 17,\n",
+ " \"message\": {\n",
+ " \"content\": \"A line parallel to any line must have the same slope of that line. Therefore, let's re-write $2x-4y = 9$ in slope-intercept form and then use the slope.$$\\n\\\\begin{array}{r r c r@{\\\\;\\\\;=\\\\;\\\\;}l}\\n2x-4y &=& 9 & 4y & 2x-9 \\\\\\\\\\n-4y & = & -2x+9 & y & \\\\displaystyle{\\\\frac{-2}{4}}x + \\\\displaystyle{\\\\frac{9}{4}}\\\\\\\\\\ny & = & \\\\displaystyle{\\\\frac{-1}{2}} x + \\\\displaystyle{\\\\frac{9}{4}}&\\n\\\\end{array}\\n$$We see that the slope is $\\\\boxed{\\\\frac{-1}{2}}$ as all other lines that our parallel to $2x-4y=9$ are of the type, in slope-intercept form, $y = -\\\\frac{1}{2} x + b$.\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 18,\n",
+ " \"message\": {\n",
+ " \"content\": \"Recall that two lines are parallel if and only if they have the same slope. Therefore, we want to find the slope of the line $2x - 4y = 9$.\\n\\nOne way to do this is to put the equation in slope-intercept form: $$ 2x - 4y = 9 \\\\Rightarrow y = \\\\frac{1}{2} x - \\\\frac{9}{4}.$$We can see that the slope of this line is $\\\\boxed{\\\\frac{1}{2}}$.\\n\\nAnother way to do this is to rewrite the equation in terms of $y$ as an explicit function of $x$, then take the derivative with respect to $x$ and evaluate the expression at any point. Doing this method is more advanced, but yields another way to check our work: $$ 2x - 4y = 9 \\\\Rightarrow y = \\\\frac{1}{2}x - \\\\frac{9}{4}.$$Differentiating with respect to $x$, we get $\\\\frac{dy}{dx} = \\\\frac{1}{2}$. Again, this confirms that the slope of the original line is $\\\\boxed{\\\\frac{1}{2}}$.\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 19,\n",
+ " \"message\": {\n",
+ " \"content\": \"The slope-intercept form of a line is $y=mx+b,$ where $m$ is the slope and $b$ is the $y$-intercept. To find the slope of a line from its equation in standard form $ax+by=c,$ solve for $y$ to get $y=-\\\\frac{a}{b}x+\\\\frac{c}{b}$ and we spot the slope of $-\\\\frac{a}{b}.$ We can find the slope-intercept form from $2x-4y=9$ as follows\\\\begin{align*}\\n2x-4y&=9 \\\\\\\\\\n-4y&=-2x+9 \\\\\\\\\\ny&=\\\\frac{1}{2}x-\\\\frac{9}{4}.\\n\\\\end{align*}The slope of this line is $\\\\frac{1}{2}.$\\n\\nEvery line parallel to this line has the same slope. Thus, the slope of a line parallel to $2x - 4y = 9$ is $\\\\boxed{\\\\frac{1}{2}}.$\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 20,\n",
+ " \"message\": {\n",
+ " \"content\": \"To find the slope of the line defined by $2x-4y=9,$ we need to solve for $y$ so that the equation is in slope-intercept form, $y=mx+b,$ where $m$ is the slope. Solving for $y$ results in \\\\begin{align*}\\n2x-4y&=9\\\\\\\\\\n\\\\Rightarrow\\\\qquad y&=\\\\frac{1}{2}x-\\\\frac{9}{4}.\\n\\\\end{align*}Therefore, the slope of the line is $\\\\displaystyle\\\\frac{1}{2}.$\\n\\nTwo non-vertical lines are parallel if and only if they have the same slope. Thus, a line parallel to $2x-4y=9$ has a slope of $\\\\boxed{\\\\frac{1}{2}}.$ (Note that, if we solve for $x$ instead of $y$, we will obtain a different value.)\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 21,\n",
+ " \"message\": {\n",
+ " \"content\": \"To find the slope of a line, we need to put the equation in slope-intercept form $y=mx+b$ where $m$ is the slope. To do this, we can start by solving the equation we're given for $y$:\\n\\n\\\\begin{align*}\\n2x - 4y &= 9\\\\\\\\\\n-4y &= -2x + 9\\\\\\\\\\ny &= \\\\frac{1}{2}x - \\\\frac{9}{4}.\\n\\\\end{align*}\\n\\nSo the slope of the line given is $\\\\frac{1}{2}$. \\n\\nFor a line to be parallel, it must have the same slope, so our answer is $\\\\boxed{\\\\frac{1}{2}}$.\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 22,\n",
+ " \"message\": {\n",
+ " \"content\": \"To obtain the slope of the given line $2x-4y=9$, we must rewrite the equation in slope-intercept form, i.e. in the form $y = mx + b$, where $m$ is the slope and $b$ is the $y$-intercept.\\n\\nSolving the given equation for $y$ yields: \\n\\n$$2x - 4y = 9 \\\\Rightarrow -4y = -2x + 9 \\\\Rightarrow y = \\\\frac{1}{2}x - \\\\frac{9}{4}$$ \\n\\nTherefore, the slope of the given line is $\\\\frac12$.\\n\\nParallel lines have the same slope. Thus, any line parallel to the given line also has a slope of $\\\\boxed{\\\\frac12}$.\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 23,\n",
+ " \"message\": {\n",
+ " \"content\": \"We can start this problem by putting the equation $2x - 4y = 9$ into slope-intercept form. Subtracting $2x$ from both sides gives $-4y = -2x + 9$, so dividing by $-4$ gives $y = \\\\frac{1}{2}x - \\\\frac{9}{4}$. Therefore, the slope of the line is $\\\\frac{1}{2}$.\\n\\nAny line that is parallel to this line will have the same slope of $\\\\frac{1}{2}$, since parallel lines have equal slopes. Therefore, the answer is $\\\\boxed{\\\\frac{1}{2}}$.\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 24,\n",
+ " \"message\": {\n",
+ " \"content\": \"We can find the slope of the line by putting $2x - 4y = 9$ into slope-intercept form, which means we need to solve for $y$:\\\\begin{align*}\\n2x - 4y &= 9\\\\\\\\\\n\\\\Rightarrow\\\\qquad 4y&=2x-9\\\\\\\\\\n\\\\Rightarrow\\\\qquad y&=\\\\frac 12x-\\\\frac 94.\\n\\\\end{align*}This line has slope $\\\\frac 12$. Any line perpendicular to this line therefore has slope the opposite reciprocal, or $-2$. $\\\\boxed{-2}$ is the slope of any line parallel to the line $2x-4y=9$.\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 25,\n",
+ " \"message\": {\n",
+ " \"content\": \"To begin, we can rearrange the given equation $2x - 4y = 9$ by subtracting $2x$ from both sides to obtain $-4y = -2x + 9$. Then, we can divide both sides by $-4$ to solve for $y$ and obtain $y = \\\\frac{1}{2}x - \\\\frac{9}{4}$.\\n\\nWe see that any line parallel to this line must have the same slope, since parallel lines have the same slope. Thus, the desired slope is $\\\\boxed{\\\\frac{1}{2}}$.\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 26,\n",
+ " \"message\": {\n",
+ " \"content\": \"To find the slope of a line parallel to $2x-4y=9,$ we need to find the slope of the given line first. To do this, we can transform it into slope-intercept form by solving for $y.$\\n\\n\\n\\\\begin{align*}\\n2x-4y &= 9\\\\\\\\\\n-4y &= -2x + 9\\\\\\\\\\ny &= \\\\frac12 x - \\\\frac94.\\n\\\\end{align*}Therefore, the slope of the given line is $\\\\frac12.$ \\n\\nAny line parallel to this line will have the same slope, so our final answer is $\\\\boxed{\\\\frac{1}{2}}.$\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 27,\n",
+ " \"message\": {\n",
+ " \"content\": \"To find the slope of the given line, we can first rewrite it in slope-intercept form: $$2x-4y=9 \\\\Rightarrow -4y=-2x+9 \\\\Rightarrow y=\\\\frac{1}{2}x-\\\\frac{9}{4}.$$Therefore, the slope of the given line is $\\\\frac{1}{2}.$\\n\\nLines that are parallel have the same slope. Therefore, any line that is parallel to this line will have a slope of $\\\\boxed{\\\\frac{1}{2}}.$\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 28,\n",
+ " \"message\": {\n",
+ " \"content\": \"To start, we solve for $y$:\\n\\n\\\\[\\\\begin{aligned} 2x - 4y &= 9 \\\\\\\\-4y &= -2x + 9 \\\\\\\\ y &= \\\\frac{1}{2}x -\\\\frac{9}{4}\\\\end{aligned}\\\\]\\n\\nThe slope of this line is $\\\\frac{1}{2}$. For lines to be parallel, they must have the same slope, so any line that is parallel to the given line also has a slope of $\\\\boxed{\\\\frac{1}{2}}$.\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 29,\n",
+ " \"message\": {\n",
+ " \"content\": \"To find the slope of the line $2x - 4y = 9$, we need to rewrite the equation in slope-intercept form, $y = mx + b,$ which means solving for $y.$\\n\\nWe start by subtracting $2x$ from both sides of the equation to get $-4y = -2x + 9.$ Dividing both sides by $-4,$ we have\\n\\\\begin{align*}\\ny &= \\\\frac{-2x+9}{-4}\\\\\\\\\\n& = \\\\frac{1}{2} \\\\cdot \\\\frac{-2x+9}{-2}\\\\\\\\\\n& = \\\\frac{1}{2}x - \\\\frac{9}{8}.\\n\\\\end{align*}Thus, the slope of the original line is $m=-\\\\frac{1}{2}$.\\n\\nA line parallel to this line must have the same slope, $\\\\boxed{-\\\\frac12}$.\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 30,\n",
+ " \"message\": {\n",
+ " \"content\": \"To find the slope of a line parallel to a given line, we must first find the slope of the given line. To do that, we solve the equation of the given line for $y$ in terms of $x$:\\n\\n\\\\begin{align*}\\n2x - 4y &= 9 \\\\\\\\\\n-4y &= -2x + 9 \\\\\\\\\\ny &= \\\\frac12 x - \\\\frac94 \\\\\\\\\\n\\\\end{align*}\\n\\nSo the slope of the given line is $\\\\frac12$, because the coefficient of $x$ in the equation $y = \\\\frac12 x - \\\\frac94$ is $\\\\frac12$.\\n\\nA line parallel to the given line must have the same slope. Therefore the slope of a line parallel to the given line is also $\\\\boxed{\\\\frac{1}{2}}$.\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 31,\n",
+ " \"message\": {\n",
+ " \"content\": \"We can find the slope of the given line by solving for $y$: $$2x-4y=9 \\\\Rightarrow -4y=-2x+9 \\\\Rightarrow y=\\\\frac{1}{2}x-\\\\frac{9}{4}.$$Hence, the slope of the given line is $\\\\frac{1}{2}$.\\n\\nFor any line parallel to this line, the slope will be the same, namely $\\\\boxed{\\\\frac{1}{2}}$.\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 32,\n",
+ " \"message\": {\n",
+ " \"content\": \"We can rearrange the given equation to get the slope-intercept form $y = \\\\frac{1}{2}x - \\\\frac{9}{4}$. We can see that any line parallel to this line will have the same slope of $\\\\boxed{\\\\frac{1}{2}}$.\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 33,\n",
+ " \"message\": {\n",
+ " \"content\": \"Converting the equation $2x - 4y = 9$ to slope-intercept form, we get $y = \\\\frac{1}{2}x - \\\\frac{9}{4}$. Therefore, lines parallel to this one will also have slope $\\\\boxed{\\\\frac{1}{2}}$.\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 34,\n",
+ " \"message\": {\n",
+ " \"content\": \"Putting the equation in slope-intercept form, we have $-4y = -2x + 9 \\\\Rightarrow y = \\\\frac{1}{2}x - \\\\frac{9}{4}$. Therefore, the slope of the original line is $\\\\frac{1}{2}$. Every line that is parallel to this one will have the same slope of $\\\\boxed{\\\\frac{1}{2}}$.\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 35,\n",
+ " \"message\": {\n",
+ " \"content\": \"In order for two lines to be parallel, they must have the same slope. To find the slope of $2x-4y=9$, we can solve for $y$ in terms of $x$ and write the equation in slope-intercept form, $y=mx+b$, where $m$ is the slope.\\n\\nFirst, we subtract $2x$ from both sides and get $-4y=-2x+9$. Then, we divide both sides by $-4$ to solve for $y$: $$-4y=-2x+9 \\\\qquad \\\\Rightarrow \\\\qquad y=\\\\frac{1}{2}x-\\\\frac{9}{4}.$$ The slope of this line is $\\\\frac{1}{2}$.\\n\\nTherefore, a line parallel to $2x-4y=9$ will also have a slope of $\\\\boxed{\\\\frac{1}{2}}$.\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 36,\n",
+ " \"message\": {\n",
+ " \"content\": \"A line parallel to $2x- 4y =9$ will have the same slope, since parallel lines have equal slopes. To find the slope, we write the equation in slope-intercept form, $ y = m x +b$, solving for $y$: $y=\\\\frac12x-\\\\frac{9}{4}$, where $m=\\\\frac12$ is the slope (coefficient of $x$). Thus a parallel line will also have a slope of $\\\\boxed{\\\\frac12}$.\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 37,\n",
+ " \"message\": {\n",
+ " \"content\": \"To determine the slope of the line given in point-slope form, we need to solve for $y$:\\n\\n$$2x - 4y = 9 \\\\Rightarrow -4y = -2x +9 \\\\Rightarrow y = \\\\frac{1}{2}x - \\\\frac{9}{4}$$\\n\\nTherefore, the line has a slope of $\\\\boxed{\\\\frac{1}{2}}$.\\n\\nTo find the slope of a line parallel to this one, note that parallel lines have the same slope. Therefore, any line parallel to the given line will also have a slope of $\\\\boxed{\\\\frac{1}{2}}$.\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 38,\n",
+ " \"message\": {\n",
+ " \"content\": \"Recall that two lines are parallel if and only if they have the same slope. To find the slope of the given line, we put it in slope-intercept form by solving for $y$: $$2x - 4y = 9.$$Isolating $y$, we get $$-4y = -2x + 9,$$which implies $$y = \\\\frac{1}{2}x - \\\\frac{9}{4}.$$Thus, the slope of the given line is $\\\\frac{1}{2}$. Any line parallel to the given line will have the same slope $\\\\frac{1}{2}$, so the answer is \\\\boxed{\\\\frac{1}{2}}.\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 39,\n",
+ " \"message\": {\n",
+ " \"content\": \"We begin by rearranging the given equation to the slope-intercept form: $$2x - 4y = 9 \\\\Rightarrow 2x - 9 = 4y \\\\Rightarrow y = \\\\frac12 x - \\\\frac94.$$The slope of the line is therefore $\\\\boxed{\\\\frac 12}$. \\n\\nRecall that all lines that are parallel have the same slope.\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 40,\n",
+ " \"message\": {\n",
+ " \"content\": \"To find the slope of a line parallel to a given line, we first need to find the slope of the given line. In order to do that, we will solve the given equation for $y$ in terms of $x$ (in other words, we will put it into slope-intercept form, $y=mx+b$).\\n\\n\\\\begin{align*}\\n2x -4y &= 9\\\\\\\\\\n-4y &= -2x + 9\\\\\\\\\\ny &= \\\\frac12 x - \\\\frac{9}{4}.\\n\\\\end{align*}\\n\\nWe now see that the slope of the given line is $\\\\frac12$. Since lines that are parallel have the same slope, any line parallel to this one will also have a slope of $\\\\frac12$. Therefore, our final answer is $\\\\boxed{\\\\frac12}$.\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 41,\n",
+ " \"message\": {\n",
+ " \"content\": \"We convert $2x - 4y = 9$ to slope-intercept form. Solving for $y$, we find that $y=\\\\frac{1}{2}x-\\\\frac{9}{4}$. Hence, if a line is parallel to this line, it will have the same slope and thus an answer of $\\\\boxed{\\\\frac{1}{2}}$.\\n\\nIn general, two lines in the plane are parallel if and only if they have the same slope.\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 42,\n",
+ " \"message\": {\n",
+ " \"content\": \"For a line to be parallel to another line, it must have the same slope. Thus, we need to find the slope of the line $2x - 4y = 9$.\\n\\nFirst, we can rearrange this equation to isolate $y$ by subtracting $2x$ from both sides and then dividing by $-4$:\\n\\n$$-4y=-2x+9$$\\n\\n$$y=\\\\frac{1}{2}x-\\\\frac{9}{4}$$\\n\\nNow we can see that the slope of this line is $\\\\frac{1}{2}$. Therefore, any line parallel to this one must also have a slope of $\\\\frac{1}{2}$. \\n\\nOur final answer is $\\\\boxed{\\\\frac{1}{2}}$.\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 43,\n",
+ " \"message\": {\n",
+ " \"content\": \"Solving for $y$ in $2x - 4y = 9$ gives $y = \\\\frac{1}{2}x - \\\\frac{9}{4}$. Thus, a line parallel to this one has slope $\\\\boxed{\\\\frac{1}{2}}$.\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 44,\n",
+ " \"message\": {\n",
+ " \"content\": \"We know that lines that are parallel have the same slope. To find the slope of $2x-4y=9$, we can rewrite the equation in slope-intercept form, $y=mx+b$, where $m$ is the slope. Solving for $y$, we get $$-4y=-2x+9\\\\qquad\\\\Rightarrow\\\\qquad y=\\\\frac{1}{2}x-\\\\frac{9}{4}.$$ Therefore, a line parallel to this line has slope $\\\\boxed{\\\\frac{1}{2}}$.\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 45,\n",
+ " \"message\": {\n",
+ " \"content\": \"First we want to get the given equation into slope-intercept form, $y=mx+b$, where $m$ is the slope. Solve this equation for $y$ explicitly to obtain $$ 2x-4y = 9 \\\\implies -4y = -2x+9 \\\\implies y = \\\\frac{1}{2}x - \\\\frac{9}{4}. $$\\n\\nComparing this equation to $y=mx+b$, we see that $m=\\\\boxed{\\\\frac{1}{2}}$ is the slope of this line. But the problem asks for the slope of a line \\\\emph{parallel} to this one. Since parallel lines have the same slope, we simply write $\\\\boxed{\\\\frac{1}{2}}$ as our answer.\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 46,\n",
+ " \"message\": {\n",
+ " \"content\": \"To say that two lines in a plane are parallel is to say that they never intersect. In the $xy$-plane, two lines that never intersect are either (1) horizontal lines (they have the same $y$-intercept, i.e. they go through the $y$-axis at the same height) or (2) vertical lines (they have the same $x$-intercept, and are, it is usually assumed, substantially less exciting!)\\n\\nPutting the equation $2x-4y=9$ in slope-intercept form $y=mx+b$ gives $y=\\\\frac{1}{2}x-\\\\frac{9}{4}$. Therefore, a parallel line would have a slope of $\\\\boxed{\\\\frac{1}{2}}$, and would have an equation of the type $y = \\\\frac{1}{2}x + b$ for some choice of $b$ (the $y$-intercept of the new line). Note that slope-intercept form identifies $m$ as the slope and $b$ as the vertical intercept of a line; try writing a few different lines in slope-intercept form and see if you agree that this form makes those geometrical concepts visually clear.\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 47,\n",
+ " \"message\": {\n",
+ " \"content\": \"To find the slope of a line parallel to $2x-4y=9$, we need to find the slope of the given line. Solving for $y$ gives $y=\\\\frac12 x - \\\\frac94$. Thus the slope of the given line is $\\\\frac12$.\\n\\nTwo lines are parallel if and only if they have the same slope, so the slope of any line parallel to $2x-4y=9$ is also $\\\\boxed{\\\\frac{1}{2}}$.\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 48,\n",
+ " \"message\": {\n",
+ " \"content\": \"To find the slope of this line, we need to put it into slope-intercept form: $$2x - 4y = 9$$ $$-4y = -2x +9$$ $$y = \\\\frac{1}{2}x - \\\\frac{9}{4}.$$ So the slope of the given line is $\\\\frac{1}{2}$. Since any line parallel to this line must have the same slope, our answer is $\\\\boxed{\\\\frac{1}{2}}$.\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 49,\n",
+ " \"message\": {\n",
+ " \"content\": \"We need to find the slope of a line that is parallel to the line $2x - 4y = 9$. First, we write this equation in slope-intercept form, $y = mx + b$. Solving for $y$ yields $y=\\\\frac{1}{2}x - \\\\frac{9}{4}$. Thus, the slope of the original line is $\\\\frac{1}{2}$.\\n\\nSince lines that are parallel have the same slope, the line we are looking for also has a slope of $\\\\frac{1}{2}$. Therefore, our answer is $\\\\boxed{\\\\frac{1}{2}}$.\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " }\n",
+ " ],\n",
+ " \"config_id\": 0,\n",
+ " \"cost\": 0.013798,\n",
+ " \"created\": 1684615715,\n",
+ " \"id\": \"chatcmpl-7INn9wZkp1kYxtJ9UVa1n0FC1NopJ\",\n",
+ " \"model\": \"gpt-3.5-turbo-0301\",\n",
+ " \"object\": \"chat.completion\",\n",
+ " \"usage\": {\n",
+ " \"completion_tokens\": 6838,\n",
+ " \"prompt_tokens\": 61,\n",
+ " \"total_tokens\": 6899\n",
+ " }\n",
+ "}\n",
+ "metric_results on the example data instance: {'expected_success': 1.0, 'success': True, 'success_vote': 1.0, 'voted_answer': 'To find the slope of a line parallel to $2x-4y=9,$ we need to find the slope of the line $2x-4y=9.$ Solving for $y$ in this first equation, we get $$\\\\begin{aligned} 2x-4y&=9\\n\\\\\\\\\\\\Rightarrow\\\\qquad 4y&=2x-9\\n\\\\\\\\\\\\Rightarrow\\\\qquad y&=\\\\frac{1}{2}x-\\\\frac{9}{4}.\\\\end{aligned}$$So the slope of this line is $\\\\frac{1}{2}.$ Therefore, the slope of any line parallel to this line will also be $\\\\boxed{\\\\frac{1}{2}}.$', 'votes': 45}\n"
+ ]
+ }
+ ],
+ "source": [
+ "response = oai.ChatCompletion.create(context=tune_data[1], config_list=config_list, **config)\n",
+ "metric_results = eval_math_responses(oai.ChatCompletion.extract_text(response), **tune_data[1])\n",
+ "print(\"response on an example data instance:\", response)\n",
+ "print(\"metric_results on the example data instance:\", metric_results)\n"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Evaluate the success rate on the test data\n",
+ "\n",
+ "You can use flaml's `oai.ChatCompletion.test` to evaluate the performance of an entire dataset with the tuned config. The following code will take a while (30 mins to 1 hour) to evaluate all the test data instances if uncommented and run. It will cost roughly $3. "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "metadata": {
+ "execution": {
+ "iopub.execute_input": "2023-02-13T23:41:56.042764Z",
+ "iopub.status.busy": "2023-02-13T23:41:56.042086Z",
+ "iopub.status.idle": "2023-02-13T23:53:05.597643Z",
+ "shell.execute_reply": "2023-02-13T23:53:05.596603Z"
+ }
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "[flaml.autogen.oai.completion: 05-20 22:21:33] {927} INFO - evaluating data instance 0\n",
+ "[flaml.autogen.oai.completion: 05-20 22:22:43] {927} INFO - evaluating data instance 1\n",
+ "[flaml.autogen.oai.completion: 05-20 22:23:03] {927} INFO - evaluating data instance 2\n",
+ "[flaml.autogen.oai.completion: 05-20 22:23:20] {927} INFO - evaluating data instance 3\n",
+ "[flaml.autogen.oai.completion: 05-20 22:23:47] {927} INFO - evaluating data instance 4\n",
+ "[flaml.autogen.oai.completion: 05-20 22:23:47] {927} INFO - evaluating data instance 5\n",
+ "[flaml.autogen.oai.completion: 05-20 22:23:48] {927} INFO - evaluating data instance 6\n",
+ "[flaml.autogen.oai.completion: 05-20 22:23:48] {927} INFO - evaluating data instance 7\n",
+ "[flaml.autogen.oai.completion: 05-20 22:23:48] {927} INFO - evaluating data instance 8\n",
+ "[flaml.autogen.oai.completion: 05-20 22:24:20] {927} INFO - evaluating data instance 9\n",
+ "[flaml.autogen.oai.completion: 05-20 22:24:20] {927} INFO - evaluating data instance 10\n",
+ "[flaml.autogen.oai.completion: 05-20 22:24:20] {927} INFO - evaluating data instance 11\n",
+ "[flaml.autogen.oai.completion: 05-20 22:25:27] {927} INFO - evaluating data instance 12\n",
+ "[flaml.autogen.oai.completion: 05-20 22:25:58] {927} INFO - evaluating data instance 13\n",
+ "[flaml.autogen.oai.completion: 05-20 22:25:58] {927} INFO - evaluating data instance 14\n",
+ "[flaml.autogen.oai.completion: 05-20 22:25:58] {927} INFO - evaluating data instance 15\n",
+ "[flaml.autogen.oai.completion: 05-20 22:25:58] {927} INFO - evaluating data instance 16\n",
+ "[flaml.autogen.oai.completion: 05-20 22:25:58] {927} INFO - evaluating data instance 17\n",
+ "[flaml.autogen.oai.completion: 05-20 22:26:21] {927} INFO - evaluating data instance 18\n",
+ "[flaml.autogen.oai.completion: 05-20 22:26:21] {927} INFO - evaluating data instance 19\n",
+ "[flaml.autogen.oai.completion: 05-20 22:26:21] {927} INFO - evaluating data instance 20\n",
+ "[flaml.autogen.oai.completion: 05-20 22:26:53] {927} INFO - evaluating data instance 21\n",
+ "[flaml.autogen.oai.completion: 05-20 22:26:53] {927} INFO - evaluating data instance 22\n",
+ "[flaml.autogen.oai.completion: 05-20 22:26:53] {927} INFO - evaluating data instance 23\n",
+ "[flaml.autogen.oai.completion: 05-20 22:26:53] {927} INFO - evaluating data instance 24\n",
+ "[flaml.autogen.oai.completion: 05-20 22:26:53] {927} INFO - evaluating data instance 25\n",
+ "[flaml.autogen.oai.completion: 05-20 22:26:53] {927} INFO - evaluating data instance 26\n",
+ "[flaml.autogen.oai.completion: 05-20 22:26:53] {927} INFO - evaluating data instance 27\n",
+ "[flaml.autogen.oai.completion: 05-20 22:26:53] {927} INFO - evaluating data instance 28\n",
+ "[flaml.autogen.oai.completion: 05-20 22:26:54] {927} INFO - evaluating data instance 29\n",
+ "[flaml.autogen.oai.completion: 05-20 22:26:54] {927} INFO - evaluating data instance 30\n",
+ "[flaml.autogen.oai.completion: 05-20 22:26:54] {927} INFO - evaluating data instance 31\n",
+ "[flaml.autogen.oai.completion: 05-20 22:26:54] {927} INFO - evaluating data instance 32\n",
+ "[flaml.autogen.oai.completion: 05-20 22:28:07] {927} INFO - evaluating data instance 33\n",
+ "[flaml.autogen.oai.completion: 05-20 22:28:07] {927} INFO - evaluating data instance 34\n",
+ "[flaml.autogen.oai.completion: 05-20 22:28:07] {927} INFO - evaluating data instance 35\n",
+ "[flaml.autogen.oai.completion: 05-20 22:28:07] {927} INFO - evaluating data instance 36\n",
+ "[flaml.autogen.oai.completion: 05-20 22:28:07] {927} INFO - evaluating data instance 37\n",
+ "[flaml.autogen.oai.completion: 05-20 22:28:07] {927} INFO - evaluating data instance 38\n",
+ "[flaml.autogen.oai.completion: 05-20 22:28:07] {927} INFO - evaluating data instance 39\n",
+ "[flaml.autogen.oai.completion: 05-20 22:28:07] {927} INFO - evaluating data instance 40\n",
+ "[flaml.autogen.oai.completion: 05-20 22:28:07] {927} INFO - evaluating data instance 41\n",
+ "[flaml.autogen.oai.completion: 05-20 22:28:22] {927} INFO - evaluating data instance 42\n",
+ "[flaml.autogen.oai.completion: 05-20 22:28:34] {927} INFO - evaluating data instance 43\n",
+ "[flaml.autogen.oai.completion: 05-20 22:28:34] {927} INFO - evaluating data instance 44\n",
+ "[flaml.autogen.oai.completion: 05-20 22:28:34] {927} INFO - evaluating data instance 45\n",
+ "[flaml.autogen.oai.completion: 05-20 22:28:34] {927} INFO - evaluating data instance 46\n",
+ "[flaml.autogen.oai.completion: 05-20 22:28:34] {927} INFO - evaluating data instance 47\n",
+ "[flaml.autogen.oai.completion: 05-20 22:28:53] {927} INFO - evaluating data instance 48\n",
+ "[flaml.autogen.oai.completion: 05-20 22:28:53] {927} INFO - evaluating data instance 49\n",
+ "[flaml.autogen.oai.completion: 05-20 22:28:53] {927} INFO - evaluating data instance 50\n",
+ "[flaml.autogen.oai.completion: 05-20 22:29:25] {927} INFO - evaluating data instance 51\n",
+ "[flaml.autogen.oai.completion: 05-20 22:29:25] {927} INFO - evaluating data instance 52\n",
+ "[flaml.autogen.oai.completion: 05-20 22:29:46] {927} INFO - evaluating data instance 53\n",
+ "[flaml.autogen.oai.completion: 05-20 22:29:46] {927} INFO - evaluating data instance 54\n",
+ "[flaml.autogen.oai.completion: 05-20 22:30:17] {927} INFO - evaluating data instance 55\n",
+ "[flaml.autogen.oai.completion: 05-20 22:30:17] {927} INFO - evaluating data instance 56\n",
+ "[flaml.autogen.oai.completion: 05-20 22:30:17] {927} INFO - evaluating data instance 57\n",
+ "[flaml.autogen.oai.completion: 05-20 22:30:17] {927} INFO - evaluating data instance 58\n",
+ "[flaml.autogen.oai.completion: 05-20 22:30:17] {927} INFO - evaluating data instance 59\n",
+ "[flaml.autogen.oai.completion: 05-20 22:30:17] {927} INFO - evaluating data instance 60\n",
+ "[flaml.autogen.oai.completion: 05-20 22:30:17] {927} INFO - evaluating data instance 61\n",
+ "[flaml.autogen.oai.completion: 05-20 22:30:41] {927} INFO - evaluating data instance 62\n",
+ "[flaml.autogen.oai.completion: 05-20 22:30:41] {927} INFO - evaluating data instance 63\n",
+ "[flaml.autogen.oai.completion: 05-20 22:30:41] {927} INFO - evaluating data instance 64\n",
+ "[flaml.autogen.oai.completion: 05-20 22:30:41] {927} INFO - evaluating data instance 65\n",
+ "[flaml.autogen.oai.completion: 05-20 22:30:41] {927} INFO - evaluating data instance 66\n",
+ "[flaml.autogen.oai.completion: 05-20 22:30:41] {927} INFO - evaluating data instance 67\n",
+ "[flaml.autogen.oai.completion: 05-20 22:30:57] {927} INFO - evaluating data instance 68\n",
+ "[flaml.autogen.oai.completion: 05-20 22:30:57] {927} INFO - evaluating data instance 69\n",
+ "[flaml.autogen.oai.completion: 05-20 22:30:57] {927} INFO - evaluating data instance 70\n",
+ "[flaml.autogen.oai.completion: 05-20 22:30:57] {927} INFO - evaluating data instance 71\n",
+ "[flaml.autogen.oai.completion: 05-20 22:30:57] {927} INFO - evaluating data instance 72\n",
+ "[flaml.autogen.oai.completion: 05-20 22:30:57] {927} INFO - evaluating data instance 73\n",
+ "[flaml.autogen.oai.completion: 05-20 22:30:57] {927} INFO - evaluating data instance 74\n",
+ "[flaml.autogen.oai.completion: 05-20 22:32:03] {927} INFO - evaluating data instance 75\n",
+ "[flaml.autogen.oai.completion: 05-20 22:32:03] {927} INFO - evaluating data instance 76\n",
+ "[flaml.autogen.oai.completion: 05-20 22:32:20] {927} INFO - evaluating data instance 77\n",
+ "[flaml.autogen.oai.completion: 05-20 22:32:20] {927} INFO - evaluating data instance 78\n",
+ "[flaml.autogen.oai.completion: 05-20 22:32:20] {927} INFO - evaluating data instance 79\n",
+ "[flaml.autogen.oai.completion: 05-20 22:32:20] {927} INFO - evaluating data instance 80\n",
+ "[flaml.autogen.oai.completion: 05-20 22:32:20] {927} INFO - evaluating data instance 81\n",
+ "[flaml.autogen.oai.completion: 05-20 22:32:20] {927} INFO - evaluating data instance 82\n",
+ "[flaml.autogen.oai.completion: 05-20 22:32:20] {927} INFO - evaluating data instance 83\n",
+ "[flaml.autogen.oai.completion: 05-20 22:32:38] {927} INFO - evaluating data instance 84\n",
+ "[flaml.autogen.oai.completion: 05-20 22:32:38] {927} INFO - evaluating data instance 85\n",
+ "[flaml.autogen.oai.completion: 05-20 22:32:38] {927} INFO - evaluating data instance 86\n",
+ "[flaml.autogen.oai.completion: 05-20 22:32:38] {927} INFO - evaluating data instance 87\n",
+ "[flaml.autogen.oai.completion: 05-20 22:33:12] {927} INFO - evaluating data instance 88\n",
+ "[flaml.autogen.oai.completion: 05-20 22:33:35] {927} INFO - evaluating data instance 89\n",
+ "[flaml.autogen.oai.completion: 05-20 22:33:57] {927} INFO - evaluating data instance 90\n",
+ "[flaml.autogen.oai.completion: 05-20 22:33:57] {927} INFO - evaluating data instance 91\n",
+ "[flaml.autogen.oai.completion: 05-20 22:33:57] {927} INFO - evaluating data instance 92\n",
+ "[flaml.autogen.oai.completion: 05-20 22:34:28] {927} INFO - evaluating data instance 93\n",
+ "[flaml.autogen.oai.completion: 05-20 22:34:28] {927} INFO - evaluating data instance 94\n",
+ "[flaml.autogen.oai.completion: 05-20 22:34:28] {927} INFO - evaluating data instance 95\n",
+ "[flaml.autogen.oai.completion: 05-20 22:34:28] {927} INFO - evaluating data instance 96\n",
+ "[flaml.autogen.oai.completion: 05-20 22:35:01] {927} INFO - evaluating data instance 97\n",
+ "[flaml.autogen.oai.completion: 05-20 22:35:01] {927} INFO - evaluating data instance 98\n",
+ "[flaml.autogen.oai.completion: 05-20 22:35:01] {927} INFO - evaluating data instance 99\n",
+ "[flaml.autogen.oai.completion: 05-20 22:35:01] {927} INFO - evaluating data instance 100\n",
+ "[flaml.autogen.oai.completion: 05-20 22:35:01] {927} INFO - evaluating data instance 101\n",
+ "[flaml.autogen.oai.completion: 05-20 22:35:01] {927} INFO - evaluating data instance 102\n",
+ "[flaml.autogen.oai.completion: 05-20 22:35:01] {927} INFO - evaluating data instance 103\n",
+ "[flaml.autogen.oai.completion: 05-20 22:35:01] {927} INFO - evaluating data instance 104\n",
+ "[flaml.autogen.oai.completion: 05-20 22:35:01] {927} INFO - evaluating data instance 105\n",
+ "[flaml.autogen.oai.completion: 05-20 22:35:01] {927} INFO - evaluating data instance 106\n",
+ "[flaml.autogen.oai.completion: 05-20 22:35:17] {927} INFO - evaluating data instance 107\n",
+ "[flaml.autogen.oai.completion: 05-20 22:35:17] {927} INFO - evaluating data instance 108\n",
+ "[flaml.autogen.oai.completion: 05-20 22:35:43] {927} INFO - evaluating data instance 109\n",
+ "[flaml.autogen.oai.completion: 05-20 22:36:16] {927} INFO - evaluating data instance 110\n",
+ "[flaml.autogen.oai.completion: 05-20 22:36:17] {927} INFO - evaluating data instance 111\n",
+ "[flaml.autogen.oai.completion: 05-20 22:37:02] {927} INFO - evaluating data instance 112\n",
+ "[flaml.autogen.oai.completion: 05-20 22:37:03] {927} INFO - evaluating data instance 113\n",
+ "[flaml.autogen.oai.completion: 05-20 22:37:03] {927} INFO - evaluating data instance 114\n",
+ "[flaml.autogen.oai.completion: 05-20 22:37:03] {927} INFO - evaluating data instance 115\n",
+ "[flaml.autogen.oai.completion: 05-20 22:37:03] {927} INFO - evaluating data instance 116\n",
+ "[flaml.autogen.oai.completion: 05-20 22:37:03] {927} INFO - evaluating data instance 117\n",
+ "[flaml.autogen.oai.completion: 05-20 22:37:03] {927} INFO - evaluating data instance 118\n",
+ "[flaml.autogen.oai.completion: 05-20 22:37:29] {927} INFO - evaluating data instance 119\n",
+ "[flaml.autogen.oai.completion: 05-20 22:37:51] {927} INFO - evaluating data instance 120\n",
+ "[flaml.autogen.oai.completion: 05-20 22:40:09] {927} INFO - evaluating data instance 121\n",
+ "[flaml.autogen.oai.completion: 05-20 22:40:40] {927} INFO - evaluating data instance 122\n",
+ "[flaml.autogen.oai.completion: 05-20 22:40:48] {927} INFO - evaluating data instance 123\n",
+ "[flaml.autogen.oai.completion: 05-20 22:40:59] {927} INFO - evaluating data instance 124\n",
+ "[flaml.autogen.oai.completion: 05-20 22:41:12] {927} INFO - evaluating data instance 125\n",
+ "[flaml.autogen.oai.completion: 05-20 22:41:27] {927} INFO - evaluating data instance 126\n",
+ "[flaml.autogen.oai.completion: 05-20 22:41:58] {927} INFO - evaluating data instance 127\n",
+ "[flaml.autogen.oai.completion: 05-20 22:42:29] {927} INFO - evaluating data instance 128\n",
+ "[flaml.autogen.oai.completion: 05-20 22:42:53] {927} INFO - evaluating data instance 129\n",
+ "[flaml.autogen.oai.completion: 05-20 22:43:06] {927} INFO - evaluating data instance 130\n",
+ "[flaml.autogen.oai.completion: 05-20 22:44:13] {927} INFO - evaluating data instance 131\n",
+ "[flaml.autogen.oai.completion: 05-20 22:44:45] {927} INFO - evaluating data instance 132\n",
+ "[flaml.autogen.oai.completion: 05-20 22:45:16] {927} INFO - evaluating data instance 133\n",
+ "[flaml.autogen.oai.completion: 05-20 22:45:35] {927} INFO - evaluating data instance 134\n",
+ "[flaml.autogen.oai.completion: 05-20 22:45:51] {927} INFO - evaluating data instance 135\n",
+ "[flaml.autogen.oai.completion: 05-20 22:46:22] {927} INFO - evaluating data instance 136\n",
+ "[flaml.autogen.oai.completion: 05-20 22:47:33] {927} INFO - evaluating data instance 137\n",
+ "[flaml.autogen.oai.completion: 05-20 22:48:38] {927} INFO - evaluating data instance 138\n",
+ "[flaml.autogen.oai.completion: 05-20 22:49:52] {927} INFO - evaluating data instance 139\n",
+ "[flaml.autogen.oai.completion: 05-20 22:50:23] {927} INFO - evaluating data instance 140\n",
+ "[flaml.autogen.oai.completion: 05-20 22:50:35] {927} INFO - evaluating data instance 141\n",
+ "[flaml.autogen.oai.completion: 05-20 22:51:49] {927} INFO - evaluating data instance 142\n",
+ "[flaml.autogen.oai.completion: 05-20 22:52:08] {927} INFO - evaluating data instance 143\n",
+ "[flaml.autogen.oai.completion: 05-20 22:53:14] {927} INFO - evaluating data instance 144\n",
+ "[flaml.autogen.oai.completion: 05-20 22:53:26] {927} INFO - evaluating data instance 145\n",
+ "[flaml.autogen.oai.completion: 05-20 22:53:58] {927} INFO - evaluating data instance 146\n",
+ "[flaml.autogen.oai.completion: 05-20 22:54:30] {927} INFO - evaluating data instance 147\n",
+ "[flaml.autogen.oai.completion: 05-20 22:55:40] {927} INFO - evaluating data instance 148\n",
+ "[flaml.autogen.oai.completion: 05-20 22:56:11] {927} INFO - evaluating data instance 149\n",
+ "[flaml.autogen.oai.completion: 05-20 22:56:24] {927} INFO - evaluating data instance 150\n",
+ "[flaml.autogen.oai.completion: 05-20 22:56:31] {927} INFO - evaluating data instance 151\n",
+ "[flaml.autogen.oai.completion: 05-20 22:57:44] {927} INFO - evaluating data instance 152\n",
+ "[flaml.autogen.oai.completion: 05-20 22:58:54] {927} INFO - evaluating data instance 153\n",
+ "[flaml.autogen.oai.completion: 05-20 22:59:26] {927} INFO - evaluating data instance 154\n",
+ "[flaml.autogen.oai.completion: 05-20 22:59:37] {927} INFO - evaluating data instance 155\n",
+ "[flaml.autogen.oai.completion: 05-20 22:59:47] {927} INFO - evaluating data instance 156\n",
+ "[flaml.autogen.oai.completion: 05-20 23:00:52] {927} INFO - evaluating data instance 157\n",
+ "[flaml.autogen.oai.completion: 05-20 23:02:02] {927} INFO - evaluating data instance 158\n",
+ "[flaml.autogen.oai.completion: 05-20 23:02:33] {927} INFO - evaluating data instance 159\n",
+ "[flaml.autogen.oai.completion: 05-20 23:03:01] {927} INFO - evaluating data instance 160\n",
+ "[flaml.autogen.oai.completion: 05-20 23:03:16] {927} INFO - evaluating data instance 161\n",
+ "[flaml.autogen.oai.completion: 05-20 23:03:25] {927} INFO - evaluating data instance 162\n",
+ "[flaml.autogen.oai.completion: 05-20 23:04:34] {927} INFO - evaluating data instance 163\n",
+ "[flaml.autogen.oai.completion: 05-20 23:05:05] {927} INFO - evaluating data instance 164\n",
+ "[flaml.autogen.oai.completion: 05-20 23:06:17] {927} INFO - evaluating data instance 165\n",
+ "[flaml.autogen.oai.completion: 05-20 23:06:38] {927} INFO - evaluating data instance 166\n",
+ "[flaml.autogen.oai.completion: 05-20 23:07:02] {927} INFO - evaluating data instance 167\n",
+ "[flaml.autogen.oai.completion: 05-20 23:07:31] {927} INFO - evaluating data instance 168\n",
+ "[flaml.autogen.oai.completion: 05-20 23:07:49] {927} INFO - evaluating data instance 169\n",
+ "[flaml.autogen.oai.completion: 05-20 23:08:02] {927} INFO - evaluating data instance 170\n",
+ "[flaml.autogen.oai.completion: 05-20 23:09:15] {927} INFO - evaluating data instance 171\n",
+ "[flaml.autogen.oai.completion: 05-20 23:09:39] {927} INFO - evaluating data instance 172\n",
+ "[flaml.autogen.oai.completion: 05-20 23:10:54] {927} INFO - evaluating data instance 173\n",
+ "[flaml.autogen.oai.completion: 05-20 23:11:13] {927} INFO - evaluating data instance 174\n",
+ "[flaml.autogen.oai.completion: 05-20 23:12:27] {927} INFO - evaluating data instance 175\n",
+ "[flaml.autogen.oai.completion: 05-20 23:13:36] {927} INFO - evaluating data instance 176\n",
+ "[flaml.autogen.oai.completion: 05-20 23:13:57] {927} INFO - evaluating data instance 177\n",
+ "[flaml.autogen.oai.completion: 05-20 23:14:27] {927} INFO - evaluating data instance 178\n",
+ "[flaml.autogen.oai.completion: 05-20 23:14:48] {927} INFO - evaluating data instance 179\n",
+ "[flaml.autogen.oai.completion: 05-20 23:15:09] {927} INFO - evaluating data instance 180\n",
+ "[flaml.autogen.oai.completion: 05-20 23:15:26] {927} INFO - evaluating data instance 181\n",
+ "[flaml.autogen.oai.completion: 05-20 23:16:41] {927} INFO - evaluating data instance 182\n",
+ "[flaml.autogen.oai.completion: 05-20 23:17:01] {927} INFO - evaluating data instance 183\n",
+ "[flaml.autogen.oai.completion: 05-20 23:18:08] {927} INFO - evaluating data instance 184\n",
+ "[flaml.autogen.oai.completion: 05-20 23:18:23] {927} INFO - evaluating data instance 185\n",
+ "[flaml.autogen.oai.completion: 05-20 23:18:44] {927} INFO - evaluating data instance 186\n",
+ "[flaml.autogen.oai.completion: 05-20 23:19:12] {927} INFO - evaluating data instance 187\n",
+ "[flaml.autogen.oai.completion: 05-20 23:20:19] {927} INFO - evaluating data instance 188\n",
+ "[flaml.autogen.oai.completion: 05-20 23:20:37] {927} INFO - evaluating data instance 189\n",
+ "[flaml.autogen.oai.completion: 05-20 23:21:43] {927} INFO - evaluating data instance 190\n",
+ "[flaml.autogen.oai.completion: 05-20 23:22:14] {927} INFO - evaluating data instance 191\n",
+ "[flaml.autogen.oai.completion: 05-20 23:22:30] {927} INFO - evaluating data instance 192\n",
+ "[flaml.autogen.oai.completion: 05-20 23:22:48] {927} INFO - evaluating data instance 193\n",
+ "[flaml.autogen.oai.completion: 05-20 23:23:18] {927} INFO - evaluating data instance 194\n",
+ "[flaml.autogen.oai.completion: 05-20 23:23:48] {927} INFO - evaluating data instance 195\n",
+ "[flaml.autogen.oai.completion: 05-20 23:24:04] {927} INFO - evaluating data instance 196\n",
+ "[flaml.autogen.oai.completion: 05-20 23:25:19] {927} INFO - evaluating data instance 197\n",
+ "[flaml.autogen.oai.completion: 05-20 23:25:45] {927} INFO - evaluating data instance 198\n",
+ "[flaml.autogen.oai.completion: 05-20 23:26:06] {927} INFO - evaluating data instance 199\n",
+ "[flaml.autogen.oai.completion: 05-20 23:26:19] {927} INFO - evaluating data instance 200\n",
+ "performance on test data with the tuned config: {'expected_success': 0.9825768303568986, 'success': 0.9900497512437811, 'success_vote': 0.9054726368159204, 'votes': 32.114427860696516, 'cost': 3.163385999999999, 'inference_cost': 0.015738238805970146}\n"
+ ]
+ }
+ ],
+ "source": [
+ "# result = oai.ChatCompletion.test(test_data, logging_level=logging.INFO, config_list=config_list, **config)\n",
+ "# print(\"performance on test data with the tuned config:\", result)"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "What about the default, untuned gpt-4 config (with the same prompt as the tuned config)? We can evaluate it and compare:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "performance on test data from gpt-4 with a default config: {'expected_success': 0.6965174129353234, 'success': 0.6965174129353234, 'success_vote': 0.6965174129353234, 'votes': 1.0, 'cost': 1.9264799999999993, 'inference_cost': 0.009584477611940295}\n"
+ ]
+ }
+ ],
+ "source": [
+ "# the following code will cost roughly $2 if uncommented and run.\n",
+ "\n",
+ "# default_config = {\"model\": 'gpt-4', \"prompt\": prompts[0]}\n",
+ "# default_result = oai.ChatCompletion.test(test_data, config_list=config_list, **default_config)\n",
+ "# print(\"performance on test data from gpt-4 with a default config:\", default_result)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "tuned config succeeds in 90.5% test cases\n",
+ "untuned config succeeds in 69.7% test cases\n"
+ ]
+ }
+ ],
+ "source": [
+ "# print(\"tuned config succeeds in {:.1f}% test cases\".format(result[\"success_vote\"] * 100))\n",
+ "# print(\"untuned config succeeds in {:.1f}% test cases\".format(default_result[\"success_vote\"] * 100))"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "The default use of GPT-4 has a much lower accuracy. Note that the default config has a lower inference cost. What if we heuristically increase the number of responses n?"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 18,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# The following evaluation costs $3 and longer than one hour if you uncomment it and run it.\n",
+ "\n",
+ "# config_n2 = {\"model\": 'gpt-4', \"prompt\": prompts[0], \"n\": 2}\n",
+ "# result_n2 = oai.ChatCompletion.test(test_data, config_list=config_list, **config_n2)\n",
+ "# print(\"performance on test data from gpt-4 with a default config and n=2:\", result_n2)\n"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "The inference cost is doubled and matches the tuned config. But the success rate doesn't improve much. What if we further increase the number of responses n to 5?"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 19,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# The following evaluation costs $8 and longer than one hour if you uncomment it and run it.\n",
+ "\n",
+ "# config_n5 = {\"model\": 'gpt-4', \"prompt\": prompts[0], \"n\": 5}\n",
+ "# result_n5 = oai.ChatCompletion.test(test_data, config_list=config_list, **config_n5)\n",
+ "# print(\"performance on test data from gpt-4 with a default config and n=5:\", result_n5)"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "We find that the 'success_vote' metric is increased at the cost of exceeding the inference budget. But the tuned configuration has both higher 'success_vote' (91% vs. 87%) and lower average inference cost ($0.015 vs. $0.037 per instance).\n",
+ "\n",
+ "A developer could use flaml to tune the configuration to satisfy the target inference budget while maximizing the value out of it."
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.9.15"
+ },
+ "vscode": {
+ "interpreter": {
+ "hash": "949777d72b0d2535278d3dc13498b2535136f6dfe0678499012e853ee9abcab1"
+ }
+ },
+ "widgets": {
+ "application/vnd.jupyter.widget-state+json": {
+ "state": {
+ "2d910cfd2d2a4fc49fc30fbbdc5576a7": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "2.0.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "2.0.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "2.0.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border_bottom": null,
+ "border_left": null,
+ "border_right": null,
+ "border_top": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "454146d0f7224f038689031002906e6f": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "2.0.0",
+ "model_name": "HBoxModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "2.0.0",
+ "_model_name": "HBoxModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "2.0.0",
+ "_view_name": "HBoxView",
+ "box_style": "",
+ "children": [
+ "IPY_MODEL_e4ae2b6f5a974fd4bafb6abb9d12ff26",
+ "IPY_MODEL_577e1e3cc4db4942b0883577b3b52755",
+ "IPY_MODEL_b40bdfb1ac1d4cffb7cefcb870c64d45"
+ ],
+ "layout": "IPY_MODEL_dc83c7bff2f241309537a8119dfc7555",
+ "tabbable": null,
+ "tooltip": null
+ }
+ },
+ "577e1e3cc4db4942b0883577b3b52755": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "2.0.0",
+ "model_name": "FloatProgressModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "2.0.0",
+ "_model_name": "FloatProgressModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "2.0.0",
+ "_view_name": "ProgressView",
+ "bar_style": "success",
+ "description": "",
+ "description_allow_html": false,
+ "layout": "IPY_MODEL_2d910cfd2d2a4fc49fc30fbbdc5576a7",
+ "max": 1,
+ "min": 0,
+ "orientation": "horizontal",
+ "style": "IPY_MODEL_74a6ba0c3cbc4051be0a83e152fe1e62",
+ "tabbable": null,
+ "tooltip": null,
+ "value": 1
+ }
+ },
+ "6086462a12d54bafa59d3c4566f06cb2": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "2.0.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "2.0.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "2.0.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border_bottom": null,
+ "border_left": null,
+ "border_right": null,
+ "border_top": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "74a6ba0c3cbc4051be0a83e152fe1e62": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "2.0.0",
+ "model_name": "ProgressStyleModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "2.0.0",
+ "_model_name": "ProgressStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "2.0.0",
+ "_view_name": "StyleView",
+ "bar_color": null,
+ "description_width": ""
+ }
+ },
+ "7d3f3d9e15894d05a4d188ff4f466554": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "2.0.0",
+ "model_name": "HTMLStyleModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "2.0.0",
+ "_model_name": "HTMLStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "2.0.0",
+ "_view_name": "StyleView",
+ "background": null,
+ "description_width": "",
+ "font_size": null,
+ "text_color": null
+ }
+ },
+ "b40bdfb1ac1d4cffb7cefcb870c64d45": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "2.0.0",
+ "model_name": "HTMLModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "2.0.0",
+ "_model_name": "HTMLModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "2.0.0",
+ "_view_name": "HTMLView",
+ "description": "",
+ "description_allow_html": false,
+ "layout": "IPY_MODEL_f1355871cc6f4dd4b50d9df5af20e5c8",
+ "placeholder": "​",
+ "style": "IPY_MODEL_ca245376fd9f4354af6b2befe4af4466",
+ "tabbable": null,
+ "tooltip": null,
+ "value": " 1/1 [00:00&lt;00:00, 44.69it/s]"
+ }
+ },
+ "ca245376fd9f4354af6b2befe4af4466": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "2.0.0",
+ "model_name": "HTMLStyleModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "2.0.0",
+ "_model_name": "HTMLStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "2.0.0",
+ "_view_name": "StyleView",
+ "background": null,
+ "description_width": "",
+ "font_size": null,
+ "text_color": null
+ }
+ },
+ "dc83c7bff2f241309537a8119dfc7555": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "2.0.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "2.0.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "2.0.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border_bottom": null,
+ "border_left": null,
+ "border_right": null,
+ "border_top": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "e4ae2b6f5a974fd4bafb6abb9d12ff26": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "2.0.0",
+ "model_name": "HTMLModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "2.0.0",
+ "_model_name": "HTMLModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "2.0.0",
+ "_view_name": "HTMLView",
+ "description": "",
+ "description_allow_html": false,
+ "layout": "IPY_MODEL_6086462a12d54bafa59d3c4566f06cb2",
+ "placeholder": "​",
+ "style": "IPY_MODEL_7d3f3d9e15894d05a4d188ff4f466554",
+ "tabbable": null,
+ "tooltip": null,
+ "value": "100%"
+ }
+ },
+ "f1355871cc6f4dd4b50d9df5af20e5c8": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "2.0.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "2.0.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "2.0.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border_bottom": null,
+ "border_left": null,
+ "border_right": null,
+ "border_top": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ }
+ },
+ "version_major": 2,
+ "version_minor": 0
+ }
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/notebook/autogen_openai.ipynb b/notebook/autogen_openai_completion.ipynb
similarity index 99%
rename from notebook/autogen_openai.ipynb
rename to notebook/autogen_openai_completion.ipynb
index 68d77935c..664f12184 100644
--- a/notebook/autogen_openai.ipynb
+++ b/notebook/autogen_openai_completion.ipynb
@@ -757,7 +757,7 @@
}
],
"source": [
- "# result = oai.Completion.test(test_data, config)\n",
+ "# result = oai.Completion.test(test_data, **config)\n",
"# print(\"performance on test data with the tuned config:\", result)"
]
},
diff --git a/notebook/research/math_level5counting.ipynb b/notebook/research/math_level5counting.ipynb
index aa712cf41..1f27036d1 100644
--- a/notebook/research/math_level5counting.ipynb
+++ b/notebook/research/math_level5counting.ipynb
@@ -313,7 +313,7 @@
"source": [
"### Evaluate the success rate on the test data\n",
"\n",
- "You can use flaml's `oai.ChatCompletion.test` to evaluate the performance of an entire dataset with the tuned config."
+ "You can use flaml's `oai.ChatCompletion.test` to evaluate the performance of an entire dataset with a config."
]
},
{
@@ -325,7 +325,7 @@
"import logging\n",
"\n",
"config_n1 = {\"model\": 'gpt-4', \"prompt\": prompt, \"max_tokens\": 600, \"n\": 1}\n",
- "n1_result = oai.ChatCompletion.test(test_data[:50], config_n1, eval_math_responses)\n",
+ "n1_result = oai.ChatCompletion.test(test_data[:50], eval_math_responses, **config_n1)\n",
"print(n1_result)"
]
},
@@ -336,8 +336,8 @@
"outputs": [],
"source": [
"oai.ChatCompletion.request_timeout = 120\n",
- "config_n10 = {\"model\": 'gpt-4', \"prompt\": prompts[0], \"max_tokens\": 600, \"n\": 10}\n",
- "n10_result = oai.ChatCompletion.test(test_data[:50], config_n10, eval_math_responses, logging_level=logging.INFO)\n",
+ "config_n10 = {\"model\": 'gpt-4', \"prompt\": prompt, \"max_tokens\": 600, \"n\": 10}\n",
+ "n10_result = oai.ChatCompletion.test(test_data[:50], eval_math_responses, logging_level=logging.INFO, **config_n10)\n",
"print(n10_result)"
]
},
@@ -347,8 +347,8 @@
"metadata": {},
"outputs": [],
"source": [
- "config_n30 = {\"model\": 'gpt-4', \"prompt\": prompts[0], \"max_tokens\": 600, \"n\": 30}\n",
- "n30_result = oai.ChatCompletion.test(test_data[:50], config_n30, eval_math_responses, logging_level=logging.INFO)\n",
+ "config_n30 = {\"model\": 'gpt-4', \"prompt\": prompt, \"max_tokens\": 600, \"n\": 30}\n",
+ "n30_result = oai.ChatCompletion.test(test_data[:50], eval_math_responses, logging_level=logging.INFO, **config_n30)\n",
"print(n30_result)"
]
},
diff --git a/test/openai/test_completion.py b/test/autogen/oai/test_completion.py
similarity index 76%
rename from test/openai/test_completion.py
rename to test/autogen/oai/test_completion.py
index cb4b821f9..80b453615 100644
--- a/test/openai/test_completion.py
+++ b/test/autogen/oai/test_completion.py
@@ -4,6 +4,7 @@ import numpy as np
import pytest
from functools import partial
import os
+import json
from flaml import oai
from flaml.autogen.code_utils import (
eval_function_completions,
@@ -17,6 +18,54 @@ from flaml.autogen.code_utils import (
)
from flaml.autogen.math_utils import eval_math_responses, solve_problem
+KEY_LOC = "test/autogen"
+here = os.path.abspath(os.path.dirname(__file__))
+
+
+def yes_or_no_filter(context, response, **_):
+ return context.get("yes_or_no_choice", False) is False or any(
+ text in ["Yes.", "No."] for text in oai.Completion.extract_text(response)
+ )
+
+
+def valid_json_filter(response, **_):
+ for text in oai.Completion.extract_text(response):
+ try:
+ json.loads(text)
+ return True
+ except ValueError:
+ pass
+ return False
+
+
+def test_filter():
+ try:
+ import openai
+ except ImportError as exc:
+ print(exc)
+ return
+ response = oai.Completion.create(
+ context={"yes_or_no_choice": True},
+ config_list=[{"model": "text-ada-001"}, {"model": "gpt-3.5-turbo"}, {"model": "text-davinci-003"}],
+ prompt="Is 37 a prime number? Please answer 'Yes.' or 'No.'",
+ filter_func=yes_or_no_filter,
+ )
+ assert oai.Completion.extract_text(response)[0] in ["Yes.", "No."]
+ response = oai.Completion.create(
+ context={"yes_or_no_choice": False},
+ config_list=[{"model": "text-ada-001"}, {"model": "gpt-3.5-turbo"}, {"model": "text-davinci-003"}],
+ prompt="Is 37 a prime number?",
+ filter_func=yes_or_no_filter,
+ )
+ assert response["model"] == "text-ada-001"
+ response = oai.Completion.create(
+ config_list=[{"model": "text-ada-001"}, {"model": "gpt-3.5-turbo"}, {"model": "text-davinci-003"}],
+ prompt="How to construct a json request to Bing API to search for 'latest AI news'? Return the JSON request.",
+ filter_func=valid_json_filter,
+ )
+ assert response["config_id"] == 2 or response["pass_filter"], "the response must pass filter unless all fail"
+ assert not response["pass_filter"] or json.loads(oai.Completion.extract_text(response)[0])
+
def test_chatcompletion():
params = oai.ChatCompletion._construct_params(
@@ -46,36 +95,7 @@ def test_multi_model():
print(exc)
return
response = oai.Completion.create(
- config_list=[
- {
- "model": "gpt-4",
- "api_key": os.environ.get("OPENAI_API_KEY"),
- "api_type": "open_ai",
- "api_base": "https://api.openai.com/v1",
- "api_version": None,
- },
- {
- "model": "gpt-4",
- "api_key": os.environ.get("AZURE_OPENAI_API_KEY"),
- "api_type": "azure",
- "api_base": os.environ.get("AZURE_OPENAI_API_BASE"),
- "api_version": "2023-03-15-preview",
- },
- {
- "model": "gpt-3.5-turbo",
- "api_key": os.environ.get("OPENAI_API_KEY"),
- "api_type": "open_ai",
- "api_base": "https://api.openai.com/v1",
- "api_version": None,
- },
- {
- "model": "gpt-3.5-turbo",
- "api_key": os.environ.get("AZURE_OPENAI_API_KEY"),
- "api_type": "azure",
- "api_base": os.environ.get("AZURE_OPENAI_API_BASE"),
- "api_version": "2023-03-15-preview",
- },
- ],
+ config_list=oai.config_list_gpt4_gpt35(KEY_LOC),
prompt="Hi",
)
print(response)
@@ -96,7 +116,7 @@ def test_execute_code():
# read a file
print(execute_code("with open('tmp/codetest.py', 'r') as f: a=f.read()"))
# create a file
- print(execute_code("with open('tmp/codetest.py', 'w') as f: f.write('b=1')", work_dir="test/openai/my_tmp"))
+ print(execute_code("with open('tmp/codetest.py', 'w') as f: f.write('b=1')", work_dir=f"{here}/my_tmp"))
# execute code in a file
print(execute_code(filename="tmp/codetest.py"))
# execute code for assertion error
@@ -116,25 +136,29 @@ def test_improve():
except ImportError as exc:
print(exc)
return
+ config_list = oai.config_list_openai_aoai(KEY_LOC)
improved, _ = improve_function(
"flaml/autogen/math_utils.py",
"solve_problem",
"Solve math problems accurately, by avoiding calculation errors and reduce reasoning errors.",
+ config_list=config_list,
)
- with open("test/openai/math_utils.py.improved", "w") as f:
+ with open(f"{here}/math_utils.py.improved", "w") as f:
f.write(improved)
suggestion, _ = improve_code(
["flaml/autogen/code_utils.py", "flaml/autogen/math_utils.py"],
"leverage generative AI smartly and cost-effectively",
+ config_list=config_list,
)
print(suggestion)
improvement, cost = improve_code(
["flaml/autogen/code_utils.py", "flaml/autogen/math_utils.py"],
"leverage generative AI smartly and cost-effectively",
suggest_only=False,
+ config_list=config_list,
)
print(cost)
- with open("test/openai/suggested_improvement.txt", "w") as f:
+ with open(f"{here}/suggested_improvement.txt", "w") as f:
f.write(improvement)
@@ -196,7 +220,7 @@ print(f"Text: {text}")
"""
)
print(code)
- solution, cost = solve_problem("1+1=")
+ solution, cost = solve_problem("1+1=", config_list=oai.config_list_gpt4_gpt35(KEY_LOC))
print(solution, cost)
@@ -226,6 +250,7 @@ def test_humaneval(num_samples=1):
}
for x in range(n_tune_data, len(data))
]
+ oai.Completion.clear_cache(cache_path_root="{here}/cache")
oai.Completion.set_cache(seed)
try:
import openai
@@ -233,6 +258,7 @@ def test_humaneval(num_samples=1):
except ImportError as exc:
print(exc)
return
+ oai.Completion.clear_cache(400)
# a minimal tuning example
config, _ = oai.Completion.tune(
data=tune_data,
@@ -254,7 +280,8 @@ def test_humaneval(num_samples=1):
prompt="{definition}",
)
responses = oai.Completion.create(context=test_data[0], **config)
- # a minimal tuning example for tuning chat completion models using the Completion class
+ # a minimal tuning example for tuning chat completion models using the ChatCompletion class
+ config_list = oai.config_list_openai_aoai(KEY_LOC)
config, _ = oai.ChatCompletion.tune(
data=tune_data,
metric="expected_success",
@@ -262,12 +289,14 @@ def test_humaneval(num_samples=1):
eval_func=eval_function_completions,
n=1,
messages=[{"role": "user", "content": "{definition}"}],
+ config_list=config_list,
)
- responses = oai.ChatCompletion.create(context=test_data[0], **config)
+ responses = oai.ChatCompletion.create(context=test_data[0], config_list=config_list, **config)
print(responses)
- code, cost, _ = implement(tune_data[1], [config])
+ code, cost, selected = implement(tune_data[1], [{**config_list[-1], **config}])
print(code)
print(cost)
+ assert selected == 0
print(eval_function_completions([code], **tune_data[1]))
# a more comprehensive tuning example
config2, analysis = oai.Completion.tune(
@@ -295,9 +324,11 @@ def test_humaneval(num_samples=1):
oai.Completion.data = test_data[:num_samples]
result = oai.Completion._eval(analysis.best_config, prune=False, eval_only=True)
print("result without pruning", result)
- result = oai.Completion.test(test_data[:num_samples], config=config2)
+ result = oai.Completion.test(test_data[:num_samples], **config2)
print(result)
code, cost, selected = implement(tune_data[1], [config2, config])
+ print(code)
+ print(cost)
print(selected)
print(eval_function_completions([code], **tune_data[1]))
@@ -352,12 +383,12 @@ def test_math(num_samples=-1):
"stop": "###",
}
test_data_sample = test_data[0:3]
- result = oai.ChatCompletion.test(test_data_sample, vanilla_config, eval_math_responses)
+ result = oai.ChatCompletion.test(test_data_sample, eval_math_responses, **vanilla_config)
result = oai.ChatCompletion.test(
test_data_sample,
- vanilla_config,
eval_math_responses,
agg_method="median",
+ **vanilla_config,
)
def my_median(results):
@@ -368,13 +399,12 @@ def test_math(num_samples=-1):
result = oai.ChatCompletion.test(
test_data_sample,
- vanilla_config,
eval_math_responses,
agg_method=my_median,
+ **vanilla_config,
)
result = oai.ChatCompletion.test(
test_data_sample,
- vanilla_config,
eval_math_responses,
agg_method={
"expected_success": my_median,
@@ -382,6 +412,7 @@ def test_math(num_samples=-1):
"success_vote": my_average,
"votes": np.mean,
},
+ **vanilla_config,
)
print(result)
@@ -399,7 +430,7 @@ def test_math(num_samples=-1):
stop="###", # the stop sequence
)
print("tuned config", config)
- result = oai.ChatCompletion.test(test_data_sample, config)
+ result = oai.ChatCompletion.test(test_data_sample, config_list=oai.config_list_openai_aoai(KEY_LOC), **config)
print("result from tuned config:", result)
print("empty responses", eval_math_responses([], None))
@@ -407,13 +438,15 @@ def test_math(num_samples=-1):
if __name__ == "__main__":
import openai
- openai.api_key = os.environ["OPENAI_API_KEY"] = open("test/openai/key.txt").read().strip()
- os.environ["AZURE_OPENAI_API_KEY"] = open("test/openai/key_azure.txt").read().strip()
- os.environ["AZURE_OPENAI_API_BASE"] = open("test/openai/base_azure.txt").read().strip()
- test_chatcompletion()
+ config_list = oai.config_list_openai_aoai(KEY_LOC)
+ assert len(config_list) >= 3, config_list
+ openai.api_key = os.environ["OPENAI_API_KEY"]
+
+ # test_filter()
+ # test_chatcompletion()
# test_multi_model()
# test_execute_code()
# test_improve()
# test_nocontext()
- # test_humaneval(1)
+ test_humaneval(1)
# test_math(1)
diff --git a/test/autogen/oai/test_notebook.py b/test/autogen/oai/test_notebook.py
new file mode 100644
index 000000000..8c76a6eef
--- /dev/null
+++ b/test/autogen/oai/test_notebook.py
@@ -0,0 +1,64 @@
+import sys
+import os
+import pytest
+
+try:
+ import openai
+
+ skip = False
+except ImportError:
+ skip = True
+
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+
+def run_notebook(input_nb, output_nb="executed_openai_notebook.ipynb", save=False):
+ import nbformat
+ from nbconvert.preprocessors import ExecutePreprocessor
+ from nbconvert.preprocessors import CellExecutionError
+
+ try:
+ file_path = os.path.join(here, os.pardir, os.pardir, os.pardir, "notebook", input_nb)
+ with open(file_path) as nb_file:
+ nb = nbformat.read(nb_file, as_version=4)
+ preprocessor = ExecutePreprocessor(timeout=4800, kernel_name="python3")
+ preprocessor.preprocess(nb, {"metadata": {"path": here}})
+
+ output_file_name = "executed_openai_notebook_output.txt"
+ output_file = os.path.join(here, output_file_name)
+ with open(output_file, "a") as nb_output_file:
+ for cell in nb.cells:
+ if cell.cell_type == "code" and "outputs" in cell:
+ for output in cell.outputs:
+ if "text" in output:
+ nb_output_file.write(output["text"].strip() + "\n")
+ elif "data" in output and "text/plain" in output["data"]:
+ nb_output_file.write(output["data"]["text/plain"].strip() + "\n")
+ except CellExecutionError:
+ raise
+ finally:
+ if save:
+ with open(os.path.join(here, output_nb), "w", encoding="utf-8") as nb_executed_file:
+ nbformat.write(nb, nb_executed_file)
+
+
+ skip or not sys.version.startswith("3.10"),
+ reason="do not run openai test if openai is not installed or py!=3.10",
+)
+def test_autogen_openai_completion(save=False):
+ run_notebook("autogen_openai_completion.ipynb", save=save)
+
+
+ skip or not sys.version.startswith("3.11"),
+ reason="do not run openai test if openai is not installed or py!=3.11",
+)
+def test_autogen_chatgpt_gpt4(save=False):
+ run_notebook("autogen_chatgpt_gpt4.ipynb", save=save)
+
+
+if __name__ == "__main__":
+ test_autogen_chatgpt_gpt4(save=True)
+ test_autogen_openai_completion(save=True)
diff --git a/test/autogen/test_agent.py b/test/autogen/test_agent.py
index e7832aaff..9695d62e7 100644
--- a/test/autogen/test_agent.py
+++ b/test/autogen/test_agent.py
@@ -1,6 +1,10 @@
+import os
from flaml.autogen.code_utils import extract_code
from flaml import oai
+KEY_LOC = "test/autogen"
+here = os.path.abspath(os.path.dirname(__file__))
+
def test_extract_code():
print(extract_code("```bash\npython temp.py\n```"))
@@ -12,12 +16,13 @@ def test_coding_agent(human_input_mode="NEVER", max_consecutive_auto_reply=10):
except ImportError:
return
from flaml.autogen.agent.coding_agent import PythonAgent
- from flaml.autogen.agent.human_proxy_agent import HumanProxyAgent
+ from flaml.autogen.agent.user_proxy_agent import UserProxyAgent
+ config_list = oai.config_list_gpt4_gpt35(key_file_path=KEY_LOC)
conversations = {}
oai.ChatCompletion.start_logging(conversations)
- agent = PythonAgent("coding_agent", request_timeout=600, seed=42)
- user = HumanProxyAgent(
+ agent = PythonAgent("coding_agent", request_timeout=600, seed=42, config_list=config_list)
+ user = UserProxyAgent(
"user",
human_input_mode=human_input_mode,
max_consecutive_auto_reply=max_consecutive_auto_reply,
@@ -48,8 +53,9 @@ def test_tsp(human_input_mode="NEVER", max_consecutive_auto_reply=10):
except ImportError:
return
from flaml.autogen.agent.coding_agent import PythonAgent
- from flaml.autogen.agent.human_proxy_agent import HumanProxyAgent
+ from flaml.autogen.agent.user_proxy_agent import UserProxyAgent
+ config_list = oai.config_list_openai_aoai(key_file_path=KEY_LOC)
hard_questions = [
"What if we must go from node 1 to node 2?",
"Can we double all distances?",
@@ -57,14 +63,14 @@ def test_tsp(human_input_mode="NEVER", max_consecutive_auto_reply=10):
]
oai.ChatCompletion.start_logging()
- agent = PythonAgent("coding_agent", temperature=0)
- user = HumanProxyAgent(
+ agent = PythonAgent("coding_agent", temperature=0, config_list=config_list)
+ user = UserProxyAgent(
"user",
- work_dir="test/autogen",
+ work_dir=f"{here}",
human_input_mode=human_input_mode,
max_consecutive_auto_reply=max_consecutive_auto_reply,
)
- with open("test/autogen/tsp_prompt.txt", "r") as f:
+ with open(f"{here}/tsp_prompt.txt", "r") as f:
prompt = f.read()
# agent.receive(prompt.format(question=hard_questions[0]), user)
# agent.receive(prompt.format(question=hard_questions[1]), user)
@@ -74,14 +80,6 @@ def test_tsp(human_input_mode="NEVER", max_consecutive_auto_reply=10):
if __name__ == "__main__":
- import openai
-
- openai.api_key_path = "test/openai/key.txt"
- # if you use Azure OpenAI, comment the above line and uncomment the following lines
- # openai.api_type = "azure"
- # openai.api_base = "https://<your_endpoint>.openai.azure.com/"
- # openai.api_version = "2023-03-15-preview" # change if necessary
- # openai.api_key = "<your_api_key>"
# test_extract_code()
test_coding_agent(human_input_mode="TERMINATE")
# when GPT-4, i.e., the DEFAULT_MODEL, is used, conversation in the following test
diff --git a/test/autogen/test_human_proxy_agent.py b/test/autogen/test_user_proxy_agent.py
similarity index 51%
rename from test/autogen/test_human_proxy_agent.py
rename to test/autogen/test_user_proxy_agent.py
index 335552f8e..d750bcf93 100644
--- a/test/autogen/test_human_proxy_agent.py
+++ b/test/autogen/test_user_proxy_agent.py
@@ -1,5 +1,7 @@
from flaml import oai
+KEY_LOC = "test/autogen"
+
def test_human_agent():
try:
@@ -7,12 +9,12 @@ def test_human_agent():
except ImportError:
return
from flaml.autogen.agent.chat_agent import ChatAgent
- from flaml.autogen.agent.human_proxy_agent import HumanProxyAgent
+ from flaml.autogen.agent.user_proxy_agent import UserProxyAgent
conversations = {}
oai.ChatCompletion.start_logging(conversations)
- agent = ChatAgent("chat_agent")
- user = HumanProxyAgent("human_user", human_input_mode="NEVER", max_consecutive_auto_reply=2)
+ agent = ChatAgent("chat_agent", config_list=oai.config_list_gpt4_gpt35(key_file_path=KEY_LOC))
+ user = UserProxyAgent("human_user", human_input_mode="NEVER", max_consecutive_auto_reply=2)
agent.receive(
"""Write python code to solve the equation x^3=125. You must write code in the following format. You must always print the result.
Wait for me to return the result.
@@ -27,13 +29,4 @@ def test_human_agent():
if __name__ == "__main__":
- import openai
-
- openai.api_key_path = "test/openai/key.txt"
- # if you use Azure OpenAI, comment the above line and uncomment the following lines
- # openai.api_type = "azure"
- # openai.api_base = "https://<your_endpoint>.openai.azure.com/"
- # openai.api_version = "2023-03-15-preview" # change if necessary
- # openai.api_key = "<your_api_key>"
- # test_extract_code()
test_human_agent()
diff --git a/test/openai/test_notebook.py b/test/openai/test_notebook.py
deleted file mode 100644
index e27540b70..000000000
--- a/test/openai/test_notebook.py
+++ /dev/null
@@ -1,62 +0,0 @@
-import nbformat
-from nbconvert.preprocessors import ExecutePreprocessor
-from nbconvert.preprocessors import CellExecutionError
-import os
-import pytest
-
-try:
- import openai
-
- skip = False
-except ImportError:
- skip = True
-
-
-here = os.path.abspath(os.path.dirname(__file__))
-
-
-def run_notebook(input_nb, output_nb="executed_openai_notebook.ipynb", save=False):
- try:
- file_path = os.path.join(here, os.pardir, os.pardir, "notebook", input_nb)
- with open(file_path) as f:
- nb = nbformat.read(f, as_version=4)
- ep = ExecutePreprocessor(timeout=4800, kernel_name="python3")
- ep.preprocess(nb, {"metadata": {"path": here}})
-
- output_file_name = "executed_openai_notebook_output.txt"
- output_file = os.path.join(here, output_file_name)
- with open(output_file, "a") as f:
- for cell in nb.cells:
- if cell.cell_type == "code" and "outputs" in cell:
- for output in cell.outputs:
- if "text" in output:
- f.write(output["text"].strip() + "\n")
- elif "data" in output and "text/plain" in output["data"]:
- f.write(output["data"]["text/plain"].strip() + "\n")
- except CellExecutionError:
- raise
- finally:
- if save:
- with open(os.path.join(here, output_nb), "w", encoding="utf-8") as f:
- nbformat.write(nb, f)
-
-
- skip,
- reason="do not run openai test if openai is not installed",
-)
-def test_autogen_openai(save=False):
- run_notebook("autogen_openai.ipynb", save=save)
-
-
- skip,
- reason="do not run openai test if openai is not installed",
-)
-def test_autogen_chatgpt(save=False):
- run_notebook("autogen_chatgpt.ipynb", save=save)
-
-
-if __name__ == "__main__":
- test_autogen_chatgpt(save=True)
- test_autogen_openai(save=True)
diff --git a/website/blog/2023-05-07-1M-milestone/index.mdx b/website/blog/2023-05-07-1M-milestone/index.mdx
index e71af4313..ba5d2a9ab 100644
--- a/website/blog/2023-05-07-1M-milestone/index.mdx
+++ b/website/blog/2023-05-07-1M-milestone/index.mdx
@@ -38,6 +38,6 @@ We invite contributions from anyone interested in this topic and look forward to
## For Further Reading
* [Documentation about `flaml.autogen`](/docs/Use-Cases/Auto-Generation)
-* [Code Example: Tune chatGPT for Math Problem Solving with FLAML](https://github.com/microsoft/FLAML/blob/main/notebook/autogen_chatgpt.ipynb)
+* [Code Example: Tune chatGPT for Math Problem Solving with FLAML](https://github.com/microsoft/FLAML/blob/main/notebook/autogen_chatgpt_gpt4.ipynb)
*Do you have any experience to share about LLM applications? Do you like to see more support or research of LLMOps? Please join our [Discord](https://discord.gg/Cppx2vSPVP) server for discussion.*
diff --git a/website/docs/Examples/AutoGen-OpenAI.md b/website/docs/Examples/AutoGen-OpenAI.md
index 18759306b..82f138867 100644
--- a/website/docs/Examples/AutoGen-OpenAI.md
+++ b/website/docs/Examples/AutoGen-OpenAI.md
@@ -128,10 +128,10 @@ print(eval_with_generated_assertions(oai.Completion.extract_text(response), **tu
You can use flaml's `oai.Completion.test` to evaluate the performance of an entire dataset with the tuned config.
```python
-result = oai.Completion.test(test_data, config)
+result = oai.Completion.test(test_data, **config)
print("performance on test data with the tuned config:", result)
```
The result will vary with the inference budget and optimization budget.
-[Link to notebook](https://github.com/microsoft/FLAML/blob/main/notebook/autogen_openai.ipynb) | [Open in colab](https://colab.research.google.com/github/microsoft/FLAML/blob/main/notebook/autogen_openai.ipynb)
+[Link to notebook](https://github.com/microsoft/FLAML/blob/main/notebook/autogen_openai_completion.ipynb) | [Open in colab](https://colab.research.google.com/github/microsoft/FLAML/blob/main/notebook/autogen_openai_completion.ipynb)
diff --git a/website/docs/Use-Cases/Auto-Generation.md b/website/docs/Use-Cases/Auto-Generation.md
index 5c8d33593..11d52f3fb 100644
--- a/website/docs/Use-Cases/Auto-Generation.md
+++ b/website/docs/Use-Cases/Auto-Generation.md
@@ -26,6 +26,9 @@ There are also complex interactions among subsets of the hyperparameters. For ex
the temperature and top_p are not recommended to be altered from their default values together because they both control the randomness of the generated text, and changing both at the same time can result in conflicting effects; n and best_of are rarely tuned together because if the application can process multiple outputs, filtering on the server side causes unnecessary information loss; both n and max_tokens will affect the total number of tokens generated, which in turn will affect the cost of the request.
These interactions and trade-offs make it difficult to manually determine the optimal hyperparameter settings for a given text generation task.
+*Do the choices matter? Check this [blog post](/blog/2023/04/21/LLM-tuning-math) for a case study.*
+
+
## Tune Hyperparameters
The tuning can be performed with the following information:
@@ -46,8 +49,9 @@ The evaluation function should take a list of responses, and other keyword argum
```python
def eval_math_responses(responses: List[str], solution: str, **args) -> Dict:
# select a response from the list of responses
+ answer = voted_answer(responses)
# check whether the answer is correct
- return {"success": True or False}
+ return {"success": is_equivalent(answer, solution)}
```
[`flaml.autogen.code_utils`](../reference/autogen/code_utils) and [`flaml.autogen.math_utils`](../reference/autogen/math_utils) offer some example evaluation functions for code generation and math problem solving.
@@ -100,6 +104,8 @@ The returned `config` contains the optimized configuration and `analysis` contai
The tuend config can be used to perform inference.
+*Refer to this [page](../Examples/AutoGen-OpenAI) for a full example.*
+
## Perform Inference
One can use [`flaml.oai.Completion.create`](../reference/autogen/oai/completion#create) to perform inference.
@@ -120,6 +126,8 @@ API call results are cached locally and reused when the same request is issued.
### Error handling
+#### Runtime error
+
It is easy to hit error when calling OpenAI APIs, due to connection, rate limit, or timeout. Some of the errors are transient. `flaml.oai.Completion.create` deals with the transient errors and retries automatically. Initial request timeout, retry timeout and retry time interval can be configured via `flaml.oai.request_timeout`, `flaml.oai.retry_timeout` and `flaml.oai.retry_time`.
Moreover, one can pass a list of configurations of different models/endpoints to mitigate the rate limits. For example,
@@ -155,6 +163,29 @@ response = oai.Completion.create(
It will try querying Azure OpenAI gpt-4, OpenAI gpt-3.5-turbo, and a locally hosted llama-7B one by one, ignoring AuthenticationError, RateLimitError and Timeout,
until a valid result is returned. This can speed up the development process where the rate limit is a bottleneck. An error will be raised if the last choice fails. So make sure the last choice in the list has the best availability.
+#### Logic error
+
+Another type of error is that the returned response does not satisfy a requirement. For example, if the response is required to be a valid json string, one would like to filter the responses that are not. This can be achieved by providing a list of configurations and a filter function. For example,
+
+```python
+def valid_json_filter(context, config, response):
+ for text in oai.Completion.extract_text(response):
+ try:
+ json.loads(text)
+ return True
+ except ValueError:
+ pass
+ return False
+
+response = oai.Completion.create(
+ config_list=[{"model": "text-ada-001"}, {"model": "gpt-3.5-turbo"}, {"model": "text-davinci-003"}],
+ prompt="How to construct a json request to Bing API to search for 'latest AI news'? Return the JSON request.",
+ filter_func=valid_json_filter,
+)
+```
+
+The example above will try to use text-ada-001, gpt-3.5-turbo, and text-davinci-003 iteratively, until a valid json string is returned or the last config is used. One can also repeat the same model in the list for multiple times to try one model multiple times for increasing the robustness of the final response.
+
### Templating
If the provided prompt or message is a template, it will be automatically materialized with a given context. For example,
@@ -359,5 +390,5 @@ The compact history is more efficient and the individual API call history contai
*Interested in trying it yourself? Please check the following notebook examples:*
-* [Optimize for Code Gen](https://github.com/microsoft/FLAML/blob/main/notebook/autogen_openai.ipynb)
-* [Optimize for Math](https://github.com/microsoft/FLAML/blob/main/notebook/autogen_chatgpt.ipynb)
+* [Optimize for Code Gen](https://github.com/microsoft/FLAML/blob/main/notebook/autogen_openai_completion.ipynb)
+* [Optimize for Math](https://github.com/microsoft/FLAML/blob/main/notebook/autogen_chatgpt_gpt4.ipynb)