A programming framework for agentic AI
Adding a test function for OpenAI completion in flaml (#951)
Qingyun Wu45641000c048
diff --git a/.github/workflows/openai.yml b/.github/workflows/openai.ymlindex d0027a5b8..5eb442d97 100644--- a/.github/workflows/openai.yml+++ b/.github/workflows/openai.yml@@ -40,6 +40,7 @@ jobs:pip install coverage pytest datasets nbconvert nbformat ipykernelcoverage run -a -m pytest test/openaicoverage xml+ cat "$(pwd)/test/openai/executed_openai_notebook_output.txt"- name: Upload coverage to Codecovuses: codecov/codecov-action@v3with:diff --git a/flaml/integrations/oai/completion.py b/flaml/integrations/oai/completion.pyindex 0a7ce3d40..4bfe01aa0 100644--- a/flaml/integrations/oai/completion.py+++ b/flaml/integrations/oai/completion.py@@ -118,16 +118,17 @@ class Completion:cls.cache_path = f"{cache_path}/{seed}"@classmethod- def _get_response(cls, config: dict, eval_only=False):+ def _get_response(cls, config: dict, eval_only=False, use_cache=True):"""Get the response from the openai api call.Try cache first. If not found, call the openai api. If the api call fails, retry after retry_time."""key = get_key(config)- response = cls._cache.get(key, None)- if response is not None and (response != -1 or not eval_only):- # print("using cached response")- return response+ if use_cache:+ response = cls._cache.get(key, None)+ if response is not None and (response != -1 or not eval_only):+ # print("using cached response")+ return responseopenai_completion = (openai.ChatCompletionif config["model"] in cls.chat_models@@ -215,7 +216,27 @@ class Completion:)@classmethod- def eval(cls, config: dict, prune=True, eval_only=False):+ def _get_prompt_messages_from_config(cls, model, config):+ prompt, messages = None, None+ if model in cls.chat_models:+ # either "prompt" should be in config (for being compatible with non-chat models)+ # or "messages" should be in config (for tuning chat models only)+ prompt = config.get("prompt")+ messages = config.get("messages")+ # either prompt or messages should be in config, but not both+ assert (prompt is None) != (+ messages is None+ ), "Either prompt or messages should be in config for chat models."+ if prompt is None:+ messages = cls._messages[messages]+ else:+ prompt = cls._prompts[prompt]+ else:+ prompt = cls._prompts[config["prompt"]]+ return prompt, messages++ @classmethod+ def _eval(cls, config: dict, prune=True, eval_only=False):"""Evaluate the given config as the hyperparameter setting for the openai api call.Args:@@ -242,22 +263,7 @@ class Completion:max_tokens = config.get("max_tokens", np.inf if model in cls.chat_models else 16)- # default value in OpenAI- if model in cls.chat_models:- # either "prompt" should be in config (for being compatible with non-chat models)- # or "messages" should be in config (for tuning chat models only)- prompt = config.get("prompt")- messages = config.get("messages")- # either prompt or messages should be in config, but not both- assert (prompt is None) != (- messages is None- ), "Either prompt or messages should be in config for chat models."- if prompt is None:- messages = cls._messages[messages]- else:- prompt = cls._prompts[prompt]- else:- prompt = cls._prompts[config["prompt"]]+ prompt, messages = cls._get_prompt_messages_from_config(model, config)stop = cls._stops and cls._stops[config["stop"]]target_output_tokens = Noneif not cls.avg_input_tokens:@@ -309,33 +315,7 @@ class Completion:f"num_completions={num_completions}, data instance={i}")data_i = data[i]- if prompt is None:- params["messages"] = [- {- "role": m["role"],- "content": m["content"].format(**data_i)- if isinstance(m["content"], str)- else m["content"](data_i),- }- for m in messages- ]- elif model in cls.chat_models:- # convert prompt to messages- params["messages"] = [- {- "role": "user",- "content": prompt.format(**data_i)- if isinstance(prompt, str)- else prompt(data_i),- },- ]- params.pop("prompt", None)- else:- params["prompt"] = (- prompt.format(**data_i)- if isinstance(prompt, str)- else prompt(data_i)- )+ params = cls._construct_params(data_i, params, prompt, messages)response = cls._get_response(params, eval_only)if response == -1: # rate limit error, treat as invalidcls._update_invalid_n(@@ -481,6 +461,7 @@ class Completion:"""Tune the parameters for the OpenAI API call.TODO: support parallel tuning with ray or spark.+ TODO: support agg_method as in testArgs:data (list): The list of data points.@@ -490,20 +471,20 @@ class Completion:The function should take a list of responses and a data point as input,and return a dict of metrics. For example,- ```python- def eval_func(responses, **data):- solution = data["solution"]- success_list = []- n = len(responses)- for i in range(n):- response = responses[i]- succeed = is_equiv_chain_of_thought(response, solution)- success_list.append(succeed)- return {- "expected_success": 1 - pow(1 - sum(success_list) / n, n),- "success": any(s for s in success_list),- }- ```+ ```python+ def eval_func(responses, **data):+ solution = data["solution"]+ success_list = []+ n = len(responses)+ for i in range(n):+ response = responses[i]+ succeed = is_equiv_chain_of_thought(response, solution)+ success_list.append(succeed)+ return {+ "expected_success": 1 - pow(1 - sum(success_list) / n, n),+ "success": any(s for s in success_list),+ }+ ```log_file_name (str, optional): The log file.inference_budget (float, optional): The inference budget.@@ -613,7 +594,7 @@ class Completion:logger.setLevel(logging_level)with diskcache.Cache(cls.cache_path) as cls._cache:analysis = tune.run(- cls.eval,+ cls._eval,search_alg=search_alg,num_samples=num_samples,log_file_name=log_file_name,@@ -650,35 +631,198 @@ class Completion:"""if ERROR:raise ERROR+ params = cls._construct_params(context, config)+ if use_cache:+ with diskcache.Cache(cls.cache_path) as cls._cache:+ return cls._get_response(params)+ return cls.openai_completion_class.create(**params)++ @classmethod+ def _construct_params(cls, data_instance, config, prompt=None, messages=None):params = config.copy()- prompt = config.get("prompt")- if "messages" in config:+ model = config["model"]+ prompt = config.get("prompt") if prompt is None else prompt+ messages = config.get("messages") if messages is None else messages+ # either "prompt" should be in config (for being compatible with non-chat models)+ # or "messages" should be in config (for tuning chat models only)+ if prompt is None and model in cls.chat_models:+ if messages is None:+ raise ValueError(+ "Either prompt or messages should be in config for chat models."+ )+ if prompt is None:params["messages"] = [{- k: v.format(**context) if isinstance(v, str) else v(context)- for k, v in message.items()+ "role": m["role"],+ "content": m["content"].format(**data_instance)+ if isinstance(m["content"], str)+ else m["content"](data_instance),}- for message in config["messages"]+ for m in messages]- params.pop("prompt", None)- elif config["model"] in cls.chat_models:+ elif model in cls.chat_models:+ # convert prompt to messages+ if isinstance(prompt, str):+ prompt_msg = prompt.format(**data_instance)+ else:+ prompt_msg = prompt(data_instance)params["messages"] = [{"role": "user",- "content": prompt.format(**context)+ "content": prompt_msgif isinstance(prompt, str)- else prompt(context),- }+ else prompt(data_instance),+ },]params.pop("prompt", None)else:params["prompt"] = (- prompt.format(**context) if isinstance(prompt, str) else prompt(context)+ prompt.format(**data_instance)+ if isinstance(prompt, str)+ else prompt(data_instance))- if use_cache:- with diskcache.Cache(cls.cache_path) as cls._cache:- return cls._get_response(params)- return cls.openai_completion_class.create(**params)+ return params++ @classmethod+ def test(+ cls,+ data,+ config,+ eval_func=None,+ use_cache=True,+ agg_method="avg",+ return_responses_and_per_instance_result=False,+ seed=41,+ cache_path=".cache",+ ):+ """Evaluate the responses created with the config for the OpenAI API call.++ Args:+ data (list): The list of test data points.+ config (dict): Hyperparameter setting for the openai api call.+ eval_func (Callable): The evaluation function for responses per data instance.+ The function should take a list of responses and a data point as input,+ and return a dict of metrics. You need to either provide a valid callable+ eval_func; or do not provide one (set None) but call the test function after+ calling the tune function in which a eval_func is provided.+ In the latter case we will use the eval_func provided via tune function.+ Defaults to None.++ ```python+ def eval_func(responses, **data):+ solution = data["solution"]+ success_list = []+ n = len(responses)+ for i in range(n):+ response = responses[i]+ succeed = is_equiv_chain_of_thought(response, solution)+ success_list.append(succeed)+ return {+ "expected_success": 1 - pow(1 - sum(success_list) / n, n),+ "success": any(s for s in success_list),+ }+ ```+ use_cache (bool, Optional): Whether to use cached responses. Defaults to True.+ agg_method (str, Callable or a dict of Callable): Result aggregation method (across+ multiple instances) for each of the metrics. Defaults to 'avg'.+ An example agg_method in str:++ ```python+ agg_method = 'median'+ ```+ An example agg_method in a Callable:++ ```python+ agg_method = np.median+ ```++ An example agg_method in a dict of Callable:++ ```python+ agg_method={'median_success': np.median, 'avg_success': np.mean}+ ```++ return_responses_and_per_instance_result (bool): Whether to also return responses+ and per instance results in addition to the aggregated results.+ seed (int): Random seed for the evaluation. Defaults to 41.+ cache_path (str): Path to the cache directory. Defaults to '.cache'.+ If a cache directory does not exist, it will be created, otherwise use the existing one.+ Returns:+ None in case of rate limit error or when a valid eval_func is not provided in either test or tune;+ Otherwise, a dict of aggregated results, responses and per instance results if `return_responses_and_per_instance_result` is True;+ Otherwise, a dict of aggregated results (responses and per instance results are not returned).+ """+ model = config["model"]+ result_agg, responses_list, result_list = {}, [], []+ metric_keys = None+ cls.set_cache(seed, cache_path)+ with diskcache.Cache(cls.cache_path) as cls._cache:+ for i, data_i in enumerate(data):+ logger.info(f"evaluating data instance {i}")+ params = cls._construct_params(data_i, config)+ response = cls._get_response(+ params, eval_only=True, use_cache=use_cache+ )+ if response == -1: # rate limit error, treat as invalid+ return None+ # evaluate the quality of the responses+ responses = (+ [r["message"]["content"].rstrip() for r in response["choices"]]+ if model in cls.chat_models+ else [r["text"].rstrip() for r in response["choices"]]+ )++ if eval_func is not None:+ metrics = eval_func(responses, **data_i)+ elif hasattr(cls, "_eval_func"):+ metrics = cls._eval_func(responses, **data_i)+ else:+ logger.warning(+ "Please either provide a valid eval_func or do the test after the tune function is called"+ )+ return+ if not metric_keys:+ metric_keys = []+ for k in metrics.keys():+ try:+ _ = float(metrics[k])+ metric_keys.append(k)+ except ValueError:+ pass+ result_list.append(metrics)+ if return_responses_and_per_instance_result:+ responses_list.append(responses)+ if isinstance(agg_method, str):+ if agg_method in ["avg", "average"]:+ for key in metric_keys:+ result_agg[key] = np.mean([r[key] for r in result_list])+ elif agg_method == "median":+ for key in metric_keys:+ result_agg[key] = np.median([r[key] for r in result_list])+ else:+ logger.warning(+ f"Aggregation method {agg_method} not supported. Please write your own aggregation method as a callable(s)."+ )+ elif callable(agg_method):+ for key in metric_keys:+ result_agg[key] = agg_method([r[key] for r in result_list])+ elif isinstance(agg_method, dict):+ for key in metric_keys:+ metric_agg_method = agg_method[key]+ assert callable(+ metric_agg_method+ ), "please provide a callable for each metric"+ result_agg[key] = metric_agg_method([r[key] for r in result_list])+ else:+ raise ValueError(+ "agg_method needs to be a string ('avg' or 'median'),\+ or a callable, or a dictionary of callable."+ )+ # should we also return the result_list and responses_list or not?+ if return_responses_and_per_instance_result:+ return result_agg, result_list, responses_list+ else:+ return result_aggclass ChatCompletion(Completion):diff --git a/notebook/integrate_chatgpt.ipynb b/notebook/integrate_chatgpt.ipynbindex 5495b37c7..57d549391 100644--- a/notebook/integrate_chatgpt.ipynb+++ b/notebook/integrate_chatgpt.ipynb@@ -743,7 +743,7 @@" # -1 means decided by the optimization budget only\n"," num_samples=-1,\n"," # model=\"chatgpt-35-turbo-0301\", # uncomment if using Azure OpenAI\n",- " # model=\"gpt-3-turbo\", # uncomment if you don't have access to gpt-4\n",+ " # model=\"gpt-3.5-turbo\", # uncomment if you don't have access to gpt-4\n"," prompt=prompts, # the prompt templates to choose from\n"," # stop=\"###\", # the stop sequence\n"," logging_level=logging.INFO, # the logging level\n",@@ -1271,8 +1271,9 @@],"source": ["responses = oai.ChatCompletion.create(context=tune_data[1], **config)\n",- "print(responses)\n",- "print(success_metrics([response[\"message\"][\"content\"].rstrip() for response in responses[\"choices\"]], **tune_data[1]))\n"+ "metric_results = success_metrics([response[\"message\"][\"content\"].rstrip() for response in responses[\"choices\"]], **tune_data[1])\n",+ "print(\"response on an example data instance:\", responses)\n",+ "print(\"metric_results on the example data instance:\", metric_results)\n"]},{@@ -1282,7 +1283,7 @@"source": ["### Evaluate the success rate on the test data\n","\n",- "You can use flaml's `oai.ChatCompletion.eval` to evaluate the performance of an entire dataset with the tuned config. To do that you need to set `oai.ChatCompletion.data` to the data to evaluate. The following code will take a while (30 mins to 1 hour) to evaluate all the test data instances if uncommented and run. It will cost roughly $3. "+ "You can use flaml's `oai.ChatCompletion.test` to evaluate the performance of an entire dataset with the tuned config. The following code will take a while (30 mins to 1 hour) to evaluate all the test data instances if uncommented and run. It will cost roughly $3. "]},{@@ -1306,9 +1307,8 @@}],"source": [- "# oai.ChatCompletion.data = test_data\n",- "# result = oai.ChatCompletion.eval(analysis.best_config, prune=False, eval_only=True)\n",- "# print(result)"+ "# result = oai.Completion.test(test_data, config, success_metrics)\n",+ "# print(\"performance on test data with the tuned config:\", result)"]},{@@ -1336,9 +1336,9 @@"# assuming you have access to gpt-4; otherwise use gpt-3.5-turbo\n","# the following code will cost roughly $2 if uncommented and run.\n","\n",- "# default_config = {\"model\": 'gpt-4', \"prompt\": 0}\n",- "# default_result = oai.ChatCompletion.eval(default_config, prune=False, eval_only=True)\n",- "# print(default_result)"+ "# default_config = {\"model\": 'gpt-4', \"prompt\": prompts[0]}\n",+ "# default_result = oai.Completion.test(test_data, default_config, success_metrics)\n",+ "# print(\"performance on test data from gpt-4 with a default config:\", default_result)"]},{@@ -1382,11 +1382,9 @@}],"source": [- "# The following evaluation costs $8 and nearly one hour if you uncomment it and run it.\n",- "\n",- "# config_larger = {\"model\": 'gpt-4', \"prompt\": 0, \"n\": 5}\n",- "# default_result = oai.ChatCompletion.eval(config_larger, prune=False, eval_only=True)\n",- "# print(default_result)"+ "# config_larger = {\"model\": 'gpt-4', \"prompt\": prompts[0], \"n\": 5}\n",+ "# default_result = oai.ChatCompletion.test(test_data, config_larger, success_metrics)\n",+ "# print(\"performance on test data from gpt-4 with a default config and n=5:\", default_result)"]},{diff --git a/notebook/integrate_openai.ipynb b/notebook/integrate_openai.ipynbindex 6b7434265..0d4d2ff20 100644--- a/notebook/integrate_openai.ipynb+++ b/notebook/integrate_openai.ipynb@@ -613,7 +613,20 @@],"source": ["print(\"optimized config\", config)\n",- "print(\"best result on tuning data\", analysis.best_result)"+ "print(\"best result on tuning data\", analysis.best_result)\n",+ "\n",+ "# save results to notebook_output.txt\n",+ "from flaml.version import __version__ as flaml_version\n",+ "import datetime\n",+ "results = {\"optimized config\": config, \"best result on tuning data\": analysis.best_result,}\n",+ "result_info_dict = {\"result_name\": \"integrate_openai.ipynb + optimized config and best result on tuning data\",\n",+ " \"flaml_version\": flaml_version, \n",+ " \"time\": datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\"),\n",+ " \"results\": results}\n",+ "result_info = \"result name: {result_name}, flaml version: {flaml_version}, time: {time}, results: {results}\".format(**result_info_dict)\n",+ "with open(\"notebook_output.txt\", \"a\") as f:\n",+ " f.write(\"\\n\")\n",+ " f.write(result_info)"]},{@@ -773,8 +786,9 @@],"source": ["responses = oai.Completion.create(context=tune_data[1], **config)\n",- "print(responses)\n",- "print(success_metrics([response[\"message\"][\"content\"] if config[\"model\"] in oai.Completion.chat_models else response[\"text\"] for response in responses[\"choices\"]], **tune_data[1]))\n"+ "metric_results = success_metrics([response[\"message\"][\"content\"] if config[\"model\"] in oai.Completion.chat_models else response[\"text\"] for response in responses[\"choices\"]], **tune_data[1])\n",+ "print(\"response on an example data instance:\", responses)\n",+ "print(\"metric_results on the example data instance:\", metric_results)\n"]},{@@ -784,7 +798,7 @@"source": ["### Evaluate the success rate on the test data\n","\n",- "You can use flaml's `oai.Completion.eval` to evaluate the performance of an entire dataset with the tuned config. To do that you need to set `oai.Completion.data` to the data to evaluate. The following code will take a while to evaluate all the 144 test data instances. The cost is about $7 if you uncomment it and run it."+ "You can use flaml's `oai.Completion.test` to evaluate the performance of an entire dataset with the tuned config. The following code will take a while to evaluate all the 144 test data instances. The cost is about $7 if you uncomment it and run it."]},{@@ -808,9 +822,8 @@}],"source": [- "# oai.Completion.data = test_data\n",- "# result = oai.Completion.eval(analysis.best_config, prune=False, eval_only=True)\n",- "# print(result)"+ "result = oai.Completion.test(test_data, config, success_metrics)\n",+ "print(\"performance on test data with the tuned config:\", result)"]},{@@ -824,9 +837,9 @@],"metadata": {"kernelspec": {- "display_name": "Python 3",+ "display_name": "tutorial","language": "python",- "name": "python3"+ "name": "tutorial"},"language_info": {"codemirror_mode": {@@ -838,7 +851,7 @@"name": "python","nbconvert_exporter": "python","pygments_lexer": "ipython3",- "version": "3.9.16"+ "version": "3.9.7"},"vscode": {"interpreter": {diff --git a/test/openai/test_completion.py b/test/openai/test_completion.pyindex 8c2cfa598..d20f0f63c 100644--- a/test/openai/test_completion.py+++ b/test/openai/test_completion.py@@ -133,14 +133,381 @@ def test_humaneval(num_samples=1):responses = oai.Completion.create(context=test_data[0], **config)print(responses)oai.Completion.data = test_data[:num_samples]- result = oai.Completion.eval(analysis.best_config, prune=False, eval_only=True)+ result = oai.Completion._eval(analysis.best_config, prune=False, eval_only=True)+ print("result without pruning", result)+ result = oai.Completion.test(test_data[:num_samples], config=config)print(result)except ImportError as exc:print(exc)+def test_math(num_samples=-1):+ from typing import Optional++ def remove_boxed(string: str) -> Optional[str]:+ """Source: https://github.com/hendrycks/math+ Extract the text within a \\boxed{...} environment.+ Example:+ >>> remove_boxed(\\boxed{\\frac{2}{3}})+ \\frac{2}{3}+ """+ left = "\\boxed{"+ try:+ assert string[: len(left)] == left+ assert string[-1] == "}"+ return string[len(left) : -1]+ except Exception:+ return None++ def last_boxed_only_string(string: str) -> Optional[str]:+ """Source: https://github.com/hendrycks/math+ Extract the last \\boxed{...} or \\fbox{...} element from a string.+ """+ idx = string.rfind("\\boxed")+ if idx < 0:+ idx = string.rfind("\\fbox")+ if idx < 0:+ return None++ i = idx+ right_brace_idx = None+ num_left_braces_open = 0+ while i < len(string):+ if string[i] == "{":+ num_left_braces_open += 1+ if string[i] == "}":+ num_left_braces_open -= 1+ if num_left_braces_open == 0:+ right_brace_idx = i+ break+ i += 1++ if right_brace_idx is None:+ retval = None+ else:+ retval = string[idx : right_brace_idx + 1]++ return retval++ def _fix_fracs(string: str) -> str:+ """Source: https://github.com/hendrycks/math+ Reformat fractions.+ Examples:+ >>> _fix_fracs("\\frac1b")+ \frac{1}{b}+ >>> _fix_fracs("\\frac12")+ \frac{1}{2}+ >>> _fix_fracs("\\frac1{72}")+ \frac{1}{72}+ """+ substrs = string.split("\\frac")+ new_str = substrs[0]+ if len(substrs) > 1:+ substrs = substrs[1:]+ for substr in substrs:+ new_str += "\\frac"+ if substr[0] == "{":+ new_str += substr+ else:+ try:+ assert len(substr) >= 2+ except Exception:+ return string+ a = substr[0]+ b = substr[1]+ if b != "{":+ if len(substr) > 2:+ post_substr = substr[2:]+ new_str += "{" + a + "}{" + b + "}" + post_substr+ else:+ new_str += "{" + a + "}{" + b + "}"+ else:+ if len(substr) > 2:+ post_substr = substr[2:]+ new_str += "{" + a + "}" + b + post_substr+ else:+ new_str += "{" + a + "}" + b+ string = new_str+ return string++ def _fix_a_slash_b(string: str) -> str:+ """Source: https://github.com/hendrycks/math+ Reformat fractions formatted as a/b to \\frac{a}{b}.+ Example:+ >>> _fix_a_slash_b("2/3")+ \frac{2}{3}+ """+ if len(string.split("/")) != 2:+ return string+ a_str = string.split("/")[0]+ b_str = string.split("/")[1]+ try:+ a = int(a_str)+ b = int(b_str)+ assert string == "{}/{}".format(a, b)+ new_string = "\\frac{" + str(a) + "}{" + str(b) + "}"+ return new_string+ except Exception:+ return string++ def _remove_right_units(string: str) -> str:+ """Source: https://github.com/hendrycks/math"""+ if "\\text{ " in string:+ splits = string.split("\\text{ ")+ assert len(splits) == 2+ return splits[0]+ else:+ return string++ def _fix_sqrt(string: str) -> str:+ """Source: https://github.com/hendrycks/math"""+ if "\\sqrt" not in string:+ return string+ splits = string.split("\\sqrt")+ new_string = splits[0]+ for split in splits[1:]:+ if split[0] != "{":+ a = split[0]+ new_substr = "\\sqrt{" + a + "}" + split[1:]+ else:+ new_substr = "\\sqrt" + split+ new_string += new_substr+ return new_string++ def _strip_string(string: str) -> str:+ """Source: https://github.com/hendrycks/math+ Apply the reformatting helper functions above.+ """+ # linebreaks+ string = string.replace("\n", "")+ # print(string)++ # remove inverse spaces+ string = string.replace("\\!", "")+ # print(string)++ # replace \\ with \+ string = string.replace("\\\\", "\\")+ # print(string)++ # replace tfrac and dfrac with frac+ string = string.replace("tfrac", "frac")+ string = string.replace("dfrac", "frac")+ # print(string)++ # remove \left and \right+ string = string.replace("\\left", "")+ string = string.replace("\\right", "")+ # print(string)++ # Remove circ (degrees)+ string = string.replace("^{\\circ}", "")+ string = string.replace("^\\circ", "")++ # remove dollar signs+ string = string.replace("\\$", "")++ # remove units (on the right)+ string = _remove_right_units(string)++ # remove percentage+ string = string.replace("\\%", "")+ string = string.replace(r"\%", "")++ # " 0." equivalent to " ." and "{0." equivalent to "{." Alternatively, add "0" if "." is the start of the string+ string = string.replace(" .", " 0.")+ string = string.replace("{.", "{0.")+ # if empty, return empty string+ if len(string) == 0:+ return string+ if string[0] == ".":+ string = "0" + string++ # to consider: get rid of e.g. "k = " or "q = " at beginning+ if len(string.split("=")) == 2:+ if len(string.split("=")[0]) <= 2:+ string = string.split("=")[1]++ # fix sqrt3 --> sqrt{3}+ string = _fix_sqrt(string)++ # remove spaces+ string = string.replace(" ", "")++ # \frac1b or \frac12 --> \frac{1}{b} and \frac{1}{2}, etc.+ # Even works with \frac1{72} (but not \frac{72}1).+ # Also does a/b --> \\frac{a}{b}+ string = _fix_fracs(string)++ # manually change 0.5 --> \frac{1}{2}+ if string == "0.5":+ string = "\\frac{1}{2}"++ # NOTE: X/Y changed to \frac{X}{Y} in dataset, but in simple cases fix in case the model output is X/Y+ string = _fix_a_slash_b(string)++ return string++ def get_answer(solution: Optional[str]) -> Optional[str]:+ if solution is None:+ return None+ last_boxed = last_boxed_only_string(solution)+ if last_boxed is None:+ return None+ answer = remove_boxed(last_boxed)+ if answer is None:+ return None+ return answer++ def is_equiv(str1: Optional[str], str2: Optional[str]) -> float:+ """Returns (as a float) whether two strings containing math are equivalent up to differences of formatting in+ - units+ - fractions+ - square roots+ - superfluous LaTeX.+ Source: https://github.com/hendrycks/math+ """+ if str1 is None and str2 is None:+ print("WARNING: Both None")+ return 1.0+ if str1 is None or str2 is None:+ return 0.0++ try:+ ss1 = _strip_string(str1)+ ss2 = _strip_string(str2)+ return float(ss1 == ss2)+ except Exception:+ return float(str1 == str2)++ def is_equiv_chain_of_thought(str1: str, str2: str) -> float:+ """Strips the solution first before calling `is_equiv`."""+ ans1 = get_answer(str1)+ ans2 = get_answer(str2)++ return is_equiv(ans1, ans2)++ def success_metrics(responses, solution, **args):+ """Check if each response is correct.++ Args:+ responses (list): The list of responses.+ solution (str): The canonical solution.++ Returns:+ dict: The success metrics.+ """+ success_list = []+ n = len(responses)+ for i in range(n):+ response = responses[i]+ succeed = is_equiv_chain_of_thought(response, solution)+ success_list.append(succeed)+ return {+ "expected_success": 1 - pow(1 - sum(success_list) / n, n),+ "success": any(s for s in success_list),+ }++ seed = 41+ data = datasets.load_dataset("competition_math")+ train_data = data["train"].shuffle(seed=seed)+ test_data = data["test"].shuffle(seed=seed)+ n_tune_data = 20+ tune_data = [+ {+ "problem": train_data[x]["problem"],+ "solution": train_data[x]["solution"],+ }+ for x in range(len(train_data))+ if train_data[x]["level"] == "Level 1"+ ][:n_tune_data]+ test_data = [+ {+ "problem": test_data[x]["problem"],+ "solution": test_data[x]["solution"],+ }+ for x in range(len(test_data))+ if test_data[x]["level"] == "Level 1"+ ]+ print(+ "max tokens in tuning data's canonical solutions",+ max([len(x["solution"].split()) for x in tune_data]),+ )+ print(len(tune_data), len(test_data))+ # prompt template+ prompts = [+ lambda data: "Given a mathematics problem, determine the answer. Simplify your answer as much as possible.\n###\nProblem: What is the value of $\\sqrt{3! \\cdot 3!}$ expressed as a positive integer?\nAnswer: $\\sqrt{3!\\cdot3!}$ is equal to $\\sqrt{(3!)^2}=3!=3\\cdot2\\cdot1=\\boxed{6}$.\n###\nProblem: %s\nAnswer:"+ + data["problem"]+ ]++ try:+ oai.ChatCompletion.set_cache(seed)+ vanilla_config = {+ "model": "gpt-3.5-turbo",+ "temperature": 1,+ "max_tokens": 2048,+ "n": 1,+ "prompt": prompts[0],+ "stop": "###",+ }+ test_data_sample = test_data[0:3]+ result = oai.ChatCompletion.test(+ test_data_sample, vanilla_config, success_metrics+ )+ test_data_sample = test_data[3:6]+ result = oai.ChatCompletion.test(+ test_data_sample,+ vanilla_config,+ success_metrics,+ use_cache=False,+ agg_method="median",+ )++ def my_median(results):+ return np.median(results)++ def my_average(results):+ return np.mean(results)++ result = oai.ChatCompletion.test(+ test_data_sample,+ vanilla_config,+ success_metrics,+ use_cache=False,+ agg_method=my_median,+ )+ result = oai.ChatCompletion.test(+ test_data_sample,+ vanilla_config,+ success_metrics,+ use_cache=False,+ agg_method={"expected_success": my_median, "success": my_average},+ )++ print(result)++ config, _ = oai.ChatCompletion.tune(+ data=tune_data, # the data for tuning+ metric="expected_success", # the metric to optimize+ mode="max", # the optimization mode+ eval_func=success_metrics, # the evaluation function to return the success metrics+ # log_file_name="logs/math.log", # the log file name+ inference_budget=0.002, # the inference budget (dollar)+ optimization_budget=0.01, # the optimization budget (dollar)+ num_samples=num_samples,+ prompt=prompts, # the prompt templates to choose from+ stop="###", # the stop sequence+ )+ print("tuned config", config)+ result = oai.ChatCompletion.test(test_data_sample, config)+ print("result from tuned config:", result)+ except (ImportError, NameError) as exc:+ print(exc)++if __name__ == "__main__":import openaiopenai.api_key_path = "test/openai/key.txt"test_humaneval(-1)+ test_math(-1)diff --git a/test/openai/test_notebook.py b/test/openai/test_notebook.pyindex 0285350c8..fa6800167 100644--- a/test/openai/test_notebook.py+++ b/test/openai/test_notebook.py@@ -15,13 +15,24 @@ except ImportError:here = os.path.abspath(os.path.dirname(__file__))-def run_notebook(input_nb, output_nb="executed_notebook.ipynb", save=False):+def run_notebook(input_nb, output_nb="executed_openai_notebook.ipynb", save=False):try:file_path = os.path.join(here, os.pardir, os.pardir, "notebook", input_nb)with open(file_path) as f:nb = nbformat.read(f, as_version=4)ep = ExecutePreprocessor(timeout=3600, kernel_name="python3")ep.preprocess(nb, {"metadata": {"path": here}})++ output_file_name = "executed_openai_notebook_output.txt"+ output_file = os.path.join(here, output_file_name)+ with open(output_file, "a") as f:+ for cell in nb.cells:+ if cell.cell_type == "code" and "outputs" in cell:+ for output in cell.outputs:+ if "text" in output:+ f.write(output["text"].strip() + "\n")+ elif "data" in output and "text/plain" in output["data"]:+ f.write(output["data"]["text/plain"].strip() + "\n")except CellExecutionError:raisefinally:@@ -48,3 +59,4 @@ def test_integrate_chatgpt(save=False):if __name__ == "__main__":test_integrate_chatgpt(save=True)+ test_integrate_openai(save=True)diff --git a/website/docs/Examples/Integrate - OpenAI.md b/website/docs/Examples/Integrate - OpenAI.mdindex f6289fd3c..a3f83c979 100644--- a/website/docs/Examples/Integrate - OpenAI.md+++ b/website/docs/Examples/Integrate - OpenAI.md@@ -173,11 +173,10 @@ print(success_metrics([response["text"].rstrip() for response in responses["choi#### Evaluate the success rate on the test data-You can use flaml's `oai.Completion.eval` to evaluate the performance of an entire dataset with the tuned config. To do that you need to set `oai.Completion.data` to the data to evaluate.+You can use flaml's `oai.Completion.test` to evaluate the performance of an entire dataset with the tuned config.```python-oai.Completion.data = test_data-result = oai.Completion.eval(analysis.best_config, prune=False, eval_only=True)+result = oai.Completion.test(test_data, config)print(result)```