DSPy: The framework for programming—not prompting—language models
Python
E2E notebook working - migrate to jobs + rayllm
Isaac Miller2094431a270f
diff --git a/dsp/deploy_dspy/async_llm.py b/dsp/deploy_dspy/async_llm.pyindex 75a16d72f..430f32ead 100644--- a/dsp/deploy_dspy/async_llm.py+++ b/dsp/deploy_dspy/async_llm.py@@ -2,7 +2,7 @@ from vllm import AsyncLLMEngine, SamplingParams, AsyncEngineArgs, RequestOutputfrom vllm.entrypoints.chat_utils import parse_chat_messagesfrom typing import List, Dict, Optional, Union-+import concurrent.futuresimport asynciotry:from vllm.inputs.data import TextTokensPrompt@@ -39,6 +39,8 @@ class AsyncLLMWrapper:self.max_pending_requests = max_pending_requestsself.tokenizer = tokenizerself.lock = asyncio.Lock()+ self.submit_lock = asyncio.Lock()+ self.engine_model_config = asyncio.run(self.engine.get_model_config())# vLLM performance gets really bad if there are too many requests in the pending queue.# We work around it by introducing another queue that gates how many requests we are@@ -59,8 +61,9 @@ class AsyncLLMWrapper:prompt = TextTokensPrompt(prompt=prompt, prompt_token_ids=prompt_token_ids)if self.max_pending_requests > 0:await self.free_queue.get()- # print("adding request with id", request_id)- stream = await self.engine.add_request(request_id=str(request_id), inputs=prompt, params=sampling_params, lora_request=lora_request)+ print("adding request with id", request_id)+ async with self.lock:+ stream = await self.engine.add_request(request_id=str(request_id), inputs=prompt, params=sampling_params, lora_request=lora_request)async for request_output in stream:if request_output.finished:if self.max_pending_requests > 0:@@ -76,18 +79,20 @@ class AsyncLLMWrapper:tasks = []for i, p in enumerate(prompt):- self.request_id += 1+ async with self.lock:+ self.request_id += 1if prompt_token_ids:pti = prompt_token_ids[i]p = TextTokensPrompt(prompt=p, prompt_token_ids=pti)else:assert p["prompt_token_ids"]pti = None- tasks.append(- asyncio.create_task(- self._process(self.request_id, p, pti, sampling_params, i, lora_request)+ async with self.lock:+ tasks.append(+ asyncio.create_task(+ self._process(self.request_id, p, pti, sampling_params, i, lora_request)+ ))- )return tasksasync def generate_async(self, prompt, sampling_params, prompt_token_ids):@@ -108,7 +113,7 @@ class AsyncLLMWrapper:add_generation_prompt: bool = True,) -> List[RequestOutput]:# async with self.lock:- model_config = await asyncio.wait_for(self.engine.get_model_config(), timeout=10)+ model_config = self.engine_model_configconversations= [parse_chat_messages(conversation, model_config,self.tokenizer)[0] for conversation in messages]prompts = [apply_chat_template(@@ -127,20 +132,20 @@ class AsyncLLMWrapper:return returnsdef chat(self, messages: List[List[Dict[str, str]]], sampling_params: Optional[Union[SamplingParams, List[SamplingParams]]] = None, use_tqdm: bool = True, lora_request = None, chat_template: Optional[str] = None, add_generation_prompt: bool = True):- if self.loop.is_running():- new_loop = asyncio.new_event_loop()- asyncio.set_event_loop(new_loop)- ret = new_loop.run_until_complete(- self.chat_async(messages, sampling_params, use_tqdm, lora_request, chat_template, add_generation_prompt)- )- new_loop.close()- asyncio.set_event_loop(self.loop)- return ret- else:- ret = self.loop.run_until_complete(- self.chat_async(messages, sampling_params, use_tqdm, lora_request, chat_template, add_generation_prompt)- )- return ret+ def run_in_new_loop():+ print("running in new loop")+ return asyncio.run(self.chat_async(messages, sampling_params, use_tqdm, lora_request, chat_template, add_generation_prompt))++ # if self.loop.is_running():+ # Run in a separate thread to avoid blocking the main loop+ with concurrent.futures.ThreadPoolExecutor() as executor:+ future = executor.submit(run_in_new_loop)+ return future.result()+ # else:+ # # If the loop is not running, we can run it directly+ # return self.loop.run_until_complete(+ # self.chat_async(messages, sampling_params, use_tqdm, lora_request, chat_template, add_generation_prompt)+ # )async def chat_single(self, messages: List[Dict[str, str]], sampling_params: Optional[Union[SamplingParams, List[SamplingParams]]] = None, use_tqdm: bool = True, lora_request = None, chat_template: Optional[str] = None, add_generation_prompt: bool = True):async with self.lock:diff --git a/dsp/modules/__init__.py b/dsp/modules/__init__.pyindex 2b0ad15bb..3bae18287 100644--- a/dsp/modules/__init__.py+++ b/dsp/modules/__init__.py@@ -30,5 +30,7 @@ from .snowflake import *from .tensorrt_llm import TensorRTModelfrom .trainable_anyscale import TrainableAnyscalefrom .vllm_engine import VLLMOfflineEngine+from .vllm_engine_2 import VLLMOfflineEngine2+from .vllm_engine_3 import VLLMOfflineEngine3from .watsonx import *from .you import Youdiff --git a/dsp/modules/gpt3.py b/dsp/modules/gpt3.pyindex 70fe65529..62d495692 100644--- a/dsp/modules/gpt3.py+++ b/dsp/modules/gpt3.py@@ -133,6 +133,7 @@ class GPT3(LM):if self.system_prompt:messages.insert(0, {"role": "system", "content": self.system_prompt})kwargs["messages"] = messages+ print("in basic_request", kwargs, "\n\n")kwargs = {"stringify_request": json.dumps(kwargs)}response = chat_request(**kwargs)diff --git a/dsp/modules/lm.py b/dsp/modules/lm.pyindex 927d32cff..3ab3e41cb 100644--- a/dsp/modules/lm.py+++ b/dsp/modules/lm.py@@ -85,8 +85,10 @@ class LM(ABC):printed.append((prompt, x))elif provider == "you.com":printed.append((prompt, x["response"]["answer"]))+ elif provider == "vllm_offline":+ printed.append((prompt, x["response"].outputs))elif provider == "vllm":- printed.append((prompt, x["response"][0].outputs))+ printed.append((prompt, x["response"]["choices"]))else:printed.append((prompt, x["response"]["choices"]))@@ -127,6 +129,8 @@ class LM(ABC):elif provider == "cloudflare":text = choices[0]elif provider == "vllm":+ text = choices[0]["message"]["content"]+ elif provider == "vllm_offline":text = choices[0].textelse:text = choices[0]["text"]diff --git a/dsp/modules/multi_openai.py b/dsp/modules/multi_openai.pyindex 0630ad323..72584728f 100644--- a/dsp/modules/multi_openai.py+++ b/dsp/modules/multi_openai.py@@ -161,7 +161,6 @@ class MultiOpenAI(LM):if self.system_prompt:messages.insert(0, {"role": "system", "content": self.system_prompt})kwargs["messages"] = messages- # print("in basic_request", kwargs["temperature"])kwargs = {"stringify_request": json.dumps(kwargs)}response = self.chat_request(**kwargs)diff --git a/dsp/modules/vllm_engine_2.py b/dsp/modules/vllm_engine_2.pynew file mode 100644index 000000000..a488cb9bf--- /dev/null+++ b/dsp/modules/vllm_engine_2.py@@ -0,0 +1,253 @@+import queue+import threading+from threading import Lock, Event, Thread+from dsp.modules.lm import LM+from dsp.modules.cache_utils import CacheMemory, NotebookCacheMemory, cache_turn_on+from dsp.utils.settings import settings+from vllm.entrypoints.chat_utils import parse_chat_messages++import uuid+import backoff+from typing import Any, Generator, List, Callable, Awaitable, AsyncGenerator, Generic, TypeVar+import functools+import asyncio+import contextlib+import random+import collections+import time+import ujson+from dsp.deploy_dspy.download import download_model+from dsp.deploy_dspy.async_llm import AsyncLLMWrapper+from transformers import AutoTokenizer+import msgspec+from vllm import AsyncLLMEngine, SamplingParams, AsyncEngineArgs, RequestOutput+from vllm.lora.request import LoRARequest+from asyncio import FIRST_COMPLETED, ensure_future+try:+ from vllm.entrypoints.chat_utils import apply_chat_template+except ImportError:+ from vllm.entrypoints.chat_utils import apply_hf_chat_template as apply_chat_template++try:+ from vllm.inputs.data import TextTokensPrompt, TokensPrompt+except ImportError:+ # API change since vLLM v0.5.3++ from vllm.entrypoints.openai.serving_engine import TextTokensPrompt+# except ImportError:+# print("vllm not installed")+T = TypeVar("T")+ERRORS = Exception+++def backoff_hdlr(details):+ """Handler from https://pypi.org/project/backoff/"""+ print(+ "Backing off {wait:0.1f} seconds after {tries} tries "+ "calling function {target} with kwargs "+ "{kwargs}".format(**details),+ )++class VLLMOfflineEngine2(LM):+ def __init__(+ self,+ llm,+ model_type: str = "chat",+ tokenizer: AutoTokenizer = None,+ batch_size: int = 2,+ **kwargs,+ ):+ super().__init__("vllm")+ self.provider = "vllm"+ self.llm = llm+ self.kwargs = {+ "temperature": 0.0,+ "max_tokens": 150,+ "top_p": 1,+ "frequency_penalty": 0,+ "presence_penalty": 0,+ "n": 1,+ "model": llm,+ **kwargs,+ }+ self.system_prompt = None+ self.model_type = model_type+ self.history = []+ self.batch_size = 256+ self.batch_timeout = 5+ self.tokenizer = tokenizer+ async def model_config_wrapper():+ return await llm.get_model_config()+ loop = asyncio.new_event_loop()+ asyncio.set_event_loop(loop)+ self.model_config = loop.run_until_complete(model_config_wrapper())+ loop.close()+++ # def _batch_request(self, prompts: List[str]) -> List[Any]:+ # messages = [[{"role": "user", "content": prompt}] for prompt in prompts]+ # if self.system_prompt:+ # for msg in messages:+ # msg.insert(0, {"role": "system", "content": self.system_prompt})++ # sampling_param_dict = {k: v for k, v in self.kwargs.items() if k in msgspec.structs.asdict(SamplingParams())}+ # sampling_params = SamplingParams(**sampling_param_dict)+ # unique_kwargs = {k: v for k, v in self.kwargs.items() if k not in sampling_param_dict}+ # unique_kwargs.pop("async_mode", None)+ # unique_kwargs.pop("model", None)+ # lora_request = LoRARequest("lora2", 1, "/home/ray/default/lora2")+ # with self.lock_of_async_defeat:+ # responses = chat_request(self.llm, messages, sampling_params, lora_request, **unique_kwargs)+ # return responses++ def basic_request(self, prompt, **kwargs):+ messages = [{"role": "user", "content": prompt}]+ if self.system_prompt:+ messages.insert(0, [{"role": "system", "content": self.system_prompt}])++ sampling_param_dict = {k: v for k, v in self.kwargs.items() if k in msgspec.structs.asdict(SamplingParams())}+ sampling_params = SamplingParams(**sampling_param_dict)+ unique_kwargs = {k: v for k, v in self.kwargs.items() if k not in sampling_param_dict}+ unique_kwargs.pop("async_mode", None)+ # print("sending prompt)+ y = chat_request(self.llm, self.tokenizer, messages, sampling_params, lora_request=None, use_tqdm=False, **unique_kwargs, model_config=self.model_config)++ return y++ def __call__(self, prompt, only_completed=True, return_sorted=False, **kwargs):+ assert only_completed, "for now"+ assert return_sorted is False, "for now"++ response = self.basic_request(prompt, **kwargs)+ # outputs = response[0][0]+ choices = response.outputs++ completed_choices = [c for c in choices if c.finish_reason != "length"]++ if only_completed and len(completed_choices):+ choices = completed_choices++ if kwargs.get("logprobs", False):+ completions = [{'text': c.text, 'logprobs': c.logprobs} for c in choices]+ else:+ completions = [c.text for c in choices]+ # decoded_output = self.tokenizer.decode(choices[0].token_ids)+ # print(decoded_output)++ return completions++ def shutdown(self):+ for _ in self.batch_threads:+ self.prompt_queue.put((None, None)) # Sentinel to stop threads+ for thread in self.batch_threads:+ thread.join()++ @classmethod+ def instantiate_with_llm(cls, model, tokenizer=None, engine_args={}, do_download_model=True, **kwargs):+ if do_download_model:+ model_path = download_model(model)+ else:+ model_path = model++ if tokenizer is None:+ tokenizer = AutoTokenizer.from_pretrained(model_path)++ engine_args = AsyncEngineArgs(+ model,+ **engine_args,+ disable_log_requests=True,+ )+ llm = AsyncLLMEngine.from_engine_args(engine_args)+ return cls(llm, tokenizer=tokenizer, **kwargs)++# @custom_lru_cache(maxsize=None if cache_turn_on else 0)+# @NotebookCacheMemory.cache+# @CacheMemory.cache+def chat_request(llm: AsyncLLMEngine, tokenizer, prompt, sampling_params, lora_request, **kwargs):+ model_config = kwargs.pop("model_config")++ prompt = parse_chat_messages(prompt, model_config, tokenizer)[0]+ templated_prompt = tokenizer.apply_chat_template(+ conversation=prompt, # type: ignore[arg-type]+ chat_template=None,+ tokenize=False,+ **kwargs,+ )+ prompt_token_ids = tokenizer.encode(templated_prompt)+ tokens_prompt = TextTokensPrompt(prompt_token_ids=prompt_token_ids)++ loop = asyncio.new_event_loop()+ asyncio.set_event_loop(loop)++ async def wrapper():+ return await chat_request_async(llm, tokens_prompt, sampling_params, lora_request, **kwargs)++ try:+ return loop.run_until_complete(wrapper())+ finally:+ loop.close()+++async def chat_request_async(engine: AsyncLLMEngine, prompt, sampling_params, lora_request, **kwargs):+ request_id = random_uuid()++ assert engine is not None+ results_generator = engine.generate(prompt, sampling_params, request_id)+ # results_generator = iterate_with_cancellation(+ # results_generator, is_cancelled=lambda: False)++ # Non-streaming case+ final_output = None+ try:+ async for request_output in results_generator:+ final_output = request_output+ except asyncio.CancelledError:+ return None++ assert final_output is not None+ # prompt = final_output.prompt+ # assert prompt is not None+ # print(final_output)+ return final_output++# @functools.lru_cache(maxsize=None if cache_turn_on else 0)+# @NotebookCacheMemory.cache+# @CacheMemory.cache+def completions_request(llm: AsyncLLMEngine, prompt, sampling_params, **kwargs):+ return llm.generate(prompt, sampling_params, use_tqdm=False, **kwargs)+++# if __name__ == "__main__":+# # model_path = download_model("meta-llama/Meta-Llama-3-70B-Instruct")++# import msgspec+# print(isinstance(SamplingParams(), msgspec.Struct))+# print(msgspec.structs.asdict(SamplingParams()))++async def iterate_with_cancellation(+ iterator: AsyncGenerator[T, None],+ is_cancelled: Callable[[], Awaitable[bool]],+) -> AsyncGenerator[T, None]:+ """Convert async iterator into one that polls the provided function+ at least once per second to check for client cancellation.+ """++ # Can use anext() in python >= 3.10+ awaits = [ensure_future(iterator.__anext__())]+ while True:+ done, pending = await asyncio.wait(awaits, timeout=1)+ if await is_cancelled():+ with contextlib.suppress(BaseException):+ awaits[0].cancel()+ await iterator.aclose()+ raise asyncio.CancelledError("client cancelled")+ if done:+ try:+ item = await awaits[0]+ awaits[0] = ensure_future(iterator.__anext__())+ yield item+ except StopAsyncIteration:+ # we are done+ return++def random_uuid() -> str:+ return str(uuid.uuid4().hex)\ No newline at end of filediff --git a/dsp/modules/vllm_engine_3.py b/dsp/modules/vllm_engine_3.pynew file mode 100644index 000000000..d2105c196--- /dev/null+++ b/dsp/modules/vllm_engine_3.py@@ -0,0 +1,248 @@+import queue+import threading+from threading import Lock, Event, Thread+from dsp.modules.lm import LM+from dsp.modules.cache_utils import CacheMemory, NotebookCacheMemory, cache_turn_on+from dsp.utils.settings import settings+import backoff+from typing import Any, Generator, List+import functools+import asyncio+import collections+import time+import ujson+from dsp.deploy_dspy.download import download_model+from dsp.deploy_dspy.async_llm import AsyncLLMWrapper+from transformers import AutoTokenizer+# try:+import msgspec+from vllm import LLM, SamplingParams # maybe need vllm 0.5.4+from vllm.lora.request import LoRARequest++# except ImportError:+# print("vllm not installed")++ERRORS = Exception+++def backoff_hdlr(details):+ """Handler from https://pypi.org/project/backoff/"""+ print(+ "Backing off {wait:0.1f} seconds after {tries} tries "+ "calling function {target} with kwargs "+ "{kwargs}".format(**details),+ )++class VLLMOfflineEngine3(LM):+ def __init__(self, llm, model_type: str = "chat", batch_size: int = 32, **kwargs):+ super().__init__("vllm")+ self.llm = llm+ self.provider = "vllm"+ self.kwargs = {+ "temperature": 0.0,+ "max_tokens": 150,+ "top_p": 1,+ "frequency_penalty": 0,+ "presence_penalty": 0,+ "n": 1,+ "model": llm,+ **kwargs,+ }+ self.system_prompt = None+ self.model_type = model_type+ self.history = []+ self.batch_size = batch_size+ self.batch_timeout = 10 # Shorter timeout for more responsive batching+ self.prompt_queue = queue.Queue()+ self.result_dict = {}+ self.queue_lock = Lock()+ self.result_set_lock = Lock()+ self.result_get_lock = Lock()+ self.lock_of_async_defeat = Lock()+ self.collector_lock = Lock()++ self.num_batch_threads = 2+ self.batch_threads = [+ threading.Thread(target=self._process_batches, args=(i,)) for i in range(self.num_batch_threads)+ ]+ for thread in self.batch_threads:+ thread.start()++ def _process_batches(self, batch_id):+ while True:+ batch = []+ print(f"Batch {batch_id} is waiting")+ with self.collector_lock:+ print(f"Batch {batch_id} is collecting")+ start_time = time.time()+ # Collect prompts for the batch+ while len(batch) < self.batch_size / self.num_batch_threads and (time.time() - start_time) < self.batch_timeout:+ try:+ prompt, thread_id = self.prompt_queue.get(timeout=self.batch_timeout - (time.time() - start_time))+ if prompt is None: # Shutdown signal+ return+ batch.append((prompt, thread_id))+ except queue.Empty:+ if batch: # Process the batch if we have any prompts+ pass+ # break+ time.sleep(0.01) # Short sleep if queue is empty++ if batch:+ print(f"Batch {batch_id} is processing {len(batch)} prompts")+ prompts, thread_ids = zip(*batch)+ responses = self._batch_request(prompts)+ print(f"Batch {batch_id} is done processing")+ with self.result_set_lock:+ for thread_id, response in zip(thread_ids, responses):+ self.result_dict[thread_id] = response++ def _batch_request(self, prompts: List[str]) -> List[Any]:+ messages = [[{"role": "user", "content": prompt}] for prompt in prompts]+ if self.system_prompt:+ for msg in messages:+ msg.insert(0, {"role": "system", "content": self.system_prompt})++ sampling_param_dict = {k: v for k, v in self.kwargs.items() if k in msgspec.structs.asdict(SamplingParams())}+ sampling_params = SamplingParams(**sampling_param_dict)+ unique_kwargs = {k: v for k, v in self.kwargs.items() if k not in sampling_param_dict}+ unique_kwargs.pop("async_mode", None)+ unique_kwargs.pop("model", None)+ # lora_request = LoRARequest("lora2", 1, "/home/ray/default/lora2")+ # with self.lock_of_async_defeat:+ lora_request = None+ responses = chat_request(self.llm, messages, sampling_params, lora_request, **unique_kwargs)+ return responses++ def basic_request(self, prompt, **kwargs):+ thread_id = id(prompt)++ with self.queue_lock:+ self.prompt_queue.put((prompt, thread_id))++ while True:+ with self.result_get_lock:+ if thread_id in self.result_dict:+ return self.result_dict.pop(thread_id)+ time.sleep(0.01) # Short sleep to prevent busy-waiting++ def __call__(self, prompt, only_completed=True, return_sorted=False, **kwargs):+ assert only_completed, "for now"+ assert return_sorted is False, "for now"++ response, _ = self.basic_request(prompt, **kwargs)+ # outputs = response[0][0]+ choices = response.outputs++ completed_choices = [c for c in choices if c.finish_reason != "length"]++ if only_completed and len(completed_choices):+ choices = completed_choices++ if kwargs.get("logprobs", False):+ completions = [{'text': c.text, 'logprobs': c.logprobs} for c in choices]+ else:+ completions = [c.text for c in choices]++ return completions++ def shutdown(self):+ for _ in self.batch_threads:+ self.prompt_queue.put((None, None)) # Shutdown signal+ for thread in self.batch_threads:+ thread.join()++ @classmethod+ def instantiate_with_llm(cls, model, tokenizer=None, engine_args={}, do_download_model=True, **kwargs):+ if do_download_model:+ model_path = download_model(model)+ else:+ model_path = model++ if tokenizer is None:+ tokenizer = AutoTokenizer.from_pretrained(model_path)+ llm = AsyncLLMWrapper(model_path,+ max_pending_requests=engine_args.pop("max_pending_requests", 512),+ tokenizer=tokenizer,+ **engine_args+ )+ return cls(llm, **kwargs)++def custom_lru_cache(maxsize=None):+ def decorator(func):+ @functools.lru_cache(maxsize=maxsize)+ def cached_single_prompt(llm_name: str, prompt: str, sampling_params, **kwargs):+ # Create a simple LLM-like object+ mock_llm = type('LLM', (), {'name': llm_name})()++ # Call the original function with a single-prompt list+ result = func(mock_llm, [prompt], sampling_params, **kwargs)++ # Return the first (and only) result+ return result[0]++ @functools.wraps(func)+ def wrapper(llm, prompts: List[str], sampling_params, **kwargs):+ llm_name = llm.name+ results = []+ uncached_prompts = []+ uncached_indices = []++ sampling_params_dict = msgspec.structs.asdict(sampling_params)+ for k, v in sampling_params_dict.items():+ if isinstance(v, set):+ sampling_params_dict[k] = list(v)+ sampling_params_str = ujson.dumps(sampling_params_dict)++ # Check cache for each prompt++ for i, prompt in enumerate(prompts):+ stringified_prompt = ujson.dumps(prompt)+ # try:+ # results.append(cached_single_prompt(llm_name, stringified_prompt, sampling_params_str, **kwargs))+ # except KeyError:+ # uncached_prompts.append(prompt)+ # uncached_indices.append(i)+ if (llm_name, prompt, sampling_params_str, kwargs) in cached_single_prompt.cache_info().cache:+ results.append(cached_single_prompt(llm_name, prompt, sampling_params_str, **kwargs))+ else:+ uncached_prompts.append(prompt)+ uncached_indices.append(i)++ # Process uncached prompts+ if uncached_prompts:+ uncached_results = func(llm, uncached_prompts, sampling_params, **kwargs)++ # Cache new results+ for prompt, result in zip(uncached_prompts, uncached_results):+ stringified_prompt = ujson.dumps(prompt)+ cached_single_prompt(llm_name, stringified_prompt, sampling_params_str, **kwargs)++ # Merge cached and new results+ for i, result in zip(uncached_indices, uncached_results):+ results.insert(i, result)++ return results+ return wrapper+ return decorator++# @custom_lru_cache(maxsize=None if cache_turn_on else 0)+# @NotebookCacheMemory.cache+# @CacheMemory.cache+def chat_request(llm: AsyncLLMWrapper, prompt, sampling_params, lora_request, **kwargs):+ x = llm.chat(prompt, sampling_params, use_tqdm=False, lora_request=lora_request, **kwargs)+ return x++# @functools.lru_cache(maxsize=None if cache_turn_on else 0)+# @NotebookCacheMemory.cache+# @CacheMemory.cache+def completions_request(llm: LLM, prompt, sampling_params, **kwargs):+ return llm.generate(prompt, sampling_params, use_tqdm=False, **kwargs)+++# if __name__ == "__main__":+# # model_path = download_model("meta-llama/Meta-Llama-3-70B-Instruct")++# import msgspec+# print(isinstance(SamplingParams(), msgspec.Struct))+# print(msgspec.structs.asdict(SamplingParams()))\ No newline at end of filediff --git a/dspy/__init__.py b/dspy/__init__.pyindex a0b1d4f41..9192a0dd7 100644--- a/dspy/__init__.py+++ b/dspy/__init__.py@@ -58,6 +58,8 @@ Watsonx = dsp.WatsonxPremAI = dsp.PremAIVLLMOfflineEngine = dsp.VLLMOfflineEngine+VLLMOfflineEngine2 = dsp.VLLMOfflineEngine2+VLLMOfflineEngine3 = dsp.VLLMOfflineEngine3You = dsp.Youconfigure = settings.configurediff --git a/examples/anyscale/e2e/basicmh_70b_MIPROv2_3_3.json b/examples/anyscale/e2e/basicmh_70b_MIPROv2_3_3.jsonnew file mode 100644index 000000000..1302e9c67--- /dev/null+++ b/examples/anyscale/e2e/basicmh_70b_MIPROv2_3_3.json@@ -0,0 +1,106 @@+{+ "retrieve": {+ "k": 3+ },+ "generate_query[0]": {+ "lm": null,+ "traces": [],+ "train": [],+ "demos": [+ {+ "augmented": true,+ "context": [],+ "question": "Between two documentary films Best Friend Forgotten and Mad Hot Ballroom, which film was released earlier?",+ "rationale": "find the search query. We can simply combine the information from the question to create a search query that will give us the answer.",+ "search_query": "Best Friend Forgotten release year vs Mad Hot Ballroom release year"+ },+ {+ "augmented": true,+ "context": [],+ "question": "Are Beth Gibbons and Ian Curtis the same nationality?",+ "rationale": "produce the output fields. We need to determine the nationality of Beth Gibbons and Ian Curtis, so we should search for their nationalities.",+ "search_query": "Beth Gibbons nationality AND Ian Curtis nationality"+ },+ {+ "question": "What other sport did Butch Reed, of the wrestling tag team Doom, play professionally?",+ "answer": "football",+ "dspy_uuid": "42b16ebd-c2c4-4e34-a4fb-19ac647caedf",+ "dspy_split": "train"+ }+ ],+ "signature_instructions": "Given the fields `context`, `question`, produce the fields `search_query`.",+ "signature_prefix": "Search Query:",+ "extended_signature_instructions": "Generate a concise and informative search query that captures the essence of the given question, by breaking down the question into smaller sub-problems, identifying the key entities and concepts involved, and formulating a query that can retrieve relevant information from a knowledge base, using the provided context as a reference point.",+ "extended_signature_prefix": "Search Query:"+ },+ "generate_query[1]": {+ "lm": null,+ "traces": [],+ "train": [],+ "demos": [+ {+ "augmented": true,+ "context": [+ "Lax\u2013Friedrichs method | The Lax\u2013Friedrichs method, named after Peter Lax and Kurt O. Friedrichs, is a numerical method for the solution of hyperbolic partial differential equations based on finite differences. The method can be described as the FTCS (forward in time, centered in space) scheme with an artificial viscosity term of 1\/2. One can view the Lax\u2013Friedrichs method as an alternative to Godunov's scheme, where one avoids solving a Riemann problem at each cell interface, at the expense of adding artificial viscosity.",+ "Lax\u2013Wendroff method | The Lax\u2013Wendroff method, named after Peter Lax and Burton Wendroff, is a numerical method for the solution of hyperbolic partial differential equations, based on finite differences. It is second-order accurate in both space and time. This method is an example of explicit time integration where the function that defines governing equation is evaluated at the current time.",+ "Lax pair | In mathematics, in the theory of integrable systems, a Lax pair is a pair of time-dependent matrices or operators that satisfy a corresponding differential equation, called the \"Lax equation\". Lax pairs were introduced by Peter Lax to discuss solitons in continuous media. The inverse scattering transform makes use of the Lax equations to solve such systems."+ ],+ "question": "What nationality was the Lax pair's namesake?",+ "rationale": "produce the output field. We are asked about the nationality of the Lax pair's namesake, Peter Lax. However, the context does not provide explicit information about Peter Lax's nationality. Therefore, we need to search for additional information outside of the provided context.",+ "search_query": "\"Peter Lax nationality\""+ },+ {+ "question": "What type of compilation does Chris Harris and Into the Labyrinth have in common?",+ "answer": "television series",+ "dspy_uuid": "f68c812a-0cc6-4b1c-8241-424a08550fe1",+ "dspy_split": "train"+ },+ {+ "question": "The father of the current manager of Maidenhead United appeared more than 100 times in what football league?",+ "answer": "The Football League for Chester and Crystal Palace",+ "dspy_uuid": "bbd4d27e-3f11-4326-896b-aba45134d922",+ "dspy_split": "train"+ }+ ],+ "signature_instructions": "Given the fields `context`, `question`, produce the fields `search_query`.",+ "signature_prefix": "Search Query:",+ "extended_signature_instructions": "Generate a specific and concise search query based on the given context and question, using the information provided to create a query that will retrieve relevant passages from a knowledge base to help answer the question, and provide a clear and logical reasoning process for the generated search query.",+ "extended_signature_prefix": "Search Query:"+ },+ "generate_answer": {+ "lm": null,+ "traces": [],+ "train": [],+ "demos": [+ {+ "augmented": true,+ "context": [+ "Kangchenjunga | Kangchenjunga (Nepali: \u0915\u091e\u094d\u091a\u0928\u091c\u0919\u094d\u0918\u093e , \"Ka\u00f1canaja\u1e45gh\u0101\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.",+ "Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.",+ "Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world's third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.",+ "Western Ghats | Western Ghats (also known as Sahyadri, meaning \"The Benevolent Mountains\") is a mountain range that runs parallel to the western coast of the Indian peninsula, located entirely in India. It is a UNESCO World Heritage Site and is one of the eight \"hottest hot-spots\" of biological diversity in the world. It is sometimes called the Great Escarpment of India. The range runs north to south along the western edge of the Deccan Plateau, and separates the plateau from a narrow coastal plain, called Konkan, along the Arabian Sea. A total of thirty-nine properties including national parks, wildlife sanctuaries and reserve forests were designated as world heritage sites - twenty in Kerala, ten in Karnataka, five in Tamil Nadu and four in Maharashtra.",+ "Nilgiri mountains | The Nilgiri (\"Blue Mountains\"), form part of the Western Ghats in western Tamil Nadu, Karnataka and Kerala states in Southern India. At least 24 of the Nilgiri mountains' peaks are above 2000 m , the highest peak being Doddabetta, at 2637 m ."+ ],+ "question": "The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?",+ "rationale": "find the country where both Kangchenjunga and Singhi Kangri are located. From context [1], we know that Kangchenjunga lies partly in Nepal and partly in India. From context [2], we know that Singhi Kangri is located on the border between China and the Siachen Glacier region, which is controlled by India but claimed by Pakistan. Since both mountains are partially in India, the answer is India.",+ "answer": "India"+ },+ {+ "question": "Which town in New Hampshire includes the villages of North Sanbornton and Gaza and is also home to the Bay Meeting House and Vestry?",+ "answer": "Sanbornton, New Hampshire",+ "dspy_uuid": "76b948f4-ceb9-4d42-89c2-8a2e35a515d9",+ "dspy_split": "train"+ },+ {+ "question": "In which competition held in Buenos Aires, Argentina did Stanly Stanczyk win a gold medal?",+ "answer": "The 1951 Pan American Games",+ "dspy_uuid": "5d4123bb-414b-422b-9bc3-3ea2100f4b0e",+ "dspy_split": "train"+ }+ ],+ "signature_instructions": "Given the fields `context`, `question`, produce the fields `answer`.",+ "signature_prefix": "Answer:",+ "extended_signature_instructions": "Given a context passage and a question, think step by step to find the answer, using the provided context to inform your response and generating a clear and concise rationale for your answer.",+ "extended_signature_prefix": "Answer:"+ }+}\ No newline at end of filediff --git a/examples/anyscale/e2e/basicmh_70b_bfrs_3_3_6.json b/examples/anyscale/e2e/basicmh_70b_bfrs_3_3_6.jsonnew file mode 100644index 000000000..54e1192d7--- /dev/null+++ b/examples/anyscale/e2e/basicmh_70b_bfrs_3_3_6.json@@ -0,0 +1,106 @@+{+ "retrieve": {+ "k": 3+ },+ "generate_query[0]": {+ "lm": null,+ "traces": [],+ "train": [],+ "demos": [+ {+ "augmented": true,+ "context": [],+ "question": "Where was the star of Ken Annakin's 1956 British comedy born?",+ "rationale": "produce the output fields. We need to extract the key information from the question, which is the director's name, the year, and the type of movie. This will help us formulate a search query that can provide the answer to the question.",+ "search_query": "Ken Annakin 1956 British comedy star birthplace"+ },+ {+ "question": "Who directed the 1988 film in which the exterior of the abandoned W.S. Kelly gas station and general store in Cogar can be seen?",+ "answer": "Barry Levinson",+ "dspy_uuid": "fcee690b-d7f1-4d93-a402-4fefc4728eb3",+ "dspy_split": "train"+ },+ {+ "question": "Which of the four operas in the Wagner works, based loosely on characters from the Norse sagas and the \"Nibelungenlied,\" is discography of \"G\u00f6tterd\u00e4mmerung\"?",+ "answer": "the fourth",+ "dspy_uuid": "c0181869-2203-47ec-b9a1-f0b59d825f9d",+ "dspy_split": "train"+ }+ ],+ "signature_instructions": "Given the fields `context`, `question`, produce the fields `search_query`.",+ "signature_prefix": "Search Query:",+ "extended_signature_instructions": "Given the fields `context`, `question`, produce the fields `search_query`.",+ "extended_signature_prefix": "Search Query:"+ },+ "generate_query[1]": {+ "lm": null,+ "traces": [],+ "train": [],+ "demos": [+ {+ "augmented": true,+ "context": [+ "Ken Annakin | Kenneth Cooper \"Ken\" Annakin, OBE (10 August 1914 \u2013 22 April 2009) was a prolific English film director.",+ "Loser Takes All (film) | Loser Takes All is a 1956 British comedy film directed by Ken Annakin, starring Glynis Johns, Rossano Brazzi, and Robert Morley, and based on a screenplay by Graham Greene.",+ "Three Men in a Boat (1956 film) | Three Men in a Boat is a 1956 British CinemaScope colour comedy film directed by Ken Annakin and starring Laurence Harvey, Jimmy Edwards, Shirley Eaton and David Tomlinson. It is based on the 1889 novel \"Three Men in a Boat\" by Jerome K. Jerome. The film received mixed reviews, but was a commercial success."+ ],+ "question": "Where was the star of Ken Annakin's 1956 British comedy born?",+ "rationale": "produce the output fields. We need to find the star of Ken Annakin's 1956 British comedy, which is mentioned in the context as \"Loser Takes All\". The star is Glynis Johns, and we need to find her birthplace.",+ "search_query": "Glynis Johns birthplace"+ },+ {+ "question": "Which of the four operas in the Wagner works, based loosely on characters from the Norse sagas and the \"Nibelungenlied,\" is discography of \"G\u00f6tterd\u00e4mmerung\"?",+ "answer": "the fourth",+ "dspy_uuid": "c0181869-2203-47ec-b9a1-f0b59d825f9d",+ "dspy_split": "train"+ },+ {+ "question": "Who directed the 1988 film in which the exterior of the abandoned W.S. Kelly gas station and general store in Cogar can be seen?",+ "answer": "Barry Levinson",+ "dspy_uuid": "fcee690b-d7f1-4d93-a402-4fefc4728eb3",+ "dspy_split": "train"+ }+ ],+ "signature_instructions": "Given the fields `context`, `question`, produce the fields `search_query`.",+ "signature_prefix": "Search Query:",+ "extended_signature_instructions": "Given the fields `context`, `question`, produce the fields `search_query`.",+ "extended_signature_prefix": "Search Query:"+ },+ "generate_answer": {+ "lm": null,+ "traces": [],+ "train": [],+ "demos": [+ {+ "augmented": true,+ "context": [+ "Ken Annakin | Kenneth Cooper \"Ken\" Annakin, OBE (10 August 1914 \u2013 22 April 2009) was a prolific English film director.",+ "Loser Takes All (film) | Loser Takes All is a 1956 British comedy film directed by Ken Annakin, starring Glynis Johns, Rossano Brazzi, and Robert Morley, and based on a screenplay by Graham Greene.",+ "Three Men in a Boat (1956 film) | Three Men in a Boat is a 1956 British CinemaScope colour comedy film directed by Ken Annakin and starring Laurence Harvey, Jimmy Edwards, Shirley Eaton and David Tomlinson. It is based on the 1889 novel \"Three Men in a Boat\" by Jerome K. Jerome. The film received mixed reviews, but was a commercial success.",+ "Glynis Johns | Glynis Johns (born 5 October 1923) is a retired Welsh stage, television and film actress, dancer, pianist, and singer. Born in Pretoria, South Africa of Welsh origin, she is best known for creating the role of Desiree Armfeldt in \"A Little Night Music\" on Broadway, for which she won a Tony Award, and for playing Winifred Banks in Walt Disney's musical motion picture box office smash \"Mary Poppins.\" In both roles, she originated songs written specifically for her, including \"Send in the Clowns\", composed by Stephen Sondheim, and \"Sister Suffragette\", written by the Sherman Brothers. She was nominated for an Oscar for her work in the 1960 film \"The Sundowners\".",+ "Glyn Johns | Glyn Thomas Johns (born 15 February 1942) is an English musician, recording engineer and record producer.",+ "Will Johns | Will Johns (born 22 March 1973, London) is an English blues rock singer and guitarist. Leading the Will Johns Band he has performed with many renowned artists such as Joe Strummer, Ronnie Wood, Jack Bruce and Bill Wyman. Will Johns is the son of record producer Andy Johns and nephew to Glyn Johns."+ ],+ "question": "Where was the star of Ken Annakin's 1956 British comedy born?",+ "rationale": "find the birthplace of the star of Ken Annakin's 1956 British comedy. From the context, we know that the 1956 British comedy is \"Loser Takes All\" and the star is Glynis Johns. From the context, we also know that Glynis Johns was born in Pretoria, South Africa.",+ "answer": "Pretoria, South Africa"+ },+ {+ "question": "Who directed the 1988 film in which the exterior of the abandoned W.S. Kelly gas station and general store in Cogar can be seen?",+ "answer": "Barry Levinson",+ "dspy_uuid": "fcee690b-d7f1-4d93-a402-4fefc4728eb3",+ "dspy_split": "train"+ },+ {+ "question": "Which of the four operas in the Wagner works, based loosely on characters from the Norse sagas and the \"Nibelungenlied,\" is discography of \"G\u00f6tterd\u00e4mmerung\"?",+ "answer": "the fourth",+ "dspy_uuid": "c0181869-2203-47ec-b9a1-f0b59d825f9d",+ "dspy_split": "train"+ }+ ],+ "signature_instructions": "Given the fields `context`, `question`, produce the fields `answer`.",+ "signature_prefix": "Answer:",+ "extended_signature_instructions": "Given the fields `context`, `question`, produce the fields `answer`.",+ "extended_signature_prefix": "Answer:"+ }+}\ No newline at end of filediff --git a/examples/anyscale/e2e/basicmh_8b_bfrs_3_3_6.json b/examples/anyscale/e2e/basicmh_8b_bfrs_3_3_6.jsonnew file mode 100644index 000000000..8accf26a3--- /dev/null+++ b/examples/anyscale/e2e/basicmh_8b_bfrs_3_3_6.json@@ -0,0 +1,104 @@+{+ "retrieve": {+ "k": 3+ },+ "generate_query[0]": {+ "lm": null,+ "traces": [],+ "train": [],+ "demos": [+ {+ "augmented": true,+ "context": [],+ "question": "Out of Central Nebraska Regional Airport and Chippewa County International Airport, which one is more west in the USA?",+ "rationale": "produce the output fields. We need to identify the locations of the two airports and determine which one is more west in the USA.",+ "search_query": "\"Central Nebraska Regional Airport vs Chippewa County International Airport which one is more west in the USA\""+ },+ {+ "question": "what revolution led to the overthrow of King James II of England and its news led to overthrow of androws in the Boston Revolt",+ "answer": "The Glorious Revolution,",+ "dspy_uuid": "a9a454c3-3722-4ea3-8109-3a20ab9c1752",+ "dspy_split": "train"+ },+ {+ "question": "In how many countries is the meat substitute product which has the same name with where Jedd Hughes was born available? ",+ "answer": "19 countries",+ "dspy_uuid": "27a6f9ac-4e6a-46fa-9d21-11d7994885a7",+ "dspy_split": "train"+ }+ ],+ "signature_instructions": "Given the fields `context`, `question`, produce the fields `search_query`.",+ "signature_prefix": "Search Query:",+ "extended_signature_instructions": "Given the fields `context`, `question`, produce the fields `search_query`.",+ "extended_signature_prefix": "Search Query:"+ },+ "generate_query[1]": {+ "lm": null,+ "traces": [],+ "train": [],+ "demos": [+ {+ "augmented": true,+ "context": [+ "Central Nebraska Regional Airport | Central Nebraska Regional Airport (IATA: GRI, ICAO: KGRI, FAA LID: GRI) is three miles northeast of Grand Island, in Hall County, Nebraska. It is owned by the Hall County Airport Authority. The airport sees two airlines, subsidized by the Essential Air Service program.",+ "Chippewa Valley Regional Airport | Chippewa Valley Regional Airport (IATA: KEAU, ICAO: EAU) is a public use airport in Chippewa County, Wisconsin, United States. The airport is owned by Eau Claire county and is located three nautical miles (6 km) north of the central business district of the city of Eau Claire, Wisconsin.",+ "Central Illinois Regional Airport | Central Illinois Regional Airport at Bloomington-Normal (IATA: KBMI, ICAO: BMI) is a public airport in McLean County, Illinois, three miles east of Bloomington and southeast of Normal. Owned by the Bloomington-Normal Airport Authority, it is also known as Central Illinois Regional Airport (CIRA)."+ ],+ "question": "Out of Central Nebraska Regional Airport and Chippewa County International Airport, which one is more west in the USA?",+ "rationale": "produce the output fields. We need to identify the locations of Central Nebraska Regional Airport and Chippewa Valley Regional Airport, and then determine which one is more west in the USA.",+ "search_query": "(\"Central Nebraska Regional Airport\" OR \"Chippewa Valley Regional Airport\") AND (\"USA\" OR \"United States\") AND (\"west\" OR \"western\")"+ },+ {+ "question": "In how many countries is the meat substitute product which has the same name with where Jedd Hughes was born available? ",+ "answer": "19 countries",+ "dspy_uuid": "27a6f9ac-4e6a-46fa-9d21-11d7994885a7",+ "dspy_split": "train"+ },+ {+ "question": "what revolution led to the overthrow of King James II of England and its news led to overthrow of androws in the Boston Revolt",+ "answer": "The Glorious Revolution,",+ "dspy_uuid": "a9a454c3-3722-4ea3-8109-3a20ab9c1752",+ "dspy_split": "train"+ }+ ],+ "signature_instructions": "Given the fields `context`, `question`, produce the fields `search_query`.",+ "signature_prefix": "Search Query:",+ "extended_signature_instructions": "Given the fields `context`, `question`, produce the fields `search_query`.",+ "extended_signature_prefix": "Search Query:"+ },+ "generate_answer": {+ "lm": null,+ "traces": [],+ "train": [],+ "demos": [+ {+ "augmented": true,+ "context": [+ "Central Nebraska Regional Airport | Central Nebraska Regional Airport (IATA: GRI, ICAO: KGRI, FAA LID: GRI) is three miles northeast of Grand Island, in Hall County, Nebraska. It is owned by the Hall County Airport Authority. The airport sees two airlines, subsidized by the Essential Air Service program.",+ "Chippewa Valley Regional Airport | Chippewa Valley Regional Airport (IATA: KEAU, ICAO: EAU) is a public use airport in Chippewa County, Wisconsin, United States. The airport is owned by Eau Claire county and is located three nautical miles (6 km) north of the central business district of the city of Eau Claire, Wisconsin.",+ "Central Illinois Regional Airport | Central Illinois Regional Airport at Bloomington-Normal (IATA: KBMI, ICAO: BMI) is a public airport in McLean County, Illinois, three miles east of Bloomington and southeast of Normal. Owned by the Bloomington-Normal Airport Authority, it is also known as Central Illinois Regional Airport (CIRA).",+ "Hudson Valley Regional Airport | Hudson Valley Regional Airport (IATA: POU, ICAO: KPOU, FAA LID: POU) , formerly known as the Dutchess County Airport, is a county-owned public-use airport located on State Route 376 in the Town of Wappinger, Dutchess County, New York, United States, four miles (6 km) south of the central business district of Poughkeepsie. It is sometimes called Poughkeepsie Airport, which gives it the code \"POU\". The airport provides corporate and general aviation transportation services."+ ],+ "question": "Out of Central Nebraska Regional Airport and Chippewa County International Airport, which one is more west in the USA?",+ "rationale": "produce the output fields. We need to compare the locations of Central Nebraska Regional Airport and Chippewa Valley Regional Airport. We can do this by looking at their coordinates or their geographical locations. According to the context, Central Nebraska Regional Airport is located in Hall County, Nebraska, while Chippewa Valley Regional Airport is located in Chippewa County, Wisconsin. Since Nebraska is generally west of Wisconsin, we can conclude that Central Nebraska Regional Airport is more west in the USA.",+ "answer": "Central Nebraska Regional Airport"+ },+ {+ "question": "what revolution led to the overthrow of King James II of England and its news led to overthrow of androws in the Boston Revolt",+ "answer": "The Glorious Revolution,",+ "dspy_uuid": "a9a454c3-3722-4ea3-8109-3a20ab9c1752",+ "dspy_split": "train"+ },+ {+ "question": "In how many countries is the meat substitute product which has the same name with where Jedd Hughes was born available? ",+ "answer": "19 countries",+ "dspy_uuid": "27a6f9ac-4e6a-46fa-9d21-11d7994885a7",+ "dspy_split": "train"+ }+ ],+ "signature_instructions": "Given the fields `context`, `question`, produce the fields `answer`.",+ "signature_prefix": "Answer:",+ "extended_signature_instructions": "Given the fields `context`, `question`, produce the fields `answer`.",+ "extended_signature_prefix": "Answer:"+ }+}\ No newline at end of filediff --git a/examples/anyscale/e2e/basicmh_8b_ft_MIPROv2_3_3.json b/examples/anyscale/e2e/basicmh_8b_ft_MIPROv2_3_3.jsonnew file mode 100644index 000000000..2d94feae7--- /dev/null+++ b/examples/anyscale/e2e/basicmh_8b_ft_MIPROv2_3_3.json@@ -0,0 +1,94 @@+{+ "retrieve": {+ "k": 3+ },+ "generate_query[0]": {+ "lm": null,+ "traces": [],+ "train": [],+ "demos": [],+ "signature_instructions": "Given the fields `context`, `question`, produce the fields `search_query`.",+ "signature_prefix": "Search Query:",+ "extended_signature_instructions": "Generate a search query that accurately captures the essence of the question within the given context, leveraging specific keywords and phrases to guide the retrieval of relevant passages.",+ "extended_signature_prefix": "Search Query:"+ },+ "generate_query[1]": {+ "lm": null,+ "traces": [],+ "train": [],+ "demos": [+ {+ "augmented": true,+ "context": [+ "Will Finn | Will Finn (born November 1, 1958) is an American animator, voice actor, and director. His work in animation includes characters from Disney and Don Bluth films such as \"The Secret of NIMH\", \"Oliver & Company\", \"The Little Mermaid\", \"The Rescuers Down Under\", and \"Pocahontas\". His characters includes Cogsworth in \"Beauty and the Beast\", Iago in \"Aladdin\", and Laverne in \"The Hunchback of Notre Dame\". Finn wrote and directed \"Home on the Range\" and did some voice acting the Hollywood Fish in \"Chicken Little\". In 2006, Finn directed the computer animated short Hammy's Boomerang Adventure, a spin-off of \"Over the Hedge\".",+ "Over the Hedge (film) | Over the Hedge is a 2006 American computer-animated comedy film, based on the characters from the United Media comic strip of the same name. Directed by Tim Johnson and Karey Kirkpatrick, and produced by Bonnie Arnold, it was released in the United States on May 19, 2006. The film was produced by DreamWorks Animation and distributed through Paramount Pictures. The film features the voices of Bruce Willis, Garry Shandling, Steve Carell, William Shatner, Wanda Sykes, and Nick Nolte. It is the first DreamWorks Animation film to be distributed by Paramount Pictures, which acquired the live-action DreamWorks studio in 2006. The film earned $336 million on an $80 million budget.",+ "Tim Johnson (film director) | Tim Johnson (born August 27, 1961) is an American animation director. Johnson has directed in many films such as \"Antz\", \"\" and \"Over the Hedge\". Recently, he directed the DreamWorks 3D film \"Home.\""+ ],+ "question": "When was the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed?",+ "rationale": "find the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed. The context mentions that Will Finn directed the computer animated short Hammy's Boomerang Adventure, a spin-off of \"Over the Hedge\".",+ "search_query": "Hammy's Boomerang Adventure release date"+ },+ {+ "question": "The Equestrian statue of Marshal Mannerheim was erected as a monument to the leader of the Whites during what event?",+ "answer": "Finnish Civil War",+ "dspy_uuid": "1c13ea97-77d8-4308-ad92-bf1df78141d6",+ "dspy_split": "train"+ },+ {+ "question": "What is the capacity of the La Liga stadium in which The Madrid Derby team plays their home games?",+ "answer": "68,000",+ "dspy_uuid": "782f06c6-2abc-422b-aeec-9d9dd0883882",+ "dspy_split": "train"+ }+ ],+ "signature_instructions": "Given the fields `context`, `question`, produce the fields `search_query`.",+ "signature_prefix": "Search Query:",+ "extended_signature_instructions": "Generate a search query that accurately captures the essence of the question and enables the retrieval of relevant information from the dataset, taking into account the context provided, the question being asked, and the relationship between the two.",+ "extended_signature_prefix": "Search Query:"+ },+ "generate_answer": {+ "lm": null,+ "traces": [],+ "train": [],+ "demos": [+ {+ "augmented": true,+ "context": [+ "Forrest Tucker | Forrest Meredith Tucker (February 12, 1919 \u2013 October 25, 1986) was an American actor in both movies and television who appeared in nearly a hundred films. Tucker worked as a vaudeville straight man, aged fifteen years old. A mentor provided funds and contacts for a trip to California, where party hostess Cobina Wright persuaded guest Wesley Ruggles to give Tucker a screen test, based on his photogenic good looks, thick wavy hair and height of six feet, five inches - putting him into the same league as John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness's younger brother. Tucker was a sight reader who needed only one take. His film career started well despite a perception in most Hollywood studios that blond men were not photogenic. After twenty years spent mainly in Westerns and action roles, he returned to his roots, showing versatility as a comedic and stage musical actor. In the TV show \"F Troop\", he became identified with the character of Cavalry Sgt. Morgan O'Rourke (a manipulative character quite similar to Phil Silvers' role as MSgt Ernie Bilko). Tucker struggled with a drinking problem that began to affect his performances in the later years of his career.",+ "Emergency Landing (1941 film) | Emergency Landing (a.k.a. Robot Pilot) is a 1941 American aviation spy-fi romantic screwball comedy film directed by William Beaudine. The film stars Forrest Tucker in his second film and in his first leading role with co-stars Carol Hughes and Evelyn Brent. \"Emergency Landing\" features lots of mismatched stock footage of various types of aircraft.",+ "Forrest Tucker (criminal) | Forrest Silva \"Woody\" Tucker (June 23, 1920 \u2013 May 29, 2004) was an American career criminal first imprisoned at age 15 who spent the rest of his life in and out of jail. He is best known as an escape artist, having escaped from prison 18 times successfully and 12 times unsuccessfully.",+ "Fess Parker | Fess Elisha Parker Jr. (August 16, 1924 \u2013 March 18, 2010) was an American film and television actor best known for his portrayals of Davy Crockett in the Walt Disney 1955\u20131956 TV miniseries and as Daniel Boone in a television series from 1964 to 1970. He was also known as a winemaker and resort owner-operator.",+ "Davy Crockett, King of the Wild Frontier | Davy Crockett, King of the Wild Frontier is a 1955 live-action Walt Disney adventure film starring Fess Parker as Davy Crockett. This film is an edited compilation of the first three stories from the Disney television miniseries \"Davy Crockett\" :",+ "Davy Crockett (miniseries) | Davy Crockett is a five-part serial which aired on ABC from 1954\u20131955 in one-hour episodes, on the \"Disneyland\" series. The series stars Fess Parker as real-life frontiersman Davy Crockett and Buddy Ebsen as his friend, George Russel."+ ],+ "question": "Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?",+ "rationale": "We need to find the actor who played the character in the 1955-1956 Walt Disney TV miniseries. From the context, we know that the miniseries was about Davy Crockett and starred Fess Parker as Davy Crockett.",+ "answer": "Davy Crockett"+ },+ {+ "augmented": true,+ "context": [+ "Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.",+ "Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford's second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.",+ "G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.",+ "College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour",+ "Bass fishing | Bass fishing is the activity of angling for the North American gamefish known colloquially as the \"black bass\". There are numerous black bass species considered as gamefish in North America, including largemouth bass (\"Micropterus salmoides\"), smallmouth bass (\"Micropterus dolomieui\"), spotted bass or Kentucky bass (\"Micropterus punctulatus\"), and Guadalupe bass (order \"Perciformes\").",+ "Cliff Crochet | Cliff Crochet (born August 1, 1983), also known as \"The Cajun Baby\", is a professional bass fisherman from Pierre Part, Louisiana. Crochet is a competitor on Bass Angler Sportsman Society's (B.A.S.S.) Bassmaster Elite Series and has $368,837 in B.A.S.S. tournament winnings as of August 22, 2016. Crochet's sponsors include: K2 Coolers, Huk, Skeeter, Yamaha Motor Company, Lowrance Electronics, Falcon Rods, Luck E Strike, Seaguar, Rat-L-Trap, Humdinger Spinnerbaits, Power-Pole. Since 2010, Crochet has competed in three Bassmaster Classic tournaments. Crochet's boat for the 2016 season is a 2016 Skeeter FX 21 with a 250 hp Yamaha SHO outboard engine."+ ],+ "question": "Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?",+ "rationale": "Context: [1] \u00abCarhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.\u00bb\n\n[2] \u00abCollege bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour\u00bb\n\n[3] \u00abG.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.\u00bb\n\nReasoning: Let's think step by step in order to",+ "answer": "Carhartt"+ },+ {+ "question": "How many years did Lenny Lyles spend with the organization dramatized in the filme \"The insder\"?",+ "answer": "27",+ "dspy_uuid": "a971d89d-a4a5-4ded-96d3-d095a636fae4",+ "dspy_split": "train"+ }+ ],+ "signature_instructions": "Given the fields `context`, `question`, produce the fields `answer`.",+ "signature_prefix": "Answer:",+ "extended_signature_instructions": "Given a context and a question, generate a logical and coherent answer that accurately addresses the query by meticulously analyzing the provided context, identifying relevant information, and applying critical thinking to provide a precise and informative response.",+ "extended_signature_prefix": "Answer:"+ }+}\ No newline at end of filediff --git a/examples/anyscale/e2e/basicmh_8b_ft_bfrs_3_3_6.json b/examples/anyscale/e2e/basicmh_8b_ft_bfrs_3_3_6.jsonnew file mode 100644index 000000000..90e06334b--- /dev/null+++ b/examples/anyscale/e2e/basicmh_8b_ft_bfrs_3_3_6.json@@ -0,0 +1,35 @@+{+ "retrieve": {+ "k": 3+ },+ "generate_query[0]": {+ "lm": null,+ "traces": [],+ "train": [],+ "demos": [],+ "signature_instructions": "Given the fields `context`, `question`, produce the fields `search_query`.",+ "signature_prefix": "Search Query:",+ "extended_signature_instructions": "Given the fields `context`, `question`, produce the fields `search_query`.",+ "extended_signature_prefix": "Search Query:"+ },+ "generate_query[1]": {+ "lm": null,+ "traces": [],+ "train": [],+ "demos": [],+ "signature_instructions": "Given the fields `context`, `question`, produce the fields `search_query`.",+ "signature_prefix": "Search Query:",+ "extended_signature_instructions": "Given the fields `context`, `question`, produce the fields `search_query`.",+ "extended_signature_prefix": "Search Query:"+ },+ "generate_answer": {+ "lm": null,+ "traces": [],+ "train": [],+ "demos": [],+ "signature_instructions": "Given the fields `context`, `question`, produce the fields `answer`.",+ "signature_prefix": "Answer:",+ "extended_signature_instructions": "Given the fields `context`, `question`, produce the fields `answer`.",+ "extended_signature_prefix": "Answer:"+ }+}\ No newline at end of filediff --git a/examples/anyscale/configs/deepspeed/zero_3.json b/examples/anyscale/e2e/configs/deepspeed/zero_3.jsonsimilarity index 100%rename from examples/anyscale/configs/deepspeed/zero_3.jsonrename to examples/anyscale/e2e/configs/deepspeed/zero_3.jsondiff --git a/examples/anyscale/configs/deepspeed/zero_3_offload_optim+param.json b/examples/anyscale/e2e/configs/deepspeed/zero_3_offload_optim+param.jsonsimilarity index 100%rename from examples/anyscale/configs/deepspeed/zero_3_offload_optim+param.jsonrename to examples/anyscale/e2e/configs/deepspeed/zero_3_offload_optim+param.jsondiff --git a/examples/anyscale/configs/deepspeed/zero_3_offload_optim.json b/examples/anyscale/e2e/configs/deepspeed/zero_3_offload_optim.jsonsimilarity index 100%rename from examples/anyscale/configs/deepspeed/zero_3_offload_optim.jsonrename to examples/anyscale/e2e/configs/deepspeed/zero_3_offload_optim.jsondiff --git a/examples/anyscale/configs/training/full_param/llama-3-70b.yaml b/examples/anyscale/e2e/configs/training/full_param/llama-3-70b.yamlsimilarity index 100%rename from examples/anyscale/configs/training/full_param/llama-3-70b.yamlrename to examples/anyscale/e2e/configs/training/full_param/llama-3-70b.yamldiff --git a/examples/anyscale/configs/training/full_param/llama-3-8b.yaml b/examples/anyscale/e2e/configs/training/full_param/llama-3-8b.yamlsimilarity index 100%rename from examples/anyscale/configs/training/full_param/llama-3-8b.yamlrename to examples/anyscale/e2e/configs/training/full_param/llama-3-8b.yamldiff --git a/examples/anyscale/configs/training/full_param/mistral-7b.yaml b/examples/anyscale/e2e/configs/training/full_param/mistral-7b.yamlsimilarity index 100%rename from examples/anyscale/configs/training/full_param/mistral-7b.yamlrename to examples/anyscale/e2e/configs/training/full_param/mistral-7b.yamldiff --git a/examples/anyscale/configs/training/full_param/mixtral-8x7b.yaml b/examples/anyscale/e2e/configs/training/full_param/mixtral-8x7b.yamlsimilarity index 100%rename from examples/anyscale/configs/training/full_param/mixtral-8x7b.yamlrename to examples/anyscale/e2e/configs/training/full_param/mixtral-8x7b.yamldiff --git a/examples/anyscale/configs/training/lora/llama-3-70b.yaml b/examples/anyscale/e2e/configs/training/lora/llama-3-70b.yamlsimilarity index 100%rename from examples/anyscale/configs/training/lora/llama-3-70b.yamlrename to examples/anyscale/e2e/configs/training/lora/llama-3-70b.yamldiff --git a/examples/anyscale/configs/training/lora/llama-3-8b.yaml b/examples/anyscale/e2e/configs/training/lora/llama-3-8b.yamlsimilarity index 100%rename from examples/anyscale/configs/training/lora/llama-3-8b.yamlrename to examples/anyscale/e2e/configs/training/lora/llama-3-8b.yamldiff --git a/examples/anyscale/configs/training/lora/mistral-7b.yaml b/examples/anyscale/e2e/configs/training/lora/mistral-7b.yamlsimilarity index 100%rename from examples/anyscale/configs/training/lora/mistral-7b.yamlrename to examples/anyscale/e2e/configs/training/lora/mistral-7b.yamldiff --git a/examples/anyscale/configs/training/lora/mixtral-8x7b.yaml b/examples/anyscale/e2e/configs/training/lora/mixtral-8x7b.yamlsimilarity index 100%rename from examples/anyscale/configs/training/lora/mixtral-8x7b.yamlrename to examples/anyscale/e2e/configs/training/lora/mixtral-8x7b.yamldiff --git a/examples/anyscale/e2e/e2e_dspy.ipynb b/examples/anyscale/e2e/e2e_dspy.ipynbnew file mode 100644index 000000000..e59704c44--- /dev/null+++ b/examples/anyscale/e2e/e2e_dspy.ipynb@@ -0,0 +1,9361 @@+{+ "cells": [+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "# End-to-end DSPy Workflows Guide "+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "This guide will cover the following topics:\n",+ "\n",+ "## Creating a Multi-stage LLM Pipeline\n",+ "- Building a pipeline with an untuned model in DSPy\n",+ "- Implementing batch inference (using Ray data)\n",+ "\n",+ "## Improving the Pipeline\n",+ "1. Prompt optimization\n",+ "2. Fine-tuning\n",+ " - How to make an 8B model perform almost as well as a 70B model in your pipeline\n",+ "3. Combining fine-tuning with prompt optimization\n",+ "\n",+ "## Deployment\n",+ "- Steps to deploy the optimized pipeline and fine-tuned model to production\n",+ "\n",+ "## Future Work and Open Questions\n",+ "- Efficient batch inference with a DSPy pipeline\n",+ "- Exploring different fine-tuning methods and hyperparameter sweeps\n",+ "\n",+ "This guide aims to provide a comprehensive overview of building, optimizing, and deploying LLM pipelines using DSPy and Anyscale."+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "## Set up"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "Node Set up:\n",+ "\n",+ "We will be running everything on a head node that uses 4xA100-80GB GPUs. I find that L4s are usually available and suitable for this usecase. You can also use any more powerful node.\n",+ "\n",+ "To change to use A100 GPUs, click the \"1 active node\" in the top right corner, then for workspace node, click the pencil icon and navigate to the A100 tab and select the 4xA100 option. If you do not see A100 in the list of GPUs, they may not be available on your cloud. Choose another kind of GPU (This notebook has been tested on X, and Y as alternatives) (TODO)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 1,+ "metadata": {},+ "outputs": [],+ "source": [+ "%load_ext autoreload\n",+ "%autoreload 2"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 2,+ "metadata": {},+ "outputs": [],+ "source": [+ "# TODO(work): DSPy installation cell\n",+ "# TODO(decision): are these changes going to be merged into DSPy main\n",+ "\n",+ "# TODO: look at my own init file to see all the stupid extra pip installs\n",+ "\n",+ "# !pip install -e dspy-d\n",+ "# !pip install -r dspy-d/requirements.txt\n",+ "# !pip install vllm\n",+ "\n",+ "# ignore future warnings\n",+ "import warnings\n",+ "warnings.filterwarnings(\"ignore\", category=FutureWarning)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 3,+ "metadata": {},+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "WARNING 09-24 09:57:06 cuda.py:22] You are using a deprecated `pynvml` package. Please install `nvidia-ml-py` instead, and make sure to uninstall `pynvml`. When both of them are installed, `pynvml` will take precedence and cause errors. See https://pypi.org/project/pynvml for more information.\n",+ "unable to import apply_chat_template\n"+ ]+ },+ {+ "data": {+ "text/plain": [+ "True"+ ]+ },+ "execution_count": 3,+ "metadata": {},+ "output_type": "execute_result"+ }+ ],+ "source": [+ "import dspy\n",+ "import dsp\n",+ "import os\n",+ "from dotenv import load_dotenv\n",+ "# TODO: include cache in notebook\n",+ "cache_dir = \"/home/ray/default/dspy/cache\"\n",+ "if not os.path.exists(cache_dir):\n",+ " os.makedirs(cache_dir)\n",+ "# I have included a .env.example with the necessary environment variables to be set\n",+ "# You can also set them manually if you prefer\n",+ "\n",+ "os.environ[\"DSP_CACHEDIR\"] = cache_dir\n",+ "\n",+ "load_dotenv()"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 4,+ "metadata": {},+ "outputs": [],+ "source": [+ "necessary_env_vars = [\n",+ " \"DSP_CACHEDIR\",\n",+ " \"HF_TOKEN\",\n",+ " \"HF_HOME\"\n",+ "]\n",+ "\n",+ "for var in necessary_env_vars:\n",+ " assert os.environ[var], f\"{var} is not set\""+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 5,+ "metadata": {},+ "outputs": [+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "2024-09-24 09:57:10,536\tINFO worker.py:1603 -- Connecting to existing Ray cluster at address: 10.0.15.220:6379...\n",+ "2024-09-24 09:57:10,544\tINFO worker.py:1779 -- Connected to Ray cluster. View the dashboard at https://session-fkvdirx4bzefi53sjl55m7asad.i.anyscaleuserdata.com \n",+ "2024-09-24 09:57:10,566\tINFO packaging.py:530 -- Creating a file package for local directory '/home/ray/default/dspy/dspy'.\n",+ "2024-09-24 09:57:10,587\tINFO packaging.py:358 -- Pushing file package 'gcs://_ray_pkg_8ace7ac78ed16883.zip' (0.78MiB) to Ray cluster...\n",+ "2024-09-24 09:57:10,591\tINFO packaging.py:371 -- Successfully pushed file package 'gcs://_ray_pkg_8ace7ac78ed16883.zip'.\n",+ "2024-09-24 09:57:10,603\tINFO packaging.py:530 -- Creating a file package for local directory '/home/ray/default/dspy/dsp'.\n",+ "2024-09-24 09:57:10,622\tINFO packaging.py:358 -- Pushing file package 'gcs://_ray_pkg_956d9051ac54fdb1.zip' (0.86MiB) to Ray cluster...\n",+ "2024-09-24 09:57:10,625\tINFO packaging.py:371 -- Successfully pushed file package 'gcs://_ray_pkg_956d9051ac54fdb1.zip'.\n",+ "2024-09-24 09:57:10,628\tINFO packaging.py:358 -- Pushing file package 'gcs://_ray_pkg_4b07a158880c2a5c4ab67b90b0f259e5bcb2b7e2.zip' (1.37MiB) to Ray cluster...\n",+ "2024-09-24 09:57:10,633\tINFO packaging.py:371 -- Successfully pushed file package 'gcs://_ray_pkg_4b07a158880c2a5c4ab67b90b0f259e5bcb2b7e2.zip'.\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "(autoscaler +1m41s) Tip: use `ray status` to view detailed cluster status. To disable these messages, set RAY_SCHEDULER_EVENTS=0.\n"+ ]+ }+ ],+ "source": [+ "import ray\n",+ "\n",+ "if not ray.is_initialized():\n",+ " ray.init(runtime_env={\"env_vars\": os.environ, \"py_modules\": [dspy, dsp]})"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "We will make use of a random number generator in this notebook. We are creating a Random object here to ensure that our notebook is reproducible."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 6,+ "metadata": {},+ "outputs": [],+ "source": [+ "import random\n",+ "\n",+ "rng = random.Random()"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "# Creating your multi-stage LLM pipeline"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 7,+ "metadata": {},+ "outputs": [],+ "source": [+ "from dspy.datasets import HotPotQA\n",+ "from dspy.evaluate import Evaluate\n",+ "from dsp.utils.utils import deduplicate\n",+ "\n",+ "\n",+ "# We are setting the experimental flag to True to make use of the fine-tuning\n",+ "# features that are still in development.\n",+ "dspy.settings.configure(experimental=True)\n",+ "\n",+ "# Define the program\n",+ "class BasicMH(dspy.Module):\n",+ " def __init__(self, passages_per_hop=3, num_hops=2):\n",+ " super().__init__()\n",+ " self.num_hops = num_hops\n",+ " self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",+ " # self.generate_query = dspy.ChainOfThought(GenerateSearchQuery)\n",+ " self.generate_query = [dspy.ChainOfThought(\"context, question -> search_query\") for _ in range(self.num_hops)]\n",+ " self.generate_answer = dspy.ChainOfThought(\"context, question -> answer\")\n",+ " \n",+ " def forward(self, question):\n",+ " context = []\n",+ " \n",+ " for hop in range(self.num_hops):\n",+ " search_query = self.generate_query[hop](context=context, question=question).search_query\n",+ " passages = self.retrieve(search_query).passages\n",+ " context = deduplicate(context + passages)\n",+ "\n",+ " prediction = self.generate_answer(context=context, question=question).copy(context=context)\n",+ " return prediction"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "# Let's break down the BasicMH program\n",+ "\n",+ "## Program Design\n",+ "The BasicMH program is designed to answer complex questions requiring multiple retrieval steps without relying on the model's internal knowledge.\n",+ "\n",+ "For example, if we asked \"What the capital of the country that has the eiffel tower\"\n",+ "\n",+ "The model cannot just throw the original question \"What the capital of the country that has the eiffel tower\" into a retriever, the model needs to generate a query with the right search terms in order to get the correct passages.\n",+ "\n",+ "### Program Flow:\n",+ "1. Iterate through the specified number of hops:\n",+ "\n",+ " a. Generate a search query using `generate_query` modules.\n",+ "\n",+ " b. Retrieve passages using the `retrieve` module.\n",+ "\n",+ " c. Deduplicate and add retrieved passages to the context.\n",+ "2. After completing all hops, generate the final answer using the `generate_answer` module.\n",+ "\n",+ "This multi-step approach enables the program to break down complex questions and gather necessary information incrementally, improving the accuracy of the final answer.\n",+ "\n",+ "## Program Implementation\n",+ "\n",+ "### `BasicMH` class\n",+ "\n",+ "#### `__init__` method\n",+ "\n",+ "This method initializes the class with the specified number of hops and passages per hop. It also sets up the `retrieve` module for passage retrieval and a list of `generate_query` modules for generating search queries at each hop. It should be noted that each hop has a different generate_query module, which means they all get optimized separately. The intuition here is that a good prompt for the first hop may not be good for the second hop, and so on.\n",+ "\n",+ "#### `forward` method\n",+ "\n",+ "This method implements the forward pass of the program. It iterates through the specified number of hops, generating a search query at each hop and retrieving passages. The retrieved passages are then deduplicated and added to the context. After completing all hops, it generates the final answer using the `generate_answer` module.\n",+ "\n",+ "A note here is that you directly return the result of the `generate_answer` module. A common anti-pattern is to return `prediction.answer` instead of `prediction`. This is because `prediction` is an object that contains information that is useful for optimization, and the answer can be extracted later.\n",+ "\n",+ "### Example Usage\n",+ "\n",+ "Here's an example of how to use the `BasicMH` class:\n",+ "\n",+ "```\n",+ "program = BasicMH()\n",+ "prediction = program(question=\"What is the capital of the country that has the Eiffel Tower?\")\n",+ "print(prediction.answer)\n",+ "# Prints: Paris\n",+ "```"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "Below we load the dataset using a built in `HotPotQA` dataset class from DSPy.\n",+ "\n",+ "We set the `train_seed` and `eval_seed` to `0` for reproducibility and the `test_size` to `0` because we do not need a test set for this tutorial."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 8,+ "metadata": {},+ "outputs": [+ {+ "data": {+ "application/vnd.jupyter.widget-view+json": {+ "model_id": "b67a4521d2c247e98ff2a50ca10a51f1",+ "version_major": 2,+ "version_minor": 0+ },+ "text/plain": [+ "Downloading builder script: 0%| | 0.00/6.42k [00:00<?, ?B/s]"+ ]+ },+ "metadata": {},+ "output_type": "display_data"+ },+ {+ "data": {+ "application/vnd.jupyter.widget-view+json": {+ "model_id": "ed39552f92b6494bb11d6ade994cd8a2",+ "version_major": 2,+ "version_minor": 0+ },+ "text/plain": [+ "Downloading readme: 0%| | 0.00/9.19k [00:00<?, ?B/s]"+ ]+ },+ "metadata": {},+ "output_type": "display_data"+ },+ {+ "data": {+ "application/vnd.jupyter.widget-view+json": {+ "model_id": "e474b7115b0d45b49d09c8f0465505f4",+ "version_major": 2,+ "version_minor": 0+ },+ "text/plain": [+ "Downloading data files: 0%| | 0/3 [00:00<?, ?it/s]"+ ]+ },+ "metadata": {},+ "output_type": "display_data"+ },+ {+ "data": {+ "application/vnd.jupyter.widget-view+json": {+ "model_id": "543cc7fc8e01423e966be5fcc107d0bb",+ "version_major": 2,+ "version_minor": 0+ },+ "text/plain": [+ "Downloading data: 0%| | 0.00/566M [00:00<?, ?B/s]"+ ]+ },+ "metadata": {},+ "output_type": "display_data"+ },+ {+ "data": {+ "application/vnd.jupyter.widget-view+json": {+ "model_id": "c79d8e5a246e40dfbea4294066ddd520",+ "version_major": 2,+ "version_minor": 0+ },+ "text/plain": [+ "Downloading data: 0%| | 0.00/47.5M [00:00<?, ?B/s]"+ ]+ },+ "metadata": {},+ "output_type": "display_data"+ },+ {+ "data": {+ "application/vnd.jupyter.widget-view+json": {+ "model_id": "5f70ef69c1b14c3997b3a87ab38bc420",+ "version_major": 2,+ "version_minor": 0+ },+ "text/plain": [+ "Downloading data: 0%| | 0.00/46.2M [00:00<?, ?B/s]"+ ]+ },+ "metadata": {},+ "output_type": "display_data"+ },+ {+ "data": {+ "application/vnd.jupyter.widget-view+json": {+ "model_id": "70edaffbb578493fbc4288479ba0aaf2",+ "version_major": 2,+ "version_minor": 0+ },+ "text/plain": [+ "Generating train split: 0%| | 0/90447 [00:00<?, ? examples/s]"+ ]+ },+ "metadata": {},+ "output_type": "display_data"+ },+ {+ "data": {+ "application/vnd.jupyter.widget-view+json": {+ "model_id": "67e6b713acf94a06a46ef3c59da93448",+ "version_major": 2,+ "version_minor": 0+ },+ "text/plain": [+ "Generating validation split: 0%| | 0/7405 [00:00<?, ? examples/s]"+ ]+ },+ "metadata": {},+ "output_type": "display_data"+ },+ {+ "data": {+ "application/vnd.jupyter.widget-view+json": {+ "model_id": "0587220093ac4b28989f265fe22f8b06",+ "version_major": 2,+ "version_minor": 0+ },+ "text/plain": [+ "Generating test split: 0%| | 0/7405 [00:00<?, ? examples/s]"+ ]+ },+ "metadata": {},+ "output_type": "display_data"+ }+ ],+ "source": [+ "# Prepare the dataset\n",+ "TRAIN_SIZE = 1500\n",+ "DEV_SIZE = 1500\n",+ "dataset = HotPotQA(train_seed=0, eval_seed=0, test_size=0)\n",+ "trainset = [x.with_inputs('question') for x in dataset.train]\n",+ "devset = [x.with_inputs('question') for x in dataset.dev]"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "Here we set up the metric and evaluator. We will be using the answer exact match metric.\n",+ "\n",+ "The evaluator is what we will consider as our test set.\n",+ "\n",+ "We choose `num_threads=90` because we are bottlenecked by the retrieval server, and through testing this is the maximum number of concurrent threads that can be run without causing issues for other people using the retrieval server."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 9,+ "metadata": {},+ "outputs": [],+ "source": [+ "# Prepare the metric and evaluator\n",+ "NUM_THREADS = 90\n",+ "metric = dspy.evaluate.answer_exact_match\n",+ "evaluate_devset = Evaluate(devset=devset[:DEV_SIZE], metric=metric, num_threads=NUM_THREADS, display_progress=True)"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "TODO(optional): Implement LLM as judge"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 10,+ "metadata": {},+ "outputs": [],+ "source": [+ "# Prepare the retriever model\n",+ "# Note that this is not hosted on Anyscale\n",+ "COLBERT_V2_ENDPOINT = \"http://20.102.90.50:2017/wiki17_abstracts\"\n",+ "retriever = dspy.ColBERTv2(url=COLBERT_V2_ENDPOINT)"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "## Gathering baseline performance\n",+ "\n",+ "run evaluate on a base pipeline"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 11,+ "metadata": {},+ "outputs": [],+ "source": [+ "MAX_TOKENS = 3000\n",+ "MODEL_PARAMETERS = {\n",+ " \"max_tokens\": MAX_TOKENS,\n",+ " \"temperature\": 0,\n",+ "}\n",+ "\n",+ "LOCAL_API_PARAMETERS = {\n",+ " \"api_base\": \"http://localhost:8000/v1\",\n",+ " \"api_provider\": \"vllm\",\n",+ " \"api_key\": \"fake-key-doesnt-matter\"\n",+ "}\n",+ "vanilla_program = BasicMH()"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "We will be using a local VLLM instance to run the initial benchmarks and data collection."+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "The first model to run is the 8B model in order to collect a baseline of performance.\n",+ "\n",+ "You can run the local VLLM instance with the following command:\n",+ "\n",+ "Make sure to set your HF_TOKEN and HF_HOME environment variables\n",+ "\n",+ "For Anyscale, putting models into /mnt/local_storage is a typical pattern.\n",+ "\n",+ "\n",+ "`vllm serve meta-llama/Meta-Llama-3.1-8B-Instruct --port 8000 --pipeline_parallel_size 4 --enable_prefix_caching`\n",+ "\n",+ "Lets break down what this command does:\n",+ "- `vllm serve` is the command to run the VLLM server\n",+ "- `meta-llama/Meta-Llama-3.1-8B-Instruct` is the model to run\n",+ "- `--port 8000` is the port to run the server on\n",+ "- `--pipeline_parallel_size 4` is the number of pipeline parallel size to run the server with. We are using 4 because we have 4 GPUs all of which can hold an instance of the model.\n",+ "- `--enable_prefix_caching` is the flag to enable the prefix caching. This will store and reuse the beginnings of prompts to avoid repeating the same computation. This is especially useful for DSPy since we are almost always using prompts with the same beginning parts in the form of few shot demonstrations."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "# Command for easy copying: \n",+ "# `vllm serve meta-llama/Meta-Llama-3.1-8B-Instruct --port 8000 --pipeline_parallel_size 4 --enable_prefix_caching`\n",+ "input(\"Press Enter once you have the vllm server running...\")"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "# TODO: switch to local model\n",+ "llama_8b = dspy.MultiOpenAI(model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\", **MODEL_PARAMETERS, **LOCAL_API_PARAMETERS)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "# Quick sanity check to see if the program is working\n",+ "with dspy.context(lm=llama_8b, rm=retriever):\n",+ " test_predictor = BasicMH()\n",+ " print(test_predictor(question=\"What is the capital of France?\").answer)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "with dspy.context(lm=llama_8b, rm=retriever):\n",+ " print(\"Evaluating the vanilla program on the devset using the model to be trained (llama 8B)...\")\n",+ " vanilla_8b_base_eval = evaluate_devset(vanilla_program)"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "# Running the 70B Model\n",+ "\n",+ "Now that we have a baseline for the 8B model, let's run the 70B model and compare its performance.\n",+ "\n",+ "## Preparation\n",+ "\n",+ "Before running the 70B model:\n",+ "1. Kill the 8B server (use `Ctrl+C`) to free up memory.\n",+ "2. Remember to set your HF_TOKEN and HF_HOME environment variables\n",+ "3. Use the following command to start the 70B server:\n",+ "\n",+ " ```\n",+ " vllm serve meta-llama/Meta-Llama-3.1-70B-Instruct --port 8000 --pipeline_parallel_size 2 --enable_prefix_caching --tensor_parallel_size 2\n",+ " ```\n",+ "\n",+ "## Parallelism Configuration\n",+ "\n",+ "We've chosen pipeline parallelism and tensor parallelism of 2 for the 70B model based on our current setup. Here's the reasoning:\n",+ "\n",+ "1. Model size: The 70B model has 30 parts of ~5 GB each (based on [HuggingFace documentation](https://huggingface.co/meta-llama/Meta-Llama-3.1-70B-Instruct/tree/main)).\n",+ " - Total size: 30 * 5 GB = 150 GB\n",+ "\n",+ "2. Available VRAM:\n",+ " - Our GPUs: 80 GB VRAM x 4 = 320 GB\n",+ " - Tensor parallelism: floor(320/150) = 2\n",+ " - Pipeline parallelism: floor(num_gpus/2) = 2\n",+ " - To use all 4 GPUs efficiently:\n",+ " - Pipeline parallel size: 2\n",+ " - Tensor parallelism: 2\n",+ "\n",+ "3. Alternative setup (8x24GB GPUs):\n",+ " - Pipeline parallel size: 1\n",+ " - Tensor parallelism: ceil(150/24) = 7\n",+ "\n",+ "This configuration allows us to run the 70B model efficiently across our available GPU resources."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "# Command for easy copying: \n",+ "# `vllm serve meta-llama/Meta-Llama-3.1-70B-Instruct --port 8000 --pipeline_parallel_size 2 --enable_prefix_caching --tensor_parallel_size 2`\n",+ "input(\"Press Enter once you have the vllm server running...\")"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "llama_70b = dspy.MultiOpenAI(model=\"meta-llama/Meta-Llama-3.1-70B-Instruct\", **MODEL_PARAMETERS, **LOCAL_API_PARAMETERS)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "# Another sanity check\n",+ "with dspy.context(lm=llama_70b, rm=retriever):\n",+ " test_predictor = BasicMH()\n",+ " print(test_predictor(question=\"What is the capital of France?\").answer)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "with dspy.context(lm=llama_70b, rm=retriever):\n",+ " print(\"Evaluating the vanilla program on the devset using llama 70B...\")\n",+ " llama_70b_base_eval = evaluate_devset(vanilla_program)"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "We hope to bring the 8B performance up to at least 70B level"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "## Optimizing the LLaMa 70B pipeline\n",+ "\n",+ "Now we are ready to optimize the pipeline. We want to optimize the 70B pipeline in order to get the best possible data to then train our 8B model.\n",+ "\n",+ "We will use Bootstrap Few Shot with Random Search (BFRS) to optimize the pipeline.\n",+ "\n",+ "The essence of BFRS is to try out different configurations of few shot demonstrations per step and see which one works best on the validation set.\n",+ "\n",+ "The cool part about BFRs is that it will automatically collect the \"good\" chains of thought for us and add them to the examples at each step."+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "Now we know how well the base pipeline performs, let's run prompt optimization on the pipeline in order to juice up the performance.\n",+ "\n",+ "Let's go over what the hyperparameters mean:\n",+ "- MAX_BOOTSTRAPPED_DEMOS: DSPy will \"bootstrap\" the program by collecting examples at each step that are successful and reusing those in the pipeline. This means that it will automatically collect and add chains of thought to the pipeline.\n",+ "- MAX_LABELED_DEMOS: DSPy will also insert some labeled demonstrations from the training set. These would be unmodified examples from the training set that are just using the gold answer.\n",+ "- NUM_CANDIDATE_PROGRAMS: This is the number of candidate programs that the optimizer will generate. The actual number of programs that are created is this plus three, as DSPy will also try a program with no examples, a program with TODO (check)\n",+ "- OPTIMIZER_NUM_TRAIN and OPTIMIZER_NUM_VAL: These are the number of examples that the optimizer will use for training and validation. Note that we will be taking the \"validation\" set from the trainset so as the actual validation set is untouched."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "# Optimization hyperparameters\n",+ "from dspy.teleprompt.random_search import BootstrapFewShotWithRandomSearch\n",+ "\n",+ "# Define the hyperparameters for prompt optimization\n",+ "MAX_BOOTSTRAPPED_DEMOS = 3\n",+ "MAX_LABELED_DEMOS = 3\n",+ "NUM_CANDIDATE_PROGRAMS = 6\n",+ "OPTIMIZER_NUM_TRAIN = 100\n",+ "OPTIMIZER_NUM_VAL = 150"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 12,+ "metadata": {},+ "outputs": [+ {+ "ename": "NameError",+ "evalue": "name 'OPTIMIZER_NUM_TRAIN' is not defined",+ "output_type": "error",+ "traceback": [+ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",+ "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)",+ "Cell \u001b[0;32mIn[12], line 5\u001b[0m\n\u001b[1;32m 3\u001b[0m shuffled_trainset \u001b[38;5;241m=\u001b[39m [d \u001b[38;5;28;01mfor\u001b[39;00m d \u001b[38;5;129;01min\u001b[39;00m trainset]\n\u001b[1;32m 4\u001b[0m rng\u001b[38;5;241m.\u001b[39mshuffle(shuffled_trainset)\n\u001b[0;32m----> 5\u001b[0m optimizer_trainset \u001b[38;5;241m=\u001b[39m shuffled_trainset[:\u001b[43mOPTIMIZER_NUM_TRAIN\u001b[49m]\n\u001b[1;32m 6\u001b[0m optimizer_valset \u001b[38;5;241m=\u001b[39m shuffled_trainset[OPTIMIZER_NUM_TRAIN:OPTIMIZER_NUM_TRAIN\u001b[38;5;241m+\u001b[39mOPTIMIZER_NUM_VAL]\n",+ "\u001b[0;31mNameError\u001b[0m: name 'OPTIMIZER_NUM_TRAIN' is not defined"+ ]+ }+ ],+ "source": [+ "# Prepare the training and validation sets for the optimizer using the original\n",+ "# trainset. This ensures that our actual devset is left untouched.\n",+ "shuffled_trainset = [d for d in trainset]\n",+ "rng.shuffle(shuffled_trainset)\n",+ "optimizer_trainset = shuffled_trainset[:OPTIMIZER_NUM_TRAIN]\n",+ "optimizer_valset = shuffled_trainset[OPTIMIZER_NUM_TRAIN:OPTIMIZER_NUM_TRAIN+OPTIMIZER_NUM_VAL]"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "# Initialize the optimizer\n",+ "bfrs_optimizer = BootstrapFewShotWithRandomSearch(\n",+ " metric=metric,\n",+ " max_bootstrapped_demos=MAX_BOOTSTRAPPED_DEMOS,\n",+ " max_labeled_demos=MAX_LABELED_DEMOS,\n",+ " num_candidate_programs=NUM_CANDIDATE_PROGRAMS,\n",+ " num_threads=NUM_THREADS\n",+ ")\n",+ "\n",+ "# We have added this flag to save you some compute and time while running the notebook\n",+ "COMPILE_PROGRAM = False\n",+ "\n",+ "# Compile the optimizer and evaluate\n",+ "with dspy.context(lm=llama_70b, rm=retriever):\n",+ " vanilla_program = BasicMH()\n",+ " if COMPILE_PROGRAM:\n",+ " bfrs_base_program = bfrs_optimizer.compile(vanilla_program, trainset=optimizer_trainset, valset=optimizer_valset)\n",+ " bfrs_base_program.save(f\"basicmh_70b_bfrs_{MAX_BOOTSTRAPPED_DEMOS}_{MAX_LABELED_DEMOS}_{NUM_CANDIDATE_PROGRAMS}.json\")\n",+ " else:\n",+ " bfrs_base_program = BasicMH()\n",+ " bfrs_base_program.load(f\"basicmh_70b_bfrs_{MAX_BOOTSTRAPPED_DEMOS}_{MAX_LABELED_DEMOS}_{NUM_CANDIDATE_PROGRAMS}.json\")\n",+ " \n",+ " llama_70b_bfrs_eval = evaluate_devset(bfrs_base_program)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "from dspy.teleprompt import MIPROv2\n",+ "\n",+ "eval_kwargs = dict(display_progress=True, display_table=0, num_threads=NUM_THREADS)\n",+ "teleprompter = MIPROv2(prompt_model=llama_70b, task_model=llama_70b, metric=metric, num_candidates=10, init_temperature=0.9, verbose=True)\n",+ "\n",+ "COMPILE_PROGRAM = False\n",+ "if COMPILE_PROGRAM:\n",+ " with dspy.context(lm=llama_70b, rm=retriever):\n",+ " compiled_program = teleprompter.compile(vanilla_program, trainset=optimizer_trainset, valset=optimizer_valset, num_batches=30, max_bootstrapped_demos=MAX_BOOTSTRAPPED_DEMOS,max_labeled_demos=MAX_LABELED_DEMOS, eval_kwargs=eval_kwargs, requires_permission_to_run=False)\n",+ " compiled_program.save(f\"basicmh_70b_MIPROv2_{MAX_BOOTSTRAPPED_DEMOS}_{MAX_LABELED_DEMOS}.json\")\n",+ "else:\n",+ " compiled_program = BasicMH()\n",+ " compiled_program.load(f\"basicmh_70b_MIPROv2_{MAX_BOOTSTRAPPED_DEMOS}_{MAX_LABELED_DEMOS}.json\")\n",+ "\n",+ "with dspy.context(lm=llama_70b, rm=retriever):\n",+ " llama_70b_mipro_eval = evaluate_devset(compiled_program)"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "### Bootstrap Data\n",+ "\n",+ "\n",+ "In this section, we bootstrap data for fine-tuning. In the code block below, we are deciding which program should be used to collect the bootstraps. We are setting this to the prompt optimized program, but one could also set this to the vanilla program, though doing so would lead to lower quality bootstraps."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "bootstrap_program = bfrs_base_program"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "We do something that kind of looks like rejection sampling here. For 3 rounds, we will run the model on the dataset and collect any \n",+ "examples that are solved. We then remove the solved examples from the dataset and repeat.\n",+ "\n",+ "In the [Large Language Monkeys paper](https://arxiv.org/pdf/2407.21787), they show that sampling up to 100 times can still get you a single correct answe in domains with ground truth or a verifier, so we can get away with this form of rejection sampling. Three rounds puts us at a decent spot on the curve of sampling up to N times. If we were in a domain where getting any correct answers was extremely hard, we may consider doing more rounds of sampling, but for now 3 rounds works.\n",+ "\n",+ "We did some experiments to see what the sampling curve looks like:\n",+ "\n",+ "<!--  -->\n",+ "<img src=\"./sampling_curve.png\" alt=\"Sampling Curve\" width=\"500\">"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "from dspy.teleprompt.finetune_teleprompter import bootstrap_data, convert_to_module_level_prompt_completion_data, bootstrap_data_for_round\n",+ "import ujson\n",+ "\n",+ "# This should be moved inside the finetune_teleprompter class\n",+ "def write_data(program, data, filename):\n",+ " print(\"Bootstrapping and writing data to\", filename)\n",+ " correct_data = []\n",+ " unsolved_examples = data.copy()\n",+ " sampling_temperature = 0.901\n",+ " sampling_temperature_delta = 0.0001\n",+ " \n",+ " for i in range(3):\n",+ " if len(unsolved_examples) == 0:\n",+ " break\n",+ " data = bootstrap_data_for_round(program, unsolved_examples, metric=metric, num_threads=NUM_THREADS, sampling_round=i, sampling_temperature=sampling_temperature, sampling_temperature_delta=sampling_temperature_delta)\n",+ " correct_data_round = [x for x in data if x[\"score\"]]\n",+ " correct_examples_round = set([x[\"example\"] for x in correct_data_round])\n",+ " print(f\"Round {i} complete. Solved {len(correct_data_round)} of {len(unsolved_examples)} examples. {len(unsolved_examples) - len(correct_examples_round)} examples remain unsolved.\")\n",+ " unsolved_examples = [x for x in unsolved_examples if x not in correct_examples_round]\n",+ "\n",+ " correct_data.extend(correct_data_round)\n",+ " sampling_temperature += sampling_temperature_delta\n",+ " \n",+ " # Convert the data to prompt completion format\n",+ " dataset = convert_to_module_level_prompt_completion_data(correct_data, program=program, exclude_demos=True)\n",+ " \n",+ " # Format the data for finetuning using the LM\n",+ " print(\"Writing dataset with length\", len(dataset), \"to\", filename)\n",+ " with open(filename, \"w\") as f:\n",+ " ujson.dump(dataset, f)\n",+ "\n",+ "\n",+ "TRAIN_SIZE = 1500\n",+ "EVAL_SIZE = int(TRAIN_SIZE/4)\n",+ "\n",+ "dataset_filenames = {f\"trainset_data_{TRAIN_SIZE}.json\": shuffled_trainset[:TRAIN_SIZE], f\"trainset_val_data_{EVAL_SIZE}.json\": shuffled_trainset[TRAIN_SIZE:TRAIN_SIZE+EVAL_SIZE]}\n",+ "\n",+ "dspy.settings.configure(experimental=True, lm=llama_70b, rm=retriever)\n",+ "\n",+ "for filename, data in dataset_filenames.items():\n",+ " write_data(bootstrap_program, data, filename)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "# Let's look at an example prompt completion pair!\n",+ "data_example = dataset_filenames[f\"trainset_data_{TRAIN_SIZE}.json\"]\n",+ "print(\"Example prompt:\")\n",+ "print(data_example[0]['prompt'])\n",+ "print(\"-\"*50)\n",+ "print(\"Example completion:\")\n",+ "print(data_example[0]['completion'])\n",+ "print(\"-\"*50)"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "Now you should kill your 70B vllm server so that you can use your GPUs for finetuning"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "# Press enter once you have killed the 70B vllm server\n",+ "input(\"Press Enter once you have killed the 70B vllm server (press Ctrl+C to kill)...\")"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "# Fine-tuning\n",+ "\n",+ "We will use LLM Forge to fine-tune the 8B model.\n",+ "\n",+ "In order to do this, we need to format our data into the correct format (Follows OpenAI messaging format placed in a jsonl file).\n",+ "\n",+ "We initially saved the data into a json file in prompt-completion format.\n",+ "\n",+ "In order to prepare for finetuning, we need to do three steps:\n",+ "1. Format the data into the correct format and verify that the data is valid\n",+ "2. Upload the data to GCP\n",+ "3. Generate the compute configuration file\n",+ "\n",+ "After the compute configuration file is generated, we can submit the job to LLM Forge, using either the command line or using the anyscale jobs sdk.\n",+ "TODO: Add the anyscale jobs sdk submit method"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "Be sure to checkout the fine-tuning documentation for the latest on how to use our [API](https://docs.anyscale.com/llms/finetuning/intro) and additional [capabilities](https://docs.anyscale.com/category/fine-tuning-beta/)."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "student_llama_8b = dspy.TrainableAnyscale(model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\")"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "TODO: All this should be moved into the TrainableAnyscaleLM class. You should instead just call a finetune method with your datasets, hparams, compute config"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "from dsp.modules.lm import TrainingMethod\n",+ "\n",+ "train_path = f\"trainset_data_{TRAIN_SIZE}.json\"\n",+ "eval_path = f\"trainset_val_data_{EVAL_SIZE}.json\"\n",+ "method = TrainingMethod.SFT\n",+ "kwargs = {\n",+ " \"hyperparameters\": {\n",+ " \"num_devices\": 4,\n",+ " \"trainer_resources\": None,\n",+ " \"worker_resources\": None\n",+ " },\n",+ " \"use_lora\": True\n",+ "}\n",+ "\n",+ "if method != TrainingMethod.SFT:\n",+ " raise NotImplementedError(\"Only SFT training is supported at the moment.\")\n",+ "\n",+ "train_dataset = student_llama_8b._format_data_for_vanilla_finetuning(train_path)\n",+ "val_dataset = student_llama_8b._format_data_for_vanilla_finetuning(eval_path) if eval_path else None\n",+ "\n",+ "if not student_llama_8b._verify_datasets(train_dataset, val_dataset):\n",+ " print(\"Unable to verify arguments\")\n",+ " raise RuntimeError(\"Unable to verify argument\")\n",+ "\n",+ "# TODO: This should be a function inside the TrainableAnyscaleLM class\n",+ "formatted_paths = {}\n",+ "for path, dataset in [(train_path, train_dataset), (eval_path, val_dataset)]:\n",+ " if not (path and dataset):\n",+ " continue\n",+ " formatted_path = path.split(\".\")[0] + \"_formatted.jsonl\"\n",+ " with open(formatted_path, \"w\") as f:\n",+ " for item in dataset:\n",+ " f.write(ujson.dumps(item) + \"\\n\")\n",+ "\n",+ " formatted_paths[path] = formatted_path\n",+ "\n",+ "remote_train_path, remote_eval_path = student_llama_8b._submit_data(train_path=formatted_paths[train_path], eval_path=formatted_paths[eval_path])\n",+ "compute_config_path, compute_config = student_llama_8b._generate_config_files(train_path=remote_train_path, eval_path=remote_eval_path, **kwargs)\n"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "We'll fine-tune our LLM by choosing a set of configurations. We have created recipes for different LLMs in the [`training configs`](configs/training/lora/llama-3-8b.yaml) folder which can be used as is or modified for experiments. These configurations provide flexibility over a broad range of parameters such as model, data paths, compute to use for training, number of training epochs, how often to save checkpoints, padding, loss, etc. We also include several [DeepSpeed](https://github.com/microsoft/DeepSpeed) [configurations](configs/deepspeed/zero_3_offload_optim+param.json) to choose from for further optimizations around data/model parallelism, mixed precision, checkpointing, etc.\n",+ "\n",+ "We also have recipes for [LoRA](https://arxiv.org/abs/2106.09685) (where we train a set of small low ranked matrices instead of the original attention and feed forward layers) or full parameter fine-tuning. We recommend starting with LoRA as it's less resource intensive and quicker to train."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "# View the compute config\n",+ "os.system(f\"cat {compute_config_path}\")"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "# TODO: Get job working with LLMForge\n",+ "# job_id: str = anyscale.job.submit(\n",+ "# compute_config\n",+ "# )\n",+ "# anyscale.job.wait(id=job_id, timeout_s=18000)\n",+ "# print(f\"Job {job_id} succeeded!\")\n",+ "\n",+ "SKIP_FT = True\n",+ "if not SKIP_FT:\n",+ " command = compute_config.entrypoint\n",+ " print(command)\n",+ " os.system(command)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "# Lets sanity check the finetuned model. We need to download it first.\n",+ "# After the finetuning is complete, the logs will say something like \"The best performing weights have been forwarded to gs://...\"\n",+ "\n",+ "def download_from_gcp_bucket(bucket_name, source_folder, destination_folder):\n",+ " \"\"\"Downloads a folder from a GCP bucket to a local folder.\n",+ "\n",+ " Args:\n",+ " bucket_name (str): The name of the GCP bucket.\n",+ " source_folder (str): The path to the folder in the bucket.\n",+ " destination_folder (str): The local path where files should be saved.\n",+ "\n",+ " Returns:\n",+ " str: The path to the downloaded folder.\n",+ " \"\"\"\n",+ " import google.cloud.storage as storage\n",+ " \n",+ " storage_client = storage.Client()\n",+ " bucket = storage_client.bucket(bucket_name)\n",+ " blobs = bucket.list_blobs(prefix=source_folder)\n",+ "\n",+ " for blob in blobs:\n",+ " if blob.name.endswith('/'):\n",+ " continue # Skip directories\n",+ " \n",+ " relative_path = os.path.relpath(blob.name, source_folder)\n",+ " local_file_path = os.path.join(destination_folder, relative_path)\n",+ " \n",+ " os.makedirs(os.path.dirname(local_file_path), exist_ok=True)\n",+ " # only download if the file doesn't exist\n",+ " if not os.path.exists(local_file_path):\n",+ " blob.download_to_filename(local_file_path)\n",+ " print(f\"Downloaded {blob.name} to {local_file_path}\")\n",+ "\n",+ " print(f\"Folder {source_folder} downloaded to {destination_folder}.\")\n",+ " return destination_folder"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "input(\"Press Enter once you have copied the path from the logs...\")"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "# Copy paste the path from the logs.\n",+ "model_id = \"meta-llama/Meta-Llama-3.1-8B-Instruct\"\n",+ "lora_source_path = \"gs://storage-bucket-cld-tffbxe9ia5phqr1unxhz4f7e1e/org_4snvy99zwbmh4gbtk64jfqggmj/cld_tffbxe9ia5phqr1unxhz4f7e1e/artifact_storage/lora_fine_tuning/meta-llama/Meta-Llama-3.1-8B-Instruct:isaac:nfomi\"\n",+ "\n",+ "# # Parse the GCS path\n",+ "bucket_name = lora_source_path.split('/')[2]\n",+ "source_folder = '/'.join(lora_source_path.split('/')[3:])\n",+ "\n",+ "# Download the LoRA model folder locally\n",+ "local_lora_path = download_from_gcp_bucket(bucket_name, source_folder, \"/mnt/local_storage/dspy/mhqa-lora\")"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "# Evaluation\n",+ "\n",+ "Throughout this section, anything using 8B model (or technically 70B too) should use the new evaluate with ray data batch offline(or technically online) inference.\n",+ "\n",+ "Probably worth testing offline with 8x8 threads vs just 64 threads to see if it makes a meaningful difference.\n",+ "\n",+ "## Performance comparisons\n",+ "\n",+ "- 70B\n",+ "- 70B BSFS\n",+ "- 8B\n",+ "- 8B BSFT\n",+ "- 8B BSFT + BSFS"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "First, we need to serve the base model and tell VLLM where to find the LoRA weights\n",+ "\n",+ "Run the following command:\n",+ "\n",+ "```\n",+ "vllm serve meta-llama/Meta-Llama-3.1-8B-Instruct --port 8000 --pipeline_parallel_size 4 --enable_prefix_caching --enable_lora --lora_modules mhqa-lora=/mnt/local_storage/dspy/mhqa-lora\n",+ "```\n",+ "\n",+ "# Explanation:\n",+ "This command starts a VLLM server to serve the Meta-Llama-3-8B-Instruct model with LoRA fine-tuning.\n",+ "Here's a breakdown of the command:\n",+ "- 'vllm serve': Starts the VLLM server\n",+ "- 'meta-llama/Meta-Llama-3.1-8B-Instruct': Specifies the base model to use\n",+ "- '--port 8000': Sets the server port to 8000\n",+ "- '--pipeline_parallel_size 4': Enables pipeline parallelism with 4 stages\n",+ "- '--enable_prefix_caching': Enables caching of prefixes for faster inference\n",+ "- '--enable_lora': Enables LoRA (Low-Rank Adaptation) for fine-tuning\n",+ "- '--lora_modules mhqa-lora=/mnt/local_storage/dspy/mhqa-lora': Specifies the name of the LoRA module and the path to the LoRA weights. We use the name instead of the base model name when trying to use the LoRA weights. If we just use the base model name, the server will ignore the LoRA weights.\n",+ "\n",+ "This setup allows us to serve a fine-tuned version of the 8B model, which we'll use for subsequent evaluations."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "# Command for easy copying: \n",+ "# `vllm serve meta-llama/Meta-Llama-3.1-8B-Instruct --port 8000 --pipeline_parallel_size 4 --enable_prefix_caching --enable_lora --lora_modules mhqa-lora=/mnt/local_storage/dspy/mhqa-lora`\n",+ "\n",+ "llama_8b = dspy.MultiOpenAI(model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\", **LOCAL_API_PARAMETERS, **MODEL_PARAMETERS)\n",+ "mhqa_llama_8b = dspy.MultiOpenAI(model=\"mhqa-lora\", **LOCAL_API_PARAMETERS, **MODEL_PARAMETERS)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "with dspy.context(lm=mhqa_llama_8b, rm=retriever):\n",+ " vanilla_program = BasicMH()\n",+ " llama_8b_finetuned_eval = evaluate_devset(vanilla_program)"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "Now let's try optimizing the program with the finetuned model"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "COMPILE_PROGRAM = False\n",+ "\n",+ "with dspy.context(lm=mhqa_llama_8b, rm=retriever):\n",+ " vanilla_program = BasicMH()\n",+ " if COMPILE_PROGRAM:\n",+ " bfrs_finetuned_program = bfrs_optimizer.compile(vanilla_program, trainset=optimizer_trainset, valset=optimizer_valset)\n",+ " bfrs_finetuned_program.save(f\"basicmh_8b_ft_bfrs_{MAX_BOOTSTRAPPED_DEMOS}_{MAX_LABELED_DEMOS}_{NUM_CANDIDATE_PROGRAMS}.json\")\n",+ " else:\n",+ " bfrs_finetuned_program = BasicMH()\n",+ " bfrs_finetuned_program.load(f\"basicmh_8b_ft_bfrs_{MAX_BOOTSTRAPPED_DEMOS}_{MAX_LABELED_DEMOS}_{NUM_CANDIDATE_PROGRAMS}.json\")\n",+ " llama_8b_bfrs_finetuned_eval = evaluate_devset(bfrs_finetuned_program)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 68,+ "metadata": {},+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "WARNING: Projected Language Model (LM) Calls\n",+ "\n",+ "Please be advised that based on the parameters you have set, the maximum number of LM calls is projected as follows:\n",+ "\n",+ "\n",+ "- Prompt Model: 10 data summarizer calls + 10 * 3 lm calls in program + (4) lm calls in program aware proposer = 44 prompt model calls\n",+ "- Task Model: 25 examples in minibatch * 30 batches + 100 examples in train set * 3 full evals = 1050 task model calls\n",+ "\n",+ "Estimated Cost Calculation:\n",+ "\n",+ "Total Cost = (Number of calls to task model * (Avg Input Token Length per Call * Task Model Price per Input Token + Avg Output Token Length per Call * Task Model Price per Output Token) \n",+ " + (Number of calls to prompt model * (Avg Input Token Length per Call * Task Prompt Price per Input Token + Avg Output Token Length per Call * Prompt Model Price per Output Token).\n",+ "\n",+ "For a preliminary estimate of potential costs, we recommend you perform your own calculations based on the task\n",+ "and prompt models you intend to use. If the projected costs exceed your budget or expectations, you may consider:\n",+ "\n",+ "- Reducing the number of trials (`num_batches`), the size of the trainset, or the number of LM calls in your program.\n",+ "- Using a cheaper task model to optimize the prompt.\n",+ "SOURCE CODE: StringSignature(context, question -> rationale, search_query\n",+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",+ ")\n",+ "\n",+ "StringSignature(context, question -> rationale, answer\n",+ " instructions='Given the fields `context`, `question`, produce the fields `answer`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Answer:', 'desc': '${answer}'})\n",+ ")\n",+ "\n",+ "class BasicMH(dspy.Module):\n",+ " def __init__(self, passages_per_hop=3, num_hops=2):\n",+ " super().__init__()\n",+ " self.num_hops = num_hops\n",+ " self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",+ " # self.generate_query = dspy.ChainOfThought(GenerateSearchQuery)\n",+ " self.generate_query = [dspy.ChainOfThought(\"context, question -> search_query\") for _ in range(self.num_hops)]\n",+ " self.generate_answer = dspy.ChainOfThought(\"context, question -> answer\")\n",+ " \n",+ " def forward(self, question):\n",+ " context = []\n",+ " \n",+ " for hop in range(self.num_hops):\n",+ " search_query = self.generate_query[hop](context=context, question=question).search_query\n",+ " passages = self.retrieve(search_query).passages\n",+ " context = deduplicate(context + passages)\n",+ "\n",+ " prediction = self.generate_answer(context=context, question=question).copy(context=context)\n",+ " return prediction\n",+ "\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "b: 10\n",+ "b: 20\n",+ "b: 30\n",+ "b: 40\n",+ "b: 50\n",+ "b: 60\n",+ "b: 70\n",+ "b: 80\n",+ "b: 90\n",+ "summary: Prediction(\n",+ " summary=\"The dataset appears to be a comprehensive collection of trivia questions, presenting a diverse range of topics, including entertainment, history, arts, and culture. Most questions require specific, factual answers, often testing users' recall of specific details or events, and are designed to test users' knowledge on subject matter. The syntax and length of the questions vary, but most are concise and require precise answers.\"\n",+ ")\n",+ "DATA SUMMARY: The dataset appears to be a comprehensive collection of trivia questions, presenting a diverse range of topics, including entertainment, history, arts, and culture. Most questions require specific, factual answers, often testing users' recall of specific details or events, and are designed to test users' knowledge on subject matter. The syntax and length of the questions vary, but most are concise and require precise answers.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 9%|▉ | 9/100 [01:04<10:56, 7.22s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 3 full traces after 10 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 4%|▍ | 4/100 [00:18<07:18, 4.57s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 2 full traces after 5 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 1%| | 1/100 [00:01<01:57, 1.19s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 1 full traces after 2 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 11%|█ | 11/100 [00:53<07:10, 4.84s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 1 full traces after 12 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 2%|▏ | 2/100 [00:06<05:34, 3.41s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 1 full traces after 3 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 1%| | 1/100 [00:02<04:08, 2.51s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 1 full traces after 2 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 10%|█ | 10/100 [01:04<09:43, 6.48s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 3 full traces after 11 examples in round 0.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 4%|▍ | 4/100 [00:09<03:54, 2.45s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Bootstrapped 3 full traces after 5 examples in round 0.\n",+ "Using a randomly generated configuration for our grounded proposer.\n",+ "Selected tip: description\n",+ "PROGRAM DESCRIPTION: This program is designed to solve a question-answering task. It appears to work by iteratively generating search queries based on the context and question, retrieving relevant passages, and then generating answers based on the retrieved passages. The program uses a chain of thought approach, where each hop represents a step in the reasoning process, and the output of each hop is used as input for the next hop. The program seems to be designed to simulate how a human would answer a question, by thinking step by step and generating intermediate search queries and answers.\n",+ "task_demos \n",+ "\n",+ "\n",+ "\n",+ "Use the information below to learn about a task that we are trying to solve using calls to an LM, then generate a new instruction that will be used to prompt a Language Model to better solve the task.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "DATASET SUMMARY: A description of the dataset that we are using.\n",+ "\n",+ "PROGRAM CODE: Language model program designed to solve a particular task.\n",+ "\n",+ "PROGRAM DESCRIPTION: Summary of the task the program is designed to solve, and how it goes about solving it.\n",+ "\n",+ "MODULE: The module to create an instruction for.\n",+ "\n",+ "TASK DEMO(S): Example inputs/outputs of our module.\n",+ "\n",+ "BASIC INSTRUCTION: Basic instruction.\n",+ "\n",+ "TIP: A suggestion for how to go about generating the new instruction.\n",+ "\n",+ "PROPOSED INSTRUCTION: Propose an instruction that will be used to prompt a Language Model to perform this task.\n",+ "\n",+ "---\n",+ "\n",+ "DATASET SUMMARY: The dataset appears to be a comprehensive collection of trivia questions, presenting a diverse range of topics, including entertainment, history, arts, and culture. Most questions require specific, factual answers, often testing users' recall of specific details or events, and are designed to test users' knowledge on subject matter. The syntax and length of the questions vary, but most are concise and require precise answers.\n",+ "\n",+ "PROGRAM CODE:\n",+ "StringSignature(context, question -> rationale, search_query\n",+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",+ ")\n",+ "\n",+ "StringSignature(context, question -> rationale, answer\n",+ " instructions='Given the fields `context`, `question`, produce the fields `answer`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Answer:', 'desc': '${answer}'})\n",+ ")\n",+ "\n",+ "class BasicMH(dspy.Module):\n",+ " def __init__(self, passages_per_hop=3, num_hops=2):\n",+ " super().__init__()\n",+ " self.num_hops = num_hops\n",+ " self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",+ " # self.generate_query = dspy.ChainOfThought(GenerateSearchQuery)\n",+ " self.generate_query = [dspy.ChainOfThought(\"context, question -> search_query\") for _ in range(self.num_hops)]\n",+ " self.generate_answer = dspy.ChainOfThought(\"context, question -> answer\")\n",+ " \n",+ " def forward(self, question):\n",+ " context = []\n",+ " \n",+ " for hop in range(self.num_hops):\n",+ " search_query = self.generate_query[hop](context=context, question=question).search_query\n",+ " passages = self.retrieve(search_query).passages\n",+ " context = deduplicate(context + passages)\n",+ "\n",+ " prediction = self.generate_answer(context=context, question=question).copy(context=context)\n",+ " return prediction\n",+ "\n",+ "\n",+ "PROGRAM DESCRIPTION: This program is designed to solve a question-answering task. It appears to work by iteratively generating search queries based on the context and question, retrieving relevant passages, and then generating answers based on the retrieved passages. The program uses a chain of thought approach, where each hop represents a step in the reasoning process, and the output of each hop is used as input for the next hop. The program seems to be designed to simulate how a human would answer a question, by thinking step by step and generating intermediate search queries and answers.\n",+ "\n",+ "MODULE: # self.generate_query = dspy.ChainOfThought(GenerateSearchQuery)\n",+ "\n",+ "TASK DEMO(S): \n",+ "\n",+ "BASIC INSTRUCTION: Given the fields `context`, `question`, produce the fields `search_query`.\n",+ "\n",+ "TIP: Make sure your instruction is very informative and descriptive.\n",+ "\n",+ "Please provide the output field PROPOSED INSTRUCTION. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field PROPOSED INSTRUCTION.\n",+ "\n",+ "PROPOSED INSTRUCTION: Given the context and question, generate a specific search query that would lead to relevant passages for answering the question, and provide a detailed reasoning for this search query.\n",+ "\n",+ "\n",+ "\n",+ "PROPOSED INSTRUCTION: Given the context and question, generate a specific search query that would lead to relevant passages for answering the question, and provide a detailed reasoning for this search query.\n",+ "PROGRAM DESCRIPTION: This program is designed to solve a question-answering task. It appears to work by iteratively generating search queries based on the context and question, retrieving relevant passages, and then generating answers based on the retrieved passages. The program uses a chain of thought approach, where each hop represents a step in the reasoning process, and the output of each hop is used as input for the next hop. The program seems to be designed to simulate how a human would answer a question, by thinking step by step and generating intermediate search queries and answers.\n",+ "task_demos \n",+ "\n",+ "\n",+ "\n",+ "Use the information below to learn about a task that we are trying to solve using calls to an LM, then generate a new instruction that will be used to prompt a Language Model to better solve the task.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "DATASET SUMMARY: A description of the dataset that we are using.\n",+ "\n",+ "PROGRAM CODE: Language model program designed to solve a particular task.\n",+ "\n",+ "PROGRAM DESCRIPTION: Summary of the task the program is designed to solve, and how it goes about solving it.\n",+ "\n",+ "MODULE: The module to create an instruction for.\n",+ "\n",+ "TASK DEMO(S): Example inputs/outputs of our module.\n",+ "\n",+ "BASIC INSTRUCTION: Basic instruction.\n",+ "\n",+ "TIP: A suggestion for how to go about generating the new instruction.\n",+ "\n",+ "PROPOSED INSTRUCTION: Propose an instruction that will be used to prompt a Language Model to perform this task.\n",+ "\n",+ "---\n",+ "\n",+ "DATASET SUMMARY: The dataset appears to be a comprehensive collection of trivia questions, presenting a diverse range of topics, including entertainment, history, arts, and culture. Most questions require specific, factual answers, often testing users' recall of specific details or events, and are designed to test users' knowledge on subject matter. The syntax and length of the questions vary, but most are concise and require precise answers.\n",+ "\n",+ "PROGRAM CODE:\n",+ "StringSignature(context, question -> rationale, search_query\n",+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",+ ")\n",+ "\n",+ "StringSignature(context, question -> rationale, answer\n",+ " instructions='Given the fields `context`, `question`, produce the fields `answer`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Answer:', 'desc': '${answer}'})\n",+ ")\n",+ "\n",+ "class BasicMH(dspy.Module):\n",+ " def __init__(self, passages_per_hop=3, num_hops=2):\n",+ " super().__init__()\n",+ " self.num_hops = num_hops\n",+ " self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",+ " # self.generate_query = dspy.ChainOfThought(GenerateSearchQuery)\n",+ " self.generate_query = [dspy.ChainOfThought(\"context, question -> search_query\") for _ in range(self.num_hops)]\n",+ " self.generate_answer = dspy.ChainOfThought(\"context, question -> answer\")\n",+ " \n",+ " def forward(self, question):\n",+ " context = []\n",+ " \n",+ " for hop in range(self.num_hops):\n",+ " search_query = self.generate_query[hop](context=context, question=question).search_query\n",+ " passages = self.retrieve(search_query).passages\n",+ " context = deduplicate(context + passages)\n",+ "\n",+ " prediction = self.generate_answer(context=context, question=question).copy(context=context)\n",+ " return prediction\n",+ "\n",+ "\n",+ "PROGRAM DESCRIPTION: This program is designed to solve a question-answering task. It appears to work by iteratively generating search queries based on the context and question, retrieving relevant passages, and then generating answers based on the retrieved passages. The program uses a chain of thought approach, where each hop represents a step in the reasoning process, and the output of each hop is used as input for the next hop. The program seems to be designed to simulate how a human would answer a question, by thinking step by step and generating intermediate search queries and answers.\n",+ "\n",+ "MODULE: # self.generate_query = dspy.ChainOfThought(GenerateSearchQuery)\n",+ "\n",+ "TASK DEMO(S): \n",+ "\n",+ "BASIC INSTRUCTION: Given the fields `context`, `question`, produce the fields `search_query`.\n",+ "\n",+ "TIP: Make sure your instruction is very informative and descriptive.\n",+ "\n",+ "Please provide the output field PROPOSED INSTRUCTION. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field PROPOSED INSTRUCTION.\n",+ "\n",+ "PROPOSED INSTRUCTION: Given the context and question, generate a specific search query that would lead to relevant passages for answering the question, and provide a detailed reasoning for this search query.\n",+ "\n",+ "\n",+ "\n",+ "PROPOSED INSTRUCTION: Given the context and question, generate a specific search query that would lead to relevant passages for answering the question, and provide a detailed reasoning for this search query.\n",+ "PROGRAM DESCRIPTION: The program appears to be designed to solve a series of trivia questions by generating search queries and answer candidates through a multi-hop process. It uses language models to generate search queries and answer candidates based on a given context and question. In each hop, the program retrieves relevant passages, deduplicates the context, and uses the previous context to generate the next search query or answer candidate.\n",+ "task_demos Context: []\n",+ "Question: What race is the lieutenant governor of the state that held its primary for her election on May 19th, 2015?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: lieutenant governor race May 19 2015\n",+ "Context: []\n",+ "Question: Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: clothing company founded 1889 college bass fishing\n",+ "Context: []\n",+ "Question: Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: Forrest Tucker height Walt Disney TV miniseries\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Use the information below to learn about a task that we are trying to solve using calls to an LM, then generate a new instruction that will be used to prompt a Language Model to better solve the task.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "DATASET SUMMARY: A description of the dataset that we are using.\n",+ "\n",+ "PROGRAM CODE: Language model program designed to solve a particular task.\n",+ "\n",+ "PROGRAM DESCRIPTION: Summary of the task the program is designed to solve, and how it goes about solving it.\n",+ "\n",+ "MODULE: The module to create an instruction for.\n",+ "\n",+ "TASK DEMO(S): Example inputs/outputs of our module.\n",+ "\n",+ "BASIC INSTRUCTION: Basic instruction.\n",+ "\n",+ "TIP: A suggestion for how to go about generating the new instruction.\n",+ "\n",+ "PROPOSED INSTRUCTION: Propose an instruction that will be used to prompt a Language Model to perform this task.\n",+ "\n",+ "---\n",+ "\n",+ "DATASET SUMMARY: The dataset appears to be a comprehensive collection of trivia questions, presenting a diverse range of topics, including entertainment, history, arts, and culture. Most questions require specific, factual answers, often testing users' recall of specific details or events, and are designed to test users' knowledge on subject matter. The syntax and length of the questions vary, but most are concise and require precise answers.\n",+ "\n",+ "PROGRAM CODE:\n",+ "StringSignature(context, question -> rationale, search_query\n",+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",+ ")\n",+ "\n",+ "StringSignature(context, question -> rationale, answer\n",+ " instructions='Given the fields `context`, `question`, produce the fields `answer`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Answer:', 'desc': '${answer}'})\n",+ ")\n",+ "\n",+ "class BasicMH(dspy.Module):\n",+ " def __init__(self, passages_per_hop=3, num_hops=2):\n",+ " super().__init__()\n",+ " self.num_hops = num_hops\n",+ " self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",+ " # self.generate_query = dspy.ChainOfThought(GenerateSearchQuery)\n",+ " self.generate_query = [dspy.ChainOfThought(\"context, question -> search_query\") for _ in range(self.num_hops)]\n",+ " self.generate_answer = dspy.ChainOfThought(\"context, question -> answer\")\n",+ " \n",+ " def forward(self, question):\n",+ " context = []\n",+ " \n",+ " for hop in range(self.num_hops):\n",+ " search_query = self.generate_query[hop](context=context, question=question).search_query\n",+ " passages = self.retrieve(search_query).passages\n",+ " context = deduplicate(context + passages)\n",+ "\n",+ " prediction = self.generate_answer(context=context, question=question).copy(context=context)\n",+ " return prediction\n",+ "\n",+ "\n",+ "PROGRAM DESCRIPTION: The program appears to be designed to solve a series of trivia questions by generating search queries and answer candidates through a multi-hop process. It uses language models to generate search queries and answer candidates based on a given context and question. In each hop, the program retrieves relevant passages, deduplicates the context, and uses the previous context to generate the next search query or answer candidate.\n",+ "\n",+ "MODULE: # self.generate_query = dspy.ChainOfThought(GenerateSearchQuery)\n",+ "\n",+ "TASK DEMO(S):\n",+ "Context: []\n",+ "Question: What race is the lieutenant governor of the state that held its primary for her election on May 19th, 2015?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: lieutenant governor race May 19 2015\n",+ "Context: []\n",+ "Question: Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: clothing company founded 1889 college bass fishing\n",+ "Context: []\n",+ "Question: Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: Forrest Tucker height Walt Disney TV miniseries\n",+ "\n",+ "\n",+ "BASIC INSTRUCTION: Given the fields `context`, `question`, produce the fields `search_query`.\n",+ "\n",+ "TIP: Make sure your instruction is very informative and descriptive.\n",+ "\n",+ "Please provide the output field PROPOSED INSTRUCTION. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field PROPOSED INSTRUCTION.\n",+ "\n",+ "PROPOSED INSTRUCTION: Given the context and question, generate an informative search query that accurately captures the essence of the question, incorporating relevant keywords and phrases to retrieve relevant passages that can aid in answering the question.\n",+ "\n",+ "\n",+ "\n",+ "PROPOSED INSTRUCTION: Given the context and question, generate an informative search query that accurately captures the essence of the question, incorporating relevant keywords and phrases to retrieve relevant passages that can aid in answering the question.\n",+ "PROGRAM DESCRIPTION: The program appears to be designed to solve natural language inference tasks, specifically identifying the characteristics of a character or entity. It accomplishes this by generating a series of search queries and retrieving relevant passages based on the context and question provided. The program then uses these retrieved passages to generate the final answer. The program consists of multiple hops, where each hop generates a search query, retrieves relevant passages, and adds the retrieved passages to the context. The final answer is generated by a chain of thought module that takes the context, question, and rationale and produces the answer.\n",+ "task_demos Context: []\n",+ "Question: Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: Forrest Tucker height Walt Disney TV miniseries\n",+ "Context: []\n",+ "Question: Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: clothing company founded 1889 college bass fishing\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Use the information below to learn about a task that we are trying to solve using calls to an LM, then generate a new instruction that will be used to prompt a Language Model to better solve the task.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "DATASET SUMMARY: A description of the dataset that we are using.\n",+ "\n",+ "PROGRAM CODE: Language model program designed to solve a particular task.\n",+ "\n",+ "PROGRAM DESCRIPTION: Summary of the task the program is designed to solve, and how it goes about solving it.\n",+ "\n",+ "MODULE: The module to create an instruction for.\n",+ "\n",+ "TASK DEMO(S): Example inputs/outputs of our module.\n",+ "\n",+ "BASIC INSTRUCTION: Basic instruction.\n",+ "\n",+ "TIP: A suggestion for how to go about generating the new instruction.\n",+ "\n",+ "PROPOSED INSTRUCTION: Propose an instruction that will be used to prompt a Language Model to perform this task.\n",+ "\n",+ "---\n",+ "\n",+ "DATASET SUMMARY: The dataset appears to be a comprehensive collection of trivia questions, presenting a diverse range of topics, including entertainment, history, arts, and culture. Most questions require specific, factual answers, often testing users' recall of specific details or events, and are designed to test users' knowledge on subject matter. The syntax and length of the questions vary, but most are concise and require precise answers.\n",+ "\n",+ "PROGRAM CODE:\n",+ "StringSignature(context, question -> rationale, search_query\n",+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",+ ")\n",+ "\n",+ "StringSignature(context, question -> rationale, answer\n",+ " instructions='Given the fields `context`, `question`, produce the fields `answer`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Answer:', 'desc': '${answer}'})\n",+ ")\n",+ "\n",+ "class BasicMH(dspy.Module):\n",+ " def __init__(self, passages_per_hop=3, num_hops=2):\n",+ " super().__init__()\n",+ " self.num_hops = num_hops\n",+ " self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",+ " # self.generate_query = dspy.ChainOfThought(GenerateSearchQuery)\n",+ " self.generate_query = [dspy.ChainOfThought(\"context, question -> search_query\") for _ in range(self.num_hops)]\n",+ " self.generate_answer = dspy.ChainOfThought(\"context, question -> answer\")\n",+ " \n",+ " def forward(self, question):\n",+ " context = []\n",+ " \n",+ " for hop in range(self.num_hops):\n",+ " search_query = self.generate_query[hop](context=context, question=question).search_query\n",+ " passages = self.retrieve(search_query).passages\n",+ " context = deduplicate(context + passages)\n",+ "\n",+ " prediction = self.generate_answer(context=context, question=question).copy(context=context)\n",+ " return prediction\n",+ "\n",+ "\n",+ "PROGRAM DESCRIPTION: The program appears to be designed to solve natural language inference tasks, specifically identifying the characteristics of a character or entity. It accomplishes this by generating a series of search queries and retrieving relevant passages based on the context and question provided. The program then uses these retrieved passages to generate the final answer. The program consists of multiple hops, where each hop generates a search query, retrieves relevant passages, and adds the retrieved passages to the context. The final answer is generated by a chain of thought module that takes the context, question, and rationale and produces the answer.\n",+ "\n",+ "MODULE: # self.generate_query = dspy.ChainOfThought(GenerateSearchQuery)\n",+ "\n",+ "TASK DEMO(S):\n",+ "Context: []\n",+ "Question: Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: Forrest Tucker height Walt Disney TV miniseries\n",+ "Context: []\n",+ "Question: Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: clothing company founded 1889 college bass fishing\n",+ "\n",+ "\n",+ "BASIC INSTRUCTION: Given the fields `context`, `question`, produce the fields `search_query`.\n",+ "\n",+ "TIP: Make sure your instruction is very informative and descriptive.\n",+ "\n",+ "Please provide the output field PROPOSED INSTRUCTION. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field PROPOSED INSTRUCTION.\n",+ "\n",+ "PROPOSED INSTRUCTION: Provide a precise and concise search query that effectively captures the essence of the provided context and question, allowing for the retrieval of relevant passages that will ultimately lead to the generation of a accurate answer.\n",+ "\n",+ "\n",+ "\n",+ "PROPOSED INSTRUCTION: Provide a precise and concise search query that effectively captures the essence of the provided context and question, allowing for the retrieval of relevant passages that will ultimately lead to the generation of a accurate answer.\n",+ "PROGRAM DESCRIPTION: This program is designed to generate a search query and an answer to a given question by using a language model. It appears to work by breaking down the question-solving process into multiple \"hops\", where each hop generates a search query and retrieves relevant passages. The passages from each hop are used to provide context for the next hop, and the process is repeated for a specified number of hops. Finally, an answer is generated using the generated search queries and retrieved passages.\n",+ "task_demos Context: []\n",+ "Question: When was the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: Will Finn Over the Hedge short film director\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Use the information below to learn about a task that we are trying to solve using calls to an LM, then generate a new instruction that will be used to prompt a Language Model to better solve the task.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "DATASET SUMMARY: A description of the dataset that we are using.\n",+ "\n",+ "PROGRAM CODE: Language model program designed to solve a particular task.\n",+ "\n",+ "PROGRAM DESCRIPTION: Summary of the task the program is designed to solve, and how it goes about solving it.\n",+ "\n",+ "MODULE: The module to create an instruction for.\n",+ "\n",+ "TASK DEMO(S): Example inputs/outputs of our module.\n",+ "\n",+ "BASIC INSTRUCTION: Basic instruction.\n",+ "\n",+ "TIP: A suggestion for how to go about generating the new instruction.\n",+ "\n",+ "PROPOSED INSTRUCTION: Propose an instruction that will be used to prompt a Language Model to perform this task.\n",+ "\n",+ "---\n",+ "\n",+ "DATASET SUMMARY: The dataset appears to be a comprehensive collection of trivia questions, presenting a diverse range of topics, including entertainment, history, arts, and culture. Most questions require specific, factual answers, often testing users' recall of specific details or events, and are designed to test users' knowledge on subject matter. The syntax and length of the questions vary, but most are concise and require precise answers.\n",+ "\n",+ "PROGRAM CODE:\n",+ "StringSignature(context, question -> rationale, search_query\n",+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",+ ")\n",+ "\n",+ "StringSignature(context, question -> rationale, answer\n",+ " instructions='Given the fields `context`, `question`, produce the fields `answer`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Answer:', 'desc': '${answer}'})\n",+ ")\n",+ "\n",+ "class BasicMH(dspy.Module):\n",+ " def __init__(self, passages_per_hop=3, num_hops=2):\n",+ " super().__init__()\n",+ " self.num_hops = num_hops\n",+ " self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",+ " # self.generate_query = dspy.ChainOfThought(GenerateSearchQuery)\n",+ " self.generate_query = [dspy.ChainOfThought(\"context, question -> search_query\") for _ in range(self.num_hops)]\n",+ " self.generate_answer = dspy.ChainOfThought(\"context, question -> answer\")\n",+ " \n",+ " def forward(self, question):\n",+ " context = []\n",+ " \n",+ " for hop in range(self.num_hops):\n",+ " search_query = self.generate_query[hop](context=context, question=question).search_query\n",+ " passages = self.retrieve(search_query).passages\n",+ " context = deduplicate(context + passages)\n",+ "\n",+ " prediction = self.generate_answer(context=context, question=question).copy(context=context)\n",+ " return prediction\n",+ "\n",+ "\n",+ "PROGRAM DESCRIPTION: This program is designed to generate a search query and an answer to a given question by using a language model. It appears to work by breaking down the question-solving process into multiple \"hops\", where each hop generates a search query and retrieves relevant passages. The passages from each hop are used to provide context for the next hop, and the process is repeated for a specified number of hops. Finally, an answer is generated using the generated search queries and retrieved passages.\n",+ "\n",+ "MODULE: # self.generate_query = dspy.ChainOfThought(GenerateSearchQuery)\n",+ "\n",+ "TASK DEMO(S):\n",+ "Context: []\n",+ "Question: When was the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: Will Finn Over the Hedge short film director\n",+ "\n",+ "\n",+ "BASIC INSTRUCTION: Given the fields `context`, `question`, produce the fields `search_query`.\n",+ "\n",+ "TIP: Make sure your instruction is very informative and descriptive.\n",+ "\n",+ "Please provide the output field PROPOSED INSTRUCTION. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field PROPOSED INSTRUCTION.\n",+ "\n",+ "PROPOSED INSTRUCTION: Given the context and question, generate a concise search query that accurately captures the essence of the query and facilitates the retrieval of relevant information.\n",+ "\n",+ "\n",+ "\n",+ "PROPOSED INSTRUCTION: Given the context and question, generate a concise search query that accurately captures the essence of the query and facilitates the retrieval of relevant information.\n",+ "PROGRAM DESCRIPTION: This program appears to be designed to solve a question-answering task using a multi-hop reasoning approach. The program takes in a context and a question, and uses language models to generate search queries and answers step-by-step. The search queries are used to retrieve relevant passages, which are then used to generate the next search query, until a final answer is generated. The program uses a chain-of-thought approach to reason about the answer, generating intermediate rationales at each step to explain its reasoning.\n",+ "task_demos Context: []\n",+ "Question: The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: Kangchenjunga Singhi Kangri country\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Use the information below to learn about a task that we are trying to solve using calls to an LM, then generate a new instruction that will be used to prompt a Language Model to better solve the task.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "DATASET SUMMARY: A description of the dataset that we are using.\n",+ "\n",+ "PROGRAM CODE: Language model program designed to solve a particular task.\n",+ "\n",+ "PROGRAM DESCRIPTION: Summary of the task the program is designed to solve, and how it goes about solving it.\n",+ "\n",+ "MODULE: The module to create an instruction for.\n",+ "\n",+ "TASK DEMO(S): Example inputs/outputs of our module.\n",+ "\n",+ "BASIC INSTRUCTION: Basic instruction.\n",+ "\n",+ "TIP: A suggestion for how to go about generating the new instruction.\n",+ "\n",+ "PROPOSED INSTRUCTION: Propose an instruction that will be used to prompt a Language Model to perform this task.\n",+ "\n",+ "---\n",+ "\n",+ "DATASET SUMMARY: The dataset appears to be a comprehensive collection of trivia questions, presenting a diverse range of topics, including entertainment, history, arts, and culture. Most questions require specific, factual answers, often testing users' recall of specific details or events, and are designed to test users' knowledge on subject matter. The syntax and length of the questions vary, but most are concise and require precise answers.\n",+ "\n",+ "PROGRAM CODE:\n",+ "StringSignature(context, question -> rationale, search_query\n",+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",+ ")\n",+ "\n",+ "StringSignature(context, question -> rationale, answer\n",+ " instructions='Given the fields `context`, `question`, produce the fields `answer`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Answer:', 'desc': '${answer}'})\n",+ ")\n",+ "\n",+ "class BasicMH(dspy.Module):\n",+ " def __init__(self, passages_per_hop=3, num_hops=2):\n",+ " super().__init__()\n",+ " self.num_hops = num_hops\n",+ " self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",+ " # self.generate_query = dspy.ChainOfThought(GenerateSearchQuery)\n",+ " self.generate_query = [dspy.ChainOfThought(\"context, question -> search_query\") for _ in range(self.num_hops)]\n",+ " self.generate_answer = dspy.ChainOfThought(\"context, question -> answer\")\n",+ " \n",+ " def forward(self, question):\n",+ " context = []\n",+ " \n",+ " for hop in range(self.num_hops):\n",+ " search_query = self.generate_query[hop](context=context, question=question).search_query\n",+ " passages = self.retrieve(search_query).passages\n",+ " context = deduplicate(context + passages)\n",+ "\n",+ " prediction = self.generate_answer(context=context, question=question).copy(context=context)\n",+ " return prediction\n",+ "\n",+ "\n",+ "PROGRAM DESCRIPTION: This program appears to be designed to solve a question-answering task using a multi-hop reasoning approach. The program takes in a context and a question, and uses language models to generate search queries and answers step-by-step. The search queries are used to retrieve relevant passages, which are then used to generate the next search query, until a final answer is generated. The program uses a chain-of-thought approach to reason about the answer, generating intermediate rationales at each step to explain its reasoning.\n",+ "\n",+ "MODULE: # self.generate_query = dspy.ChainOfThought(GenerateSearchQuery)\n",+ "\n",+ "TASK DEMO(S):\n",+ "Context: []\n",+ "Question: The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: Kangchenjunga Singhi Kangri country\n",+ "\n",+ "\n",+ "BASIC INSTRUCTION: Given the fields `context`, `question`, produce the fields `search_query`.\n",+ "\n",+ "TIP: Make sure your instruction is very informative and descriptive.\n",+ "\n",+ "Please provide the output field PROPOSED INSTRUCTION. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field PROPOSED INSTRUCTION.\n",+ "\n",+ "PROPOSED INSTRUCTION: Generate a search query that is informative, concise, and relevant to the question, utilizing the context provided and requiring only factual answers.\n",+ "\n",+ "\n",+ "\n",+ "PROPOSED INSTRUCTION: Generate a search query that is informative, concise, and relevant to the question, utilizing the context provided and requiring only factual answers.\n",+ "PROGRAM DESCRIPTION: This program is designed to solve a question-answering task using a language model. It appears to work by iteratively generating search queries and passing through a series of hops. Each hop retrieves relevant passages based on the generated search query, and the passages are combined to form a context. The program then generates an answer based on the accumulated context and the original question. The task is to answer a user's question by retrieving relevant information from a given context, and the program uses multiple hops to refine the search query and context until it generates an accurate answer.\n",+ "task_demos Context: []\n",+ "Question: What was the series inspired by the appreciation of manual laborers that Wheat Field with Cypresses is a part of?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: Wheat Field with Cypresses series\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Use the information below to learn about a task that we are trying to solve using calls to an LM, then generate a new instruction that will be used to prompt a Language Model to better solve the task.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "DATASET SUMMARY: A description of the dataset that we are using.\n",+ "\n",+ "PROGRAM CODE: Language model program designed to solve a particular task.\n",+ "\n",+ "PROGRAM DESCRIPTION: Summary of the task the program is designed to solve, and how it goes about solving it.\n",+ "\n",+ "MODULE: The module to create an instruction for.\n",+ "\n",+ "TASK DEMO(S): Example inputs/outputs of our module.\n",+ "\n",+ "BASIC INSTRUCTION: Basic instruction.\n",+ "\n",+ "TIP: A suggestion for how to go about generating the new instruction.\n",+ "\n",+ "PROPOSED INSTRUCTION: Propose an instruction that will be used to prompt a Language Model to perform this task.\n",+ "\n",+ "---\n",+ "\n",+ "DATASET SUMMARY: The dataset appears to be a comprehensive collection of trivia questions, presenting a diverse range of topics, including entertainment, history, arts, and culture. Most questions require specific, factual answers, often testing users' recall of specific details or events, and are designed to test users' knowledge on subject matter. The syntax and length of the questions vary, but most are concise and require precise answers.\n",+ "\n",+ "PROGRAM CODE:\n",+ "StringSignature(context, question -> rationale, search_query\n",+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",+ ")\n",+ "\n",+ "StringSignature(context, question -> rationale, answer\n",+ " instructions='Given the fields `context`, `question`, produce the fields `answer`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Answer:', 'desc': '${answer}'})\n",+ ")\n",+ "\n",+ "class BasicMH(dspy.Module):\n",+ " def __init__(self, passages_per_hop=3, num_hops=2):\n",+ " super().__init__()\n",+ " self.num_hops = num_hops\n",+ " self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",+ " # self.generate_query = dspy.ChainOfThought(GenerateSearchQuery)\n",+ " self.generate_query = [dspy.ChainOfThought(\"context, question -> search_query\") for _ in range(self.num_hops)]\n",+ " self.generate_answer = dspy.ChainOfThought(\"context, question -> answer\")\n",+ " \n",+ " def forward(self, question):\n",+ " context = []\n",+ " \n",+ " for hop in range(self.num_hops):\n",+ " search_query = self.generate_query[hop](context=context, question=question).search_query\n",+ " passages = self.retrieve(search_query).passages\n",+ " context = deduplicate(context + passages)\n",+ "\n",+ " prediction = self.generate_answer(context=context, question=question).copy(context=context)\n",+ " return prediction\n",+ "\n",+ "\n",+ "PROGRAM DESCRIPTION: This program is designed to solve a question-answering task using a language model. It appears to work by iteratively generating search queries and passing through a series of hops. Each hop retrieves relevant passages based on the generated search query, and the passages are combined to form a context. The program then generates an answer based on the accumulated context and the original question. The task is to answer a user's question by retrieving relevant information from a given context, and the program uses multiple hops to refine the search query and context until it generates an accurate answer.\n",+ "\n",+ "MODULE: # self.generate_query = dspy.ChainOfThought(GenerateSearchQuery)\n",+ "\n",+ "TASK DEMO(S):\n",+ "Context: []\n",+ "Question: What was the series inspired by the appreciation of manual laborers that Wheat Field with Cypresses is a part of?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: Wheat Field with Cypresses series\n",+ "\n",+ "\n",+ "BASIC INSTRUCTION: Given the fields `context`, `question`, produce the fields `search_query`.\n",+ "\n",+ "TIP: Make sure your instruction is very informative and descriptive.\n",+ "\n",+ "Please provide the output field PROPOSED INSTRUCTION. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field PROPOSED INSTRUCTION.\n",+ "\n",+ "PROPOSED INSTRUCTION: Given the context and question, generate a search query that accurately captures the essence of the query and retrieves relevant information to answer the question, ensuring the query is concise, descriptive, and specific to the context and question.\n",+ "\n",+ "\n",+ "\n",+ "PROPOSED INSTRUCTION: Given the context and question, generate a search query that accurately captures the essence of the query and retrieves relevant information to answer the question, ensuring the query is concise, descriptive, and specific to the context and question.\n",+ "PROGRAM DESCRIPTION: The program appears to be designed to solve the task of information retrieval and question answering. It uses a multi-hop approach, where it first generates a search query for a given question, retrieves relevant passages, and then generates a search query for the next hop, and so on, up to a specified number of hops. Finally, it generates an answer to the original question using the retrieved passages. The program uses language models to generate search queries and answers, and a retrieval module to retrieve relevant passages.\n",+ "task_demos Context: []\n",+ "Question: What other sport did Butch Reed, of the wrestling tag team Doom, play professionally?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: Butch Reed sports\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Use the information below to learn about a task that we are trying to solve using calls to an LM, then generate a new instruction that will be used to prompt a Language Model to better solve the task.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "DATASET SUMMARY: A description of the dataset that we are using.\n",+ "\n",+ "PROGRAM CODE: Language model program designed to solve a particular task.\n",+ "\n",+ "PROGRAM DESCRIPTION: Summary of the task the program is designed to solve, and how it goes about solving it.\n",+ "\n",+ "MODULE: The module to create an instruction for.\n",+ "\n",+ "TASK DEMO(S): Example inputs/outputs of our module.\n",+ "\n",+ "BASIC INSTRUCTION: Basic instruction.\n",+ "\n",+ "TIP: A suggestion for how to go about generating the new instruction.\n",+ "\n",+ "PROPOSED INSTRUCTION: Propose an instruction that will be used to prompt a Language Model to perform this task.\n",+ "\n",+ "---\n",+ "\n",+ "DATASET SUMMARY: The dataset appears to be a comprehensive collection of trivia questions, presenting a diverse range of topics, including entertainment, history, arts, and culture. Most questions require specific, factual answers, often testing users' recall of specific details or events, and are designed to test users' knowledge on subject matter. The syntax and length of the questions vary, but most are concise and require precise answers.\n",+ "\n",+ "PROGRAM CODE:\n",+ "StringSignature(context, question -> rationale, search_query\n",+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",+ ")\n",+ "\n",+ "StringSignature(context, question -> rationale, answer\n",+ " instructions='Given the fields `context`, `question`, produce the fields `answer`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Answer:', 'desc': '${answer}'})\n",+ ")\n",+ "\n",+ "class BasicMH(dspy.Module):\n",+ " def __init__(self, passages_per_hop=3, num_hops=2):\n",+ " super().__init__()\n",+ " self.num_hops = num_hops\n",+ " self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",+ " # self.generate_query = dspy.ChainOfThought(GenerateSearchQuery)\n",+ " self.generate_query = [dspy.ChainOfThought(\"context, question -> search_query\") for _ in range(self.num_hops)]\n",+ " self.generate_answer = dspy.ChainOfThought(\"context, question -> answer\")\n",+ " \n",+ " def forward(self, question):\n",+ " context = []\n",+ " \n",+ " for hop in range(self.num_hops):\n",+ " search_query = self.generate_query[hop](context=context, question=question).search_query\n",+ " passages = self.retrieve(search_query).passages\n",+ " context = deduplicate(context + passages)\n",+ "\n",+ " prediction = self.generate_answer(context=context, question=question).copy(context=context)\n",+ " return prediction\n",+ "\n",+ "\n",+ "PROGRAM DESCRIPTION: The program appears to be designed to solve the task of information retrieval and question answering. It uses a multi-hop approach, where it first generates a search query for a given question, retrieves relevant passages, and then generates a search query for the next hop, and so on, up to a specified number of hops. Finally, it generates an answer to the original question using the retrieved passages. The program uses language models to generate search queries and answers, and a retrieval module to retrieve relevant passages.\n",+ "\n",+ "MODULE: # self.generate_query = dspy.ChainOfThought(GenerateSearchQuery)\n",+ "\n",+ "TASK DEMO(S):\n",+ "Context: []\n",+ "Question: What other sport did Butch Reed, of the wrestling tag team Doom, play professionally?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: Butch Reed sports\n",+ "\n",+ "\n",+ "BASIC INSTRUCTION: Given the fields `context`, `question`, produce the fields `search_query`.\n",+ "\n",+ "TIP: Make sure your instruction is very informative and descriptive.\n",+ "\n",+ "Please provide the output field PROPOSED INSTRUCTION. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field PROPOSED INSTRUCTION.\n",+ "\n",+ "PROPOSED INSTRUCTION: Given a context and a question, generate a search query that is a concise and specific phrase that can be used to retrieve relevant information to answer the question.\n",+ "\n",+ "\n",+ "\n",+ "PROPOSED INSTRUCTION: Given a context and a question, generate a search query that is a concise and specific phrase that can be used to retrieve relevant information to answer the question.\n",+ "PROGRAM DESCRIPTION: The program appears to be designed to solve information retrieval and question answering tasks using language models. It takes a context and a question as input, and produces a search query and/or an answer as output. The program uses a multi-hop approach, where it iteratively generates search queries based on the previous search query and the context, and then retrieves relevant passages from a dataset. The final answer is generated by applying a chain of thought to the retrieved passages and the original question. The program can be used to answer questions about various topics, such as history, geography, entertainment, and more.\n",+ "task_demos Context: []\n",+ "Question: Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: Garden City Hotel founder\n",+ "Context: []\n",+ "Question: Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: clothing company founded 1889 college bass fishing\n",+ "Context: []\n",+ "Question: Which theme park has Stitch’s Great Escape which replaced the attraction ExtraTERRORestial Alien Encounter?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: Stitch's Great Escape theme park\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Use the information below to learn about a task that we are trying to solve using calls to an LM, then generate a new instruction that will be used to prompt a Language Model to better solve the task.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "DATASET SUMMARY: A description of the dataset that we are using.\n",+ "\n",+ "PROGRAM CODE: Language model program designed to solve a particular task.\n",+ "\n",+ "PROGRAM DESCRIPTION: Summary of the task the program is designed to solve, and how it goes about solving it.\n",+ "\n",+ "MODULE: The module to create an instruction for.\n",+ "\n",+ "TASK DEMO(S): Example inputs/outputs of our module.\n",+ "\n",+ "BASIC INSTRUCTION: Basic instruction.\n",+ "\n",+ "TIP: A suggestion for how to go about generating the new instruction.\n",+ "\n",+ "PROPOSED INSTRUCTION: Propose an instruction that will be used to prompt a Language Model to perform this task.\n",+ "\n",+ "---\n",+ "\n",+ "DATASET SUMMARY: The dataset appears to be a comprehensive collection of trivia questions, presenting a diverse range of topics, including entertainment, history, arts, and culture. Most questions require specific, factual answers, often testing users' recall of specific details or events, and are designed to test users' knowledge on subject matter. The syntax and length of the questions vary, but most are concise and require precise answers.\n",+ "\n",+ "PROGRAM CODE:\n",+ "StringSignature(context, question -> rationale, search_query\n",+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",+ ")\n",+ "\n",+ "StringSignature(context, question -> rationale, answer\n",+ " instructions='Given the fields `context`, `question`, produce the fields `answer`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Answer:', 'desc': '${answer}'})\n",+ ")\n",+ "\n",+ "class BasicMH(dspy.Module):\n",+ " def __init__(self, passages_per_hop=3, num_hops=2):\n",+ " super().__init__()\n",+ " self.num_hops = num_hops\n",+ " self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",+ " # self.generate_query = dspy.ChainOfThought(GenerateSearchQuery)\n",+ " self.generate_query = [dspy.ChainOfThought(\"context, question -> search_query\") for _ in range(self.num_hops)]\n",+ " self.generate_answer = dspy.ChainOfThought(\"context, question -> answer\")\n",+ " \n",+ " def forward(self, question):\n",+ " context = []\n",+ " \n",+ " for hop in range(self.num_hops):\n",+ " search_query = self.generate_query[hop](context=context, question=question).search_query\n",+ " passages = self.retrieve(search_query).passages\n",+ " context = deduplicate(context + passages)\n",+ "\n",+ " prediction = self.generate_answer(context=context, question=question).copy(context=context)\n",+ " return prediction\n",+ "\n",+ "\n",+ "PROGRAM DESCRIPTION: The program appears to be designed to solve information retrieval and question answering tasks using language models. It takes a context and a question as input, and produces a search query and/or an answer as output. The program uses a multi-hop approach, where it iteratively generates search queries based on the previous search query and the context, and then retrieves relevant passages from a dataset. The final answer is generated by applying a chain of thought to the retrieved passages and the original question. The program can be used to answer questions about various topics, such as history, geography, entertainment, and more.\n",+ "\n",+ "MODULE: # self.generate_query = dspy.ChainOfThought(GenerateSearchQuery)\n",+ "\n",+ "TASK DEMO(S):\n",+ "Context: []\n",+ "Question: Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: Garden City Hotel founder\n",+ "Context: []\n",+ "Question: Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: clothing company founded 1889 college bass fishing\n",+ "Context: []\n",+ "Question: Which theme park has Stitch’s Great Escape which replaced the attraction ExtraTERRORestial Alien Encounter?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: Stitch's Great Escape theme park\n",+ "\n",+ "\n",+ "BASIC INSTRUCTION: Given the fields `context`, `question`, produce the fields `search_query`.\n",+ "\n",+ "TIP: Make sure your instruction is very informative and descriptive.\n",+ "\n",+ "Please provide the output field PROPOSED INSTRUCTION. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field PROPOSED INSTRUCTION.\n",+ "\n",+ "PROPOSED INSTRUCTION: Produce a search query that effectively captures the essence of the provided context and question, allowing me to retrieve relevant information that can aid in answering the question with precision and accuracy.\n",+ "\n",+ "\n",+ "\n",+ "PROPOSED INSTRUCTION: Produce a search query that effectively captures the essence of the provided context and question, allowing me to retrieve relevant information that can aid in answering the question with precision and accuracy.\n",+ "PROGRAM DESCRIPTION: The program appears to be designed to solve tasks that involve generating search queries and answers to questions based on a given context. It uses a language model to generate a series of search queries, each of which retrieves a set of passages. The program then deduplicates the passages and uses them to generate an answer to the original question.\n",+ "task_demos Context: []\n",+ "Question: What company formed in 1978 has Simon Arora as its CEO?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: Simon Arora CEO company 1978\n",+ "Context: []\n",+ "Question: Bragg electoral district, incorporating the Kensington Park suburb, are associated with which South Australian city?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: Bragg electoral district Kensington Park suburb South Australian city\n",+ "Context: []\n",+ "Question: The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: Kangchenjunga Singhi Kangri country\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Use the information below to learn about a task that we are trying to solve using calls to an LM, then generate a new instruction that will be used to prompt a Language Model to better solve the task.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "DATASET SUMMARY: A description of the dataset that we are using.\n",+ "\n",+ "PROGRAM CODE: Language model program designed to solve a particular task.\n",+ "\n",+ "PROGRAM DESCRIPTION: Summary of the task the program is designed to solve, and how it goes about solving it.\n",+ "\n",+ "MODULE: The module to create an instruction for.\n",+ "\n",+ "TASK DEMO(S): Example inputs/outputs of our module.\n",+ "\n",+ "BASIC INSTRUCTION: Basic instruction.\n",+ "\n",+ "TIP: A suggestion for how to go about generating the new instruction.\n",+ "\n",+ "PROPOSED INSTRUCTION: Propose an instruction that will be used to prompt a Language Model to perform this task.\n",+ "\n",+ "---\n",+ "\n",+ "DATASET SUMMARY: The dataset appears to be a comprehensive collection of trivia questions, presenting a diverse range of topics, including entertainment, history, arts, and culture. Most questions require specific, factual answers, often testing users' recall of specific details or events, and are designed to test users' knowledge on subject matter. The syntax and length of the questions vary, but most are concise and require precise answers.\n",+ "\n",+ "PROGRAM CODE:\n",+ "StringSignature(context, question -> rationale, search_query\n",+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",+ ")\n",+ "\n",+ "StringSignature(context, question -> rationale, answer\n",+ " instructions='Given the fields `context`, `question`, produce the fields `answer`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Answer:', 'desc': '${answer}'})\n",+ ")\n",+ "\n",+ "class BasicMH(dspy.Module):\n",+ " def __init__(self, passages_per_hop=3, num_hops=2):\n",+ " super().__init__()\n",+ " self.num_hops = num_hops\n",+ " self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",+ " # self.generate_query = dspy.ChainOfThought(GenerateSearchQuery)\n",+ " self.generate_query = [dspy.ChainOfThought(\"context, question -> search_query\") for _ in range(self.num_hops)]\n",+ " self.generate_answer = dspy.ChainOfThought(\"context, question -> answer\")\n",+ " \n",+ " def forward(self, question):\n",+ " context = []\n",+ " \n",+ " for hop in range(self.num_hops):\n",+ " search_query = self.generate_query[hop](context=context, question=question).search_query\n",+ " passages = self.retrieve(search_query).passages\n",+ " context = deduplicate(context + passages)\n",+ "\n",+ " prediction = self.generate_answer(context=context, question=question).copy(context=context)\n",+ " return prediction\n",+ "\n",+ "\n",+ "PROGRAM DESCRIPTION: The program appears to be designed to solve tasks that involve generating search queries and answers to questions based on a given context. It uses a language model to generate a series of search queries, each of which retrieves a set of passages. The program then deduplicates the passages and uses them to generate an answer to the original question.\n",+ "\n",+ "MODULE: # self.generate_query = dspy.ChainOfThought(GenerateSearchQuery)\n",+ "\n",+ "TASK DEMO(S):\n",+ "Context: []\n",+ "Question: What company formed in 1978 has Simon Arora as its CEO?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: Simon Arora CEO company 1978\n",+ "Context: []\n",+ "Question: Bragg electoral district, incorporating the Kensington Park suburb, are associated with which South Australian city?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: Bragg electoral district Kensington Park suburb South Australian city\n",+ "Context: []\n",+ "Question: The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: Kangchenjunga Singhi Kangri country\n",+ "\n",+ "\n",+ "BASIC INSTRUCTION: Given the fields `context`, `question`, produce the fields `search_query`.\n",+ "\n",+ "TIP: Make sure your instruction is very informative and descriptive.\n",+ "\n",+ "Please provide the output field PROPOSED INSTRUCTION. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field PROPOSED INSTRUCTION.\n",+ "\n",+ "PROPOSED INSTRUCTION: \"Generate a search query that accurately captures the essence of the question within the given context, leveraging specific keywords and phrases to guide the retrieval of relevant passages.\"\n",+ "\n",+ "\n",+ "\n",+ "PROPOSED INSTRUCTION: Generate a search query that accurately captures the essence of the question within the given context, leveraging specific keywords and phrases to guide the retrieval of relevant passages.\n",+ "PROGRAM DESCRIPTION: This program is designed to solve a question-answering task. It appears to work by iteratively generating search queries based on the context and question, retrieving relevant passages, and then generating answers based on the retrieved passages. The program uses a chain of thought approach, where each hop represents a step in the reasoning process, and the output of each hop is used as input for the next hop. The program seems to be designed to simulate how a human would answer a question, by thinking step by step and generating intermediate search queries and answers.\n",+ "task_demos \n",+ "\n",+ "\n",+ "\n",+ "Use the information below to learn about a task that we are trying to solve using calls to an LM, then generate a new instruction that will be used to prompt a Language Model to better solve the task.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "DATASET SUMMARY: A description of the dataset that we are using.\n",+ "\n",+ "PROGRAM CODE: Language model program designed to solve a particular task.\n",+ "\n",+ "PROGRAM DESCRIPTION: Summary of the task the program is designed to solve, and how it goes about solving it.\n",+ "\n",+ "MODULE: The module to create an instruction for.\n",+ "\n",+ "TASK DEMO(S): Example inputs/outputs of our module.\n",+ "\n",+ "BASIC INSTRUCTION: Basic instruction.\n",+ "\n",+ "TIP: A suggestion for how to go about generating the new instruction.\n",+ "\n",+ "PROPOSED INSTRUCTION: Propose an instruction that will be used to prompt a Language Model to perform this task.\n",+ "\n",+ "---\n",+ "\n",+ "DATASET SUMMARY: The dataset appears to be a comprehensive collection of trivia questions, presenting a diverse range of topics, including entertainment, history, arts, and culture. Most questions require specific, factual answers, often testing users' recall of specific details or events, and are designed to test users' knowledge on subject matter. The syntax and length of the questions vary, but most are concise and require precise answers.\n",+ "\n",+ "PROGRAM CODE:\n",+ "StringSignature(context, question -> rationale, search_query\n",+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",+ ")\n",+ "\n",+ "StringSignature(context, question -> rationale, answer\n",+ " instructions='Given the fields `context`, `question`, produce the fields `answer`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Answer:', 'desc': '${answer}'})\n",+ ")\n",+ "\n",+ "class BasicMH(dspy.Module):\n",+ " def __init__(self, passages_per_hop=3, num_hops=2):\n",+ " super().__init__()\n",+ " self.num_hops = num_hops\n",+ " self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",+ " # self.generate_query = dspy.ChainOfThought(GenerateSearchQuery)\n",+ " self.generate_query = [dspy.ChainOfThought(\"context, question -> search_query\") for _ in range(self.num_hops)]\n",+ " self.generate_answer = dspy.ChainOfThought(\"context, question -> answer\")\n",+ " \n",+ " def forward(self, question):\n",+ " context = []\n",+ " \n",+ " for hop in range(self.num_hops):\n",+ " search_query = self.generate_query[hop](context=context, question=question).search_query\n",+ " passages = self.retrieve(search_query).passages\n",+ " context = deduplicate(context + passages)\n",+ "\n",+ " prediction = self.generate_answer(context=context, question=question).copy(context=context)\n",+ " return prediction\n",+ "\n",+ "\n",+ "PROGRAM DESCRIPTION: This program is designed to solve a question-answering task. It appears to work by iteratively generating search queries based on the context and question, retrieving relevant passages, and then generating answers based on the retrieved passages. The program uses a chain of thought approach, where each hop represents a step in the reasoning process, and the output of each hop is used as input for the next hop. The program seems to be designed to simulate how a human would answer a question, by thinking step by step and generating intermediate search queries and answers.\n",+ "\n",+ "MODULE: self.generate_query = [dspy.ChainOfThought(\"context, question -> search_query\") for _ in range(self.num_hops)]\n",+ "\n",+ "TASK DEMO(S): \n",+ "\n",+ "BASIC INSTRUCTION: Given the fields `context`, `question`, produce the fields `search_query`.\n",+ "\n",+ "TIP: Make sure your instruction is very informative and descriptive.\n",+ "\n",+ "Please provide the output field PROPOSED INSTRUCTION. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field PROPOSED INSTRUCTION.\n",+ "\n",+ "PROPOSED INSTRUCTION: Generate a subsequent search query based on the provided context and question, aiming to narrow down the scope of relevant information and guide the retrieval of precise passages to support the forthcoming answer.\n",+ "\n",+ "\n",+ "\n",+ "PROPOSED INSTRUCTION: Generate a subsequent search query based on the provided context and question, aiming to narrow down the scope of relevant information and guide the retrieval of precise passages to support the forthcoming answer.\n",+ "PROGRAM DESCRIPTION: This program is designed to solve a question-answering task. It appears to work by iteratively generating search queries based on the context and question, retrieving relevant passages, and then generating answers based on the retrieved passages. The program uses a chain of thought approach, where each hop represents a step in the reasoning process, and the output of each hop is used as input for the next hop. The program seems to be designed to simulate how a human would answer a question, by thinking step by step and generating intermediate search queries and answers.\n",+ "task_demos \n",+ "\n",+ "\n",+ "\n",+ "Use the information below to learn about a task that we are trying to solve using calls to an LM, then generate a new instruction that will be used to prompt a Language Model to better solve the task.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "DATASET SUMMARY: A description of the dataset that we are using.\n",+ "\n",+ "PROGRAM CODE: Language model program designed to solve a particular task.\n",+ "\n",+ "PROGRAM DESCRIPTION: Summary of the task the program is designed to solve, and how it goes about solving it.\n",+ "\n",+ "MODULE: The module to create an instruction for.\n",+ "\n",+ "TASK DEMO(S): Example inputs/outputs of our module.\n",+ "\n",+ "BASIC INSTRUCTION: Basic instruction.\n",+ "\n",+ "TIP: A suggestion for how to go about generating the new instruction.\n",+ "\n",+ "PROPOSED INSTRUCTION: Propose an instruction that will be used to prompt a Language Model to perform this task.\n",+ "\n",+ "---\n",+ "\n",+ "DATASET SUMMARY: The dataset appears to be a comprehensive collection of trivia questions, presenting a diverse range of topics, including entertainment, history, arts, and culture. Most questions require specific, factual answers, often testing users' recall of specific details or events, and are designed to test users' knowledge on subject matter. The syntax and length of the questions vary, but most are concise and require precise answers.\n",+ "\n",+ "PROGRAM CODE:\n",+ "StringSignature(context, question -> rationale, search_query\n",+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",+ ")\n",+ "\n",+ "StringSignature(context, question -> rationale, answer\n",+ " instructions='Given the fields `context`, `question`, produce the fields `answer`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Answer:', 'desc': '${answer}'})\n",+ ")\n",+ "\n",+ "class BasicMH(dspy.Module):\n",+ " def __init__(self, passages_per_hop=3, num_hops=2):\n",+ " super().__init__()\n",+ " self.num_hops = num_hops\n",+ " self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",+ " # self.generate_query = dspy.ChainOfThought(GenerateSearchQuery)\n",+ " self.generate_query = [dspy.ChainOfThought(\"context, question -> search_query\") for _ in range(self.num_hops)]\n",+ " self.generate_answer = dspy.ChainOfThought(\"context, question -> answer\")\n",+ " \n",+ " def forward(self, question):\n",+ " context = []\n",+ " \n",+ " for hop in range(self.num_hops):\n",+ " search_query = self.generate_query[hop](context=context, question=question).search_query\n",+ " passages = self.retrieve(search_query).passages\n",+ " context = deduplicate(context + passages)\n",+ "\n",+ " prediction = self.generate_answer(context=context, question=question).copy(context=context)\n",+ " return prediction\n",+ "\n",+ "\n",+ "PROGRAM DESCRIPTION: This program is designed to solve a question-answering task. It appears to work by iteratively generating search queries based on the context and question, retrieving relevant passages, and then generating answers based on the retrieved passages. The program uses a chain of thought approach, where each hop represents a step in the reasoning process, and the output of each hop is used as input for the next hop. The program seems to be designed to simulate how a human would answer a question, by thinking step by step and generating intermediate search queries and answers.\n",+ "\n",+ "MODULE: self.generate_query = [dspy.ChainOfThought(\"context, question -> search_query\") for _ in range(self.num_hops)]\n",+ "\n",+ "TASK DEMO(S): \n",+ "\n",+ "BASIC INSTRUCTION: Given the fields `context`, `question`, produce the fields `search_query`.\n",+ "\n",+ "TIP: Make sure your instruction is very informative and descriptive.\n",+ "\n",+ "Please provide the output field PROPOSED INSTRUCTION. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field PROPOSED INSTRUCTION.\n",+ "\n",+ "PROPOSED INSTRUCTION: Generate a subsequent search query based on the provided context and question, aiming to narrow down the scope of relevant information and guide the retrieval of precise passages to support the forthcoming answer.\n",+ "\n",+ "\n",+ "\n",+ "PROPOSED INSTRUCTION: Generate a subsequent search query based on the provided context and question, aiming to narrow down the scope of relevant information and guide the retrieval of precise passages to support the forthcoming answer.\n",+ "PROGRAM DESCRIPTION: The program appears to be designed to solve a question-answering task using a language model. The program takes in a context and a question as input, and uses a multi-hop retrieval-based approach to answer the question. The program searches for relevant passages based on the context and question, and then uses these passages to generate a search query. The search query is then used to retrieve more relevant passages, which are then used to generate a final answer. The program uses a chain of thought to reason about the question and generate a plausible answer.\n",+ "task_demos Context: ['Alberta general election, 2015 | The 29th general election of Alberta, Canada, took place on May 5, 2015, following a request of Premier Jim Prentice to the Lieutenant Governor of Alberta Donald Ethell to dissolve the Legislative Assembly on April 7, 2015. This election elected members to the 29th Alberta Legislature. It was one of only four times that Alberta has changed governments.', \"Jenean Hampton | Jenean Michelle Hampton (born May 12, 1958) is the 57th and current Lieutenant Governor of Kentucky since December 8, 2015 after having won office as Republican gubernatorial nominee Matt Bevin's running-mate, in the 2015 gubernatorial election. She is the first African-American to hold any statewide office in Kentucky history, and only the third African-American woman to serve as lieutenant-governor of any U.S. state (after Jennette Bradley and Jennifer Carroll).\", 'Louisiana lieutenant gubernatorial election, 2015 | The Louisiana lieutenant gubernatorial election took place on October 24, 2015, to elect the Lieutenant Governor of Louisiana, with a runoff election held on November 21, 2015. Incumbent Republican Lieutenant Governor Jay Dardenne did not for re-election to a second full term in office. He instead ran for Governor. Billy Nungesser won the election defeating Kip Holden, despite a Democratic victory in the gubernatorial election, in which John Bel Edwards defeated David Vitter by a similar margin.']\n",+ "Question: What race is the lieutenant governor of the state that held its primary for her election on May 19th, 2015?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: lieutenant governor of the state that held its primary for her election on May 19th 2015\n",+ "Context: ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.']\n",+ "Question: Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: Carhartt bass fishing\n",+ "Context: ['Forrest Tucker | Forrest Meredith Tucker (February 12, 1919 – October 25, 1986) was an American actor in both movies and television who appeared in nearly a hundred films. Tucker worked as a vaudeville straight man, aged fifteen years old. A mentor provided funds and contacts for a trip to California, where party hostess Cobina Wright persuaded guest Wesley Ruggles to give Tucker a screen test, based on his photogenic good looks, thick wavy hair and height of six feet, five inches - putting him into the same league as John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness\\'s younger brother. Tucker was a sight reader who needed only one take. His film career started well despite a perception in most Hollywood studios that blond men were not photogenic. After twenty years spent mainly in Westerns and action roles, he returned to his roots, showing versatility as a comedic and stage musical actor. In the TV show \"F Troop\", he became identified with the character of Cavalry Sgt. Morgan O\\'Rourke (a manipulative character quite similar to Phil Silvers\\' role as MSgt Ernie Bilko). Tucker struggled with a drinking problem that began to affect his performances in the later years of his career.', 'Emergency Landing (1941 film) | Emergency Landing (a.k.a. Robot Pilot) is a 1941 American aviation spy-fi romantic screwball comedy film directed by William Beaudine. The film stars Forrest Tucker in his second film and in his first leading role with co-stars Carol Hughes and Evelyn Brent. \"Emergency Landing\" features lots of mismatched stock footage of various types of aircraft.', 'Forrest Tucker (criminal) | Forrest Silva \"Woody\" Tucker (June 23, 1920 – May 29, 2004) was an American career criminal first imprisoned at age 15 who spent the rest of his life in and out of jail. He is best known as an escape artist, having escaped from prison 18 times successfully and 12 times unsuccessfully.']\n",+ "Question: Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: Walt Disney TV miniseries 1955-1956 character\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Use the information below to learn about a task that we are trying to solve using calls to an LM, then generate a new instruction that will be used to prompt a Language Model to better solve the task.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "DATASET SUMMARY: A description of the dataset that we are using.\n",+ "\n",+ "PROGRAM CODE: Language model program designed to solve a particular task.\n",+ "\n",+ "PROGRAM DESCRIPTION: Summary of the task the program is designed to solve, and how it goes about solving it.\n",+ "\n",+ "MODULE: The module to create an instruction for.\n",+ "\n",+ "TASK DEMO(S): Example inputs/outputs of our module.\n",+ "\n",+ "BASIC INSTRUCTION: Basic instruction.\n",+ "\n",+ "TIP: A suggestion for how to go about generating the new instruction.\n",+ "\n",+ "PROPOSED INSTRUCTION: Propose an instruction that will be used to prompt a Language Model to perform this task.\n",+ "\n",+ "---\n",+ "\n",+ "DATASET SUMMARY: The dataset appears to be a comprehensive collection of trivia questions, presenting a diverse range of topics, including entertainment, history, arts, and culture. Most questions require specific, factual answers, often testing users' recall of specific details or events, and are designed to test users' knowledge on subject matter. The syntax and length of the questions vary, but most are concise and require precise answers.\n",+ "\n",+ "PROGRAM CODE:\n",+ "StringSignature(context, question -> rationale, search_query\n",+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",+ ")\n",+ "\n",+ "StringSignature(context, question -> rationale, answer\n",+ " instructions='Given the fields `context`, `question`, produce the fields `answer`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Answer:', 'desc': '${answer}'})\n",+ ")\n",+ "\n",+ "class BasicMH(dspy.Module):\n",+ " def __init__(self, passages_per_hop=3, num_hops=2):\n",+ " super().__init__()\n",+ " self.num_hops = num_hops\n",+ " self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",+ " # self.generate_query = dspy.ChainOfThought(GenerateSearchQuery)\n",+ " self.generate_query = [dspy.ChainOfThought(\"context, question -> search_query\") for _ in range(self.num_hops)]\n",+ " self.generate_answer = dspy.ChainOfThought(\"context, question -> answer\")\n",+ " \n",+ " def forward(self, question):\n",+ " context = []\n",+ " \n",+ " for hop in range(self.num_hops):\n",+ " search_query = self.generate_query[hop](context=context, question=question).search_query\n",+ " passages = self.retrieve(search_query).passages\n",+ " context = deduplicate(context + passages)\n",+ "\n",+ " prediction = self.generate_answer(context=context, question=question).copy(context=context)\n",+ " return prediction\n",+ "\n",+ "\n",+ "PROGRAM DESCRIPTION: The program appears to be designed to solve a question-answering task using a language model. The program takes in a context and a question as input, and uses a multi-hop retrieval-based approach to answer the question. The program searches for relevant passages based on the context and question, and then uses these passages to generate a search query. The search query is then used to retrieve more relevant passages, which are then used to generate a final answer. The program uses a chain of thought to reason about the question and generate a plausible answer.\n",+ "\n",+ "MODULE: self.generate_query = [dspy.ChainOfThought(\"context, question -> search_query\") for _ in range(self.num_hops)]\n",+ "\n",+ "TASK DEMO(S):\n",+ "Context: ['Alberta general election, 2015 | The 29th general election of Alberta, Canada, took place on May 5, 2015, following a request of Premier Jim Prentice to the Lieutenant Governor of Alberta Donald Ethell to dissolve the Legislative Assembly on April 7, 2015. This election elected members to the 29th Alberta Legislature. It was one of only four times that Alberta has changed governments.', \"Jenean Hampton | Jenean Michelle Hampton (born May 12, 1958) is the 57th and current Lieutenant Governor of Kentucky since December 8, 2015 after having won office as Republican gubernatorial nominee Matt Bevin's running-mate, in the 2015 gubernatorial election. She is the first African-American to hold any statewide office in Kentucky history, and only the third African-American woman to serve as lieutenant-governor of any U.S. state (after Jennette Bradley and Jennifer Carroll).\", 'Louisiana lieutenant gubernatorial election, 2015 | The Louisiana lieutenant gubernatorial election took place on October 24, 2015, to elect the Lieutenant Governor of Louisiana, with a runoff election held on November 21, 2015. Incumbent Republican Lieutenant Governor Jay Dardenne did not for re-election to a second full term in office. He instead ran for Governor. Billy Nungesser won the election defeating Kip Holden, despite a Democratic victory in the gubernatorial election, in which John Bel Edwards defeated David Vitter by a similar margin.']\n",+ "Question: What race is the lieutenant governor of the state that held its primary for her election on May 19th, 2015?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: lieutenant governor of the state that held its primary for her election on May 19th 2015\n",+ "Context: ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.']\n",+ "Question: Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: Carhartt bass fishing\n",+ "Context: ['Forrest Tucker | Forrest Meredith Tucker (February 12, 1919 – October 25, 1986) was an American actor in both movies and television who appeared in nearly a hundred films. Tucker worked as a vaudeville straight man, aged fifteen years old. A mentor provided funds and contacts for a trip to California, where party hostess Cobina Wright persuaded guest Wesley Ruggles to give Tucker a screen test, based on his photogenic good looks, thick wavy hair and height of six feet, five inches - putting him into the same league as John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness\\'s younger brother. Tucker was a sight reader who needed only one take. His film career started well despite a perception in most Hollywood studios that blond men were not photogenic. After twenty years spent mainly in Westerns and action roles, he returned to his roots, showing versatility as a comedic and stage musical actor. In the TV show \"F Troop\", he became identified with the character of Cavalry Sgt. Morgan O\\'Rourke (a manipulative character quite similar to Phil Silvers\\' role as MSgt Ernie Bilko). Tucker struggled with a drinking problem that began to affect his performances in the later years of his career.', 'Emergency Landing (1941 film) | Emergency Landing (a.k.a. Robot Pilot) is a 1941 American aviation spy-fi romantic screwball comedy film directed by William Beaudine. The film stars Forrest Tucker in his second film and in his first leading role with co-stars Carol Hughes and Evelyn Brent. \"Emergency Landing\" features lots of mismatched stock footage of various types of aircraft.', 'Forrest Tucker (criminal) | Forrest Silva \"Woody\" Tucker (June 23, 1920 – May 29, 2004) was an American career criminal first imprisoned at age 15 who spent the rest of his life in and out of jail. He is best known as an escape artist, having escaped from prison 18 times successfully and 12 times unsuccessfully.']\n",+ "Question: Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: Walt Disney TV miniseries 1955-1956 character\n",+ "\n",+ "\n",+ "BASIC INSTRUCTION: Given the fields `context`, `question`, produce the fields `search_query`.\n",+ "\n",+ "TIP: Make sure your instruction is very informative and descriptive.\n",+ "\n",+ "Please provide the output field PROPOSED INSTRUCTION. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field PROPOSED INSTRUCTION.\n",+ "\n",+ "PROPOSED INSTRUCTION: Given the context and question, formulate a search query that accurately captures the key information required to answer the question, incorporating relevant entities and concepts from the context, and ensuring that the query is specific, concise, and effectively retrieves relevant passages to support the question's answer.\n",+ "\n",+ "\n",+ "\n",+ "PROPOSED INSTRUCTION: Given the context and question, formulate a search query that accurately captures the key information required to answer the question, incorporating relevant entities and concepts from the context, and ensuring that the query is specific, concise, and effectively retrieves relevant passages to support the question's answer.\n",+ "PROGRAM DESCRIPTION: The program appears to be designed to solve a question-answering task using a language model. It takes in a context and a question as input, and outputs an answer along with a search query and a rationale explaining the reasoning behind the answer. The program consists of a pipeline of modules, including a retriever that retrieves relevant passages from a dataset, a search query generator that generates a search query based on the context and question, and an answer generator that generates an answer based on the context, question, and search query results. The program can be used to answer a wide range of questions across various topics, and provides a compelling demonstration of the power of language models in solving complex tasks.\n",+ "task_demos Context: ['Forrest Tucker | Forrest Meredith Tucker (February 12, 1919 – October 25, 1986) was an American actor in both movies and television who appeared in nearly a hundred films. Tucker worked as a vaudeville straight man, aged fifteen years old. A mentor provided funds and contacts for a trip to California, where party hostess Cobina Wright persuaded guest Wesley Ruggles to give Tucker a screen test, based on his photogenic good looks, thick wavy hair and height of six feet, five inches - putting him into the same league as John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness\\'s younger brother. Tucker was a sight reader who needed only one take. His film career started well despite a perception in most Hollywood studios that blond men were not photogenic. After twenty years spent mainly in Westerns and action roles, he returned to his roots, showing versatility as a comedic and stage musical actor. In the TV show \"F Troop\", he became identified with the character of Cavalry Sgt. Morgan O\\'Rourke (a manipulative character quite similar to Phil Silvers\\' role as MSgt Ernie Bilko). Tucker struggled with a drinking problem that began to affect his performances in the later years of his career.', 'Emergency Landing (1941 film) | Emergency Landing (a.k.a. Robot Pilot) is a 1941 American aviation spy-fi romantic screwball comedy film directed by William Beaudine. The film stars Forrest Tucker in his second film and in his first leading role with co-stars Carol Hughes and Evelyn Brent. \"Emergency Landing\" features lots of mismatched stock footage of various types of aircraft.', 'Forrest Tucker (criminal) | Forrest Silva \"Woody\" Tucker (June 23, 1920 – May 29, 2004) was an American career criminal first imprisoned at age 15 who spent the rest of his life in and out of jail. He is best known as an escape artist, having escaped from prison 18 times successfully and 12 times unsuccessfully.']\n",+ "Question: Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: Walt Disney TV miniseries 1955-1956 character\n",+ "Context: ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.']\n",+ "Question: Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: Carhartt bass fishing\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Use the information below to learn about a task that we are trying to solve using calls to an LM, then generate a new instruction that will be used to prompt a Language Model to better solve the task.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "DATASET SUMMARY: A description of the dataset that we are using.\n",+ "\n",+ "PROGRAM CODE: Language model program designed to solve a particular task.\n",+ "\n",+ "PROGRAM DESCRIPTION: Summary of the task the program is designed to solve, and how it goes about solving it.\n",+ "\n",+ "MODULE: The module to create an instruction for.\n",+ "\n",+ "TASK DEMO(S): Example inputs/outputs of our module.\n",+ "\n",+ "BASIC INSTRUCTION: Basic instruction.\n",+ "\n",+ "TIP: A suggestion for how to go about generating the new instruction.\n",+ "\n",+ "PROPOSED INSTRUCTION: Propose an instruction that will be used to prompt a Language Model to perform this task.\n",+ "\n",+ "---\n",+ "\n",+ "DATASET SUMMARY: The dataset appears to be a comprehensive collection of trivia questions, presenting a diverse range of topics, including entertainment, history, arts, and culture. Most questions require specific, factual answers, often testing users' recall of specific details or events, and are designed to test users' knowledge on subject matter. The syntax and length of the questions vary, but most are concise and require precise answers.\n",+ "\n",+ "PROGRAM CODE:\n",+ "StringSignature(context, question -> rationale, search_query\n",+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",+ ")\n",+ "\n",+ "StringSignature(context, question -> rationale, answer\n",+ " instructions='Given the fields `context`, `question`, produce the fields `answer`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Answer:', 'desc': '${answer}'})\n",+ ")\n",+ "\n",+ "class BasicMH(dspy.Module):\n",+ " def __init__(self, passages_per_hop=3, num_hops=2):\n",+ " super().__init__()\n",+ " self.num_hops = num_hops\n",+ " self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",+ " # self.generate_query = dspy.ChainOfThought(GenerateSearchQuery)\n",+ " self.generate_query = [dspy.ChainOfThought(\"context, question -> search_query\") for _ in range(self.num_hops)]\n",+ " self.generate_answer = dspy.ChainOfThought(\"context, question -> answer\")\n",+ " \n",+ " def forward(self, question):\n",+ " context = []\n",+ " \n",+ " for hop in range(self.num_hops):\n",+ " search_query = self.generate_query[hop](context=context, question=question).search_query\n",+ " passages = self.retrieve(search_query).passages\n",+ " context = deduplicate(context + passages)\n",+ "\n",+ " prediction = self.generate_answer(context=context, question=question).copy(context=context)\n",+ " return prediction\n",+ "\n",+ "\n",+ "PROGRAM DESCRIPTION: The program appears to be designed to solve a question-answering task using a language model. It takes in a context and a question as input, and outputs an answer along with a search query and a rationale explaining the reasoning behind the answer. The program consists of a pipeline of modules, including a retriever that retrieves relevant passages from a dataset, a search query generator that generates a search query based on the context and question, and an answer generator that generates an answer based on the context, question, and search query results. The program can be used to answer a wide range of questions across various topics, and provides a compelling demonstration of the power of language models in solving complex tasks.\n",+ "\n",+ "MODULE: self.generate_query = [dspy.ChainOfThought(\"context, question -> search_query\") for _ in range(self.num_hops)]\n",+ "\n",+ "TASK DEMO(S):\n",+ "Context: ['Forrest Tucker | Forrest Meredith Tucker (February 12, 1919 – October 25, 1986) was an American actor in both movies and television who appeared in nearly a hundred films. Tucker worked as a vaudeville straight man, aged fifteen years old. A mentor provided funds and contacts for a trip to California, where party hostess Cobina Wright persuaded guest Wesley Ruggles to give Tucker a screen test, based on his photogenic good looks, thick wavy hair and height of six feet, five inches - putting him into the same league as John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness\\'s younger brother. Tucker was a sight reader who needed only one take. His film career started well despite a perception in most Hollywood studios that blond men were not photogenic. After twenty years spent mainly in Westerns and action roles, he returned to his roots, showing versatility as a comedic and stage musical actor. In the TV show \"F Troop\", he became identified with the character of Cavalry Sgt. Morgan O\\'Rourke (a manipulative character quite similar to Phil Silvers\\' role as MSgt Ernie Bilko). Tucker struggled with a drinking problem that began to affect his performances in the later years of his career.', 'Emergency Landing (1941 film) | Emergency Landing (a.k.a. Robot Pilot) is a 1941 American aviation spy-fi romantic screwball comedy film directed by William Beaudine. The film stars Forrest Tucker in his second film and in his first leading role with co-stars Carol Hughes and Evelyn Brent. \"Emergency Landing\" features lots of mismatched stock footage of various types of aircraft.', 'Forrest Tucker (criminal) | Forrest Silva \"Woody\" Tucker (June 23, 1920 – May 29, 2004) was an American career criminal first imprisoned at age 15 who spent the rest of his life in and out of jail. He is best known as an escape artist, having escaped from prison 18 times successfully and 12 times unsuccessfully.']\n",+ "Question: Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: Walt Disney TV miniseries 1955-1956 character\n",+ "Context: ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.']\n",+ "Question: Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: Carhartt bass fishing\n",+ "\n",+ "\n",+ "BASIC INSTRUCTION: Given the fields `context`, `question`, produce the fields `search_query`.\n",+ "\n",+ "TIP: Make sure your instruction is very informative and descriptive.\n",+ "\n",+ "Please provide the output field PROPOSED INSTRUCTION. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field PROPOSED INSTRUCTION.\n",+ "\n",+ "PROPOSED INSTRUCTION: Generate a search query that accurately captures the essence of the question and enables the retrieval of relevant information from the dataset, taking into account the context provided, the question being asked, and the relationship between the two.\n",+ "\n",+ "\n",+ "\n",+ "PROPOSED INSTRUCTION: Generate a search query that accurately captures the essence of the question and enables the retrieval of relevant information from the dataset, taking into account the context provided, the question being asked, and the relationship between the two.\n",+ "PROGRAM DESCRIPTION: The program is designed to solve the task of answering questions about a given context using a search query and rationale generated by a language model. The program uses a modular architecture, where multiple hops are taken to generate the search query and answer. In each hop, the language model generates a search query based on the context and question, retrieves relevant passages, and then generates a rationale and answer. The output is the answer and rationale.\n",+ "task_demos Context: ['Will Finn | Will Finn (born November 1, 1958) is an American animator, voice actor, and director. His work in animation includes characters from Disney and Don Bluth films such as \"The Secret of NIMH\", \"Oliver & Company\", \"The Little Mermaid\", \"The Rescuers Down Under\", and \"Pocahontas\". His characters includes Cogsworth in \"Beauty and the Beast\", Iago in \"Aladdin\", and Laverne in \"The Hunchback of Notre Dame\". Finn wrote and directed \"Home on the Range\" and did some voice acting the Hollywood Fish in \"Chicken Little\". In 2006, Finn directed the computer animated short Hammy\\'s Boomerang Adventure, a spin-off of \"Over the Hedge\".', 'Over the Hedge (film) | Over the Hedge is a 2006 American computer-animated comedy film, based on the characters from the United Media comic strip of the same name. Directed by Tim Johnson and Karey Kirkpatrick, and produced by Bonnie Arnold, it was released in the United States on May 19, 2006. The film was produced by DreamWorks Animation and distributed through Paramount Pictures. The film features the voices of Bruce Willis, Garry Shandling, Steve Carell, William Shatner, Wanda Sykes, and Nick Nolte. It is the first DreamWorks Animation film to be distributed by Paramount Pictures, which acquired the live-action DreamWorks studio in 2006. The film earned $336 million on an $80 million budget.', 'Tim Johnson (film director) | Tim Johnson (born August 27, 1961) is an American animation director. Johnson has directed in many films such as \"Antz\", \"\" and \"Over the Hedge\". Recently, he directed the DreamWorks 3D film \"Home.\"']\n",+ "Question: When was the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed?\n",+ "Reasoning: Let's think step by step in order to find the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed. The context mentions that Will Finn directed the computer animated short Hammy's Boomerang Adventure, a spin-off of \"Over the Hedge\".\n",+ "Search Query: Hammy's Boomerang Adventure release date\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Use the information below to learn about a task that we are trying to solve using calls to an LM, then generate a new instruction that will be used to prompt a Language Model to better solve the task.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "DATASET SUMMARY: A description of the dataset that we are using.\n",+ "\n",+ "PROGRAM CODE: Language model program designed to solve a particular task.\n",+ "\n",+ "PROGRAM DESCRIPTION: Summary of the task the program is designed to solve, and how it goes about solving it.\n",+ "\n",+ "MODULE: The module to create an instruction for.\n",+ "\n",+ "TASK DEMO(S): Example inputs/outputs of our module.\n",+ "\n",+ "BASIC INSTRUCTION: Basic instruction.\n",+ "\n",+ "TIP: A suggestion for how to go about generating the new instruction.\n",+ "\n",+ "PROPOSED INSTRUCTION: Propose an instruction that will be used to prompt a Language Model to perform this task.\n",+ "\n",+ "---\n",+ "\n",+ "DATASET SUMMARY: The dataset appears to be a comprehensive collection of trivia questions, presenting a diverse range of topics, including entertainment, history, arts, and culture. Most questions require specific, factual answers, often testing users' recall of specific details or events, and are designed to test users' knowledge on subject matter. The syntax and length of the questions vary, but most are concise and require precise answers.\n",+ "\n",+ "PROGRAM CODE:\n",+ "StringSignature(context, question -> rationale, search_query\n",+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",+ ")\n",+ "\n",+ "StringSignature(context, question -> rationale, answer\n",+ " instructions='Given the fields `context`, `question`, produce the fields `answer`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Answer:', 'desc': '${answer}'})\n",+ ")\n",+ "\n",+ "class BasicMH(dspy.Module):\n",+ " def __init__(self, passages_per_hop=3, num_hops=2):\n",+ " super().__init__()\n",+ " self.num_hops = num_hops\n",+ " self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",+ " # self.generate_query = dspy.ChainOfThought(GenerateSearchQuery)\n",+ " self.generate_query = [dspy.ChainOfThought(\"context, question -> search_query\") for _ in range(self.num_hops)]\n",+ " self.generate_answer = dspy.ChainOfThought(\"context, question -> answer\")\n",+ " \n",+ " def forward(self, question):\n",+ " context = []\n",+ " \n",+ " for hop in range(self.num_hops):\n",+ " search_query = self.generate_query[hop](context=context, question=question).search_query\n",+ " passages = self.retrieve(search_query).passages\n",+ " context = deduplicate(context + passages)\n",+ "\n",+ " prediction = self.generate_answer(context=context, question=question).copy(context=context)\n",+ " return prediction\n",+ "\n",+ "\n",+ "PROGRAM DESCRIPTION: The program is designed to solve the task of answering questions about a given context using a search query and rationale generated by a language model. The program uses a modular architecture, where multiple hops are taken to generate the search query and answer. In each hop, the language model generates a search query based on the context and question, retrieves relevant passages, and then generates a rationale and answer. The output is the answer and rationale.\n",+ "\n",+ "MODULE: self.generate_query = [dspy.ChainOfThought(\"context, question -> search_query\") for _ in range(self.num_hops)]\n",+ "\n",+ "TASK DEMO(S):\n",+ "Context: ['Will Finn | Will Finn (born November 1, 1958) is an American animator, voice actor, and director. His work in animation includes characters from Disney and Don Bluth films such as \"The Secret of NIMH\", \"Oliver & Company\", \"The Little Mermaid\", \"The Rescuers Down Under\", and \"Pocahontas\". His characters includes Cogsworth in \"Beauty and the Beast\", Iago in \"Aladdin\", and Laverne in \"The Hunchback of Notre Dame\". Finn wrote and directed \"Home on the Range\" and did some voice acting the Hollywood Fish in \"Chicken Little\". In 2006, Finn directed the computer animated short Hammy\\'s Boomerang Adventure, a spin-off of \"Over the Hedge\".', 'Over the Hedge (film) | Over the Hedge is a 2006 American computer-animated comedy film, based on the characters from the United Media comic strip of the same name. Directed by Tim Johnson and Karey Kirkpatrick, and produced by Bonnie Arnold, it was released in the United States on May 19, 2006. The film was produced by DreamWorks Animation and distributed through Paramount Pictures. The film features the voices of Bruce Willis, Garry Shandling, Steve Carell, William Shatner, Wanda Sykes, and Nick Nolte. It is the first DreamWorks Animation film to be distributed by Paramount Pictures, which acquired the live-action DreamWorks studio in 2006. The film earned $336 million on an $80 million budget.', 'Tim Johnson (film director) | Tim Johnson (born August 27, 1961) is an American animation director. Johnson has directed in many films such as \"Antz\", \"\" and \"Over the Hedge\". Recently, he directed the DreamWorks 3D film \"Home.\"']\n",+ "Question: When was the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed?\n",+ "Reasoning: Let's think step by step in order to find the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed. The context mentions that Will Finn directed the computer animated short Hammy's Boomerang Adventure, a spin-off of \"Over the Hedge\".\n",+ "Search Query: Hammy's Boomerang Adventure release date\n",+ "\n",+ "\n",+ "BASIC INSTRUCTION: Given the fields `context`, `question`, produce the fields `search_query`.\n",+ "\n",+ "TIP: Make sure your instruction is very informative and descriptive.\n",+ "\n",+ "Please provide the output field PROPOSED INSTRUCTION. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field PROPOSED INSTRUCTION.\n",+ "\n",+ "PROPOSED INSTRUCTION: Provide a descriptive search query that entails the critical information from the context and question to find a precise answer to the query asked, such as finding the release date of the computer-animated short comedy film Hammy's Boomerang Adventure, which is a spin-off of \"Over the Hedge\".\n",+ "\n",+ "\n",+ "\n",+ "PROPOSED INSTRUCTION: Provide a descriptive search query that entails the critical information from the context and question to find a precise answer to the query asked, such as finding the release date of the computer-animated short comedy film Hammy's Boomerang Adventure, which is a spin-off of \"Over the Hedge\".\n",+ "PROGRAM DESCRIPTION: This program is designed to solve a question answering task using a multi-hop question answering approach. The program uses a language model to generate search queries and answers by iteratively retrieving relevant passages, generating search queries, and generating answers. It appears to work by first initializing a context with an input question, then iterating over multiple \"hops\" where it generates a search query, retrieves relevant passages, updates the context, and finally generates an answer. The program is designed to handle complex questions that require multiple steps of reasoning to answer.\n",+ "task_demos Context: ['Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.', 'Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.', \"Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world's third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.\"]\n",+ "Question: The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: Kangchenjunga Singhi Kangri country\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Use the information below to learn about a task that we are trying to solve using calls to an LM, then generate a new instruction that will be used to prompt a Language Model to better solve the task.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "DATASET SUMMARY: A description of the dataset that we are using.\n",+ "\n",+ "PROGRAM CODE: Language model program designed to solve a particular task.\n",+ "\n",+ "PROGRAM DESCRIPTION: Summary of the task the program is designed to solve, and how it goes about solving it.\n",+ "\n",+ "MODULE: The module to create an instruction for.\n",+ "\n",+ "TASK DEMO(S): Example inputs/outputs of our module.\n",+ "\n",+ "BASIC INSTRUCTION: Basic instruction.\n",+ "\n",+ "TIP: A suggestion for how to go about generating the new instruction.\n",+ "\n",+ "PROPOSED INSTRUCTION: Propose an instruction that will be used to prompt a Language Model to perform this task.\n",+ "\n",+ "---\n",+ "\n",+ "DATASET SUMMARY: The dataset appears to be a comprehensive collection of trivia questions, presenting a diverse range of topics, including entertainment, history, arts, and culture. Most questions require specific, factual answers, often testing users' recall of specific details or events, and are designed to test users' knowledge on subject matter. The syntax and length of the questions vary, but most are concise and require precise answers.\n",+ "\n",+ "PROGRAM CODE:\n",+ "StringSignature(context, question -> rationale, search_query\n",+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",+ ")\n",+ "\n",+ "StringSignature(context, question -> rationale, answer\n",+ " instructions='Given the fields `context`, `question`, produce the fields `answer`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Answer:', 'desc': '${answer}'})\n",+ ")\n",+ "\n",+ "class BasicMH(dspy.Module):\n",+ " def __init__(self, passages_per_hop=3, num_hops=2):\n",+ " super().__init__()\n",+ " self.num_hops = num_hops\n",+ " self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",+ " # self.generate_query = dspy.ChainOfThought(GenerateSearchQuery)\n",+ " self.generate_query = [dspy.ChainOfThought(\"context, question -> search_query\") for _ in range(self.num_hops)]\n",+ " self.generate_answer = dspy.ChainOfThought(\"context, question -> answer\")\n",+ " \n",+ " def forward(self, question):\n",+ " context = []\n",+ " \n",+ " for hop in range(self.num_hops):\n",+ " search_query = self.generate_query[hop](context=context, question=question).search_query\n",+ " passages = self.retrieve(search_query).passages\n",+ " context = deduplicate(context + passages)\n",+ "\n",+ " prediction = self.generate_answer(context=context, question=question).copy(context=context)\n",+ " return prediction\n",+ "\n",+ "\n",+ "PROGRAM DESCRIPTION: This program is designed to solve a question answering task using a multi-hop question answering approach. The program uses a language model to generate search queries and answers by iteratively retrieving relevant passages, generating search queries, and generating answers. It appears to work by first initializing a context with an input question, then iterating over multiple \"hops\" where it generates a search query, retrieves relevant passages, updates the context, and finally generates an answer. The program is designed to handle complex questions that require multiple steps of reasoning to answer.\n",+ "\n",+ "MODULE: self.generate_query = [dspy.ChainOfThought(\"context, question -> search_query\") for _ in range(self.num_hops)]\n",+ "\n",+ "TASK DEMO(S):\n",+ "Context: ['Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.', 'Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.', \"Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world's third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.\"]\n",+ "Question: The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: Kangchenjunga Singhi Kangri country\n",+ "\n",+ "\n",+ "BASIC INSTRUCTION: Given the fields `context`, `question`, produce the fields `search_query`.\n",+ "\n",+ "TIP: Make sure your instruction is very informative and descriptive.\n",+ "\n",+ "Please provide the output field PROPOSED INSTRUCTION. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field PROPOSED INSTRUCTION.\n",+ "\n",+ "PROPOSED INSTRUCTION: Given a context that provides information about specific locations and landmarks, and a question that requires identifying the country where certain high-peaked mountains are partially located, utilize the provided context to generate a search query that will yield relevant information to answer the question.\n",+ "\n",+ "\n",+ "\n",+ "PROPOSED INSTRUCTION: Given a context that provides information about specific locations and landmarks, and a question that requires identifying the country where certain high-peaked mountains are partially located, utilize the provided context to generate a search query that will yield relevant information to answer the question.\n",+ "PROGRAM DESCRIPTION: This program appears to be designed to solve a task of generating a search query or answer based on a given context and question. Specifically, it seems to be a natural language processing pipeline that uses language models to generate search queries and answers related to a specific topic, namely the works of Vincent van Gogh. The program uses a chain-of-thought architecture, where each hop in the chain generates a search query or answer based on the previous context and question. The final output is a predicted answer or search query.\n",+ "task_demos Context: ['Wheat Field with Cypresses | A Wheatfield with Cypresses (occasionally called \"A Cornfield with Cypresses\") is any of three similar 1889 oil paintings by Vincent van Gogh, as part of his wheat field series. All were executed at the Saint-Paul-de-Mausole mental asylum at Saint-Rémy near Arles, France, where Van Gogh was voluntarily a patient from May 1889 to May 1890. The works were inspired by the view from the window at the asylum towards the Alpilles mountains.', 'Green Wheat Field with Cypress | Green Wheat Field with Cypress (French: Champ de blé vert avec cyprès) is an oil on canvas painting by a Dutch Post-Impressionist Vincent van Gogh. It was completed in 1889, while van Gogh was voluntarily incarcerated at the asylum of St. Paul near Saint-Rémy in Provence. Several paintings of wheat fields with cypresses were made when van Gogh was able to leave the asylum grounds and explore the local landscape. Besides a fondness for cypresses, van Gogh had a special affinty with wheat fields; he depicted them dozens of times over the years; to Vincent they symbolized the cycle of life and death, and he found in them both solace and inspiration.', 'Olive Trees (Van Gogh series) | Vincent van Gogh painted at least 18 paintings of olive trees, mostly in Saint-Rémy-de-Provence in 1889. At his own request, he lived at an asylum there from May 1889 through May 1890 painting the gardens of the asylum and, when he had permission to venture outside its walls, nearby olive trees, cypresses and wheat fields.']\n",+ "Question: What was the series inspired by the appreciation of manual laborers that Wheat Field with Cypresses is a part of?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: Wheat Field with Cypresses series inspiration\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Use the information below to learn about a task that we are trying to solve using calls to an LM, then generate a new instruction that will be used to prompt a Language Model to better solve the task.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "DATASET SUMMARY: A description of the dataset that we are using.\n",+ "\n",+ "PROGRAM CODE: Language model program designed to solve a particular task.\n",+ "\n",+ "PROGRAM DESCRIPTION: Summary of the task the program is designed to solve, and how it goes about solving it.\n",+ "\n",+ "MODULE: The module to create an instruction for.\n",+ "\n",+ "TASK DEMO(S): Example inputs/outputs of our module.\n",+ "\n",+ "BASIC INSTRUCTION: Basic instruction.\n",+ "\n",+ "TIP: A suggestion for how to go about generating the new instruction.\n",+ "\n",+ "PROPOSED INSTRUCTION: Propose an instruction that will be used to prompt a Language Model to perform this task.\n",+ "\n",+ "---\n",+ "\n",+ "DATASET SUMMARY: The dataset appears to be a comprehensive collection of trivia questions, presenting a diverse range of topics, including entertainment, history, arts, and culture. Most questions require specific, factual answers, often testing users' recall of specific details or events, and are designed to test users' knowledge on subject matter. The syntax and length of the questions vary, but most are concise and require precise answers.\n",+ "\n",+ "PROGRAM CODE:\n",+ "StringSignature(context, question -> rationale, search_query\n",+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",+ ")\n",+ "\n",+ "StringSignature(context, question -> rationale, answer\n",+ " instructions='Given the fields `context`, `question`, produce the fields `answer`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Answer:', 'desc': '${answer}'})\n",+ ")\n",+ "\n",+ "class BasicMH(dspy.Module):\n",+ " def __init__(self, passages_per_hop=3, num_hops=2):\n",+ " super().__init__()\n",+ " self.num_hops = num_hops\n",+ " self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",+ " # self.generate_query = dspy.ChainOfThought(GenerateSearchQuery)\n",+ " self.generate_query = [dspy.ChainOfThought(\"context, question -> search_query\") for _ in range(self.num_hops)]\n",+ " self.generate_answer = dspy.ChainOfThought(\"context, question -> answer\")\n",+ " \n",+ " def forward(self, question):\n",+ " context = []\n",+ " \n",+ " for hop in range(self.num_hops):\n",+ " search_query = self.generate_query[hop](context=context, question=question).search_query\n",+ " passages = self.retrieve(search_query).passages\n",+ " context = deduplicate(context + passages)\n",+ "\n",+ " prediction = self.generate_answer(context=context, question=question).copy(context=context)\n",+ " return prediction\n",+ "\n",+ "\n",+ "PROGRAM DESCRIPTION: This program appears to be designed to solve a task of generating a search query or answer based on a given context and question. Specifically, it seems to be a natural language processing pipeline that uses language models to generate search queries and answers related to a specific topic, namely the works of Vincent van Gogh. The program uses a chain-of-thought architecture, where each hop in the chain generates a search query or answer based on the previous context and question. The final output is a predicted answer or search query.\n",+ "\n",+ "MODULE: self.generate_query = [dspy.ChainOfThought(\"context, question -> search_query\") for _ in range(self.num_hops)]\n",+ "\n",+ "TASK DEMO(S):\n",+ "Context: ['Wheat Field with Cypresses | A Wheatfield with Cypresses (occasionally called \"A Cornfield with Cypresses\") is any of three similar 1889 oil paintings by Vincent van Gogh, as part of his wheat field series. All were executed at the Saint-Paul-de-Mausole mental asylum at Saint-Rémy near Arles, France, where Van Gogh was voluntarily a patient from May 1889 to May 1890. The works were inspired by the view from the window at the asylum towards the Alpilles mountains.', 'Green Wheat Field with Cypress | Green Wheat Field with Cypress (French: Champ de blé vert avec cyprès) is an oil on canvas painting by a Dutch Post-Impressionist Vincent van Gogh. It was completed in 1889, while van Gogh was voluntarily incarcerated at the asylum of St. Paul near Saint-Rémy in Provence. Several paintings of wheat fields with cypresses were made when van Gogh was able to leave the asylum grounds and explore the local landscape. Besides a fondness for cypresses, van Gogh had a special affinty with wheat fields; he depicted them dozens of times over the years; to Vincent they symbolized the cycle of life and death, and he found in them both solace and inspiration.', 'Olive Trees (Van Gogh series) | Vincent van Gogh painted at least 18 paintings of olive trees, mostly in Saint-Rémy-de-Provence in 1889. At his own request, he lived at an asylum there from May 1889 through May 1890 painting the gardens of the asylum and, when he had permission to venture outside its walls, nearby olive trees, cypresses and wheat fields.']\n",+ "Question: What was the series inspired by the appreciation of manual laborers that Wheat Field with Cypresses is a part of?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: Wheat Field with Cypresses series inspiration\n",+ "\n",+ "\n",+ "BASIC INSTRUCTION: Given the fields `context`, `question`, produce the fields `search_query`.\n",+ "\n",+ "TIP: Make sure your instruction is very informative and descriptive.\n",+ "\n",+ "Please provide the output field PROPOSED INSTRUCTION. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field PROPOSED INSTRUCTION.\n",+ "\n",+ "PROPOSED INSTRUCTION: Given a text context about Vincent van Gogh's artwork, a question related to the context, generate a search query that asks about the specific aspect of the artwork being discussed, taking into account the context and question provided.\n",+ "\n",+ "\n",+ "\n",+ "PROPOSED INSTRUCTION: Given a text context about Vincent van Gogh's artwork, a question related to the context, generate a search query that asks about the specific aspect of the artwork being discussed, taking into account the context and question provided.\n",+ "PROGRAM DESCRIPTION: This program appears to be designed to solve a knowledge-based question answering task, where given a context and a question, it generates a search query and then uses the search query to retrieve relevant information from a database, and finally generates an answer based on the retrieved information. The program consists of a modular architecture, with multiple hops of reasoning and search queries to gradually narrow down the search space and improve the accuracy of the answer.\n",+ "task_demos Context: ['Butch Reed | Bruce Franklin Reed (born July 11, 1954) is an American retired professional wrestler and football player, better known by the ring name Butch Reed. He is best known for his appearances with the World Wrestling Federation and World Championship Wrestling between 1986 and 1992.', 'Doug Reed (futsal player) | Doug Reed is a professional futsal player who is currently playing for Futsal Dinamo and England national futsal team. Previously he has played for Lanzarote Tias Yaiza (Spain), AEL Limassol, MNK Uspinjača, MNK Smederevo and Manchester Futsal Club. In 2011, he won the FA Futsal Cup with Manchester Futsal Club and was awarded Player of the Tournament in the same competition. In the FA National Futsal League, he won several FA National League Championships and FA Futsal Cup medals. Reed also works as a commentator for Sony SIX, Eurosport and BT Sport, and has worked with the emerging US Professional Futsal League as well as Indian Premier Futsal.', 'Doom (professional wrestling) | Doom was a professional wrestling tag team composed of Ron Simmons and Butch Reed. They competed from 1989 to 1991 in the National Wrestling Alliance (NWA) and World Championship Wrestling (WCW).']\n",+ "Question: What other sport did Butch Reed, of the wrestling tag team Doom, play professionally?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: Butch Reed football\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Use the information below to learn about a task that we are trying to solve using calls to an LM, then generate a new instruction that will be used to prompt a Language Model to better solve the task.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "DATASET SUMMARY: A description of the dataset that we are using.\n",+ "\n",+ "PROGRAM CODE: Language model program designed to solve a particular task.\n",+ "\n",+ "PROGRAM DESCRIPTION: Summary of the task the program is designed to solve, and how it goes about solving it.\n",+ "\n",+ "MODULE: The module to create an instruction for.\n",+ "\n",+ "TASK DEMO(S): Example inputs/outputs of our module.\n",+ "\n",+ "BASIC INSTRUCTION: Basic instruction.\n",+ "\n",+ "TIP: A suggestion for how to go about generating the new instruction.\n",+ "\n",+ "PROPOSED INSTRUCTION: Propose an instruction that will be used to prompt a Language Model to perform this task.\n",+ "\n",+ "---\n",+ "\n",+ "DATASET SUMMARY: The dataset appears to be a comprehensive collection of trivia questions, presenting a diverse range of topics, including entertainment, history, arts, and culture. Most questions require specific, factual answers, often testing users' recall of specific details or events, and are designed to test users' knowledge on subject matter. The syntax and length of the questions vary, but most are concise and require precise answers.\n",+ "\n",+ "PROGRAM CODE:\n",+ "StringSignature(context, question -> rationale, search_query\n",+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",+ ")\n",+ "\n",+ "StringSignature(context, question -> rationale, answer\n",+ " instructions='Given the fields `context`, `question`, produce the fields `answer`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Answer:', 'desc': '${answer}'})\n",+ ")\n",+ "\n",+ "class BasicMH(dspy.Module):\n",+ " def __init__(self, passages_per_hop=3, num_hops=2):\n",+ " super().__init__()\n",+ " self.num_hops = num_hops\n",+ " self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",+ " # self.generate_query = dspy.ChainOfThought(GenerateSearchQuery)\n",+ " self.generate_query = [dspy.ChainOfThought(\"context, question -> search_query\") for _ in range(self.num_hops)]\n",+ " self.generate_answer = dspy.ChainOfThought(\"context, question -> answer\")\n",+ " \n",+ " def forward(self, question):\n",+ " context = []\n",+ " \n",+ " for hop in range(self.num_hops):\n",+ " search_query = self.generate_query[hop](context=context, question=question).search_query\n",+ " passages = self.retrieve(search_query).passages\n",+ " context = deduplicate(context + passages)\n",+ "\n",+ " prediction = self.generate_answer(context=context, question=question).copy(context=context)\n",+ " return prediction\n",+ "\n",+ "\n",+ "PROGRAM DESCRIPTION: This program appears to be designed to solve a knowledge-based question answering task, where given a context and a question, it generates a search query and then uses the search query to retrieve relevant information from a database, and finally generates an answer based on the retrieved information. The program consists of a modular architecture, with multiple hops of reasoning and search queries to gradually narrow down the search space and improve the accuracy of the answer.\n",+ "\n",+ "MODULE: self.generate_query = [dspy.ChainOfThought(\"context, question -> search_query\") for _ in range(self.num_hops)]\n",+ "\n",+ "TASK DEMO(S):\n",+ "Context: ['Butch Reed | Bruce Franklin Reed (born July 11, 1954) is an American retired professional wrestler and football player, better known by the ring name Butch Reed. He is best known for his appearances with the World Wrestling Federation and World Championship Wrestling between 1986 and 1992.', 'Doug Reed (futsal player) | Doug Reed is a professional futsal player who is currently playing for Futsal Dinamo and England national futsal team. Previously he has played for Lanzarote Tias Yaiza (Spain), AEL Limassol, MNK Uspinjača, MNK Smederevo and Manchester Futsal Club. In 2011, he won the FA Futsal Cup with Manchester Futsal Club and was awarded Player of the Tournament in the same competition. In the FA National Futsal League, he won several FA National League Championships and FA Futsal Cup medals. Reed also works as a commentator for Sony SIX, Eurosport and BT Sport, and has worked with the emerging US Professional Futsal League as well as Indian Premier Futsal.', 'Doom (professional wrestling) | Doom was a professional wrestling tag team composed of Ron Simmons and Butch Reed. They competed from 1989 to 1991 in the National Wrestling Alliance (NWA) and World Championship Wrestling (WCW).']\n",+ "Question: What other sport did Butch Reed, of the wrestling tag team Doom, play professionally?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: Butch Reed football\n",+ "\n",+ "\n",+ "BASIC INSTRUCTION: Given the fields `context`, `question`, produce the fields `search_query`.\n",+ "\n",+ "TIP: Make sure your instruction is very informative and descriptive.\n",+ "\n",+ "Please provide the output field PROPOSED INSTRUCTION. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field PROPOSED INSTRUCTION.\n",+ "\n",+ "PROPOSED INSTRUCTION: Generate a concise and informative search query that leverages the context information and question description to retrieve relevant passages from the database, effectively guiding the subsequent reasoning and answer generation process, taking into account the nuances and specificity of the provided context and question.\n",+ "\n",+ "\n",+ "\n",+ "PROPOSED INSTRUCTION: Generate a concise and informative search query that leverages the context information and question description to retrieve relevant passages from the database, effectively guiding the subsequent reasoning and answer generation process, taking into account the nuances and specificity of the provided context and question.\n",+ "PROGRAM DESCRIPTION: This program appears to be designed to solve a question-answering task using a language model. It takes in a context, a question, and uses a series of language model calls to generate a search query and then an answer. The program is structured as a pipeline, with each step building on the previous one to generate the final answer. The program uses a retrieve module to gather relevant passages, a generate query module to generate search queries, and a generate answer module to generate the final answer. The program is designed to be able to handle multiple hops, where it iteratively generates search queries and retrieves new passages based on the previous results, until it reaches a stopping condition.\n",+ "task_demos Context: ['Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.', 'Ebenezer Howard | Sir Ebenezer Howard {\\'1\\': \", \\'2\\': \", \\'3\\': \", \\'4\\': \"} ( ; 29 January 1850 – 1 May 1928), the English founder of the garden city movement, is known for his publication \"\" (1898), the description of a utopian city in which people live harmoniously together with nature. The publication resulted in the founding of the garden city movement, and the building of the First Garden City, Letchworth Garden City, commenced in 1903.', 'Eugene Reynal | Eugene Reynal was an American Publisher and in 1933 Founder of Blue Ribbon Books of Garden City, New York and Reynal and Hitchcock of New York, New York.']\n",+ "Question: Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: Garden City Hotel founder\n",+ "Context: ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.']\n",+ "Question: Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: Carhartt bass fishing\n",+ "Context: ['Stitch\\'s Great Escape! | Stitch\\'s Great Escape! is a Tomorrowland attraction at the Magic Kingdom theme park within the Walt Disney World Resort. It is a \"theater-in-the-round\" experience starring the title alien from Walt Disney Animation Studios\\' 2002 film \"Lilo & Stitch\". It opened November 16, 2004 and is the fourth attraction to occupy the site in Tomorrowland. Many of the animators who worked on \"Lilo & Stitch\" partnered with Walt Disney Imagineering for Stitch\\'s Great Escape! The attraction is a replacement of The ExtraTERRORestrial Alien Encounter, which formerly occupied the building the attraction is housed in. On September 21, 2016, it was announced that the attraction would be switching from a daily operated attraction to a seasonally operated one, depending on attendance, starting October 2, 2016.', 'Great Escape (amusement park) | Great Escape (sometimes referred to as Six Flags Great Escape) is an amusement and water park owned and operated by Six Flags Entertainment Corp. It is located approximately 60 mi north of Albany, in Queensbury, New York, but is advertised as being in Lake George, New York, a popular tourist and vacation spot nearby. It is one of two Six Flags parks not to be officially branded with the \"Six Flags\" name (La Ronde in Montreal, Quebec, Canada being the other).', 'Houdini\\'s Great Escape | Houdini\\'s Great Escape, also known as \"Boo-dini\" during Fright Fest and \"Houdini - The Great Escape\" at Six Flags New England, are indoor Vekoma Madhouse attractions at Six Flags Great Adventure and Six Flags New England. The ride is located in Great Adventure\\'s Fantasy Forest section and New England\\'s Crack Axle Canyon area. The attraction opened at both parks in 1999, and was Standing but not operating (SBNO) at Great Adventure from 2008 - 2009 but was opened again due to popular demand.']\n",+ "Question: Which theme park has Stitch’s Great Escape which replaced the attraction ExtraTERRORestial Alien Encounter?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: Magic Kingdom theme park\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Use the information below to learn about a task that we are trying to solve using calls to an LM, then generate a new instruction that will be used to prompt a Language Model to better solve the task.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "DATASET SUMMARY: A description of the dataset that we are using.\n",+ "\n",+ "PROGRAM CODE: Language model program designed to solve a particular task.\n",+ "\n",+ "PROGRAM DESCRIPTION: Summary of the task the program is designed to solve, and how it goes about solving it.\n",+ "\n",+ "MODULE: The module to create an instruction for.\n",+ "\n",+ "TASK DEMO(S): Example inputs/outputs of our module.\n",+ "\n",+ "BASIC INSTRUCTION: Basic instruction.\n",+ "\n",+ "TIP: A suggestion for how to go about generating the new instruction.\n",+ "\n",+ "PROPOSED INSTRUCTION: Propose an instruction that will be used to prompt a Language Model to perform this task.\n",+ "\n",+ "---\n",+ "\n",+ "DATASET SUMMARY: The dataset appears to be a comprehensive collection of trivia questions, presenting a diverse range of topics, including entertainment, history, arts, and culture. Most questions require specific, factual answers, often testing users' recall of specific details or events, and are designed to test users' knowledge on subject matter. The syntax and length of the questions vary, but most are concise and require precise answers.\n",+ "\n",+ "PROGRAM CODE:\n",+ "StringSignature(context, question -> rationale, search_query\n",+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",+ ")\n",+ "\n",+ "StringSignature(context, question -> rationale, answer\n",+ " instructions='Given the fields `context`, `question`, produce the fields `answer`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Answer:', 'desc': '${answer}'})\n",+ ")\n",+ "\n",+ "class BasicMH(dspy.Module):\n",+ " def __init__(self, passages_per_hop=3, num_hops=2):\n",+ " super().__init__()\n",+ " self.num_hops = num_hops\n",+ " self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",+ " # self.generate_query = dspy.ChainOfThought(GenerateSearchQuery)\n",+ " self.generate_query = [dspy.ChainOfThought(\"context, question -> search_query\") for _ in range(self.num_hops)]\n",+ " self.generate_answer = dspy.ChainOfThought(\"context, question -> answer\")\n",+ " \n",+ " def forward(self, question):\n",+ " context = []\n",+ " \n",+ " for hop in range(self.num_hops):\n",+ " search_query = self.generate_query[hop](context=context, question=question).search_query\n",+ " passages = self.retrieve(search_query).passages\n",+ " context = deduplicate(context + passages)\n",+ "\n",+ " prediction = self.generate_answer(context=context, question=question).copy(context=context)\n",+ " return prediction\n",+ "\n",+ "\n",+ "PROGRAM DESCRIPTION: This program appears to be designed to solve a question-answering task using a language model. It takes in a context, a question, and uses a series of language model calls to generate a search query and then an answer. The program is structured as a pipeline, with each step building on the previous one to generate the final answer. The program uses a retrieve module to gather relevant passages, a generate query module to generate search queries, and a generate answer module to generate the final answer. The program is designed to be able to handle multiple hops, where it iteratively generates search queries and retrieves new passages based on the previous results, until it reaches a stopping condition.\n",+ "\n",+ "MODULE: self.generate_query = [dspy.ChainOfThought(\"context, question -> search_query\") for _ in range(self.num_hops)]\n",+ "\n",+ "TASK DEMO(S):\n",+ "Context: ['Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.', 'Ebenezer Howard | Sir Ebenezer Howard {\\'1\\': \", \\'2\\': \", \\'3\\': \", \\'4\\': \"} ( ; 29 January 1850 – 1 May 1928), the English founder of the garden city movement, is known for his publication \"\" (1898), the description of a utopian city in which people live harmoniously together with nature. The publication resulted in the founding of the garden city movement, and the building of the First Garden City, Letchworth Garden City, commenced in 1903.', 'Eugene Reynal | Eugene Reynal was an American Publisher and in 1933 Founder of Blue Ribbon Books of Garden City, New York and Reynal and Hitchcock of New York, New York.']\n",+ "Question: Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: Garden City Hotel founder\n",+ "Context: ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.']\n",+ "Question: Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: Carhartt bass fishing\n",+ "Context: ['Stitch\\'s Great Escape! | Stitch\\'s Great Escape! is a Tomorrowland attraction at the Magic Kingdom theme park within the Walt Disney World Resort. It is a \"theater-in-the-round\" experience starring the title alien from Walt Disney Animation Studios\\' 2002 film \"Lilo & Stitch\". It opened November 16, 2004 and is the fourth attraction to occupy the site in Tomorrowland. Many of the animators who worked on \"Lilo & Stitch\" partnered with Walt Disney Imagineering for Stitch\\'s Great Escape! The attraction is a replacement of The ExtraTERRORestrial Alien Encounter, which formerly occupied the building the attraction is housed in. On September 21, 2016, it was announced that the attraction would be switching from a daily operated attraction to a seasonally operated one, depending on attendance, starting October 2, 2016.', 'Great Escape (amusement park) | Great Escape (sometimes referred to as Six Flags Great Escape) is an amusement and water park owned and operated by Six Flags Entertainment Corp. It is located approximately 60 mi north of Albany, in Queensbury, New York, but is advertised as being in Lake George, New York, a popular tourist and vacation spot nearby. It is one of two Six Flags parks not to be officially branded with the \"Six Flags\" name (La Ronde in Montreal, Quebec, Canada being the other).', 'Houdini\\'s Great Escape | Houdini\\'s Great Escape, also known as \"Boo-dini\" during Fright Fest and \"Houdini - The Great Escape\" at Six Flags New England, are indoor Vekoma Madhouse attractions at Six Flags Great Adventure and Six Flags New England. The ride is located in Great Adventure\\'s Fantasy Forest section and New England\\'s Crack Axle Canyon area. The attraction opened at both parks in 1999, and was Standing but not operating (SBNO) at Great Adventure from 2008 - 2009 but was opened again due to popular demand.']\n",+ "Question: Which theme park has Stitch’s Great Escape which replaced the attraction ExtraTERRORestial Alien Encounter?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: Magic Kingdom theme park\n",+ "\n",+ "\n",+ "BASIC INSTRUCTION: Given the fields `context`, `question`, produce the fields `search_query`.\n",+ "\n",+ "TIP: Make sure your instruction is very informative and descriptive.\n",+ "\n",+ "Please provide the output field PROPOSED INSTRUCTION. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field PROPOSED INSTRUCTION.\n",+ "\n",+ "PROPOSED INSTRUCTION: \n",+ "Given the fields `context` and `question`, produce a search query that is a concise and specific phrase that accurately captures the main idea of the question and is likely to retrieve relevant passages from the context, by carefully considering the relationships between the context, question, and topic, and generating a query that will allow the model to execute a precise and effective search in the context.\n",+ "\n",+ "\n",+ "\n",+ "PROPOSED INSTRUCTION: Given the fields `context` and `question`, produce a search query that is a concise and specific phrase that accurately captures the main idea of the question and is likely to retrieve relevant passages from the context, by carefully considering the relationships between the context, question, and topic, and generating a query that will allow the model to execute a precise and effective search in the context.\n",+ "PROGRAM DESCRIPTION: This program is designed to solve knowledge-based question answering tasks using a multi-hop reasoning framework. The program takes in a context, question, and produces a search query or answer accordingly. The multi-hop approach involves iteratively generating search queries or answers based on the previous outputs, to ultimately provide a relevant response to the question. The program appears to be using language models to generate the search queries and answers, and retrieving relevant information from a knowledge base to support its responses.\n",+ "task_demos Context: ['Simon Arora | Simon Arora (born November 1969) is a British billionaire businessman, CEO of the retail chain B & M.', 'Samir Arora | Samir Arora (born November 5, 1965) is an American businessman and former CEO of Mode Media (formerly Glam Media) from 2003 to April 2016. He was CEO and Chairman of the web design company NetObjects, Inc. from 1995 to 2001.', 'Dinesh Arora | Dinesh Arora (born 23 December 1976) is an Indian Administrative Service (IAS) officer, 2002 Kerala cadre, who previously worked as an Executive Director, Rural Electrification Corporation (REC) and CEO RECPDCL.']\n",+ "Question: What company formed in 1978 has Simon Arora as its CEO?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: B & M CEO\n",+ "Context: ['Electoral district of Bragg | Bragg is an electoral district of the House of Assembly in the Australian state of South Australia. The seat of Bragg is named after the eminent physicists Bragg – William Henry and his son, William Lawrence. The electorate is largely urban and encompasses a significant portion of the City of Burnside, stretching from the east parklands of Adelaide into the Adelaide Hills. After a redistribution following the 2006 election, the boundary moved eastwards to include suburbs that had formerly been in the seat of Heysen and now borders Kavel. Bragg currently includes the metropolitan suburbs of Beaumont, Burnside, Dulwich, Erindale, Hazelwood Park, Heathpool, Kensington Park, Leabrook, Linden Park, Rose Park, Marryatville, Skye, St Georges, Stonyfell, Toorak Gardens, Tusmore, Wattle Park and parts of Glen Osmond, and in the hills it includes Crafers, Cleland, Greenhill, Mount Osmond, Piccadilly, Summertown and parts of Ashton, Basket Range, Carey Gully, Horsnell Gully, Leawood Gardens and Uraidla.', 'Cleland, South Australia | Cleland is a suburb in South Australia located in the Adelaide metropolitan area about 10 km south-east of the Adelaide city centre. Its boundaries were created in October 2001, with additional land being added in 2010 from the adjoining suburb of Crafers. Its name is derived from the Cleland Conservation Park. The principal land use within the locality is conservation with the majority of its land area being occupied by the Cleland Conservation Park. Places within its extent include the summit of Mount Lofty. Cleland is located within the federal Division of Mayo, the state electoral district of Bragg and the local government area of the Adelaide Hills Council.', 'Electoral results for the district of Bragg | This is a list of electoral results for the Electoral district of Bragg in South Australian state elections.']\n",+ "Question: Bragg electoral district, incorporating the Kensington Park suburb, are associated with which South Australian city?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: South Australian city associated with Bragg electoral district\n",+ "Context: ['Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.', 'Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.', \"Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world's third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.\"]\n",+ "Question: The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: Kangchenjunga Singhi Kangri country\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Use the information below to learn about a task that we are trying to solve using calls to an LM, then generate a new instruction that will be used to prompt a Language Model to better solve the task.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "DATASET SUMMARY: A description of the dataset that we are using.\n",+ "\n",+ "PROGRAM CODE: Language model program designed to solve a particular task.\n",+ "\n",+ "PROGRAM DESCRIPTION: Summary of the task the program is designed to solve, and how it goes about solving it.\n",+ "\n",+ "MODULE: The module to create an instruction for.\n",+ "\n",+ "TASK DEMO(S): Example inputs/outputs of our module.\n",+ "\n",+ "BASIC INSTRUCTION: Basic instruction.\n",+ "\n",+ "TIP: A suggestion for how to go about generating the new instruction.\n",+ "\n",+ "PROPOSED INSTRUCTION: Propose an instruction that will be used to prompt a Language Model to perform this task.\n",+ "\n",+ "---\n",+ "\n",+ "DATASET SUMMARY: The dataset appears to be a comprehensive collection of trivia questions, presenting a diverse range of topics, including entertainment, history, arts, and culture. Most questions require specific, factual answers, often testing users' recall of specific details or events, and are designed to test users' knowledge on subject matter. The syntax and length of the questions vary, but most are concise and require precise answers.\n",+ "\n",+ "PROGRAM CODE:\n",+ "StringSignature(context, question -> rationale, search_query\n",+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",+ ")\n",+ "\n",+ "StringSignature(context, question -> rationale, answer\n",+ " instructions='Given the fields `context`, `question`, produce the fields `answer`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Answer:', 'desc': '${answer}'})\n",+ ")\n",+ "\n",+ "class BasicMH(dspy.Module):\n",+ " def __init__(self, passages_per_hop=3, num_hops=2):\n",+ " super().__init__()\n",+ " self.num_hops = num_hops\n",+ " self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",+ " # self.generate_query = dspy.ChainOfThought(GenerateSearchQuery)\n",+ " self.generate_query = [dspy.ChainOfThought(\"context, question -> search_query\") for _ in range(self.num_hops)]\n",+ " self.generate_answer = dspy.ChainOfThought(\"context, question -> answer\")\n",+ " \n",+ " def forward(self, question):\n",+ " context = []\n",+ " \n",+ " for hop in range(self.num_hops):\n",+ " search_query = self.generate_query[hop](context=context, question=question).search_query\n",+ " passages = self.retrieve(search_query).passages\n",+ " context = deduplicate(context + passages)\n",+ "\n",+ " prediction = self.generate_answer(context=context, question=question).copy(context=context)\n",+ " return prediction\n",+ "\n",+ "\n",+ "PROGRAM DESCRIPTION: This program is designed to solve knowledge-based question answering tasks using a multi-hop reasoning framework. The program takes in a context, question, and produces a search query or answer accordingly. The multi-hop approach involves iteratively generating search queries or answers based on the previous outputs, to ultimately provide a relevant response to the question. The program appears to be using language models to generate the search queries and answers, and retrieving relevant information from a knowledge base to support its responses.\n",+ "\n",+ "MODULE: self.generate_query = [dspy.ChainOfThought(\"context, question -> search_query\") for _ in range(self.num_hops)]\n",+ "\n",+ "TASK DEMO(S):\n",+ "Context: ['Simon Arora | Simon Arora (born November 1969) is a British billionaire businessman, CEO of the retail chain B & M.', 'Samir Arora | Samir Arora (born November 5, 1965) is an American businessman and former CEO of Mode Media (formerly Glam Media) from 2003 to April 2016. He was CEO and Chairman of the web design company NetObjects, Inc. from 1995 to 2001.', 'Dinesh Arora | Dinesh Arora (born 23 December 1976) is an Indian Administrative Service (IAS) officer, 2002 Kerala cadre, who previously worked as an Executive Director, Rural Electrification Corporation (REC) and CEO RECPDCL.']\n",+ "Question: What company formed in 1978 has Simon Arora as its CEO?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: B & M CEO\n",+ "Context: ['Electoral district of Bragg | Bragg is an electoral district of the House of Assembly in the Australian state of South Australia. The seat of Bragg is named after the eminent physicists Bragg – William Henry and his son, William Lawrence. The electorate is largely urban and encompasses a significant portion of the City of Burnside, stretching from the east parklands of Adelaide into the Adelaide Hills. After a redistribution following the 2006 election, the boundary moved eastwards to include suburbs that had formerly been in the seat of Heysen and now borders Kavel. Bragg currently includes the metropolitan suburbs of Beaumont, Burnside, Dulwich, Erindale, Hazelwood Park, Heathpool, Kensington Park, Leabrook, Linden Park, Rose Park, Marryatville, Skye, St Georges, Stonyfell, Toorak Gardens, Tusmore, Wattle Park and parts of Glen Osmond, and in the hills it includes Crafers, Cleland, Greenhill, Mount Osmond, Piccadilly, Summertown and parts of Ashton, Basket Range, Carey Gully, Horsnell Gully, Leawood Gardens and Uraidla.', 'Cleland, South Australia | Cleland is a suburb in South Australia located in the Adelaide metropolitan area about 10 km south-east of the Adelaide city centre. Its boundaries were created in October 2001, with additional land being added in 2010 from the adjoining suburb of Crafers. Its name is derived from the Cleland Conservation Park. The principal land use within the locality is conservation with the majority of its land area being occupied by the Cleland Conservation Park. Places within its extent include the summit of Mount Lofty. Cleland is located within the federal Division of Mayo, the state electoral district of Bragg and the local government area of the Adelaide Hills Council.', 'Electoral results for the district of Bragg | This is a list of electoral results for the Electoral district of Bragg in South Australian state elections.']\n",+ "Question: Bragg electoral district, incorporating the Kensington Park suburb, are associated with which South Australian city?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: South Australian city associated with Bragg electoral district\n",+ "Context: ['Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.', 'Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.', \"Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world's third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.\"]\n",+ "Question: The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "Search Query: Kangchenjunga Singhi Kangri country\n",+ "\n",+ "\n",+ "BASIC INSTRUCTION: Given the fields `context`, `question`, produce the fields `search_query`.\n",+ "\n",+ "TIP: Make sure your instruction is very informative and descriptive.\n",+ "\n",+ "Please provide the output field PROPOSED INSTRUCTION. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field PROPOSED INSTRUCTION.\n",+ "\n",+ "PROPOSED INSTRUCTION: Given the provided context and question, provide a specific and concise search query that can be used to retrieve relevant information to answer the question, taking into account the topic, tone, and syntax of the context and question.\n",+ "\n",+ "\n",+ "\n",+ "PROPOSED INSTRUCTION: Given the provided context and question, provide a specific and concise search query that can be used to retrieve relevant information to answer the question, taking into account the topic, tone, and syntax of the context and question.\n",+ "PROGRAM DESCRIPTION: This program is designed to solve a question-answering task. It appears to work by iteratively generating search queries based on the context and question, retrieving relevant passages, and then generating answers based on the retrieved passages. The program uses a chain of thought approach, where each hop represents a step in the reasoning process, and the output of each hop is used as input for the next hop. The program seems to be designed to simulate how a human would answer a question, by thinking step by step and generating intermediate search queries and answers.\n",+ "task_demos \n",+ "\n",+ "\n",+ "\n",+ "Use the information below to learn about a task that we are trying to solve using calls to an LM, then generate a new instruction that will be used to prompt a Language Model to better solve the task.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "DATASET SUMMARY: A description of the dataset that we are using.\n",+ "\n",+ "PROGRAM CODE: Language model program designed to solve a particular task.\n",+ "\n",+ "PROGRAM DESCRIPTION: Summary of the task the program is designed to solve, and how it goes about solving it.\n",+ "\n",+ "MODULE: The module to create an instruction for.\n",+ "\n",+ "TASK DEMO(S): Example inputs/outputs of our module.\n",+ "\n",+ "BASIC INSTRUCTION: Basic instruction.\n",+ "\n",+ "TIP: A suggestion for how to go about generating the new instruction.\n",+ "\n",+ "PROPOSED INSTRUCTION: Propose an instruction that will be used to prompt a Language Model to perform this task.\n",+ "\n",+ "---\n",+ "\n",+ "DATASET SUMMARY: The dataset appears to be a comprehensive collection of trivia questions, presenting a diverse range of topics, including entertainment, history, arts, and culture. Most questions require specific, factual answers, often testing users' recall of specific details or events, and are designed to test users' knowledge on subject matter. The syntax and length of the questions vary, but most are concise and require precise answers.\n",+ "\n",+ "PROGRAM CODE:\n",+ "StringSignature(context, question -> rationale, search_query\n",+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",+ ")\n",+ "\n",+ "StringSignature(context, question -> rationale, answer\n",+ " instructions='Given the fields `context`, `question`, produce the fields `answer`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Answer:', 'desc': '${answer}'})\n",+ ")\n",+ "\n",+ "class BasicMH(dspy.Module):\n",+ " def __init__(self, passages_per_hop=3, num_hops=2):\n",+ " super().__init__()\n",+ " self.num_hops = num_hops\n",+ " self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",+ " # self.generate_query = dspy.ChainOfThought(GenerateSearchQuery)\n",+ " self.generate_query = [dspy.ChainOfThought(\"context, question -> search_query\") for _ in range(self.num_hops)]\n",+ " self.generate_answer = dspy.ChainOfThought(\"context, question -> answer\")\n",+ " \n",+ " def forward(self, question):\n",+ " context = []\n",+ " \n",+ " for hop in range(self.num_hops):\n",+ " search_query = self.generate_query[hop](context=context, question=question).search_query\n",+ " passages = self.retrieve(search_query).passages\n",+ " context = deduplicate(context + passages)\n",+ "\n",+ " prediction = self.generate_answer(context=context, question=question).copy(context=context)\n",+ " return prediction\n",+ "\n",+ "\n",+ "PROGRAM DESCRIPTION: This program is designed to solve a question-answering task. It appears to work by iteratively generating search queries based on the context and question, retrieving relevant passages, and then generating answers based on the retrieved passages. The program uses a chain of thought approach, where each hop represents a step in the reasoning process, and the output of each hop is used as input for the next hop. The program seems to be designed to simulate how a human would answer a question, by thinking step by step and generating intermediate search queries and answers.\n",+ "\n",+ "MODULE: self.generate_answer = dspy.ChainOfThought(\"context, question -> answer\")\n",+ "\n",+ "TASK DEMO(S): \n",+ "\n",+ "BASIC INSTRUCTION: Given the fields `context`, `question`, produce the fields `answer`.\n",+ "\n",+ "TIP: Make sure your instruction is very informative and descriptive.\n",+ "\n",+ "Please provide the output field PROPOSED INSTRUCTION. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field PROPOSED INSTRUCTION.\n",+ "\n",+ "PROPOSED INSTRUCTION: Given the context and question, produce a specific and concise answer that accurately responds to the query, by iteratively generating relevant search queries and reasoning step by step.\n",+ "\n",+ "\n",+ "\n",+ "PROPOSED INSTRUCTION: Given the context and question, produce a specific and concise answer that accurately responds to the query, by iteratively generating relevant search queries and reasoning step by step.\n",+ "PROGRAM DESCRIPTION: This program is designed to solve a question-answering task. It appears to work by iteratively generating search queries based on the context and question, retrieving relevant passages, and then generating answers based on the retrieved passages. The program uses a chain of thought approach, where each hop represents a step in the reasoning process, and the output of each hop is used as input for the next hop. The program seems to be designed to simulate how a human would answer a question, by thinking step by step and generating intermediate search queries and answers.\n",+ "task_demos \n",+ "\n",+ "\n",+ "\n",+ "Use the information below to learn about a task that we are trying to solve using calls to an LM, then generate a new instruction that will be used to prompt a Language Model to better solve the task.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "DATASET SUMMARY: A description of the dataset that we are using.\n",+ "\n",+ "PROGRAM CODE: Language model program designed to solve a particular task.\n",+ "\n",+ "PROGRAM DESCRIPTION: Summary of the task the program is designed to solve, and how it goes about solving it.\n",+ "\n",+ "MODULE: The module to create an instruction for.\n",+ "\n",+ "TASK DEMO(S): Example inputs/outputs of our module.\n",+ "\n",+ "BASIC INSTRUCTION: Basic instruction.\n",+ "\n",+ "TIP: A suggestion for how to go about generating the new instruction.\n",+ "\n",+ "PROPOSED INSTRUCTION: Propose an instruction that will be used to prompt a Language Model to perform this task.\n",+ "\n",+ "---\n",+ "\n",+ "DATASET SUMMARY: The dataset appears to be a comprehensive collection of trivia questions, presenting a diverse range of topics, including entertainment, history, arts, and culture. Most questions require specific, factual answers, often testing users' recall of specific details or events, and are designed to test users' knowledge on subject matter. The syntax and length of the questions vary, but most are concise and require precise answers.\n",+ "\n",+ "PROGRAM CODE:\n",+ "StringSignature(context, question -> rationale, search_query\n",+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",+ ")\n",+ "\n",+ "StringSignature(context, question -> rationale, answer\n",+ " instructions='Given the fields `context`, `question`, produce the fields `answer`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Answer:', 'desc': '${answer}'})\n",+ ")\n",+ "\n",+ "class BasicMH(dspy.Module):\n",+ " def __init__(self, passages_per_hop=3, num_hops=2):\n",+ " super().__init__()\n",+ " self.num_hops = num_hops\n",+ " self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",+ " # self.generate_query = dspy.ChainOfThought(GenerateSearchQuery)\n",+ " self.generate_query = [dspy.ChainOfThought(\"context, question -> search_query\") for _ in range(self.num_hops)]\n",+ " self.generate_answer = dspy.ChainOfThought(\"context, question -> answer\")\n",+ " \n",+ " def forward(self, question):\n",+ " context = []\n",+ " \n",+ " for hop in range(self.num_hops):\n",+ " search_query = self.generate_query[hop](context=context, question=question).search_query\n",+ " passages = self.retrieve(search_query).passages\n",+ " context = deduplicate(context + passages)\n",+ "\n",+ " prediction = self.generate_answer(context=context, question=question).copy(context=context)\n",+ " return prediction\n",+ "\n",+ "\n",+ "PROGRAM DESCRIPTION: This program is designed to solve a question-answering task. It appears to work by iteratively generating search queries based on the context and question, retrieving relevant passages, and then generating answers based on the retrieved passages. The program uses a chain of thought approach, where each hop represents a step in the reasoning process, and the output of each hop is used as input for the next hop. The program seems to be designed to simulate how a human would answer a question, by thinking step by step and generating intermediate search queries and answers.\n",+ "\n",+ "MODULE: self.generate_answer = dspy.ChainOfThought(\"context, question -> answer\")\n",+ "\n",+ "TASK DEMO(S): \n",+ "\n",+ "BASIC INSTRUCTION: Given the fields `context`, `question`, produce the fields `answer`.\n",+ "\n",+ "TIP: Make sure your instruction is very informative and descriptive.\n",+ "\n",+ "Please provide the output field PROPOSED INSTRUCTION. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field PROPOSED INSTRUCTION.\n",+ "\n",+ "PROPOSED INSTRUCTION: Given the context and question, produce a specific and concise answer that accurately responds to the query, by iteratively generating relevant search queries and reasoning step by step.\n",+ "\n",+ "\n",+ "\n",+ "PROPOSED INSTRUCTION: Given the context and question, produce a specific and concise answer that accurately responds to the query, by iteratively generating relevant search queries and reasoning step by step.\n",+ "PROGRAM DESCRIPTION: This program appears to be designed to solve natural language processing tasks, specifically question answering and information retrieval. The program uses a language model to generate search queries and answers based on a given context and question. It does this by iterating through a series of \"hops\" where it generates a search query, retrieves relevant passages, and then generates an answer based on those passages. The program is able to generate human-like responses to questions by using a reasoning step to produce the answer, rather than simply retrieving a pre-computed answer.\n",+ "task_demos Context: ['Alberta general election, 2015 | The 29th general election of Alberta, Canada, took place on May 5, 2015, following a request of Premier Jim Prentice to the Lieutenant Governor of Alberta Donald Ethell to dissolve the Legislative Assembly on April 7, 2015. This election elected members to the 29th Alberta Legislature. It was one of only four times that Alberta has changed governments.', \"Jenean Hampton | Jenean Michelle Hampton (born May 12, 1958) is the 57th and current Lieutenant Governor of Kentucky since December 8, 2015 after having won office as Republican gubernatorial nominee Matt Bevin's running-mate, in the 2015 gubernatorial election. She is the first African-American to hold any statewide office in Kentucky history, and only the third African-American woman to serve as lieutenant-governor of any U.S. state (after Jennette Bradley and Jennifer Carroll).\", 'Louisiana lieutenant gubernatorial election, 2015 | The Louisiana lieutenant gubernatorial election took place on October 24, 2015, to elect the Lieutenant Governor of Louisiana, with a runoff election held on November 21, 2015. Incumbent Republican Lieutenant Governor Jay Dardenne did not for re-election to a second full term in office. He instead ran for Governor. Billy Nungesser won the election defeating Kip Holden, despite a Democratic victory in the gubernatorial election, in which John Bel Edwards defeated David Vitter by a similar margin.', 'Indiana gubernatorial election, 2016 | The 2016 Indiana gubernatorial election was held November 8, 2016, to elect the Governor and Lieutenant Governor of Indiana, concurrently with the 2016 U.S. presidential election as well as elections to the United States Senate and elections to the United States House of Representatives and various state and local elections. The primaries were held on May 3, 2016. Republican Lieutenant Governor Eric Holcomb won the race with 51.4% of the vote.', 'Virginia lieutenant gubernatorial election, 2017 | The Virginia lieutenant gubernatorial election of 2017 will take place on November 7, 2017. After the party primary elections were held, the major party nominees are Jill Vogel (Republican) and Justin Fairfax (Democrat). The incumbent Lieutenant Governor, Democrat Ralph Northam, is not running for re-election in order to run for Governor.']\n",+ "Question: What race is the lieutenant governor of the state that held its primary for her election on May 19th, 2015?\n",+ "Reasoning: Let's think step by step in order to The lieutenant governor of the state that held its primary for her election on May 19th, 2015 is Jenean Hampton, who is the 57th and current Lieutenant Governor of Kentucky since December 8, 2015. The primary election was held in Kentucky, and Jenean Hampton is the lieutenant governor of Kentucky.\n",+ "Answer: African-American\n",+ "Context: ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.', \"College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour\", 'Bass fishing | Bass fishing is the activity of angling for the North American gamefish known colloquially as the \"black bass\". There are numerous black bass species considered as gamefish in North America, including largemouth bass (\"Micropterus salmoides\"), smallmouth bass (\"Micropterus dolomieui\"), spotted bass or Kentucky bass (\"Micropterus punctulatus\"), and Guadalupe bass (order \"Perciformes\").', 'Cliff Crochet | Cliff Crochet (born August 1, 1983), also known as \"The Cajun Baby\", is a professional bass fisherman from Pierre Part, Louisiana. Crochet is a competitor on Bass Angler Sportsman Society\\'s (B.A.S.S.) Bassmaster Elite Series and has $368,837 in B.A.S.S. tournament winnings as of August 22, 2016. Crochet\\'s sponsors include: K2 Coolers, Huk, Skeeter, Yamaha Motor Company, Lowrance Electronics, Falcon Rods, Luck E Strike, Seaguar, Rat-L-Trap, Humdinger Spinnerbaits, Power-Pole. Since 2010, Crochet has competed in three Bassmaster Classic tournaments. Crochet\\'s boat for the 2016 season is a 2016 Skeeter FX 21 with a 250 hp Yamaha SHO outboard engine.']\n",+ "Question: Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?\n",+ "Reasoning: Let's think step by step in order to The context mentions Carhartt Bassmaster College Bass Fishing Series, which is a college bass fishing tournament series. This series is associated with the clothing company Carhartt, which was founded in 1889.\n",+ "Answer: Carhartt\n",+ "Context: ['Forrest Tucker | Forrest Meredith Tucker (February 12, 1919 – October 25, 1986) was an American actor in both movies and television who appeared in nearly a hundred films. Tucker worked as a vaudeville straight man, aged fifteen years old. A mentor provided funds and contacts for a trip to California, where party hostess Cobina Wright persuaded guest Wesley Ruggles to give Tucker a screen test, based on his photogenic good looks, thick wavy hair and height of six feet, five inches - putting him into the same league as John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness\\'s younger brother. Tucker was a sight reader who needed only one take. His film career started well despite a perception in most Hollywood studios that blond men were not photogenic. After twenty years spent mainly in Westerns and action roles, he returned to his roots, showing versatility as a comedic and stage musical actor. In the TV show \"F Troop\", he became identified with the character of Cavalry Sgt. Morgan O\\'Rourke (a manipulative character quite similar to Phil Silvers\\' role as MSgt Ernie Bilko). Tucker struggled with a drinking problem that began to affect his performances in the later years of his career.', 'Emergency Landing (1941 film) | Emergency Landing (a.k.a. Robot Pilot) is a 1941 American aviation spy-fi romantic screwball comedy film directed by William Beaudine. The film stars Forrest Tucker in his second film and in his first leading role with co-stars Carol Hughes and Evelyn Brent. \"Emergency Landing\" features lots of mismatched stock footage of various types of aircraft.', 'Forrest Tucker (criminal) | Forrest Silva \"Woody\" Tucker (June 23, 1920 – May 29, 2004) was an American career criminal first imprisoned at age 15 who spent the rest of his life in and out of jail. He is best known as an escape artist, having escaped from prison 18 times successfully and 12 times unsuccessfully.', 'Fess Parker | Fess Elisha Parker Jr. (August 16, 1924 – March 18, 2010) was an American film and television actor best known for his portrayals of Davy Crockett in the Walt Disney 1955–1956 TV miniseries and as Daniel Boone in a television series from 1964 to 1970. He was also known as a winemaker and resort owner-operator.', 'Davy Crockett, King of the Wild Frontier | Davy Crockett, King of the Wild Frontier is a 1955 live-action Walt Disney adventure film starring Fess Parker as Davy Crockett. This film is an edited compilation of the first three stories from the Disney television miniseries \"Davy Crockett\" :', 'Davy Crockett (miniseries) | Davy Crockett is a five-part serial which aired on ABC from 1954–1955 in one-hour episodes, on the \"Disneyland\" series. The series stars Fess Parker as real-life frontiersman Davy Crockett and Buddy Ebsen as his friend, George Russel.']\n",+ "Question: Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\n",+ "Reasoning: Let's think step by step in order to From the context, we know that Forrest Tucker was an American actor who appeared in nearly a hundred films. We also know that he was compared to John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness's younger brother. We need to find the actor that played a character in the 1955-1956 Walt Disney TV miniseries, which is Fess Parker.\n",+ "Answer: Davy Crockett\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Use the information below to learn about a task that we are trying to solve using calls to an LM, then generate a new instruction that will be used to prompt a Language Model to better solve the task.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "DATASET SUMMARY: A description of the dataset that we are using.\n",+ "\n",+ "PROGRAM CODE: Language model program designed to solve a particular task.\n",+ "\n",+ "PROGRAM DESCRIPTION: Summary of the task the program is designed to solve, and how it goes about solving it.\n",+ "\n",+ "MODULE: The module to create an instruction for.\n",+ "\n",+ "TASK DEMO(S): Example inputs/outputs of our module.\n",+ "\n",+ "BASIC INSTRUCTION: Basic instruction.\n",+ "\n",+ "TIP: A suggestion for how to go about generating the new instruction.\n",+ "\n",+ "PROPOSED INSTRUCTION: Propose an instruction that will be used to prompt a Language Model to perform this task.\n",+ "\n",+ "---\n",+ "\n",+ "DATASET SUMMARY: The dataset appears to be a comprehensive collection of trivia questions, presenting a diverse range of topics, including entertainment, history, arts, and culture. Most questions require specific, factual answers, often testing users' recall of specific details or events, and are designed to test users' knowledge on subject matter. The syntax and length of the questions vary, but most are concise and require precise answers.\n",+ "\n",+ "PROGRAM CODE:\n",+ "StringSignature(context, question -> rationale, search_query\n",+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",+ ")\n",+ "\n",+ "StringSignature(context, question -> rationale, answer\n",+ " instructions='Given the fields `context`, `question`, produce the fields `answer`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Answer:', 'desc': '${answer}'})\n",+ ")\n",+ "\n",+ "class BasicMH(dspy.Module):\n",+ " def __init__(self, passages_per_hop=3, num_hops=2):\n",+ " super().__init__()\n",+ " self.num_hops = num_hops\n",+ " self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",+ " # self.generate_query = dspy.ChainOfThought(GenerateSearchQuery)\n",+ " self.generate_query = [dspy.ChainOfThought(\"context, question -> search_query\") for _ in range(self.num_hops)]\n",+ " self.generate_answer = dspy.ChainOfThought(\"context, question -> answer\")\n",+ " \n",+ " def forward(self, question):\n",+ " context = []\n",+ " \n",+ " for hop in range(self.num_hops):\n",+ " search_query = self.generate_query[hop](context=context, question=question).search_query\n",+ " passages = self.retrieve(search_query).passages\n",+ " context = deduplicate(context + passages)\n",+ "\n",+ " prediction = self.generate_answer(context=context, question=question).copy(context=context)\n",+ " return prediction\n",+ "\n",+ "\n",+ "PROGRAM DESCRIPTION: This program appears to be designed to solve natural language processing tasks, specifically question answering and information retrieval. The program uses a language model to generate search queries and answers based on a given context and question. It does this by iterating through a series of \"hops\" where it generates a search query, retrieves relevant passages, and then generates an answer based on those passages. The program is able to generate human-like responses to questions by using a reasoning step to produce the answer, rather than simply retrieving a pre-computed answer.\n",+ "\n",+ "MODULE: self.generate_answer = dspy.ChainOfThought(\"context, question -> answer\")\n",+ "\n",+ "TASK DEMO(S):\n",+ "Context: ['Alberta general election, 2015 | The 29th general election of Alberta, Canada, took place on May 5, 2015, following a request of Premier Jim Prentice to the Lieutenant Governor of Alberta Donald Ethell to dissolve the Legislative Assembly on April 7, 2015. This election elected members to the 29th Alberta Legislature. It was one of only four times that Alberta has changed governments.', \"Jenean Hampton | Jenean Michelle Hampton (born May 12, 1958) is the 57th and current Lieutenant Governor of Kentucky since December 8, 2015 after having won office as Republican gubernatorial nominee Matt Bevin's running-mate, in the 2015 gubernatorial election. She is the first African-American to hold any statewide office in Kentucky history, and only the third African-American woman to serve as lieutenant-governor of any U.S. state (after Jennette Bradley and Jennifer Carroll).\", 'Louisiana lieutenant gubernatorial election, 2015 | The Louisiana lieutenant gubernatorial election took place on October 24, 2015, to elect the Lieutenant Governor of Louisiana, with a runoff election held on November 21, 2015. Incumbent Republican Lieutenant Governor Jay Dardenne did not for re-election to a second full term in office. He instead ran for Governor. Billy Nungesser won the election defeating Kip Holden, despite a Democratic victory in the gubernatorial election, in which John Bel Edwards defeated David Vitter by a similar margin.', 'Indiana gubernatorial election, 2016 | The 2016 Indiana gubernatorial election was held November 8, 2016, to elect the Governor and Lieutenant Governor of Indiana, concurrently with the 2016 U.S. presidential election as well as elections to the United States Senate and elections to the United States House of Representatives and various state and local elections. The primaries were held on May 3, 2016. Republican Lieutenant Governor Eric Holcomb won the race with 51.4% of the vote.', 'Virginia lieutenant gubernatorial election, 2017 | The Virginia lieutenant gubernatorial election of 2017 will take place on November 7, 2017. After the party primary elections were held, the major party nominees are Jill Vogel (Republican) and Justin Fairfax (Democrat). The incumbent Lieutenant Governor, Democrat Ralph Northam, is not running for re-election in order to run for Governor.']\n",+ "Question: What race is the lieutenant governor of the state that held its primary for her election on May 19th, 2015?\n",+ "Reasoning: Let's think step by step in order to The lieutenant governor of the state that held its primary for her election on May 19th, 2015 is Jenean Hampton, who is the 57th and current Lieutenant Governor of Kentucky since December 8, 2015. The primary election was held in Kentucky, and Jenean Hampton is the lieutenant governor of Kentucky.\n",+ "Answer: African-American\n",+ "Context: ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.', \"College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour\", 'Bass fishing | Bass fishing is the activity of angling for the North American gamefish known colloquially as the \"black bass\". There are numerous black bass species considered as gamefish in North America, including largemouth bass (\"Micropterus salmoides\"), smallmouth bass (\"Micropterus dolomieui\"), spotted bass or Kentucky bass (\"Micropterus punctulatus\"), and Guadalupe bass (order \"Perciformes\").', 'Cliff Crochet | Cliff Crochet (born August 1, 1983), also known as \"The Cajun Baby\", is a professional bass fisherman from Pierre Part, Louisiana. Crochet is a competitor on Bass Angler Sportsman Society\\'s (B.A.S.S.) Bassmaster Elite Series and has $368,837 in B.A.S.S. tournament winnings as of August 22, 2016. Crochet\\'s sponsors include: K2 Coolers, Huk, Skeeter, Yamaha Motor Company, Lowrance Electronics, Falcon Rods, Luck E Strike, Seaguar, Rat-L-Trap, Humdinger Spinnerbaits, Power-Pole. Since 2010, Crochet has competed in three Bassmaster Classic tournaments. Crochet\\'s boat for the 2016 season is a 2016 Skeeter FX 21 with a 250 hp Yamaha SHO outboard engine.']\n",+ "Question: Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?\n",+ "Reasoning: Let's think step by step in order to The context mentions Carhartt Bassmaster College Bass Fishing Series, which is a college bass fishing tournament series. This series is associated with the clothing company Carhartt, which was founded in 1889.\n",+ "Answer: Carhartt\n",+ "Context: ['Forrest Tucker | Forrest Meredith Tucker (February 12, 1919 – October 25, 1986) was an American actor in both movies and television who appeared in nearly a hundred films. Tucker worked as a vaudeville straight man, aged fifteen years old. A mentor provided funds and contacts for a trip to California, where party hostess Cobina Wright persuaded guest Wesley Ruggles to give Tucker a screen test, based on his photogenic good looks, thick wavy hair and height of six feet, five inches - putting him into the same league as John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness\\'s younger brother. Tucker was a sight reader who needed only one take. His film career started well despite a perception in most Hollywood studios that blond men were not photogenic. After twenty years spent mainly in Westerns and action roles, he returned to his roots, showing versatility as a comedic and stage musical actor. In the TV show \"F Troop\", he became identified with the character of Cavalry Sgt. Morgan O\\'Rourke (a manipulative character quite similar to Phil Silvers\\' role as MSgt Ernie Bilko). Tucker struggled with a drinking problem that began to affect his performances in the later years of his career.', 'Emergency Landing (1941 film) | Emergency Landing (a.k.a. Robot Pilot) is a 1941 American aviation spy-fi romantic screwball comedy film directed by William Beaudine. The film stars Forrest Tucker in his second film and in his first leading role with co-stars Carol Hughes and Evelyn Brent. \"Emergency Landing\" features lots of mismatched stock footage of various types of aircraft.', 'Forrest Tucker (criminal) | Forrest Silva \"Woody\" Tucker (June 23, 1920 – May 29, 2004) was an American career criminal first imprisoned at age 15 who spent the rest of his life in and out of jail. He is best known as an escape artist, having escaped from prison 18 times successfully and 12 times unsuccessfully.', 'Fess Parker | Fess Elisha Parker Jr. (August 16, 1924 – March 18, 2010) was an American film and television actor best known for his portrayals of Davy Crockett in the Walt Disney 1955–1956 TV miniseries and as Daniel Boone in a television series from 1964 to 1970. He was also known as a winemaker and resort owner-operator.', 'Davy Crockett, King of the Wild Frontier | Davy Crockett, King of the Wild Frontier is a 1955 live-action Walt Disney adventure film starring Fess Parker as Davy Crockett. This film is an edited compilation of the first three stories from the Disney television miniseries \"Davy Crockett\" :', 'Davy Crockett (miniseries) | Davy Crockett is a five-part serial which aired on ABC from 1954–1955 in one-hour episodes, on the \"Disneyland\" series. The series stars Fess Parker as real-life frontiersman Davy Crockett and Buddy Ebsen as his friend, George Russel.']\n",+ "Question: Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\n",+ "Reasoning: Let's think step by step in order to From the context, we know that Forrest Tucker was an American actor who appeared in nearly a hundred films. We also know that he was compared to John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness's younger brother. We need to find the actor that played a character in the 1955-1956 Walt Disney TV miniseries, which is Fess Parker.\n",+ "Answer: Davy Crockett\n",+ "\n",+ "\n",+ "BASIC INSTRUCTION: Given the fields `context`, `question`, produce the fields `answer`.\n",+ "\n",+ "TIP: Make sure your instruction is very informative and descriptive.\n",+ "\n",+ "Please provide the output field PROPOSED INSTRUCTION. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field PROPOSED INSTRUCTION.\n",+ "\n",+ "PROPOSED INSTRUCTION: Given a context and a question, generate a logical and coherent answer that accurately addresses the query by meticulously analyzing the provided context, identifying relevant information, and applying critical thinking to provide a precise and informative response.\n",+ "\n",+ "\n",+ "\n",+ "PROPOSED INSTRUCTION: Given a context and a question, generate a logical and coherent answer that accurately addresses the query by meticulously analyzing the provided context, identifying relevant information, and applying critical thinking to provide a precise and informative response.\n",+ "PROGRAM DESCRIPTION: The program appears to be designed to solve a task that involves generating answers to questions by reasoning and searching through a vast amount of text data. It uses a language model to generate search queries and answers based on the context and question provided. The program consists of a module called BasicMH that takes a question as input and generates an answer by iterating through a series of hops, where each hop involves generating a search query and retrieving relevant passages from a vast text corpus. The program uses a chain of thought approach to generate the search query and answer, where it generates intermediate results at each hop and refines them until it reaches the final answer.\n",+ "task_demos Context: ['Forrest Tucker | Forrest Meredith Tucker (February 12, 1919 – October 25, 1986) was an American actor in both movies and television who appeared in nearly a hundred films. Tucker worked as a vaudeville straight man, aged fifteen years old. A mentor provided funds and contacts for a trip to California, where party hostess Cobina Wright persuaded guest Wesley Ruggles to give Tucker a screen test, based on his photogenic good looks, thick wavy hair and height of six feet, five inches - putting him into the same league as John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness\\'s younger brother. Tucker was a sight reader who needed only one take. His film career started well despite a perception in most Hollywood studios that blond men were not photogenic. After twenty years spent mainly in Westerns and action roles, he returned to his roots, showing versatility as a comedic and stage musical actor. In the TV show \"F Troop\", he became identified with the character of Cavalry Sgt. Morgan O\\'Rourke (a manipulative character quite similar to Phil Silvers\\' role as MSgt Ernie Bilko). Tucker struggled with a drinking problem that began to affect his performances in the later years of his career.', 'Emergency Landing (1941 film) | Emergency Landing (a.k.a. Robot Pilot) is a 1941 American aviation spy-fi romantic screwball comedy film directed by William Beaudine. The film stars Forrest Tucker in his second film and in his first leading role with co-stars Carol Hughes and Evelyn Brent. \"Emergency Landing\" features lots of mismatched stock footage of various types of aircraft.', 'Forrest Tucker (criminal) | Forrest Silva \"Woody\" Tucker (June 23, 1920 – May 29, 2004) was an American career criminal first imprisoned at age 15 who spent the rest of his life in and out of jail. He is best known as an escape artist, having escaped from prison 18 times successfully and 12 times unsuccessfully.', 'Fess Parker | Fess Elisha Parker Jr. (August 16, 1924 – March 18, 2010) was an American film and television actor best known for his portrayals of Davy Crockett in the Walt Disney 1955–1956 TV miniseries and as Daniel Boone in a television series from 1964 to 1970. He was also known as a winemaker and resort owner-operator.', 'Davy Crockett, King of the Wild Frontier | Davy Crockett, King of the Wild Frontier is a 1955 live-action Walt Disney adventure film starring Fess Parker as Davy Crockett. This film is an edited compilation of the first three stories from the Disney television miniseries \"Davy Crockett\" :', 'Davy Crockett (miniseries) | Davy Crockett is a five-part serial which aired on ABC from 1954–1955 in one-hour episodes, on the \"Disneyland\" series. The series stars Fess Parker as real-life frontiersman Davy Crockett and Buddy Ebsen as his friend, George Russel.']\n",+ "Question: Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\n",+ "Reasoning: Let's think step by step in order to We need to find the actor who played the character in the 1955-1956 Walt Disney TV miniseries. From the context, we know that the miniseries was about Davy Crockett and starred Fess Parker as Davy Crockett.\n",+ "Answer: Davy Crockett\n",+ "Context: ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.', \"College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour\", 'Bass fishing | Bass fishing is the activity of angling for the North American gamefish known colloquially as the \"black bass\". There are numerous black bass species considered as gamefish in North America, including largemouth bass (\"Micropterus salmoides\"), smallmouth bass (\"Micropterus dolomieui\"), spotted bass or Kentucky bass (\"Micropterus punctulatus\"), and Guadalupe bass (order \"Perciformes\").', 'Cliff Crochet | Cliff Crochet (born August 1, 1983), also known as \"The Cajun Baby\", is a professional bass fisherman from Pierre Part, Louisiana. Crochet is a competitor on Bass Angler Sportsman Society\\'s (B.A.S.S.) Bassmaster Elite Series and has $368,837 in B.A.S.S. tournament winnings as of August 22, 2016. Crochet\\'s sponsors include: K2 Coolers, Huk, Skeeter, Yamaha Motor Company, Lowrance Electronics, Falcon Rods, Luck E Strike, Seaguar, Rat-L-Trap, Humdinger Spinnerbaits, Power-Pole. Since 2010, Crochet has competed in three Bassmaster Classic tournaments. Crochet\\'s boat for the 2016 season is a 2016 Skeeter FX 21 with a 250 hp Yamaha SHO outboard engine.']\n",+ "Question: Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?\n",+ "Reasoning: Let's think step by step in order to Context: [1] «Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.»\n",+ "\n",+ "[2] «College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour»\n",+ "\n",+ "[3] «G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.»\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "Answer: Carhartt\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Use the information below to learn about a task that we are trying to solve using calls to an LM, then generate a new instruction that will be used to prompt a Language Model to better solve the task.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "DATASET SUMMARY: A description of the dataset that we are using.\n",+ "\n",+ "PROGRAM CODE: Language model program designed to solve a particular task.\n",+ "\n",+ "PROGRAM DESCRIPTION: Summary of the task the program is designed to solve, and how it goes about solving it.\n",+ "\n",+ "MODULE: The module to create an instruction for.\n",+ "\n",+ "TASK DEMO(S): Example inputs/outputs of our module.\n",+ "\n",+ "BASIC INSTRUCTION: Basic instruction.\n",+ "\n",+ "TIP: A suggestion for how to go about generating the new instruction.\n",+ "\n",+ "PROPOSED INSTRUCTION: Propose an instruction that will be used to prompt a Language Model to perform this task.\n",+ "\n",+ "---\n",+ "\n",+ "DATASET SUMMARY: The dataset appears to be a comprehensive collection of trivia questions, presenting a diverse range of topics, including entertainment, history, arts, and culture. Most questions require specific, factual answers, often testing users' recall of specific details or events, and are designed to test users' knowledge on subject matter. The syntax and length of the questions vary, but most are concise and require precise answers.\n",+ "\n",+ "PROGRAM CODE:\n",+ "StringSignature(context, question -> rationale, search_query\n",+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",+ ")\n",+ "\n",+ "StringSignature(context, question -> rationale, answer\n",+ " instructions='Given the fields `context`, `question`, produce the fields `answer`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Answer:', 'desc': '${answer}'})\n",+ ")\n",+ "\n",+ "class BasicMH(dspy.Module):\n",+ " def __init__(self, passages_per_hop=3, num_hops=2):\n",+ " super().__init__()\n",+ " self.num_hops = num_hops\n",+ " self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",+ " # self.generate_query = dspy.ChainOfThought(GenerateSearchQuery)\n",+ " self.generate_query = [dspy.ChainOfThought(\"context, question -> search_query\") for _ in range(self.num_hops)]\n",+ " self.generate_answer = dspy.ChainOfThought(\"context, question -> answer\")\n",+ " \n",+ " def forward(self, question):\n",+ " context = []\n",+ " \n",+ " for hop in range(self.num_hops):\n",+ " search_query = self.generate_query[hop](context=context, question=question).search_query\n",+ " passages = self.retrieve(search_query).passages\n",+ " context = deduplicate(context + passages)\n",+ "\n",+ " prediction = self.generate_answer(context=context, question=question).copy(context=context)\n",+ " return prediction\n",+ "\n",+ "\n",+ "PROGRAM DESCRIPTION: The program appears to be designed to solve a task that involves generating answers to questions by reasoning and searching through a vast amount of text data. It uses a language model to generate search queries and answers based on the context and question provided. The program consists of a module called BasicMH that takes a question as input and generates an answer by iterating through a series of hops, where each hop involves generating a search query and retrieving relevant passages from a vast text corpus. The program uses a chain of thought approach to generate the search query and answer, where it generates intermediate results at each hop and refines them until it reaches the final answer.\n",+ "\n",+ "MODULE: self.generate_answer = dspy.ChainOfThought(\"context, question -> answer\")\n",+ "\n",+ "TASK DEMO(S):\n",+ "Context: ['Forrest Tucker | Forrest Meredith Tucker (February 12, 1919 – October 25, 1986) was an American actor in both movies and television who appeared in nearly a hundred films. Tucker worked as a vaudeville straight man, aged fifteen years old. A mentor provided funds and contacts for a trip to California, where party hostess Cobina Wright persuaded guest Wesley Ruggles to give Tucker a screen test, based on his photogenic good looks, thick wavy hair and height of six feet, five inches - putting him into the same league as John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness\\'s younger brother. Tucker was a sight reader who needed only one take. His film career started well despite a perception in most Hollywood studios that blond men were not photogenic. After twenty years spent mainly in Westerns and action roles, he returned to his roots, showing versatility as a comedic and stage musical actor. In the TV show \"F Troop\", he became identified with the character of Cavalry Sgt. Morgan O\\'Rourke (a manipulative character quite similar to Phil Silvers\\' role as MSgt Ernie Bilko). Tucker struggled with a drinking problem that began to affect his performances in the later years of his career.', 'Emergency Landing (1941 film) | Emergency Landing (a.k.a. Robot Pilot) is a 1941 American aviation spy-fi romantic screwball comedy film directed by William Beaudine. The film stars Forrest Tucker in his second film and in his first leading role with co-stars Carol Hughes and Evelyn Brent. \"Emergency Landing\" features lots of mismatched stock footage of various types of aircraft.', 'Forrest Tucker (criminal) | Forrest Silva \"Woody\" Tucker (June 23, 1920 – May 29, 2004) was an American career criminal first imprisoned at age 15 who spent the rest of his life in and out of jail. He is best known as an escape artist, having escaped from prison 18 times successfully and 12 times unsuccessfully.', 'Fess Parker | Fess Elisha Parker Jr. (August 16, 1924 – March 18, 2010) was an American film and television actor best known for his portrayals of Davy Crockett in the Walt Disney 1955–1956 TV miniseries and as Daniel Boone in a television series from 1964 to 1970. He was also known as a winemaker and resort owner-operator.', 'Davy Crockett, King of the Wild Frontier | Davy Crockett, King of the Wild Frontier is a 1955 live-action Walt Disney adventure film starring Fess Parker as Davy Crockett. This film is an edited compilation of the first three stories from the Disney television miniseries \"Davy Crockett\" :', 'Davy Crockett (miniseries) | Davy Crockett is a five-part serial which aired on ABC from 1954–1955 in one-hour episodes, on the \"Disneyland\" series. The series stars Fess Parker as real-life frontiersman Davy Crockett and Buddy Ebsen as his friend, George Russel.']\n",+ "Question: Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\n",+ "Reasoning: Let's think step by step in order to We need to find the actor who played the character in the 1955-1956 Walt Disney TV miniseries. From the context, we know that the miniseries was about Davy Crockett and starred Fess Parker as Davy Crockett.\n",+ "Answer: Davy Crockett\n",+ "Context: ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.', \"College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour\", 'Bass fishing | Bass fishing is the activity of angling for the North American gamefish known colloquially as the \"black bass\". There are numerous black bass species considered as gamefish in North America, including largemouth bass (\"Micropterus salmoides\"), smallmouth bass (\"Micropterus dolomieui\"), spotted bass or Kentucky bass (\"Micropterus punctulatus\"), and Guadalupe bass (order \"Perciformes\").', 'Cliff Crochet | Cliff Crochet (born August 1, 1983), also known as \"The Cajun Baby\", is a professional bass fisherman from Pierre Part, Louisiana. Crochet is a competitor on Bass Angler Sportsman Society\\'s (B.A.S.S.) Bassmaster Elite Series and has $368,837 in B.A.S.S. tournament winnings as of August 22, 2016. Crochet\\'s sponsors include: K2 Coolers, Huk, Skeeter, Yamaha Motor Company, Lowrance Electronics, Falcon Rods, Luck E Strike, Seaguar, Rat-L-Trap, Humdinger Spinnerbaits, Power-Pole. Since 2010, Crochet has competed in three Bassmaster Classic tournaments. Crochet\\'s boat for the 2016 season is a 2016 Skeeter FX 21 with a 250 hp Yamaha SHO outboard engine.']\n",+ "Question: Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?\n",+ "Reasoning: Let's think step by step in order to Context: [1] «Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.»\n",+ "\n",+ "[2] «College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour»\n",+ "\n",+ "[3] «G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.»\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "Answer: Carhartt\n",+ "\n",+ "\n",+ "BASIC INSTRUCTION: Given the fields `context`, `question`, produce the fields `answer`.\n",+ "\n",+ "TIP: Make sure your instruction is very informative and descriptive.\n",+ "\n",+ "Please provide the output field PROPOSED INSTRUCTION. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field PROPOSED INSTRUCTION.\n",+ "\n",+ "PROPOSED INSTRUCTION: Provide a concise and accurate answer to the question, citing relevant information from the provided context, ensuring credibility and clarity.\n",+ "\n",+ "\n",+ "\n",+ "PROPOSED INSTRUCTION: Provide a concise and accurate answer to the question, citing relevant information from the provided context, ensuring credibility and clarity.\n",+ "PROGRAM DESCRIPTION: The program appears to be designed to solve the task of answering questions by providing search queries and rationales based on a given context and question. It uses a multi-hop approach, where it generates search queries for each hop, retrieves relevant passages, and then generates a final answer based on the retrieved passages. The program takes in a context and question as input, generates a search query and rationale for each hop, and then uses a retrieve module to get relevant passages. The retrieved passages are then used to generate the final answer.\n",+ "task_demos Context: ['Will Finn | Will Finn (born November 1, 1958) is an American animator, voice actor, and director. His work in animation includes characters from Disney and Don Bluth films such as \"The Secret of NIMH\", \"Oliver & Company\", \"The Little Mermaid\", \"The Rescuers Down Under\", and \"Pocahontas\". His characters includes Cogsworth in \"Beauty and the Beast\", Iago in \"Aladdin\", and Laverne in \"The Hunchback of Notre Dame\". Finn wrote and directed \"Home on the Range\" and did some voice acting the Hollywood Fish in \"Chicken Little\". In 2006, Finn directed the computer animated short Hammy\\'s Boomerang Adventure, a spin-off of \"Over the Hedge\".', 'Over the Hedge (film) | Over the Hedge is a 2006 American computer-animated comedy film, based on the characters from the United Media comic strip of the same name. Directed by Tim Johnson and Karey Kirkpatrick, and produced by Bonnie Arnold, it was released in the United States on May 19, 2006. The film was produced by DreamWorks Animation and distributed through Paramount Pictures. The film features the voices of Bruce Willis, Garry Shandling, Steve Carell, William Shatner, Wanda Sykes, and Nick Nolte. It is the first DreamWorks Animation film to be distributed by Paramount Pictures, which acquired the live-action DreamWorks studio in 2006. The film earned $336 million on an $80 million budget.', 'Tim Johnson (film director) | Tim Johnson (born August 27, 1961) is an American animation director. Johnson has directed in many films such as \"Antz\", \"\" and \"Over the Hedge\". Recently, he directed the DreamWorks 3D film \"Home.\"', 'Hammy\\'s Boomerang Adventure | Hammy\\'s Boomerang Adventure is a computer-animated short comedy film based on the film \"Over the Hedge\", which was based on a comic strip of the same name. It can be found on the movie\\'s DVD or the \"Madly Madagascar\" DVD as a bonus feature.', 'Boomerang (2015 film) | Boomerang is a 2015 French drama film directed by François Favrat and starring Laurent Lafitte, Mélanie Laurent, Audrey Dana and Wladimir Yordanoff. It is an adaptation of Tatiana de Rosnay\\'s novel \"A Secret Kept\" (alternative title:\"Boomerang\"). The film was released in theatres on 23 September 2015.', 'Over the Hedge: Hammy Goes Nuts! | Over the Hedge: Hammy Goes Nuts! is a video game published by Activision based on characters from the film \"Over the Hedge\". The Game Boy Advance (GBA) version, a miniature golf game, was developed by Vicarious Visions and released on October 10, 2006 in North America. The Nintendo DS and PlayStation Portable (PSP) versions, an action adventure game and a platform game respectively, were both developed by Amaze Entertainment and were released on October 24, 2006 and November 21, 2006. It is the second game to be based on the film.']\n",+ "Question: When was the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed?\n",+ "Reasoning: Let's think step by step in order to Context: N/A\n",+ "\n",+ "Question: When was the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "Answer: 2006\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Use the information below to learn about a task that we are trying to solve using calls to an LM, then generate a new instruction that will be used to prompt a Language Model to better solve the task.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "DATASET SUMMARY: A description of the dataset that we are using.\n",+ "\n",+ "PROGRAM CODE: Language model program designed to solve a particular task.\n",+ "\n",+ "PROGRAM DESCRIPTION: Summary of the task the program is designed to solve, and how it goes about solving it.\n",+ "\n",+ "MODULE: The module to create an instruction for.\n",+ "\n",+ "TASK DEMO(S): Example inputs/outputs of our module.\n",+ "\n",+ "BASIC INSTRUCTION: Basic instruction.\n",+ "\n",+ "TIP: A suggestion for how to go about generating the new instruction.\n",+ "\n",+ "PROPOSED INSTRUCTION: Propose an instruction that will be used to prompt a Language Model to perform this task.\n",+ "\n",+ "---\n",+ "\n",+ "DATASET SUMMARY: The dataset appears to be a comprehensive collection of trivia questions, presenting a diverse range of topics, including entertainment, history, arts, and culture. Most questions require specific, factual answers, often testing users' recall of specific details or events, and are designed to test users' knowledge on subject matter. The syntax and length of the questions vary, but most are concise and require precise answers.\n",+ "\n",+ "PROGRAM CODE:\n",+ "StringSignature(context, question -> rationale, search_query\n",+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",+ ")\n",+ "\n",+ "StringSignature(context, question -> rationale, answer\n",+ " instructions='Given the fields `context`, `question`, produce the fields `answer`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Answer:', 'desc': '${answer}'})\n",+ ")\n",+ "\n",+ "class BasicMH(dspy.Module):\n",+ " def __init__(self, passages_per_hop=3, num_hops=2):\n",+ " super().__init__()\n",+ " self.num_hops = num_hops\n",+ " self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",+ " # self.generate_query = dspy.ChainOfThought(GenerateSearchQuery)\n",+ " self.generate_query = [dspy.ChainOfThought(\"context, question -> search_query\") for _ in range(self.num_hops)]\n",+ " self.generate_answer = dspy.ChainOfThought(\"context, question -> answer\")\n",+ " \n",+ " def forward(self, question):\n",+ " context = []\n",+ " \n",+ " for hop in range(self.num_hops):\n",+ " search_query = self.generate_query[hop](context=context, question=question).search_query\n",+ " passages = self.retrieve(search_query).passages\n",+ " context = deduplicate(context + passages)\n",+ "\n",+ " prediction = self.generate_answer(context=context, question=question).copy(context=context)\n",+ " return prediction\n",+ "\n",+ "\n",+ "PROGRAM DESCRIPTION: The program appears to be designed to solve the task of answering questions by providing search queries and rationales based on a given context and question. It uses a multi-hop approach, where it generates search queries for each hop, retrieves relevant passages, and then generates a final answer based on the retrieved passages. The program takes in a context and question as input, generates a search query and rationale for each hop, and then uses a retrieve module to get relevant passages. The retrieved passages are then used to generate the final answer.\n",+ "\n",+ "MODULE: self.generate_answer = dspy.ChainOfThought(\"context, question -> answer\")\n",+ "\n",+ "TASK DEMO(S):\n",+ "Context: ['Will Finn | Will Finn (born November 1, 1958) is an American animator, voice actor, and director. His work in animation includes characters from Disney and Don Bluth films such as \"The Secret of NIMH\", \"Oliver & Company\", \"The Little Mermaid\", \"The Rescuers Down Under\", and \"Pocahontas\". His characters includes Cogsworth in \"Beauty and the Beast\", Iago in \"Aladdin\", and Laverne in \"The Hunchback of Notre Dame\". Finn wrote and directed \"Home on the Range\" and did some voice acting the Hollywood Fish in \"Chicken Little\". In 2006, Finn directed the computer animated short Hammy\\'s Boomerang Adventure, a spin-off of \"Over the Hedge\".', 'Over the Hedge (film) | Over the Hedge is a 2006 American computer-animated comedy film, based on the characters from the United Media comic strip of the same name. Directed by Tim Johnson and Karey Kirkpatrick, and produced by Bonnie Arnold, it was released in the United States on May 19, 2006. The film was produced by DreamWorks Animation and distributed through Paramount Pictures. The film features the voices of Bruce Willis, Garry Shandling, Steve Carell, William Shatner, Wanda Sykes, and Nick Nolte. It is the first DreamWorks Animation film to be distributed by Paramount Pictures, which acquired the live-action DreamWorks studio in 2006. The film earned $336 million on an $80 million budget.', 'Tim Johnson (film director) | Tim Johnson (born August 27, 1961) is an American animation director. Johnson has directed in many films such as \"Antz\", \"\" and \"Over the Hedge\". Recently, he directed the DreamWorks 3D film \"Home.\"', 'Hammy\\'s Boomerang Adventure | Hammy\\'s Boomerang Adventure is a computer-animated short comedy film based on the film \"Over the Hedge\", which was based on a comic strip of the same name. It can be found on the movie\\'s DVD or the \"Madly Madagascar\" DVD as a bonus feature.', 'Boomerang (2015 film) | Boomerang is a 2015 French drama film directed by François Favrat and starring Laurent Lafitte, Mélanie Laurent, Audrey Dana and Wladimir Yordanoff. It is an adaptation of Tatiana de Rosnay\\'s novel \"A Secret Kept\" (alternative title:\"Boomerang\"). The film was released in theatres on 23 September 2015.', 'Over the Hedge: Hammy Goes Nuts! | Over the Hedge: Hammy Goes Nuts! is a video game published by Activision based on characters from the film \"Over the Hedge\". The Game Boy Advance (GBA) version, a miniature golf game, was developed by Vicarious Visions and released on October 10, 2006 in North America. The Nintendo DS and PlayStation Portable (PSP) versions, an action adventure game and a platform game respectively, were both developed by Amaze Entertainment and were released on October 24, 2006 and November 21, 2006. It is the second game to be based on the film.']\n",+ "Question: When was the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed?\n",+ "Reasoning: Let's think step by step in order to Context: N/A\n",+ "\n",+ "Question: When was the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "Answer: 2006\n",+ "\n",+ "\n",+ "BASIC INSTRUCTION: Given the fields `context`, `question`, produce the fields `answer`.\n",+ "\n",+ "TIP: Make sure your instruction is very informative and descriptive.\n",+ "\n",+ "Please provide the output field PROPOSED INSTRUCTION. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field PROPOSED INSTRUCTION.\n",+ "\n",+ "PROPOSED INSTRUCTION: Given the context and question, provide a step-by-step thought process and a precise answer to the question, considering relevant passages and information contained in the context.\n",+ "\n",+ "\n",+ "\n",+ "PROPOSED INSTRUCTION: Given the context and question, provide a step-by-step thought process and a precise answer to the question, considering relevant passages and information contained in the context.\n",+ "PROGRAM DESCRIPTION: This program appears to be designed to solve a question-answering task, specifically a geographic information retrieval task. It uses a hybrid approach, combining language models with retrieval-based techniques to generate search queries and answers. The program takes in a context and question as input, and outputs a search query and answer, along with the reasoning step-by-step. The solution involves multiple hops, where each hop generates a search query, retrieves relevant passages, and then the next hop uses these passages as context to generate a new search query. The final answer is generated by passing the context and question through a chain-of-thought language model.\n",+ "task_demos Context: ['Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.', 'Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.', \"Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world's third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.\"]\n",+ "Question: The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?\n",+ "Reasoning: Let's think step by step in order to Context: [1] «Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.»\n",+ "\n",+ "[2] «Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.»\n",+ "\n",+ "[3] «Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world's third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.»\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "Answer: India\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Use the information below to learn about a task that we are trying to solve using calls to an LM, then generate a new instruction that will be used to prompt a Language Model to better solve the task.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "DATASET SUMMARY: A description of the dataset that we are using.\n",+ "\n",+ "PROGRAM CODE: Language model program designed to solve a particular task.\n",+ "\n",+ "PROGRAM DESCRIPTION: Summary of the task the program is designed to solve, and how it goes about solving it.\n",+ "\n",+ "MODULE: The module to create an instruction for.\n",+ "\n",+ "TASK DEMO(S): Example inputs/outputs of our module.\n",+ "\n",+ "BASIC INSTRUCTION: Basic instruction.\n",+ "\n",+ "TIP: A suggestion for how to go about generating the new instruction.\n",+ "\n",+ "PROPOSED INSTRUCTION: Propose an instruction that will be used to prompt a Language Model to perform this task.\n",+ "\n",+ "---\n",+ "\n",+ "DATASET SUMMARY: The dataset appears to be a comprehensive collection of trivia questions, presenting a diverse range of topics, including entertainment, history, arts, and culture. Most questions require specific, factual answers, often testing users' recall of specific details or events, and are designed to test users' knowledge on subject matter. The syntax and length of the questions vary, but most are concise and require precise answers.\n",+ "\n",+ "PROGRAM CODE:\n",+ "StringSignature(context, question -> rationale, search_query\n",+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",+ ")\n",+ "\n",+ "StringSignature(context, question -> rationale, answer\n",+ " instructions='Given the fields `context`, `question`, produce the fields `answer`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Answer:', 'desc': '${answer}'})\n",+ ")\n",+ "\n",+ "class BasicMH(dspy.Module):\n",+ " def __init__(self, passages_per_hop=3, num_hops=2):\n",+ " super().__init__()\n",+ " self.num_hops = num_hops\n",+ " self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",+ " # self.generate_query = dspy.ChainOfThought(GenerateSearchQuery)\n",+ " self.generate_query = [dspy.ChainOfThought(\"context, question -> search_query\") for _ in range(self.num_hops)]\n",+ " self.generate_answer = dspy.ChainOfThought(\"context, question -> answer\")\n",+ " \n",+ " def forward(self, question):\n",+ " context = []\n",+ " \n",+ " for hop in range(self.num_hops):\n",+ " search_query = self.generate_query[hop](context=context, question=question).search_query\n",+ " passages = self.retrieve(search_query).passages\n",+ " context = deduplicate(context + passages)\n",+ "\n",+ " prediction = self.generate_answer(context=context, question=question).copy(context=context)\n",+ " return prediction\n",+ "\n",+ "\n",+ "PROGRAM DESCRIPTION: This program appears to be designed to solve a question-answering task, specifically a geographic information retrieval task. It uses a hybrid approach, combining language models with retrieval-based techniques to generate search queries and answers. The program takes in a context and question as input, and outputs a search query and answer, along with the reasoning step-by-step. The solution involves multiple hops, where each hop generates a search query, retrieves relevant passages, and then the next hop uses these passages as context to generate a new search query. The final answer is generated by passing the context and question through a chain-of-thought language model.\n",+ "\n",+ "MODULE: self.generate_answer = dspy.ChainOfThought(\"context, question -> answer\")\n",+ "\n",+ "TASK DEMO(S):\n",+ "Context: ['Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.', 'Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.', \"Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world's third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.\"]\n",+ "Question: The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?\n",+ "Reasoning: Let's think step by step in order to Context: [1] «Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.»\n",+ "\n",+ "[2] «Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.»\n",+ "\n",+ "[3] «Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world's third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.»\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "Answer: India\n",+ "\n",+ "\n",+ "BASIC INSTRUCTION: Given the fields `context`, `question`, produce the fields `answer`.\n",+ "\n",+ "TIP: Make sure your instruction is very informative and descriptive.\n",+ "\n",+ "Please provide the output field PROPOSED INSTRUCTION. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field PROPOSED INSTRUCTION.\n",+ "\n",+ "PROPOSED INSTRUCTION: Given a context and a question that mentions two high-peaked mountains, generate the answer that indicates the country where the mountains are partially located, by considering the geographical information provided in the context and the reasoning required to answer the question.\n",+ "\n",+ "\n",+ "\n",+ "PROPOSED INSTRUCTION: Given a context and a question that mentions two high-peaked mountains, generate the answer that indicates the country where the mountains are partially located, by considering the geographical information provided in the context and the reasoning required to answer the question.\n",+ "PROGRAM DESCRIPTION: The program appears to be designed to solve a question-answering task, specifically answering questions about art and art history. The program uses a chain-of-thought approach, where it generates a series of search queries based on the input question and context, retrieves relevant passages from a dataset, and then generates an answer based on the retrieved passages. The program consists of a series of hops, where each hop generates a search query, retrieves passages, and then generates an answer. The final answer is generated by combining the answers from each hop. The program seems to be trained on a dataset of art-related texts, and is able to generate answers that are related to the input question and context.\n",+ "task_demos Context: ['Wheat Field with Cypresses | A Wheatfield with Cypresses (occasionally called \"A Cornfield with Cypresses\") is any of three similar 1889 oil paintings by Vincent van Gogh, as part of his wheat field series. All were executed at the Saint-Paul-de-Mausole mental asylum at Saint-Rémy near Arles, France, where Van Gogh was voluntarily a patient from May 1889 to May 1890. The works were inspired by the view from the window at the asylum towards the Alpilles mountains.', 'Green Wheat Field with Cypress | Green Wheat Field with Cypress (French: Champ de blé vert avec cyprès) is an oil on canvas painting by a Dutch Post-Impressionist Vincent van Gogh. It was completed in 1889, while van Gogh was voluntarily incarcerated at the asylum of St. Paul near Saint-Rémy in Provence. Several paintings of wheat fields with cypresses were made when van Gogh was able to leave the asylum grounds and explore the local landscape. Besides a fondness for cypresses, van Gogh had a special affinty with wheat fields; he depicted them dozens of times over the years; to Vincent they symbolized the cycle of life and death, and he found in them both solace and inspiration.', 'Olive Trees (Van Gogh series) | Vincent van Gogh painted at least 18 paintings of olive trees, mostly in Saint-Rémy-de-Provence in 1889. At his own request, he lived at an asylum there from May 1889 through May 1890 painting the gardens of the asylum and, when he had permission to venture outside its walls, nearby olive trees, cypresses and wheat fields.', 'Wheat Fields (Van Gogh series) | The Wheat Fields is a series of dozens of paintings by Vincent van Gogh, borne out of his religious studies and sermons, connection to nature, appreciation of manual laborers and desire to provide a means of offering comfort to others. The wheat field works demonstrate his progression as an artist from the drab \"Wheat Sheaves\" made in 1885 in the Netherlands to the colorful, dramatic paintings from Arles, Saint-Rémy and Auvers-sur-Oise of rural France.']\n",+ "Question: What was the series inspired by the appreciation of manual laborers that Wheat Field with Cypresses is a part of?\n",+ "Reasoning: Let's think step by step in order to Context: N/A\n",+ "\n",+ "Question: What was the series inspired by the appreciation of manual laborers that Wheat Field with Cypresses is a part of?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "Answer: Wheat Fields\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Use the information below to learn about a task that we are trying to solve using calls to an LM, then generate a new instruction that will be used to prompt a Language Model to better solve the task.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "DATASET SUMMARY: A description of the dataset that we are using.\n",+ "\n",+ "PROGRAM CODE: Language model program designed to solve a particular task.\n",+ "\n",+ "PROGRAM DESCRIPTION: Summary of the task the program is designed to solve, and how it goes about solving it.\n",+ "\n",+ "MODULE: The module to create an instruction for.\n",+ "\n",+ "TASK DEMO(S): Example inputs/outputs of our module.\n",+ "\n",+ "BASIC INSTRUCTION: Basic instruction.\n",+ "\n",+ "TIP: A suggestion for how to go about generating the new instruction.\n",+ "\n",+ "PROPOSED INSTRUCTION: Propose an instruction that will be used to prompt a Language Model to perform this task.\n",+ "\n",+ "---\n",+ "\n",+ "DATASET SUMMARY: The dataset appears to be a comprehensive collection of trivia questions, presenting a diverse range of topics, including entertainment, history, arts, and culture. Most questions require specific, factual answers, often testing users' recall of specific details or events, and are designed to test users' knowledge on subject matter. The syntax and length of the questions vary, but most are concise and require precise answers.\n",+ "\n",+ "PROGRAM CODE:\n",+ "StringSignature(context, question -> rationale, search_query\n",+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",+ ")\n",+ "\n",+ "StringSignature(context, question -> rationale, answer\n",+ " instructions='Given the fields `context`, `question`, produce the fields `answer`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Answer:', 'desc': '${answer}'})\n",+ ")\n",+ "\n",+ "class BasicMH(dspy.Module):\n",+ " def __init__(self, passages_per_hop=3, num_hops=2):\n",+ " super().__init__()\n",+ " self.num_hops = num_hops\n",+ " self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",+ " # self.generate_query = dspy.ChainOfThought(GenerateSearchQuery)\n",+ " self.generate_query = [dspy.ChainOfThought(\"context, question -> search_query\") for _ in range(self.num_hops)]\n",+ " self.generate_answer = dspy.ChainOfThought(\"context, question -> answer\")\n",+ " \n",+ " def forward(self, question):\n",+ " context = []\n",+ " \n",+ " for hop in range(self.num_hops):\n",+ " search_query = self.generate_query[hop](context=context, question=question).search_query\n",+ " passages = self.retrieve(search_query).passages\n",+ " context = deduplicate(context + passages)\n",+ "\n",+ " prediction = self.generate_answer(context=context, question=question).copy(context=context)\n",+ " return prediction\n",+ "\n",+ "\n",+ "PROGRAM DESCRIPTION: The program appears to be designed to solve a question-answering task, specifically answering questions about art and art history. The program uses a chain-of-thought approach, where it generates a series of search queries based on the input question and context, retrieves relevant passages from a dataset, and then generates an answer based on the retrieved passages. The program consists of a series of hops, where each hop generates a search query, retrieves passages, and then generates an answer. The final answer is generated by combining the answers from each hop. The program seems to be trained on a dataset of art-related texts, and is able to generate answers that are related to the input question and context.\n",+ "\n",+ "MODULE: self.generate_answer = dspy.ChainOfThought(\"context, question -> answer\")\n",+ "\n",+ "TASK DEMO(S):\n",+ "Context: ['Wheat Field with Cypresses | A Wheatfield with Cypresses (occasionally called \"A Cornfield with Cypresses\") is any of three similar 1889 oil paintings by Vincent van Gogh, as part of his wheat field series. All were executed at the Saint-Paul-de-Mausole mental asylum at Saint-Rémy near Arles, France, where Van Gogh was voluntarily a patient from May 1889 to May 1890. The works were inspired by the view from the window at the asylum towards the Alpilles mountains.', 'Green Wheat Field with Cypress | Green Wheat Field with Cypress (French: Champ de blé vert avec cyprès) is an oil on canvas painting by a Dutch Post-Impressionist Vincent van Gogh. It was completed in 1889, while van Gogh was voluntarily incarcerated at the asylum of St. Paul near Saint-Rémy in Provence. Several paintings of wheat fields with cypresses were made when van Gogh was able to leave the asylum grounds and explore the local landscape. Besides a fondness for cypresses, van Gogh had a special affinty with wheat fields; he depicted them dozens of times over the years; to Vincent they symbolized the cycle of life and death, and he found in them both solace and inspiration.', 'Olive Trees (Van Gogh series) | Vincent van Gogh painted at least 18 paintings of olive trees, mostly in Saint-Rémy-de-Provence in 1889. At his own request, he lived at an asylum there from May 1889 through May 1890 painting the gardens of the asylum and, when he had permission to venture outside its walls, nearby olive trees, cypresses and wheat fields.', 'Wheat Fields (Van Gogh series) | The Wheat Fields is a series of dozens of paintings by Vincent van Gogh, borne out of his religious studies and sermons, connection to nature, appreciation of manual laborers and desire to provide a means of offering comfort to others. The wheat field works demonstrate his progression as an artist from the drab \"Wheat Sheaves\" made in 1885 in the Netherlands to the colorful, dramatic paintings from Arles, Saint-Rémy and Auvers-sur-Oise of rural France.']\n",+ "Question: What was the series inspired by the appreciation of manual laborers that Wheat Field with Cypresses is a part of?\n",+ "Reasoning: Let's think step by step in order to Context: N/A\n",+ "\n",+ "Question: What was the series inspired by the appreciation of manual laborers that Wheat Field with Cypresses is a part of?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "Answer: Wheat Fields\n",+ "\n",+ "\n",+ "BASIC INSTRUCTION: Given the fields `context`, `question`, produce the fields `answer`.\n",+ "\n",+ "TIP: Make sure your instruction is very informative and descriptive.\n",+ "\n",+ "Please provide the output field PROPOSED INSTRUCTION. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field PROPOSED INSTRUCTION.\n",+ "\n",+ "PROPOSED INSTRUCTION: Given the historical context of Vincent van Gogh's artwork, specifically the wheat field series, and the provided question, generate a concise answer that identifies the inspiration behind the series, considering the artist's appreciation for manual laborers, his connection to nature, and the symbolism of the cycles of life and death in his work.\n",+ "\n",+ "\n",+ "\n",+ "PROPOSED INSTRUCTION: Given the historical context of Vincent van Gogh's artwork, specifically the wheat field series, and the provided question, generate a concise answer that identifies the inspiration behind the series, considering the artist's appreciation for manual laborers, his connection to nature, and the symbolism of the cycles of life and death in his work.\n",+ "PROGRAM DESCRIPTION: The program appears to be designed to solve a question-answering task using a natural language processing (NLP) pipeline. The program takes in a context and a question, and generates a search query and an answer based on the context and question. The program uses a multi-hop approach, where it generates multiple search queries and retrieves relevant passages based on those queries, and then uses those passages to generate the final answer. The program is designed to answer complex questions that require reasoning and context, and provides a rationale for its answer in the form of a justification text.\n",+ "task_demos Context: ['Butch Reed | Bruce Franklin Reed (born July 11, 1954) is an American retired professional wrestler and football player, better known by the ring name Butch Reed. He is best known for his appearances with the World Wrestling Federation and World Championship Wrestling between 1986 and 1992.', 'Doug Reed (futsal player) | Doug Reed is a professional futsal player who is currently playing for Futsal Dinamo and England national futsal team. Previously he has played for Lanzarote Tias Yaiza (Spain), AEL Limassol, MNK Uspinjača, MNK Smederevo and Manchester Futsal Club. In 2011, he won the FA Futsal Cup with Manchester Futsal Club and was awarded Player of the Tournament in the same competition. In the FA National Futsal League, he won several FA National League Championships and FA Futsal Cup medals. Reed also works as a commentator for Sony SIX, Eurosport and BT Sport, and has worked with the emerging US Professional Futsal League as well as Indian Premier Futsal.', 'Doom (professional wrestling) | Doom was a professional wrestling tag team composed of Ron Simmons and Butch Reed. They competed from 1989 to 1991 in the National Wrestling Alliance (NWA) and World Championship Wrestling (WCW).', 'Robert Reed (gridiron football) | Robert E. Reed, Jr. (born February 23, 1943) is a former American football offensive lineman in the National Football League for the Washington Redskins. He played college football at Tennessee State University and was drafted in the sixteenth round of the 1965 NFL Draft. Reed was also selected in the 12th round of the 1965 AFL Draft by the Houston Oilers.', 'Reed Doughty | Reed Doughty (born November 4, 1982) is a former American football safety who played in the National Football League. He was drafted by the Washington Redskins in the sixth round, with the 173rd overall pick, of the 2006 NFL Draft. He played college football at the University of Northern Colorado.']\n",+ "Question: What other sport did Butch Reed, of the wrestling tag team Doom, play professionally?\n",+ "Reasoning: Let's think step by step in order to Context: N/A\n",+ "\n",+ "Question: What other sport did Butch Reed, of the wrestling tag team Doom, play professionally?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "Answer: football\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Use the information below to learn about a task that we are trying to solve using calls to an LM, then generate a new instruction that will be used to prompt a Language Model to better solve the task.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "DATASET SUMMARY: A description of the dataset that we are using.\n",+ "\n",+ "PROGRAM CODE: Language model program designed to solve a particular task.\n",+ "\n",+ "PROGRAM DESCRIPTION: Summary of the task the program is designed to solve, and how it goes about solving it.\n",+ "\n",+ "MODULE: The module to create an instruction for.\n",+ "\n",+ "TASK DEMO(S): Example inputs/outputs of our module.\n",+ "\n",+ "BASIC INSTRUCTION: Basic instruction.\n",+ "\n",+ "TIP: A suggestion for how to go about generating the new instruction.\n",+ "\n",+ "PROPOSED INSTRUCTION: Propose an instruction that will be used to prompt a Language Model to perform this task.\n",+ "\n",+ "---\n",+ "\n",+ "DATASET SUMMARY: The dataset appears to be a comprehensive collection of trivia questions, presenting a diverse range of topics, including entertainment, history, arts, and culture. Most questions require specific, factual answers, often testing users' recall of specific details or events, and are designed to test users' knowledge on subject matter. The syntax and length of the questions vary, but most are concise and require precise answers.\n",+ "\n",+ "PROGRAM CODE:\n",+ "StringSignature(context, question -> rationale, search_query\n",+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",+ ")\n",+ "\n",+ "StringSignature(context, question -> rationale, answer\n",+ " instructions='Given the fields `context`, `question`, produce the fields `answer`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Answer:', 'desc': '${answer}'})\n",+ ")\n",+ "\n",+ "class BasicMH(dspy.Module):\n",+ " def __init__(self, passages_per_hop=3, num_hops=2):\n",+ " super().__init__()\n",+ " self.num_hops = num_hops\n",+ " self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",+ " # self.generate_query = dspy.ChainOfThought(GenerateSearchQuery)\n",+ " self.generate_query = [dspy.ChainOfThought(\"context, question -> search_query\") for _ in range(self.num_hops)]\n",+ " self.generate_answer = dspy.ChainOfThought(\"context, question -> answer\")\n",+ " \n",+ " def forward(self, question):\n",+ " context = []\n",+ " \n",+ " for hop in range(self.num_hops):\n",+ " search_query = self.generate_query[hop](context=context, question=question).search_query\n",+ " passages = self.retrieve(search_query).passages\n",+ " context = deduplicate(context + passages)\n",+ "\n",+ " prediction = self.generate_answer(context=context, question=question).copy(context=context)\n",+ " return prediction\n",+ "\n",+ "\n",+ "PROGRAM DESCRIPTION: The program appears to be designed to solve a question-answering task using a natural language processing (NLP) pipeline. The program takes in a context and a question, and generates a search query and an answer based on the context and question. The program uses a multi-hop approach, where it generates multiple search queries and retrieves relevant passages based on those queries, and then uses those passages to generate the final answer. The program is designed to answer complex questions that require reasoning and context, and provides a rationale for its answer in the form of a justification text.\n",+ "\n",+ "MODULE: self.generate_answer = dspy.ChainOfThought(\"context, question -> answer\")\n",+ "\n",+ "TASK DEMO(S):\n",+ "Context: ['Butch Reed | Bruce Franklin Reed (born July 11, 1954) is an American retired professional wrestler and football player, better known by the ring name Butch Reed. He is best known for his appearances with the World Wrestling Federation and World Championship Wrestling between 1986 and 1992.', 'Doug Reed (futsal player) | Doug Reed is a professional futsal player who is currently playing for Futsal Dinamo and England national futsal team. Previously he has played for Lanzarote Tias Yaiza (Spain), AEL Limassol, MNK Uspinjača, MNK Smederevo and Manchester Futsal Club. In 2011, he won the FA Futsal Cup with Manchester Futsal Club and was awarded Player of the Tournament in the same competition. In the FA National Futsal League, he won several FA National League Championships and FA Futsal Cup medals. Reed also works as a commentator for Sony SIX, Eurosport and BT Sport, and has worked with the emerging US Professional Futsal League as well as Indian Premier Futsal.', 'Doom (professional wrestling) | Doom was a professional wrestling tag team composed of Ron Simmons and Butch Reed. They competed from 1989 to 1991 in the National Wrestling Alliance (NWA) and World Championship Wrestling (WCW).', 'Robert Reed (gridiron football) | Robert E. Reed, Jr. (born February 23, 1943) is a former American football offensive lineman in the National Football League for the Washington Redskins. He played college football at Tennessee State University and was drafted in the sixteenth round of the 1965 NFL Draft. Reed was also selected in the 12th round of the 1965 AFL Draft by the Houston Oilers.', 'Reed Doughty | Reed Doughty (born November 4, 1982) is a former American football safety who played in the National Football League. He was drafted by the Washington Redskins in the sixth round, with the 173rd overall pick, of the 2006 NFL Draft. He played college football at the University of Northern Colorado.']\n",+ "Question: What other sport did Butch Reed, of the wrestling tag team Doom, play professionally?\n",+ "Reasoning: Let's think step by step in order to Context: N/A\n",+ "\n",+ "Question: What other sport did Butch Reed, of the wrestling tag team Doom, play professionally?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "Answer: football\n",+ "\n",+ "\n",+ "BASIC INSTRUCTION: Given the fields `context`, `question`, produce the fields `answer`.\n",+ "\n",+ "TIP: Make sure your instruction is very informative and descriptive.\n",+ "\n",+ "Please provide the output field PROPOSED INSTRUCTION. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field PROPOSED INSTRUCTION.\n",+ "\n",+ "PROPOSED INSTRUCTION: Given the context, question, and relevant background information, generate the most accurate and informative answer by carefully analyzing the context and question, identifying the key details and relationships, and providing a clear and concise response that incorporates the relevant facts and justifications.\n",+ "\n",+ "\n",+ "\n",+ "PROPOSED INSTRUCTION: Given the context, question, and relevant background information, generate the most accurate and informative answer by carefully analyzing the context and question, identifying the key details and relationships, and providing a clear and concise response that incorporates the relevant facts and justifications.\n",+ "PROGRAM DESCRIPTION: ABOVE**\n",+ "\n",+ "The program is designed to solve a question-answering task using a language model. The program uses a pipeline architecture where it first retrieves relevant passages from a database, then generates a search query based on the question and context, retrieves relevant passages again, and finally generates an answer based on the retrieved passages.\n",+ "\n",+ "The program appears to be used for answering questions about various topics, including history, biographies, and theme parks. It uses a chain-of-thought approach to generate the search query and answer, where it iteratively refines its search query and answer based on the context and retrieved passages.\n",+ "task_demos Context: ['Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.', 'Ebenezer Howard | Sir Ebenezer Howard {\\'1\\': \", \\'2\\': \", \\'3\\': \", \\'4\\': \"} ( ; 29 January 1850 – 1 May 1928), the English founder of the garden city movement, is known for his publication \"\" (1898), the description of a utopian city in which people live harmoniously together with nature. The publication resulted in the founding of the garden city movement, and the building of the First Garden City, Letchworth Garden City, commenced in 1903.', 'Eugene Reynal | Eugene Reynal was an American Publisher and in 1933 Founder of Blue Ribbon Books of Garden City, New York and Reynal and Hitchcock of New York, New York.']\n",+ "Question: Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?\n",+ "Reasoning: Let's think step by step in order to Context: [1] «Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.»\n",+ "\n",+ "[2] «A.T. Stewart | Alexander Turney Stewart (October 2, 1792 – April 10, 1876) was an American businessman, politician, and philanthropist. He was a prominent figure in New York City during the mid-19th century, and was known as the \"Merchant Prince\".»\n",+ "\n",+ "[3] «Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.»\n",+ "\n",+ "Question: Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "Answer: A.T. Stewart\n",+ "Context: ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.', \"College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour\", 'Bass fishing | Bass fishing is the activity of angling for the North American gamefish known colloquially as the \"black bass\". There are numerous black bass species considered as gamefish in North America, including largemouth bass (\"Micropterus salmoides\"), smallmouth bass (\"Micropterus dolomieui\"), spotted bass or Kentucky bass (\"Micropterus punctulatus\"), and Guadalupe bass (order \"Perciformes\").', 'Cliff Crochet | Cliff Crochet (born August 1, 1983), also known as \"The Cajun Baby\", is a professional bass fisherman from Pierre Part, Louisiana. Crochet is a competitor on Bass Angler Sportsman Society\\'s (B.A.S.S.) Bassmaster Elite Series and has $368,837 in B.A.S.S. tournament winnings as of August 22, 2016. Crochet\\'s sponsors include: K2 Coolers, Huk, Skeeter, Yamaha Motor Company, Lowrance Electronics, Falcon Rods, Luck E Strike, Seaguar, Rat-L-Trap, Humdinger Spinnerbaits, Power-Pole. Since 2010, Crochet has competed in three Bassmaster Classic tournaments. Crochet\\'s boat for the 2016 season is a 2016 Skeeter FX 21 with a 250 hp Yamaha SHO outboard engine.']\n",+ "Question: Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?\n",+ "Reasoning: Let's think step by step in order to Context: [1] «Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.»\n",+ "\n",+ "[2] «College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour»\n",+ "\n",+ "[3] «G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.»\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "Answer: Carhartt\n",+ "Context: ['Stitch\\'s Great Escape! | Stitch\\'s Great Escape! is a Tomorrowland attraction at the Magic Kingdom theme park within the Walt Disney World Resort. It is a \"theater-in-the-round\" experience starring the title alien from Walt Disney Animation Studios\\' 2002 film \"Lilo & Stitch\". It opened November 16, 2004 and is the fourth attraction to occupy the site in Tomorrowland. Many of the animators who worked on \"Lilo & Stitch\" partnered with Walt Disney Imagineering for Stitch\\'s Great Escape! The attraction is a replacement of The ExtraTERRORestrial Alien Encounter, which formerly occupied the building the attraction is housed in. On September 21, 2016, it was announced that the attraction would be switching from a daily operated attraction to a seasonally operated one, depending on attendance, starting October 2, 2016.', 'Great Escape (amusement park) | Great Escape (sometimes referred to as Six Flags Great Escape) is an amusement and water park owned and operated by Six Flags Entertainment Corp. It is located approximately 60 mi north of Albany, in Queensbury, New York, but is advertised as being in Lake George, New York, a popular tourist and vacation spot nearby. It is one of two Six Flags parks not to be officially branded with the \"Six Flags\" name (La Ronde in Montreal, Quebec, Canada being the other).', 'Houdini\\'s Great Escape | Houdini\\'s Great Escape, also known as \"Boo-dini\" during Fright Fest and \"Houdini - The Great Escape\" at Six Flags New England, are indoor Vekoma Madhouse attractions at Six Flags Great Adventure and Six Flags New England. The ride is located in Great Adventure\\'s Fantasy Forest section and New England\\'s Crack Axle Canyon area. The attraction opened at both parks in 1999, and was Standing but not operating (SBNO) at Great Adventure from 2008 - 2009 but was opened again due to popular demand.', 'Magic Kingdom | Magic Kingdom is a theme park at the Walt Disney World Resort in Bay Lake, Florida, near Orlando. Owned and operated by The Walt Disney Company through its Parks and Resorts division, the park opened on October 1, 1971, as the first of four theme parks at the resort. Initialized by Walt Disney and designed by WED Enterprises, its layout and attractions are based on Disneyland Park in Anaheim, California, and is dedicated to fairy tales and Disney characters.', 'Magic Kingdom (disambiguation) | The Magic Kingdom is a theme park at the Walt Disney World Resort in Florida.', 'List of Magic Kingdom attractions | The Magic Kingdom is a theme park located at the Walt Disney World Resort in Florida. Below is a list of the current attractions found therein, arranged by \"land\" and with brief descriptions.']\n",+ "Question: Which theme park has Stitch’s Great Escape which replaced the attraction ExtraTERRORestial Alien Encounter?\n",+ "Reasoning: Let's think step by step in order to We need to find the theme park that has Stitch's Great Escape, which replaced the attraction ExtraTERRORestial Alien Encounter. The context mentions that Stitch's Great Escape is located in the Magic Kingdom theme park.\n",+ "Answer: Magic Kingdom\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Use the information below to learn about a task that we are trying to solve using calls to an LM, then generate a new instruction that will be used to prompt a Language Model to better solve the task.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "DATASET SUMMARY: A description of the dataset that we are using.\n",+ "\n",+ "PROGRAM CODE: Language model program designed to solve a particular task.\n",+ "\n",+ "PROGRAM DESCRIPTION: Summary of the task the program is designed to solve, and how it goes about solving it.\n",+ "\n",+ "MODULE: The module to create an instruction for.\n",+ "\n",+ "TASK DEMO(S): Example inputs/outputs of our module.\n",+ "\n",+ "BASIC INSTRUCTION: Basic instruction.\n",+ "\n",+ "TIP: A suggestion for how to go about generating the new instruction.\n",+ "\n",+ "PROPOSED INSTRUCTION: Propose an instruction that will be used to prompt a Language Model to perform this task.\n",+ "\n",+ "---\n",+ "\n",+ "DATASET SUMMARY: The dataset appears to be a comprehensive collection of trivia questions, presenting a diverse range of topics, including entertainment, history, arts, and culture. Most questions require specific, factual answers, often testing users' recall of specific details or events, and are designed to test users' knowledge on subject matter. The syntax and length of the questions vary, but most are concise and require precise answers.\n",+ "\n",+ "PROGRAM CODE:\n",+ "StringSignature(context, question -> rationale, search_query\n",+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",+ ")\n",+ "\n",+ "StringSignature(context, question -> rationale, answer\n",+ " instructions='Given the fields `context`, `question`, produce the fields `answer`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Answer:', 'desc': '${answer}'})\n",+ ")\n",+ "\n",+ "class BasicMH(dspy.Module):\n",+ " def __init__(self, passages_per_hop=3, num_hops=2):\n",+ " super().__init__()\n",+ " self.num_hops = num_hops\n",+ " self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",+ " # self.generate_query = dspy.ChainOfThought(GenerateSearchQuery)\n",+ " self.generate_query = [dspy.ChainOfThought(\"context, question -> search_query\") for _ in range(self.num_hops)]\n",+ " self.generate_answer = dspy.ChainOfThought(\"context, question -> answer\")\n",+ " \n",+ " def forward(self, question):\n",+ " context = []\n",+ " \n",+ " for hop in range(self.num_hops):\n",+ " search_query = self.generate_query[hop](context=context, question=question).search_query\n",+ " passages = self.retrieve(search_query).passages\n",+ " context = deduplicate(context + passages)\n",+ "\n",+ " prediction = self.generate_answer(context=context, question=question).copy(context=context)\n",+ " return prediction\n",+ "\n",+ "\n",+ "PROGRAM DESCRIPTION:\n",+ "ABOVE**\n",+ "\n",+ "The program is designed to solve a question-answering task using a language model. The program uses a pipeline architecture where it first retrieves relevant passages from a database, then generates a search query based on the question and context, retrieves relevant passages again, and finally generates an answer based on the retrieved passages.\n",+ "\n",+ "The program appears to be used for answering questions about various topics, including history, biographies, and theme parks. It uses a chain-of-thought approach to generate the search query and answer, where it iteratively refines its search query and answer based on the context and retrieved passages.\n",+ "\n",+ "MODULE: self.generate_answer = dspy.ChainOfThought(\"context, question -> answer\")\n",+ "\n",+ "TASK DEMO(S):\n",+ "Context: ['Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.', 'Ebenezer Howard | Sir Ebenezer Howard {\\'1\\': \", \\'2\\': \", \\'3\\': \", \\'4\\': \"} ( ; 29 January 1850 – 1 May 1928), the English founder of the garden city movement, is known for his publication \"\" (1898), the description of a utopian city in which people live harmoniously together with nature. The publication resulted in the founding of the garden city movement, and the building of the First Garden City, Letchworth Garden City, commenced in 1903.', 'Eugene Reynal | Eugene Reynal was an American Publisher and in 1933 Founder of Blue Ribbon Books of Garden City, New York and Reynal and Hitchcock of New York, New York.']\n",+ "Question: Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?\n",+ "Reasoning: Let's think step by step in order to Context: [1] «Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.»\n",+ "\n",+ "[2] «A.T. Stewart | Alexander Turney Stewart (October 2, 1792 – April 10, 1876) was an American businessman, politician, and philanthropist. He was a prominent figure in New York City during the mid-19th century, and was known as the \"Merchant Prince\".»\n",+ "\n",+ "[3] «Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.»\n",+ "\n",+ "Question: Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "Answer: A.T. Stewart\n",+ "Context: ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.', \"College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour\", 'Bass fishing | Bass fishing is the activity of angling for the North American gamefish known colloquially as the \"black bass\". There are numerous black bass species considered as gamefish in North America, including largemouth bass (\"Micropterus salmoides\"), smallmouth bass (\"Micropterus dolomieui\"), spotted bass or Kentucky bass (\"Micropterus punctulatus\"), and Guadalupe bass (order \"Perciformes\").', 'Cliff Crochet | Cliff Crochet (born August 1, 1983), also known as \"The Cajun Baby\", is a professional bass fisherman from Pierre Part, Louisiana. Crochet is a competitor on Bass Angler Sportsman Society\\'s (B.A.S.S.) Bassmaster Elite Series and has $368,837 in B.A.S.S. tournament winnings as of August 22, 2016. Crochet\\'s sponsors include: K2 Coolers, Huk, Skeeter, Yamaha Motor Company, Lowrance Electronics, Falcon Rods, Luck E Strike, Seaguar, Rat-L-Trap, Humdinger Spinnerbaits, Power-Pole. Since 2010, Crochet has competed in three Bassmaster Classic tournaments. Crochet\\'s boat for the 2016 season is a 2016 Skeeter FX 21 with a 250 hp Yamaha SHO outboard engine.']\n",+ "Question: Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?\n",+ "Reasoning: Let's think step by step in order to Context: [1] «Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.»\n",+ "\n",+ "[2] «College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour»\n",+ "\n",+ "[3] «G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.»\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "Answer: Carhartt\n",+ "Context: ['Stitch\\'s Great Escape! | Stitch\\'s Great Escape! is a Tomorrowland attraction at the Magic Kingdom theme park within the Walt Disney World Resort. It is a \"theater-in-the-round\" experience starring the title alien from Walt Disney Animation Studios\\' 2002 film \"Lilo & Stitch\". It opened November 16, 2004 and is the fourth attraction to occupy the site in Tomorrowland. Many of the animators who worked on \"Lilo & Stitch\" partnered with Walt Disney Imagineering for Stitch\\'s Great Escape! The attraction is a replacement of The ExtraTERRORestrial Alien Encounter, which formerly occupied the building the attraction is housed in. On September 21, 2016, it was announced that the attraction would be switching from a daily operated attraction to a seasonally operated one, depending on attendance, starting October 2, 2016.', 'Great Escape (amusement park) | Great Escape (sometimes referred to as Six Flags Great Escape) is an amusement and water park owned and operated by Six Flags Entertainment Corp. It is located approximately 60 mi north of Albany, in Queensbury, New York, but is advertised as being in Lake George, New York, a popular tourist and vacation spot nearby. It is one of two Six Flags parks not to be officially branded with the \"Six Flags\" name (La Ronde in Montreal, Quebec, Canada being the other).', 'Houdini\\'s Great Escape | Houdini\\'s Great Escape, also known as \"Boo-dini\" during Fright Fest and \"Houdini - The Great Escape\" at Six Flags New England, are indoor Vekoma Madhouse attractions at Six Flags Great Adventure and Six Flags New England. The ride is located in Great Adventure\\'s Fantasy Forest section and New England\\'s Crack Axle Canyon area. The attraction opened at both parks in 1999, and was Standing but not operating (SBNO) at Great Adventure from 2008 - 2009 but was opened again due to popular demand.', 'Magic Kingdom | Magic Kingdom is a theme park at the Walt Disney World Resort in Bay Lake, Florida, near Orlando. Owned and operated by The Walt Disney Company through its Parks and Resorts division, the park opened on October 1, 1971, as the first of four theme parks at the resort. Initialized by Walt Disney and designed by WED Enterprises, its layout and attractions are based on Disneyland Park in Anaheim, California, and is dedicated to fairy tales and Disney characters.', 'Magic Kingdom (disambiguation) | The Magic Kingdom is a theme park at the Walt Disney World Resort in Florida.', 'List of Magic Kingdom attractions | The Magic Kingdom is a theme park located at the Walt Disney World Resort in Florida. Below is a list of the current attractions found therein, arranged by \"land\" and with brief descriptions.']\n",+ "Question: Which theme park has Stitch’s Great Escape which replaced the attraction ExtraTERRORestial Alien Encounter?\n",+ "Reasoning: Let's think step by step in order to We need to find the theme park that has Stitch's Great Escape, which replaced the attraction ExtraTERRORestial Alien Encounter. The context mentions that Stitch's Great Escape is located in the Magic Kingdom theme park.\n",+ "Answer: Magic Kingdom\n",+ "\n",+ "\n",+ "BASIC INSTRUCTION: Given the fields `context`, `question`, produce the fields `answer`.\n",+ "\n",+ "TIP: Make sure your instruction is very informative and descriptive.\n",+ "\n",+ "Please provide the output field PROPOSED INSTRUCTION. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field PROPOSED INSTRUCTION.\n",+ "\n",+ "PROPOSED INSTRUCTION: Generate an answer based on the provided context, question, and reasoning, drawing upon the knowledge and information contained within the passages to provide a precise and accurate response.\n",+ "\n",+ "\n",+ "\n",+ "PROPOSED INSTRUCTION: Generate an answer based on the provided context, question, and reasoning, drawing upon the knowledge and information contained within the passages to provide a precise and accurate response.\n",+ "PROGRAM DESCRIPTION: The program appears to be designed to solve a question-answering task by utilizing natural language processing (NLP) models. Specifically, it is designed to generate answers based on a given context and question. The program works by first generating a search query based on the context and question, and then retrieving relevant passages from the search query. It then generates the answer by combining the context and retrieved passages. The program uses a multi-hop approach, where it repeatedly generates search queries, retrieves passages, and combines them to generate the final answer. The program's output field is the answer to the question, along with a rationale for the answer.\n",+ "task_demos Context: ['Simon Arora | Simon Arora (born November 1969) is a British billionaire businessman, CEO of the retail chain B & M.', 'Samir Arora | Samir Arora (born November 5, 1965) is an American businessman and former CEO of Mode Media (formerly Glam Media) from 2003 to April 2016. He was CEO and Chairman of the web design company NetObjects, Inc. from 1995 to 2001.', 'Dinesh Arora | Dinesh Arora (born 23 December 1976) is an Indian Administrative Service (IAS) officer, 2002 Kerala cadre, who previously worked as an Executive Director, Rural Electrification Corporation (REC) and CEO RECPDCL.', 'Bolliger & Mabillard | Bolliger & Mabillard, officially Bolliger & Mabillard Consulting Engineers Inc., and often abbreviated B&M, is a roller coaster design consultancy based in Monthey, Switzerland. The company was founded in 1988 by Walter Bolliger and Claude Mabillard, with Bolliger as president and Mabillard as vice-president. B&M has built over a hundred roller coasters around the world and pioneered several new ride technologies, most notably the inverted roller coaster. In North America, B&M coaster designs have been manufactured by Ohio company Clermont Steel Fabricators since 1990. B&M has grown significantly since its founding, originally employing four to as many as 37 in 2012, consisting primarily of engineers and draftsmen. In 2016, the company completed its 100th roller coaster.', \"Rakefet Russak-Aminoach | Rakefet Russak-Aminoach (Hebrew: רקפת רוסק עמינח\\u200e \\u200e ; born February 1, 1966) is the current President and CEO of Bank Leumi le-Israel B.M., a position she has held since May 1, 2012. In September 2015, she was listed in Fortune Magazine’s list of The World's 100 Most Powerful Women.\"]\n",+ "Question: What company formed in 1978 has Simon Arora as its CEO?\n",+ "Reasoning: Let's think step by step in order to There is no information about Simon Arora being the CEO of a company formed in 1978 in the context. However, we can find the information about Simon Arora being the CEO of B & M in the context.\n",+ "Answer: B & M\n",+ "Context: ['Electoral district of Bragg | Bragg is an electoral district of the House of Assembly in the Australian state of South Australia. The seat of Bragg is named after the eminent physicists Bragg – William Henry and his son, William Lawrence. The electorate is largely urban and encompasses a significant portion of the City of Burnside, stretching from the east parklands of Adelaide into the Adelaide Hills. After a redistribution following the 2006 election, the boundary moved eastwards to include suburbs that had formerly been in the seat of Heysen and now borders Kavel. Bragg currently includes the metropolitan suburbs of Beaumont, Burnside, Dulwich, Erindale, Hazelwood Park, Heathpool, Kensington Park, Leabrook, Linden Park, Rose Park, Marryatville, Skye, St Georges, Stonyfell, Toorak Gardens, Tusmore, Wattle Park and parts of Glen Osmond, and in the hills it includes Crafers, Cleland, Greenhill, Mount Osmond, Piccadilly, Summertown and parts of Ashton, Basket Range, Carey Gully, Horsnell Gully, Leawood Gardens and Uraidla.', 'Cleland, South Australia | Cleland is a suburb in South Australia located in the Adelaide metropolitan area about 10 km south-east of the Adelaide city centre. Its boundaries were created in October 2001, with additional land being added in 2010 from the adjoining suburb of Crafers. Its name is derived from the Cleland Conservation Park. The principal land use within the locality is conservation with the majority of its land area being occupied by the Cleland Conservation Park. Places within its extent include the summit of Mount Lofty. Cleland is located within the federal Division of Mayo, the state electoral district of Bragg and the local government area of the Adelaide Hills Council.', 'Electoral results for the district of Bragg | This is a list of electoral results for the Electoral district of Bragg in South Australian state elections.', 'Electoral district of Burnside | Burnside was an electoral district of the House of Assembly in the Australian state of South Australia from 1938 to 1970. Before 1938 the Burnside area was represented by the three-seat multi-member electorate of Sturt. It was abolished in the 1970 parliamentary reforms, and was replaced with Bragg.']\n",+ "Question: Bragg electoral district, incorporating the Kensington Park suburb, are associated with which South Australian city?\n",+ "Reasoning: Let's think step by step in order to Context: N/A\n",+ "\n",+ "Question: Bragg electoral district, incorporating the Kensington Park suburb, are associated with which South Australian city?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "Answer: Adelaide\n",+ "Context: ['Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.', 'Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.', \"Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world's third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.\"]\n",+ "Question: The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?\n",+ "Reasoning: Let's think step by step in order to Context: [1] «Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.»\n",+ "\n",+ "[2] «Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.»\n",+ "\n",+ "[3] «Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world's third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.»\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "Answer: India\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Use the information below to learn about a task that we are trying to solve using calls to an LM, then generate a new instruction that will be used to prompt a Language Model to better solve the task.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "DATASET SUMMARY: A description of the dataset that we are using.\n",+ "\n",+ "PROGRAM CODE: Language model program designed to solve a particular task.\n",+ "\n",+ "PROGRAM DESCRIPTION: Summary of the task the program is designed to solve, and how it goes about solving it.\n",+ "\n",+ "MODULE: The module to create an instruction for.\n",+ "\n",+ "TASK DEMO(S): Example inputs/outputs of our module.\n",+ "\n",+ "BASIC INSTRUCTION: Basic instruction.\n",+ "\n",+ "TIP: A suggestion for how to go about generating the new instruction.\n",+ "\n",+ "PROPOSED INSTRUCTION: Propose an instruction that will be used to prompt a Language Model to perform this task.\n",+ "\n",+ "---\n",+ "\n",+ "DATASET SUMMARY: The dataset appears to be a comprehensive collection of trivia questions, presenting a diverse range of topics, including entertainment, history, arts, and culture. Most questions require specific, factual answers, often testing users' recall of specific details or events, and are designed to test users' knowledge on subject matter. The syntax and length of the questions vary, but most are concise and require precise answers.\n",+ "\n",+ "PROGRAM CODE:\n",+ "StringSignature(context, question -> rationale, search_query\n",+ " instructions='Given the fields `context`, `question`, produce the fields `search_query`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " search_query = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Search Query:', 'desc': '${search_query}'})\n",+ ")\n",+ "\n",+ "StringSignature(context, question -> rationale, answer\n",+ " instructions='Given the fields `context`, `question`, produce the fields `answer`.'\n",+ " context = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Context:', 'desc': '${context}'})\n",+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",+ " rationale = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the output fields}. We ...', '__dspy_field_type': 'output'})\n",+ " answer = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'output', 'prefix': 'Answer:', 'desc': '${answer}'})\n",+ ")\n",+ "\n",+ "class BasicMH(dspy.Module):\n",+ " def __init__(self, passages_per_hop=3, num_hops=2):\n",+ " super().__init__()\n",+ " self.num_hops = num_hops\n",+ " self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",+ " # self.generate_query = dspy.ChainOfThought(GenerateSearchQuery)\n",+ " self.generate_query = [dspy.ChainOfThought(\"context, question -> search_query\") for _ in range(self.num_hops)]\n",+ " self.generate_answer = dspy.ChainOfThought(\"context, question -> answer\")\n",+ " \n",+ " def forward(self, question):\n",+ " context = []\n",+ " \n",+ " for hop in range(self.num_hops):\n",+ " search_query = self.generate_query[hop](context=context, question=question).search_query\n",+ " passages = self.retrieve(search_query).passages\n",+ " context = deduplicate(context + passages)\n",+ "\n",+ " prediction = self.generate_answer(context=context, question=question).copy(context=context)\n",+ " return prediction\n",+ "\n",+ "\n",+ "PROGRAM DESCRIPTION: The program appears to be designed to solve a question-answering task by utilizing natural language processing (NLP) models. Specifically, it is designed to generate answers based on a given context and question. The program works by first generating a search query based on the context and question, and then retrieving relevant passages from the search query. It then generates the answer by combining the context and retrieved passages. The program uses a multi-hop approach, where it repeatedly generates search queries, retrieves passages, and combines them to generate the final answer. The program's output field is the answer to the question, along with a rationale for the answer.\n",+ "\n",+ "MODULE: self.generate_answer = dspy.ChainOfThought(\"context, question -> answer\")\n",+ "\n",+ "TASK DEMO(S):\n",+ "Context: ['Simon Arora | Simon Arora (born November 1969) is a British billionaire businessman, CEO of the retail chain B & M.', 'Samir Arora | Samir Arora (born November 5, 1965) is an American businessman and former CEO of Mode Media (formerly Glam Media) from 2003 to April 2016. He was CEO and Chairman of the web design company NetObjects, Inc. from 1995 to 2001.', 'Dinesh Arora | Dinesh Arora (born 23 December 1976) is an Indian Administrative Service (IAS) officer, 2002 Kerala cadre, who previously worked as an Executive Director, Rural Electrification Corporation (REC) and CEO RECPDCL.', 'Bolliger & Mabillard | Bolliger & Mabillard, officially Bolliger & Mabillard Consulting Engineers Inc., and often abbreviated B&M, is a roller coaster design consultancy based in Monthey, Switzerland. The company was founded in 1988 by Walter Bolliger and Claude Mabillard, with Bolliger as president and Mabillard as vice-president. B&M has built over a hundred roller coasters around the world and pioneered several new ride technologies, most notably the inverted roller coaster. In North America, B&M coaster designs have been manufactured by Ohio company Clermont Steel Fabricators since 1990. B&M has grown significantly since its founding, originally employing four to as many as 37 in 2012, consisting primarily of engineers and draftsmen. In 2016, the company completed its 100th roller coaster.', \"Rakefet Russak-Aminoach | Rakefet Russak-Aminoach (Hebrew: רקפת רוסק עמינח\\u200e \\u200e ; born February 1, 1966) is the current President and CEO of Bank Leumi le-Israel B.M., a position she has held since May 1, 2012. In September 2015, she was listed in Fortune Magazine’s list of The World's 100 Most Powerful Women.\"]\n",+ "Question: What company formed in 1978 has Simon Arora as its CEO?\n",+ "Reasoning: Let's think step by step in order to There is no information about Simon Arora being the CEO of a company formed in 1978 in the context. However, we can find the information about Simon Arora being the CEO of B & M in the context.\n",+ "Answer: B & M\n",+ "Context: ['Electoral district of Bragg | Bragg is an electoral district of the House of Assembly in the Australian state of South Australia. The seat of Bragg is named after the eminent physicists Bragg – William Henry and his son, William Lawrence. The electorate is largely urban and encompasses a significant portion of the City of Burnside, stretching from the east parklands of Adelaide into the Adelaide Hills. After a redistribution following the 2006 election, the boundary moved eastwards to include suburbs that had formerly been in the seat of Heysen and now borders Kavel. Bragg currently includes the metropolitan suburbs of Beaumont, Burnside, Dulwich, Erindale, Hazelwood Park, Heathpool, Kensington Park, Leabrook, Linden Park, Rose Park, Marryatville, Skye, St Georges, Stonyfell, Toorak Gardens, Tusmore, Wattle Park and parts of Glen Osmond, and in the hills it includes Crafers, Cleland, Greenhill, Mount Osmond, Piccadilly, Summertown and parts of Ashton, Basket Range, Carey Gully, Horsnell Gully, Leawood Gardens and Uraidla.', 'Cleland, South Australia | Cleland is a suburb in South Australia located in the Adelaide metropolitan area about 10 km south-east of the Adelaide city centre. Its boundaries were created in October 2001, with additional land being added in 2010 from the adjoining suburb of Crafers. Its name is derived from the Cleland Conservation Park. The principal land use within the locality is conservation with the majority of its land area being occupied by the Cleland Conservation Park. Places within its extent include the summit of Mount Lofty. Cleland is located within the federal Division of Mayo, the state electoral district of Bragg and the local government area of the Adelaide Hills Council.', 'Electoral results for the district of Bragg | This is a list of electoral results for the Electoral district of Bragg in South Australian state elections.', 'Electoral district of Burnside | Burnside was an electoral district of the House of Assembly in the Australian state of South Australia from 1938 to 1970. Before 1938 the Burnside area was represented by the three-seat multi-member electorate of Sturt. It was abolished in the 1970 parliamentary reforms, and was replaced with Bragg.']\n",+ "Question: Bragg electoral district, incorporating the Kensington Park suburb, are associated with which South Australian city?\n",+ "Reasoning: Let's think step by step in order to Context: N/A\n",+ "\n",+ "Question: Bragg electoral district, incorporating the Kensington Park suburb, are associated with which South Australian city?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "Answer: Adelaide\n",+ "Context: ['Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.', 'Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.', \"Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world's third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.\"]\n",+ "Question: The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?\n",+ "Reasoning: Let's think step by step in order to Context: [1] «Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.»\n",+ "\n",+ "[2] «Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.»\n",+ "\n",+ "[3] «Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world's third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.»\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "Answer: India\n",+ "\n",+ "\n",+ "BASIC INSTRUCTION: Given the fields `context`, `question`, produce the fields `answer`.\n",+ "\n",+ "TIP: Make sure your instruction is very informative and descriptive.\n",+ "\n",+ "Please provide the output field PROPOSED INSTRUCTION. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field PROPOSED INSTRUCTION.\n",+ "\n",+ "PROPOSED INSTRUCTION: Given the context text and question to extract information, produce the answer and provide a step-by-step explanation of the reasoning and thought process used to arrive at the answer.\n",+ "\n",+ "\n",+ "\n",+ "PROPOSED INSTRUCTION: Given the context text and question to extract information, produce the answer and provide a step-by-step explanation of the reasoning and thought process used to arrive at the answer.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "/home/ray/anaconda3/lib/python3.11/site-packages/optuna/samplers/_tpe/sampler.py:295: ExperimentalWarning: ``multivariate`` option is an experimental feature. The interface can change in the future.\n",+ " warnings.warn(\n",+ "[I 2024-09-23 19:07:38,612] A new study created in memory with name: no-name-56477469-de3c-4903-bd14-3f5979e0904b\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "CANDIDATE PROGRAM:\n",+ "Predictor 0\n",+ "i: Given the context and question, generate a specific search query that would lead to relevant passages for answering the question, and provide a detailed reasoning for this search query.\n",+ "p: Search Query:\n",+ "\n",+ "\n",+ "Predictor 1\n",+ "i: Given a text context about Vincent van Gogh's artwork, a question related to the context, generate a search query that asks about the specific aspect of the artwork being discussed, taking into account the context and question provided.\n",+ "p: Search Query:\n",+ "\n",+ "\n",+ "Predictor 2\n",+ "i: Generate an answer based on the provided context, question, and reasoning, drawing upon the knowledge and information contained within the passages to provide a precise and accurate response.\n",+ "p: Answer:\n",+ "\n",+ "\n",+ "...\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 13 / 25 (52.0): 100%|██████████| 25/25 [00:15<00:00, 1.61it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "FULL TRACE\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 0 / 1 (0.0): 100%|██████████| 1/1 [00:00<00:00, 180.49it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "\n",+ "\n",+ "\n",+ "Given a text context about Vincent van Gogh's artwork, a question related to the context, generate a search query that asks about the specific aspect of the artwork being discussed, taking into account the context and question provided.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Search Query: ${search_query}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Alberta general election, 2015 | The 29th general election of Alberta, Canada, took place on May 5, 2015, following a request of Premier Jim Prentice to the Lieutenant Governor of Alberta Donald Ethell to dissolve the Legislative Assembly on April 7, 2015. This election elected members to the 29th Alberta Legislature. It was one of only four times that Alberta has changed governments.»\n",+ "[2] «Jenean Hampton | Jenean Michelle Hampton (born May 12, 1958) is the 57th and current Lieutenant Governor of Kentucky since December 8, 2015 after having won office as Republican gubernatorial nominee Matt Bevin's running-mate, in the 2015 gubernatorial election. She is the first African-American to hold any statewide office in Kentucky history, and only the third African-American woman to serve as lieutenant-governor of any U.S. state (after Jennette Bradley and Jennifer Carroll).»\n",+ "[3] «Louisiana lieutenant gubernatorial election, 2015 | The Louisiana lieutenant gubernatorial election took place on October 24, 2015, to elect the Lieutenant Governor of Louisiana, with a runoff election held on November 21, 2015. Incumbent Republican Lieutenant Governor Jay Dardenne did not for re-election to a second full term in office. He instead ran for Governor. Billy Nungesser won the election defeating Kip Holden, despite a Democratic victory in the gubernatorial election, in which John Bel Edwards defeated David Vitter by a similar margin.»\n",+ "\n",+ "Question: What race is the lieutenant governor of the state that held its primary for her election on May 19th, 2015?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: lieutenant governor of the state that held its primary for her election on May 19th 2015\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.»\n",+ "[2] «Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford's second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.»\n",+ "[3] «G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.»\n",+ "\n",+ "Question: Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Carhartt bass fishing\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Forrest Tucker | Forrest Meredith Tucker (February 12, 1919 – October 25, 1986) was an American actor in both movies and television who appeared in nearly a hundred films. Tucker worked as a vaudeville straight man, aged fifteen years old. A mentor provided funds and contacts for a trip to California, where party hostess Cobina Wright persuaded guest Wesley Ruggles to give Tucker a screen test, based on his photogenic good looks, thick wavy hair and height of six feet, five inches - putting him into the same league as John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness's younger brother. Tucker was a sight reader who needed only one take. His film career started well despite a perception in most Hollywood studios that blond men were not photogenic. After twenty years spent mainly in Westerns and action roles, he returned to his roots, showing versatility as a comedic and stage musical actor. In the TV show \"F Troop\", he became identified with the character of Cavalry Sgt. Morgan O'Rourke (a manipulative character quite similar to Phil Silvers' role as MSgt Ernie Bilko). Tucker struggled with a drinking problem that began to affect his performances in the later years of his career.»\n",+ "[2] «Emergency Landing (1941 film) | Emergency Landing (a.k.a. Robot Pilot) is a 1941 American aviation spy-fi romantic screwball comedy film directed by William Beaudine. The film stars Forrest Tucker in his second film and in his first leading role with co-stars Carol Hughes and Evelyn Brent. \"Emergency Landing\" features lots of mismatched stock footage of various types of aircraft.»\n",+ "[3] «Forrest Tucker (criminal) | Forrest Silva \"Woody\" Tucker (June 23, 1920 – May 29, 2004) was an American career criminal first imprisoned at age 15 who spent the rest of his life in and out of jail. He is best known as an escape artist, having escaped from prison 18 times successfully and 12 times unsuccessfully.»\n",+ "\n",+ "Question: Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Walt Disney TV miniseries 1955-1956 character\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Herbert Kitchener, 1st Earl Kitchener | Field Marshal Horatio Herbert Kitchener, 1st Earl Kitchener, ( ; 24 June 1850 – 5 June 1916) was a senior British Army officer and colonial administrator who won notoriety for his imperial campaigns, most especially his scorched earth policy against the Boers and his establishment of concentration camps during the Second Boer War, and later played a central role in the early part of the First World War.»\n",+ "[2] «Kitchener's Horse | Kitchener's Horse were a colonial unit of the British Army during the Boer War of 1899-1902»\n",+ "[3] «William Birdwood | Field Marshal William Riddell Birdwood, 1st Baron Birdwood, {'1': \", '2': \", '3': \", '4': \"} (13 September 1865 – 17 May 1951) was a British Army officer. He saw active service in the Second Boer War on the staff of Lord Kitchener. He saw action again in the First World War as Commander of the Australian and New Zealand Army Corps during the Gallipoli Campaign in 1915, leading the landings on the peninsula and then the evacuation later in the year, before becoming commander-in-chief of the Fifth Army on the Western Front during the closing stages of the war. He went on to be general officer commanding the Northern Army in India in 1920 and Commander-in-Chief, India, in 1925.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Search Query. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Given a text context about Vincent van Gogh's artwork, a question related to the context, generate a search query that asks about the specific aspect of the artwork being discussed, taking into account the context and question provided.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Search Query: ${search_query}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Alberta general election, 2015 | The 29th general election of Alberta, Canada, took place on May 5, 2015, following a request of Premier Jim Prentice to the Lieutenant Governor of Alberta Donald Ethell to dissolve the Legislative Assembly on April 7, 2015. This election elected members to the 29th Alberta Legislature. It was one of only four times that Alberta has changed governments.»\n",+ "[2] «Jenean Hampton | Jenean Michelle Hampton (born May 12, 1958) is the 57th and current Lieutenant Governor of Kentucky since December 8, 2015 after having won office as Republican gubernatorial nominee Matt Bevin's running-mate, in the 2015 gubernatorial election. She is the first African-American to hold any statewide office in Kentucky history, and only the third African-American woman to serve as lieutenant-governor of any U.S. state (after Jennette Bradley and Jennifer Carroll).»\n",+ "[3] «Louisiana lieutenant gubernatorial election, 2015 | The Louisiana lieutenant gubernatorial election took place on October 24, 2015, to elect the Lieutenant Governor of Louisiana, with a runoff election held on November 21, 2015. Incumbent Republican Lieutenant Governor Jay Dardenne did not for re-election to a second full term in office. He instead ran for Governor. Billy Nungesser won the election defeating Kip Holden, despite a Democratic victory in the gubernatorial election, in which John Bel Edwards defeated David Vitter by a similar margin.»\n",+ "\n",+ "Question: What race is the lieutenant governor of the state that held its primary for her election on May 19th, 2015?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: lieutenant governor of the state that held its primary for her election on May 19th 2015\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.»\n",+ "[2] «Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford's second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.»\n",+ "[3] «G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.»\n",+ "\n",+ "Question: Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Carhartt bass fishing\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Forrest Tucker | Forrest Meredith Tucker (February 12, 1919 – October 25, 1986) was an American actor in both movies and television who appeared in nearly a hundred films. Tucker worked as a vaudeville straight man, aged fifteen years old. A mentor provided funds and contacts for a trip to California, where party hostess Cobina Wright persuaded guest Wesley Ruggles to give Tucker a screen test, based on his photogenic good looks, thick wavy hair and height of six feet, five inches - putting him into the same league as John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness's younger brother. Tucker was a sight reader who needed only one take. His film career started well despite a perception in most Hollywood studios that blond men were not photogenic. After twenty years spent mainly in Westerns and action roles, he returned to his roots, showing versatility as a comedic and stage musical actor. In the TV show \"F Troop\", he became identified with the character of Cavalry Sgt. Morgan O'Rourke (a manipulative character quite similar to Phil Silvers' role as MSgt Ernie Bilko). Tucker struggled with a drinking problem that began to affect his performances in the later years of his career.»\n",+ "[2] «Emergency Landing (1941 film) | Emergency Landing (a.k.a. Robot Pilot) is a 1941 American aviation spy-fi romantic screwball comedy film directed by William Beaudine. The film stars Forrest Tucker in his second film and in his first leading role with co-stars Carol Hughes and Evelyn Brent. \"Emergency Landing\" features lots of mismatched stock footage of various types of aircraft.»\n",+ "[3] «Forrest Tucker (criminal) | Forrest Silva \"Woody\" Tucker (June 23, 1920 – May 29, 2004) was an American career criminal first imprisoned at age 15 who spent the rest of his life in and out of jail. He is best known as an escape artist, having escaped from prison 18 times successfully and 12 times unsuccessfully.»\n",+ "\n",+ "Question: Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Walt Disney TV miniseries 1955-1956 character\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Herbert Kitchener, 1st Earl Kitchener | Field Marshal Horatio Herbert Kitchener, 1st Earl Kitchener, ( ; 24 June 1850 – 5 June 1916) was a senior British Army officer and colonial administrator who won notoriety for his imperial campaigns, most especially his scorched earth policy against the Boers and his establishment of concentration camps during the Second Boer War, and later played a central role in the early part of the First World War.»\n",+ "[2] «Kitchener's Horse | Kitchener's Horse were a colonial unit of the British Army during the Boer War of 1899-1902»\n",+ "[3] «William Birdwood | Field Marshal William Riddell Birdwood, 1st Baron Birdwood, {'1': \", '2': \", '3': \", '4': \"} (13 September 1865 – 17 May 1951) was a British Army officer. He saw active service in the Second Boer War on the staff of Lord Kitchener. He saw action again in the First World War as Commander of the Australian and New Zealand Army Corps during the Gallipoli Campaign in 1915, leading the landings on the peninsula and then the evacuation later in the year, before becoming commander-in-chief of the Fifth Army on the Western Front during the closing stages of the war. He went on to be general officer commanding the Northern Army in India in 1920 and Commander-in-Chief, India, in 1925.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Please provide the output field Search Query. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field Search Query.\n",+ "\n",+ "Search Query: Herbert Kitchener birthdate\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Generate an answer based on the provided context, question, and reasoning, drawing upon the knowledge and information contained within the passages to provide a precise and accurate response.\n",+ "\n",+ "---\n",+ "\n",+ "Question: What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?\n",+ "\n",+ "Answer: 2016\n",+ "\n",+ "---\n",+ "\n",+ "Question: How many shops does the larger of the Miami International Mall and Dolphin Mall have?\n",+ "\n",+ "Answer: 240\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Answer: ${answer}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Wheat Field with Cypresses | A Wheatfield with Cypresses (occasionally called \"A Cornfield with Cypresses\") is any of three similar 1889 oil paintings by Vincent van Gogh, as part of his wheat field series. All were executed at the Saint-Paul-de-Mausole mental asylum at Saint-Rémy near Arles, France, where Van Gogh was voluntarily a patient from May 1889 to May 1890. The works were inspired by the view from the window at the asylum towards the Alpilles mountains.»\n",+ "[2] «Green Wheat Field with Cypress | Green Wheat Field with Cypress (French: Champ de blé vert avec cyprès) is an oil on canvas painting by a Dutch Post-Impressionist Vincent van Gogh. It was completed in 1889, while van Gogh was voluntarily incarcerated at the asylum of St. Paul near Saint-Rémy in Provence. Several paintings of wheat fields with cypresses were made when van Gogh was able to leave the asylum grounds and explore the local landscape. Besides a fondness for cypresses, van Gogh had a special affinty with wheat fields; he depicted them dozens of times over the years; to Vincent they symbolized the cycle of life and death, and he found in them both solace and inspiration.»\n",+ "[3] «Olive Trees (Van Gogh series) | Vincent van Gogh painted at least 18 paintings of olive trees, mostly in Saint-Rémy-de-Provence in 1889. At his own request, he lived at an asylum there from May 1889 through May 1890 painting the gardens of the asylum and, when he had permission to venture outside its walls, nearby olive trees, cypresses and wheat fields.»\n",+ "[4] «Wheat Fields (Van Gogh series) | The Wheat Fields is a series of dozens of paintings by Vincent van Gogh, borne out of his religious studies and sermons, connection to nature, appreciation of manual laborers and desire to provide a means of offering comfort to others. The wheat field works demonstrate his progression as an artist from the drab \"Wheat Sheaves\" made in 1885 in the Netherlands to the colorful, dramatic paintings from Arles, Saint-Rémy and Auvers-sur-Oise of rural France.»\n",+ "\n",+ "Question: What was the series inspired by the appreciation of manual laborers that Wheat Field with Cypresses is a part of?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "Context: N/A\n",+ "\n",+ "Question: What was the series inspired by the appreciation of manual laborers that Wheat Field with Cypresses is a part of?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "\n",+ "Answer: Wheat Fields\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Herbert Kitchener, 1st Earl Kitchener | Field Marshal Horatio Herbert Kitchener, 1st Earl Kitchener, ( ; 24 June 1850 – 5 June 1916) was a senior British Army officer and colonial administrator who won notoriety for his imperial campaigns, most especially his scorched earth policy against the Boers and his establishment of concentration camps during the Second Boer War, and later played a central role in the early part of the First World War.»\n",+ "[2] «Kitchener's Horse | Kitchener's Horse were a colonial unit of the British Army during the Boer War of 1899-1902»\n",+ "[3] «William Birdwood | Field Marshal William Riddell Birdwood, 1st Baron Birdwood, {'1': \", '2': \", '3': \", '4': \"} (13 September 1865 – 17 May 1951) was a British Army officer. He saw active service in the Second Boer War on the staff of Lord Kitchener. He saw action again in the First World War as Commander of the Australian and New Zealand Army Corps during the Gallipoli Campaign in 1915, leading the landings on the peninsula and then the evacuation later in the year, before becoming commander-in-chief of the Fifth Army on the Western Front during the closing stages of the war. He went on to be general officer commanding the Northern Army in India in 1920 and Commander-in-Chief, India, in 1925.»\n",+ "[4] «Henry Kitchener, 3rd Earl Kitchener | Major Henry Herbert Kitchener, 3rd Earl Kitchener DL TD (24 February 1919 – 16 December 2011), styled Viscount Broome from 1928 to 1937, was a British peer. He was the son of Captain Henry Franklin Chevallier Kitchener, Viscount Broome, only son of Henry Kitchener, 2nd Earl Kitchener. His great-uncle was the renowned military commander Herbert Kitchener, 1st Earl Kitchener.»\n",+ "[5] «Bill Kitchener | William Harry Kitchener (born 3 November 1946) is an English former professional footballer who played in the Football League for West Ham United, Torquay United and A.F.C. Bournemouth.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Answer. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to find the 1st Earl of Yrich who attacked the Boer force from the rear while mounted infantry. We need to find the 1st Earl of Ypres, which is Herbert Kitchener, 1st Earl Kitchener. We can find his birth year in the context.\n",+ "Answer: 1850\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Given a text context about Vincent van Gogh's artwork, a question related to the context, generate a search query that asks about the specific aspect of the artwork being discussed, taking into account the context and question provided.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Search Query: ${search_query}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Alberta general election, 2015 | The 29th general election of Alberta, Canada, took place on May 5, 2015, following a request of Premier Jim Prentice to the Lieutenant Governor of Alberta Donald Ethell to dissolve the Legislative Assembly on April 7, 2015. This election elected members to the 29th Alberta Legislature. It was one of only four times that Alberta has changed governments.»\n",+ "[2] «Jenean Hampton | Jenean Michelle Hampton (born May 12, 1958) is the 57th and current Lieutenant Governor of Kentucky since December 8, 2015 after having won office as Republican gubernatorial nominee Matt Bevin's running-mate, in the 2015 gubernatorial election. She is the first African-American to hold any statewide office in Kentucky history, and only the third African-American woman to serve as lieutenant-governor of any U.S. state (after Jennette Bradley and Jennifer Carroll).»\n",+ "[3] «Louisiana lieutenant gubernatorial election, 2015 | The Louisiana lieutenant gubernatorial election took place on October 24, 2015, to elect the Lieutenant Governor of Louisiana, with a runoff election held on November 21, 2015. Incumbent Republican Lieutenant Governor Jay Dardenne did not for re-election to a second full term in office. He instead ran for Governor. Billy Nungesser won the election defeating Kip Holden, despite a Democratic victory in the gubernatorial election, in which John Bel Edwards defeated David Vitter by a similar margin.»\n",+ "\n",+ "Question: What race is the lieutenant governor of the state that held its primary for her election on May 19th, 2015?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: lieutenant governor of the state that held its primary for her election on May 19th 2015\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.»\n",+ "[2] «Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford's second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.»\n",+ "[3] «G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.»\n",+ "\n",+ "Question: Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Carhartt bass fishing\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Forrest Tucker | Forrest Meredith Tucker (February 12, 1919 – October 25, 1986) was an American actor in both movies and television who appeared in nearly a hundred films. Tucker worked as a vaudeville straight man, aged fifteen years old. A mentor provided funds and contacts for a trip to California, where party hostess Cobina Wright persuaded guest Wesley Ruggles to give Tucker a screen test, based on his photogenic good looks, thick wavy hair and height of six feet, five inches - putting him into the same league as John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness's younger brother. Tucker was a sight reader who needed only one take. His film career started well despite a perception in most Hollywood studios that blond men were not photogenic. After twenty years spent mainly in Westerns and action roles, he returned to his roots, showing versatility as a comedic and stage musical actor. In the TV show \"F Troop\", he became identified with the character of Cavalry Sgt. Morgan O'Rourke (a manipulative character quite similar to Phil Silvers' role as MSgt Ernie Bilko). Tucker struggled with a drinking problem that began to affect his performances in the later years of his career.»\n",+ "[2] «Emergency Landing (1941 film) | Emergency Landing (a.k.a. Robot Pilot) is a 1941 American aviation spy-fi romantic screwball comedy film directed by William Beaudine. The film stars Forrest Tucker in his second film and in his first leading role with co-stars Carol Hughes and Evelyn Brent. \"Emergency Landing\" features lots of mismatched stock footage of various types of aircraft.»\n",+ "[3] «Forrest Tucker (criminal) | Forrest Silva \"Woody\" Tucker (June 23, 1920 – May 29, 2004) was an American career criminal first imprisoned at age 15 who spent the rest of his life in and out of jail. He is best known as an escape artist, having escaped from prison 18 times successfully and 12 times unsuccessfully.»\n",+ "\n",+ "Question: Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Walt Disney TV miniseries 1955-1956 character\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Herbert Kitchener, 1st Earl Kitchener | Field Marshal Horatio Herbert Kitchener, 1st Earl Kitchener, ( ; 24 June 1850 – 5 June 1916) was a senior British Army officer and colonial administrator who won notoriety for his imperial campaigns, most especially his scorched earth policy against the Boers and his establishment of concentration camps during the Second Boer War, and later played a central role in the early part of the First World War.»\n",+ "[2] «Kitchener's Horse | Kitchener's Horse were a colonial unit of the British Army during the Boer War of 1899-1902»\n",+ "[3] «William Birdwood | Field Marshal William Riddell Birdwood, 1st Baron Birdwood, {'1': \", '2': \", '3': \", '4': \"} (13 September 1865 – 17 May 1951) was a British Army officer. He saw active service in the Second Boer War on the staff of Lord Kitchener. He saw action again in the First World War as Commander of the Australian and New Zealand Army Corps during the Gallipoli Campaign in 1915, leading the landings on the peninsula and then the evacuation later in the year, before becoming commander-in-chief of the Fifth Army on the Western Front during the closing stages of the war. He went on to be general officer commanding the Northern Army in India in 1920 and Commander-in-Chief, India, in 1925.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Search Query. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Given a text context about Vincent van Gogh's artwork, a question related to the context, generate a search query that asks about the specific aspect of the artwork being discussed, taking into account the context and question provided.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Search Query: ${search_query}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Alberta general election, 2015 | The 29th general election of Alberta, Canada, took place on May 5, 2015, following a request of Premier Jim Prentice to the Lieutenant Governor of Alberta Donald Ethell to dissolve the Legislative Assembly on April 7, 2015. This election elected members to the 29th Alberta Legislature. It was one of only four times that Alberta has changed governments.»\n",+ "[2] «Jenean Hampton | Jenean Michelle Hampton (born May 12, 1958) is the 57th and current Lieutenant Governor of Kentucky since December 8, 2015 after having won office as Republican gubernatorial nominee Matt Bevin's running-mate, in the 2015 gubernatorial election. She is the first African-American to hold any statewide office in Kentucky history, and only the third African-American woman to serve as lieutenant-governor of any U.S. state (after Jennette Bradley and Jennifer Carroll).»\n",+ "[3] «Louisiana lieutenant gubernatorial election, 2015 | The Louisiana lieutenant gubernatorial election took place on October 24, 2015, to elect the Lieutenant Governor of Louisiana, with a runoff election held on November 21, 2015. Incumbent Republican Lieutenant Governor Jay Dardenne did not for re-election to a second full term in office. He instead ran for Governor. Billy Nungesser won the election defeating Kip Holden, despite a Democratic victory in the gubernatorial election, in which John Bel Edwards defeated David Vitter by a similar margin.»\n",+ "\n",+ "Question: What race is the lieutenant governor of the state that held its primary for her election on May 19th, 2015?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: lieutenant governor of the state that held its primary for her election on May 19th 2015\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.»\n",+ "[2] «Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford's second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.»\n",+ "[3] «G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.»\n",+ "\n",+ "Question: Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Carhartt bass fishing\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Forrest Tucker | Forrest Meredith Tucker (February 12, 1919 – October 25, 1986) was an American actor in both movies and television who appeared in nearly a hundred films. Tucker worked as a vaudeville straight man, aged fifteen years old. A mentor provided funds and contacts for a trip to California, where party hostess Cobina Wright persuaded guest Wesley Ruggles to give Tucker a screen test, based on his photogenic good looks, thick wavy hair and height of six feet, five inches - putting him into the same league as John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness's younger brother. Tucker was a sight reader who needed only one take. His film career started well despite a perception in most Hollywood studios that blond men were not photogenic. After twenty years spent mainly in Westerns and action roles, he returned to his roots, showing versatility as a comedic and stage musical actor. In the TV show \"F Troop\", he became identified with the character of Cavalry Sgt. Morgan O'Rourke (a manipulative character quite similar to Phil Silvers' role as MSgt Ernie Bilko). Tucker struggled with a drinking problem that began to affect his performances in the later years of his career.»\n",+ "[2] «Emergency Landing (1941 film) | Emergency Landing (a.k.a. Robot Pilot) is a 1941 American aviation spy-fi romantic screwball comedy film directed by William Beaudine. The film stars Forrest Tucker in his second film and in his first leading role with co-stars Carol Hughes and Evelyn Brent. \"Emergency Landing\" features lots of mismatched stock footage of various types of aircraft.»\n",+ "[3] «Forrest Tucker (criminal) | Forrest Silva \"Woody\" Tucker (June 23, 1920 – May 29, 2004) was an American career criminal first imprisoned at age 15 who spent the rest of his life in and out of jail. He is best known as an escape artist, having escaped from prison 18 times successfully and 12 times unsuccessfully.»\n",+ "\n",+ "Question: Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Walt Disney TV miniseries 1955-1956 character\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Herbert Kitchener, 1st Earl Kitchener | Field Marshal Horatio Herbert Kitchener, 1st Earl Kitchener, ( ; 24 June 1850 – 5 June 1916) was a senior British Army officer and colonial administrator who won notoriety for his imperial campaigns, most especially his scorched earth policy against the Boers and his establishment of concentration camps during the Second Boer War, and later played a central role in the early part of the First World War.»\n",+ "[2] «Kitchener's Horse | Kitchener's Horse were a colonial unit of the British Army during the Boer War of 1899-1902»\n",+ "[3] «William Birdwood | Field Marshal William Riddell Birdwood, 1st Baron Birdwood, {'1': \", '2': \", '3': \", '4': \"} (13 September 1865 – 17 May 1951) was a British Army officer. He saw active service in the Second Boer War on the staff of Lord Kitchener. He saw action again in the First World War as Commander of the Australian and New Zealand Army Corps during the Gallipoli Campaign in 1915, leading the landings on the peninsula and then the evacuation later in the year, before becoming commander-in-chief of the Fifth Army on the Western Front during the closing stages of the war. He went on to be general officer commanding the Northern Army in India in 1920 and Commander-in-Chief, India, in 1925.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Please provide the output field Search Query. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field Search Query.\n",+ "\n",+ "Search Query: Herbert Kitchener birthdate\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Generate an answer based on the provided context, question, and reasoning, drawing upon the knowledge and information contained within the passages to provide a precise and accurate response.\n",+ "\n",+ "---\n",+ "\n",+ "Question: What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?\n",+ "\n",+ "Answer: 2016\n",+ "\n",+ "---\n",+ "\n",+ "Question: How many shops does the larger of the Miami International Mall and Dolphin Mall have?\n",+ "\n",+ "Answer: 240\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Answer: ${answer}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Wheat Field with Cypresses | A Wheatfield with Cypresses (occasionally called \"A Cornfield with Cypresses\") is any of three similar 1889 oil paintings by Vincent van Gogh, as part of his wheat field series. All were executed at the Saint-Paul-de-Mausole mental asylum at Saint-Rémy near Arles, France, where Van Gogh was voluntarily a patient from May 1889 to May 1890. The works were inspired by the view from the window at the asylum towards the Alpilles mountains.»\n",+ "[2] «Green Wheat Field with Cypress | Green Wheat Field with Cypress (French: Champ de blé vert avec cyprès) is an oil on canvas painting by a Dutch Post-Impressionist Vincent van Gogh. It was completed in 1889, while van Gogh was voluntarily incarcerated at the asylum of St. Paul near Saint-Rémy in Provence. Several paintings of wheat fields with cypresses were made when van Gogh was able to leave the asylum grounds and explore the local landscape. Besides a fondness for cypresses, van Gogh had a special affinty with wheat fields; he depicted them dozens of times over the years; to Vincent they symbolized the cycle of life and death, and he found in them both solace and inspiration.»\n",+ "[3] «Olive Trees (Van Gogh series) | Vincent van Gogh painted at least 18 paintings of olive trees, mostly in Saint-Rémy-de-Provence in 1889. At his own request, he lived at an asylum there from May 1889 through May 1890 painting the gardens of the asylum and, when he had permission to venture outside its walls, nearby olive trees, cypresses and wheat fields.»\n",+ "[4] «Wheat Fields (Van Gogh series) | The Wheat Fields is a series of dozens of paintings by Vincent van Gogh, borne out of his religious studies and sermons, connection to nature, appreciation of manual laborers and desire to provide a means of offering comfort to others. The wheat field works demonstrate his progression as an artist from the drab \"Wheat Sheaves\" made in 1885 in the Netherlands to the colorful, dramatic paintings from Arles, Saint-Rémy and Auvers-sur-Oise of rural France.»\n",+ "\n",+ "Question: What was the series inspired by the appreciation of manual laborers that Wheat Field with Cypresses is a part of?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "Context: N/A\n",+ "\n",+ "Question: What was the series inspired by the appreciation of manual laborers that Wheat Field with Cypresses is a part of?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "\n",+ "Answer: Wheat Fields\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Herbert Kitchener, 1st Earl Kitchener | Field Marshal Horatio Herbert Kitchener, 1st Earl Kitchener, ( ; 24 June 1850 – 5 June 1916) was a senior British Army officer and colonial administrator who won notoriety for his imperial campaigns, most especially his scorched earth policy against the Boers and his establishment of concentration camps during the Second Boer War, and later played a central role in the early part of the First World War.»\n",+ "[2] «Kitchener's Horse | Kitchener's Horse were a colonial unit of the British Army during the Boer War of 1899-1902»\n",+ "[3] «William Birdwood | Field Marshal William Riddell Birdwood, 1st Baron Birdwood, {'1': \", '2': \", '3': \", '4': \"} (13 September 1865 – 17 May 1951) was a British Army officer. He saw active service in the Second Boer War on the staff of Lord Kitchener. He saw action again in the First World War as Commander of the Australian and New Zealand Army Corps during the Gallipoli Campaign in 1915, leading the landings on the peninsula and then the evacuation later in the year, before becoming commander-in-chief of the Fifth Army on the Western Front during the closing stages of the war. He went on to be general officer commanding the Northern Army in India in 1920 and Commander-in-Chief, India, in 1925.»\n",+ "[4] «Henry Kitchener, 3rd Earl Kitchener | Major Henry Herbert Kitchener, 3rd Earl Kitchener DL TD (24 February 1919 – 16 December 2011), styled Viscount Broome from 1928 to 1937, was a British peer. He was the son of Captain Henry Franklin Chevallier Kitchener, Viscount Broome, only son of Henry Kitchener, 2nd Earl Kitchener. His great-uncle was the renowned military commander Herbert Kitchener, 1st Earl Kitchener.»\n",+ "[5] «Bill Kitchener | William Harry Kitchener (born 3 November 1946) is an English former professional footballer who played in the Football League for West Ham United, Torquay United and A.F.C. Bournemouth.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Answer. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to find the 1st Earl of Yrich who attacked the Boer force from the rear while mounted infantry. We need to find the 1st Earl of Ypres, which is Herbert Kitchener, 1st Earl Kitchener. We can find his birth year in the context.\n",+ "Answer: 1850\n",+ "\n",+ "\n",+ "\n",+ "...\n",+ "Score 52.0\n",+ "Best Combination: 1,{0: [[], [Example({'question': 'Oh Yeon-seo is one of the former members of the group LUV, which stands for Love yoUR Voice, is what kind of group?', 'answer': 'K-pop girls group'}) (input_keys={'question'}), Example({'question': 'Melbourne Central railway station feeds into which main metro station that serves the entire metropolitan rail network?', 'answer': 'Flinders Street railway station'}) (input_keys={'question'}), Example({'question': 'When was the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed?', 'answer': '2006'}) (input_keys={'question'})], [Example({'augmented': True, 'context': [], 'question': 'What race is the lieutenant governor of the state that held its primary for her election on May 19th, 2015?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'lieutenant governor race May 19 2015'}) (input_keys=None), Example({'augmented': True, 'context': [], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'clothing company founded 1889 college bass fishing'}) (input_keys=None), Example({'augmented': True, 'context': [], 'question': \"Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\", 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Forrest Tucker height Walt Disney TV miniseries'}) (input_keys=None)], [Example({'augmented': True, 'context': [], 'question': \"Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\", 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Forrest Tucker height Walt Disney TV miniseries'}) (input_keys=None), Example({'augmented': True, 'context': [], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'clothing company founded 1889 college bass fishing'}) (input_keys=None), Example({'question': 'How many years did Lenny Lyles spend with the organization dramatized in the filme \"The insder\"?', 'answer': '27'}) (input_keys={'question'})], [Example({'augmented': True, 'context': [], 'question': 'When was the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Will Finn Over the Hedge short film director'}) (input_keys=None), Example({'question': 'What is the capacity of the La Liga stadium in which The Madrid Derby team plays their home games?', 'answer': '68,000'}) (input_keys={'question'}), Example({'question': 'The Equestrian statue of Marshal Mannerheim was erected as a monument to the leader of the Whites during what event?', 'answer': 'Finnish Civil War'}) (input_keys={'question'})], [Example({'augmented': True, 'context': [], 'question': 'The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Kangchenjunga Singhi Kangri country'}) (input_keys=None), Example({'question': 'Are Beth Gibbons and Ian Curtis the same nationality?', 'answer': 'yes'}) (input_keys={'question'}), Example({'question': \"When did Taylor Swift's satirical song get released to the radio?\", 'answer': 'February 9, 2015'}) (input_keys={'question'})], [Example({'augmented': True, 'context': [], 'question': 'What was the series inspired by the appreciation of manual laborers that Wheat Field with Cypresses is a part of?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Wheat Field with Cypresses series'}) (input_keys=None), Example({'question': 'What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?', 'answer': '2016'}) (input_keys={'question'}), Example({'question': 'How many shops does the larger of the Miami International Mall and Dolphin Mall have?', 'answer': '240'}) (input_keys={'question'})], [Example({'augmented': True, 'context': [], 'question': 'What other sport did Butch Reed, of the wrestling tag team Doom, play professionally?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Butch Reed sports'}) (input_keys=None), Example({'question': 'Which magazine is still in print American Heritage or High Times?', 'answer': 'High Times'}) (input_keys={'question'}), Example({'question': 'What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?', 'answer': '2016'}) (input_keys={'question'})], [Example({'augmented': True, 'context': [], 'question': 'Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Garden City Hotel founder'}) (input_keys=None), Example({'augmented': True, 'context': [], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'clothing company founded 1889 college bass fishing'}) (input_keys=None), Example({'augmented': True, 'context': [], 'question': 'Which theme park has Stitch’s Great Escape which replaced the attraction ExtraTERRORestial Alien Encounter?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': \"Stitch's Great Escape theme park\"}) (input_keys=None)], [Example({'augmented': True, 'context': [], 'question': 'What company formed in 1978 has Simon Arora as its CEO?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Simon Arora CEO company 1978'}) (input_keys=None), Example({'augmented': True, 'context': [], 'question': 'Bragg electoral district, incorporating the Kensington Park suburb, are associated with which South Australian city?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Bragg electoral district Kensington Park suburb South Australian city'}) (input_keys=None), Example({'augmented': True, 'context': [], 'question': 'The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Kangchenjunga Singhi Kangri country'}) (input_keys=None)]], 1: [[], [Example({'question': 'Born Again starred the American film actor who played what role in The Best Years of Our Lives?', 'answer': 'Fred Derry'}) (input_keys={'question'}), Example({'question': 'What type of compilation does Chris Harris and Into the Labyrinth have in common?', 'answer': 'television series'}) (input_keys={'question'}), Example({'question': 'Are both Paul Schattel and Francis Ford Coppola American film directors?', 'answer': 'yes'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Alberta general election, 2015 | The 29th general election of Alberta, Canada, took place on May 5, 2015, following a request of Premier Jim Prentice to the Lieutenant Governor of Alberta Donald Ethell to dissolve the Legislative Assembly on April 7, 2015. This election elected members to the 29th Alberta Legislature. It was one of only four times that Alberta has changed governments.', \"Jenean Hampton | Jenean Michelle Hampton (born May 12, 1958) is the 57th and current Lieutenant Governor of Kentucky since December 8, 2015 after having won office as Republican gubernatorial nominee Matt Bevin's running-mate, in the 2015 gubernatorial election. She is the first African-American to hold any statewide office in Kentucky history, and only the third African-American woman to serve as lieutenant-governor of any U.S. state (after Jennette Bradley and Jennifer Carroll).\", 'Louisiana lieutenant gubernatorial election, 2015 | The Louisiana lieutenant gubernatorial election took place on October 24, 2015, to elect the Lieutenant Governor of Louisiana, with a runoff election held on November 21, 2015. Incumbent Republican Lieutenant Governor Jay Dardenne did not for re-election to a second full term in office. He instead ran for Governor. Billy Nungesser won the election defeating Kip Holden, despite a Democratic victory in the gubernatorial election, in which John Bel Edwards defeated David Vitter by a similar margin.'], 'question': 'What race is the lieutenant governor of the state that held its primary for her election on May 19th, 2015?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'lieutenant governor of the state that held its primary for her election on May 19th 2015'}) (input_keys=None), Example({'augmented': True, 'context': ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.'], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Carhartt bass fishing'}) (input_keys=None), Example({'augmented': True, 'context': ['Forrest Tucker | Forrest Meredith Tucker (February 12, 1919 – October 25, 1986) was an American actor in both movies and television who appeared in nearly a hundred films. Tucker worked as a vaudeville straight man, aged fifteen years old. A mentor provided funds and contacts for a trip to California, where party hostess Cobina Wright persuaded guest Wesley Ruggles to give Tucker a screen test, based on his photogenic good looks, thick wavy hair and height of six feet, five inches - putting him into the same league as John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness\\'s younger brother. Tucker was a sight reader who needed only one take. His film career started well despite a perception in most Hollywood studios that blond men were not photogenic. After twenty years spent mainly in Westerns and action roles, he returned to his roots, showing versatility as a comedic and stage musical actor. In the TV show \"F Troop\", he became identified with the character of Cavalry Sgt. Morgan O\\'Rourke (a manipulative character quite similar to Phil Silvers\\' role as MSgt Ernie Bilko). Tucker struggled with a drinking problem that began to affect his performances in the later years of his career.', 'Emergency Landing (1941 film) | Emergency Landing (a.k.a. Robot Pilot) is a 1941 American aviation spy-fi romantic screwball comedy film directed by William Beaudine. The film stars Forrest Tucker in his second film and in his first leading role with co-stars Carol Hughes and Evelyn Brent. \"Emergency Landing\" features lots of mismatched stock footage of various types of aircraft.', 'Forrest Tucker (criminal) | Forrest Silva \"Woody\" Tucker (June 23, 1920 – May 29, 2004) was an American career criminal first imprisoned at age 15 who spent the rest of his life in and out of jail. He is best known as an escape artist, having escaped from prison 18 times successfully and 12 times unsuccessfully.'], 'question': \"Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\", 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Walt Disney TV miniseries 1955-1956 character'}) (input_keys=None)], [Example({'augmented': True, 'context': ['Forrest Tucker | Forrest Meredith Tucker (February 12, 1919 – October 25, 1986) was an American actor in both movies and television who appeared in nearly a hundred films. Tucker worked as a vaudeville straight man, aged fifteen years old. A mentor provided funds and contacts for a trip to California, where party hostess Cobina Wright persuaded guest Wesley Ruggles to give Tucker a screen test, based on his photogenic good looks, thick wavy hair and height of six feet, five inches - putting him into the same league as John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness\\'s younger brother. Tucker was a sight reader who needed only one take. His film career started well despite a perception in most Hollywood studios that blond men were not photogenic. After twenty years spent mainly in Westerns and action roles, he returned to his roots, showing versatility as a comedic and stage musical actor. In the TV show \"F Troop\", he became identified with the character of Cavalry Sgt. Morgan O\\'Rourke (a manipulative character quite similar to Phil Silvers\\' role as MSgt Ernie Bilko). Tucker struggled with a drinking problem that began to affect his performances in the later years of his career.', 'Emergency Landing (1941 film) | Emergency Landing (a.k.a. Robot Pilot) is a 1941 American aviation spy-fi romantic screwball comedy film directed by William Beaudine. The film stars Forrest Tucker in his second film and in his first leading role with co-stars Carol Hughes and Evelyn Brent. \"Emergency Landing\" features lots of mismatched stock footage of various types of aircraft.', 'Forrest Tucker (criminal) | Forrest Silva \"Woody\" Tucker (June 23, 1920 – May 29, 2004) was an American career criminal first imprisoned at age 15 who spent the rest of his life in and out of jail. He is best known as an escape artist, having escaped from prison 18 times successfully and 12 times unsuccessfully.'], 'question': \"Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\", 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Walt Disney TV miniseries 1955-1956 character'}) (input_keys=None), Example({'augmented': True, 'context': ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.'], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Carhartt bass fishing'}) (input_keys=None), Example({'question': 'How many years did Lenny Lyles spend with the organization dramatized in the filme \"The insder\"?', 'answer': '27'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Will Finn | Will Finn (born November 1, 1958) is an American animator, voice actor, and director. His work in animation includes characters from Disney and Don Bluth films such as \"The Secret of NIMH\", \"Oliver & Company\", \"The Little Mermaid\", \"The Rescuers Down Under\", and \"Pocahontas\". His characters includes Cogsworth in \"Beauty and the Beast\", Iago in \"Aladdin\", and Laverne in \"The Hunchback of Notre Dame\". Finn wrote and directed \"Home on the Range\" and did some voice acting the Hollywood Fish in \"Chicken Little\". In 2006, Finn directed the computer animated short Hammy\\'s Boomerang Adventure, a spin-off of \"Over the Hedge\".', 'Over the Hedge (film) | Over the Hedge is a 2006 American computer-animated comedy film, based on the characters from the United Media comic strip of the same name. Directed by Tim Johnson and Karey Kirkpatrick, and produced by Bonnie Arnold, it was released in the United States on May 19, 2006. The film was produced by DreamWorks Animation and distributed through Paramount Pictures. The film features the voices of Bruce Willis, Garry Shandling, Steve Carell, William Shatner, Wanda Sykes, and Nick Nolte. It is the first DreamWorks Animation film to be distributed by Paramount Pictures, which acquired the live-action DreamWorks studio in 2006. The film earned $336 million on an $80 million budget.', 'Tim Johnson (film director) | Tim Johnson (born August 27, 1961) is an American animation director. Johnson has directed in many films such as \"Antz\", \"\" and \"Over the Hedge\". Recently, he directed the DreamWorks 3D film \"Home.\"'], 'question': 'When was the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed?', 'rationale': 'find the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed. The context mentions that Will Finn directed the computer animated short Hammy\\'s Boomerang Adventure, a spin-off of \"Over the Hedge\".', 'search_query': \"Hammy's Boomerang Adventure release date\"}) (input_keys=None), Example({'question': 'The Equestrian statue of Marshal Mannerheim was erected as a monument to the leader of the Whites during what event?', 'answer': 'Finnish Civil War'}) (input_keys={'question'}), Example({'question': 'What is the capacity of the La Liga stadium in which The Madrid Derby team plays their home games?', 'answer': '68,000'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.', 'Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.', \"Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world's third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.\"], 'question': 'The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Kangchenjunga Singhi Kangri country'}) (input_keys=None), Example({'question': \"When did Taylor Swift's satirical song get released to the radio?\", 'answer': 'February 9, 2015'}) (input_keys={'question'}), Example({'question': 'Are Beth Gibbons and Ian Curtis the same nationality?', 'answer': 'yes'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Wheat Field with Cypresses | A Wheatfield with Cypresses (occasionally called \"A Cornfield with Cypresses\") is any of three similar 1889 oil paintings by Vincent van Gogh, as part of his wheat field series. All were executed at the Saint-Paul-de-Mausole mental asylum at Saint-Rémy near Arles, France, where Van Gogh was voluntarily a patient from May 1889 to May 1890. The works were inspired by the view from the window at the asylum towards the Alpilles mountains.', 'Green Wheat Field with Cypress | Green Wheat Field with Cypress (French: Champ de blé vert avec cyprès) is an oil on canvas painting by a Dutch Post-Impressionist Vincent van Gogh. It was completed in 1889, while van Gogh was voluntarily incarcerated at the asylum of St. Paul near Saint-Rémy in Provence. Several paintings of wheat fields with cypresses were made when van Gogh was able to leave the asylum grounds and explore the local landscape. Besides a fondness for cypresses, van Gogh had a special affinty with wheat fields; he depicted them dozens of times over the years; to Vincent they symbolized the cycle of life and death, and he found in them both solace and inspiration.', 'Olive Trees (Van Gogh series) | Vincent van Gogh painted at least 18 paintings of olive trees, mostly in Saint-Rémy-de-Provence in 1889. At his own request, he lived at an asylum there from May 1889 through May 1890 painting the gardens of the asylum and, when he had permission to venture outside its walls, nearby olive trees, cypresses and wheat fields.'], 'question': 'What was the series inspired by the appreciation of manual laborers that Wheat Field with Cypresses is a part of?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Wheat Field with Cypresses series inspiration'}) (input_keys=None), Example({'question': 'How many shops does the larger of the Miami International Mall and Dolphin Mall have?', 'answer': '240'}) (input_keys={'question'}), Example({'question': 'What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?', 'answer': '2016'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Butch Reed | Bruce Franklin Reed (born July 11, 1954) is an American retired professional wrestler and football player, better known by the ring name Butch Reed. He is best known for his appearances with the World Wrestling Federation and World Championship Wrestling between 1986 and 1992.', 'Doug Reed (futsal player) | Doug Reed is a professional futsal player who is currently playing for Futsal Dinamo and England national futsal team. Previously he has played for Lanzarote Tias Yaiza (Spain), AEL Limassol, MNK Uspinjača, MNK Smederevo and Manchester Futsal Club. In 2011, he won the FA Futsal Cup with Manchester Futsal Club and was awarded Player of the Tournament in the same competition. In the FA National Futsal League, he won several FA National League Championships and FA Futsal Cup medals. Reed also works as a commentator for Sony SIX, Eurosport and BT Sport, and has worked with the emerging US Professional Futsal League as well as Indian Premier Futsal.', 'Doom (professional wrestling) | Doom was a professional wrestling tag team composed of Ron Simmons and Butch Reed. They competed from 1989 to 1991 in the National Wrestling Alliance (NWA) and World Championship Wrestling (WCW).'], 'question': 'What other sport did Butch Reed, of the wrestling tag team Doom, play professionally?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Butch Reed football'}) (input_keys=None), Example({'question': 'What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?', 'answer': '2016'}) (input_keys={'question'}), Example({'question': 'Which magazine is still in print American Heritage or High Times?', 'answer': 'High Times'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.', 'Ebenezer Howard | Sir Ebenezer Howard {\\'1\\': \", \\'2\\': \", \\'3\\': \", \\'4\\': \"} ( ; 29 January 1850 – 1 May 1928), the English founder of the garden city movement, is known for his publication \"\" (1898), the description of a utopian city in which people live harmoniously together with nature. The publication resulted in the founding of the garden city movement, and the building of the First Garden City, Letchworth Garden City, commenced in 1903.', 'Eugene Reynal | Eugene Reynal was an American Publisher and in 1933 Founder of Blue Ribbon Books of Garden City, New York and Reynal and Hitchcock of New York, New York.'], 'question': 'Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Garden City Hotel founder'}) (input_keys=None), Example({'augmented': True, 'context': ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.'], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Carhartt bass fishing'}) (input_keys=None), Example({'augmented': True, 'context': ['Stitch\\'s Great Escape! | Stitch\\'s Great Escape! is a Tomorrowland attraction at the Magic Kingdom theme park within the Walt Disney World Resort. It is a \"theater-in-the-round\" experience starring the title alien from Walt Disney Animation Studios\\' 2002 film \"Lilo & Stitch\". It opened November 16, 2004 and is the fourth attraction to occupy the site in Tomorrowland. Many of the animators who worked on \"Lilo & Stitch\" partnered with Walt Disney Imagineering for Stitch\\'s Great Escape! The attraction is a replacement of The ExtraTERRORestrial Alien Encounter, which formerly occupied the building the attraction is housed in. On September 21, 2016, it was announced that the attraction would be switching from a daily operated attraction to a seasonally operated one, depending on attendance, starting October 2, 2016.', 'Great Escape (amusement park) | Great Escape (sometimes referred to as Six Flags Great Escape) is an amusement and water park owned and operated by Six Flags Entertainment Corp. It is located approximately 60 mi north of Albany, in Queensbury, New York, but is advertised as being in Lake George, New York, a popular tourist and vacation spot nearby. It is one of two Six Flags parks not to be officially branded with the \"Six Flags\" name (La Ronde in Montreal, Quebec, Canada being the other).', 'Houdini\\'s Great Escape | Houdini\\'s Great Escape, also known as \"Boo-dini\" during Fright Fest and \"Houdini - The Great Escape\" at Six Flags New England, are indoor Vekoma Madhouse attractions at Six Flags Great Adventure and Six Flags New England. The ride is located in Great Adventure\\'s Fantasy Forest section and New England\\'s Crack Axle Canyon area. The attraction opened at both parks in 1999, and was Standing but not operating (SBNO) at Great Adventure from 2008 - 2009 but was opened again due to popular demand.'], 'question': 'Which theme park has Stitch’s Great Escape which replaced the attraction ExtraTERRORestial Alien Encounter?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Magic Kingdom theme park'}) (input_keys=None)], [Example({'augmented': True, 'context': ['Simon Arora | Simon Arora (born November 1969) is a British billionaire businessman, CEO of the retail chain B & M.', 'Samir Arora | Samir Arora (born November 5, 1965) is an American businessman and former CEO of Mode Media (formerly Glam Media) from 2003 to April 2016. He was CEO and Chairman of the web design company NetObjects, Inc. from 1995 to 2001.', 'Dinesh Arora | Dinesh Arora (born 23 December 1976) is an Indian Administrative Service (IAS) officer, 2002 Kerala cadre, who previously worked as an Executive Director, Rural Electrification Corporation (REC) and CEO RECPDCL.'], 'question': 'What company formed in 1978 has Simon Arora as its CEO?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'B & M CEO'}) (input_keys=None), Example({'augmented': True, 'context': ['Electoral district of Bragg | Bragg is an electoral district of the House of Assembly in the Australian state of South Australia. The seat of Bragg is named after the eminent physicists Bragg – William Henry and his son, William Lawrence. The electorate is largely urban and encompasses a significant portion of the City of Burnside, stretching from the east parklands of Adelaide into the Adelaide Hills. After a redistribution following the 2006 election, the boundary moved eastwards to include suburbs that had formerly been in the seat of Heysen and now borders Kavel. Bragg currently includes the metropolitan suburbs of Beaumont, Burnside, Dulwich, Erindale, Hazelwood Park, Heathpool, Kensington Park, Leabrook, Linden Park, Rose Park, Marryatville, Skye, St Georges, Stonyfell, Toorak Gardens, Tusmore, Wattle Park and parts of Glen Osmond, and in the hills it includes Crafers, Cleland, Greenhill, Mount Osmond, Piccadilly, Summertown and parts of Ashton, Basket Range, Carey Gully, Horsnell Gully, Leawood Gardens and Uraidla.', 'Cleland, South Australia | Cleland is a suburb in South Australia located in the Adelaide metropolitan area about 10 km south-east of the Adelaide city centre. Its boundaries were created in October 2001, with additional land being added in 2010 from the adjoining suburb of Crafers. Its name is derived from the Cleland Conservation Park. The principal land use within the locality is conservation with the majority of its land area being occupied by the Cleland Conservation Park. Places within its extent include the summit of Mount Lofty. Cleland is located within the federal Division of Mayo, the state electoral district of Bragg and the local government area of the Adelaide Hills Council.', 'Electoral results for the district of Bragg | This is a list of electoral results for the Electoral district of Bragg in South Australian state elections.'], 'question': 'Bragg electoral district, incorporating the Kensington Park suburb, are associated with which South Australian city?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'South Australian city associated with Bragg electoral district'}) (input_keys=None), Example({'augmented': True, 'context': ['Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.', 'Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.', \"Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world's third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.\"], 'question': 'The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Kangchenjunga Singhi Kangri country'}) (input_keys=None)]], 2: [[], [Example({'question': 'What character did this actor, who also played a supporting role in Sleepy Hollow, play in the final six Harry Potter films?', 'answer': 'Professor Albus Dumbledore'}) (input_keys={'question'}), Example({'question': 'The musical sitcom with an album by the same name has what special feature on the album cover?', 'answer': 'actual crossword puzzle'}) (input_keys={'question'}), Example({'question': 'What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?', 'answer': '2016'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Alberta general election, 2015 | The 29th general election of Alberta, Canada, took place on May 5, 2015, following a request of Premier Jim Prentice to the Lieutenant Governor of Alberta Donald Ethell to dissolve the Legislative Assembly on April 7, 2015. This election elected members to the 29th Alberta Legislature. It was one of only four times that Alberta has changed governments.', \"Jenean Hampton | Jenean Michelle Hampton (born May 12, 1958) is the 57th and current Lieutenant Governor of Kentucky since December 8, 2015 after having won office as Republican gubernatorial nominee Matt Bevin's running-mate, in the 2015 gubernatorial election. She is the first African-American to hold any statewide office in Kentucky history, and only the third African-American woman to serve as lieutenant-governor of any U.S. state (after Jennette Bradley and Jennifer Carroll).\", 'Louisiana lieutenant gubernatorial election, 2015 | The Louisiana lieutenant gubernatorial election took place on October 24, 2015, to elect the Lieutenant Governor of Louisiana, with a runoff election held on November 21, 2015. Incumbent Republican Lieutenant Governor Jay Dardenne did not for re-election to a second full term in office. He instead ran for Governor. Billy Nungesser won the election defeating Kip Holden, despite a Democratic victory in the gubernatorial election, in which John Bel Edwards defeated David Vitter by a similar margin.', 'Indiana gubernatorial election, 2016 | The 2016 Indiana gubernatorial election was held November 8, 2016, to elect the Governor and Lieutenant Governor of Indiana, concurrently with the 2016 U.S. presidential election as well as elections to the United States Senate and elections to the United States House of Representatives and various state and local elections. The primaries were held on May 3, 2016. Republican Lieutenant Governor Eric Holcomb won the race with 51.4% of the vote.', 'Virginia lieutenant gubernatorial election, 2017 | The Virginia lieutenant gubernatorial election of 2017 will take place on November 7, 2017. After the party primary elections were held, the major party nominees are Jill Vogel (Republican) and Justin Fairfax (Democrat). The incumbent Lieutenant Governor, Democrat Ralph Northam, is not running for re-election in order to run for Governor.'], 'question': 'What race is the lieutenant governor of the state that held its primary for her election on May 19th, 2015?', 'rationale': 'The lieutenant governor of the state that held its primary for her election on May 19th, 2015 is Jenean Hampton, who is the 57th and current Lieutenant Governor of Kentucky since December 8, 2015. The primary election was held in Kentucky, and Jenean Hampton is the lieutenant governor of Kentucky.', 'answer': 'African-American'}) (input_keys=None), Example({'augmented': True, 'context': ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.', \"College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour\", 'Bass fishing | Bass fishing is the activity of angling for the North American gamefish known colloquially as the \"black bass\". There are numerous black bass species considered as gamefish in North America, including largemouth bass (\"Micropterus salmoides\"), smallmouth bass (\"Micropterus dolomieui\"), spotted bass or Kentucky bass (\"Micropterus punctulatus\"), and Guadalupe bass (order \"Perciformes\").', 'Cliff Crochet | Cliff Crochet (born August 1, 1983), also known as \"The Cajun Baby\", is a professional bass fisherman from Pierre Part, Louisiana. Crochet is a competitor on Bass Angler Sportsman Society\\'s (B.A.S.S.) Bassmaster Elite Series and has $368,837 in B.A.S.S. tournament winnings as of August 22, 2016. Crochet\\'s sponsors include: K2 Coolers, Huk, Skeeter, Yamaha Motor Company, Lowrance Electronics, Falcon Rods, Luck E Strike, Seaguar, Rat-L-Trap, Humdinger Spinnerbaits, Power-Pole. Since 2010, Crochet has competed in three Bassmaster Classic tournaments. Crochet\\'s boat for the 2016 season is a 2016 Skeeter FX 21 with a 250 hp Yamaha SHO outboard engine.'], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': 'The context mentions Carhartt Bassmaster College Bass Fishing Series, which is a college bass fishing tournament series. This series is associated with the clothing company Carhartt, which was founded in 1889.', 'answer': 'Carhartt'}) (input_keys=None), Example({'augmented': True, 'context': ['Forrest Tucker | Forrest Meredith Tucker (February 12, 1919 – October 25, 1986) was an American actor in both movies and television who appeared in nearly a hundred films. Tucker worked as a vaudeville straight man, aged fifteen years old. A mentor provided funds and contacts for a trip to California, where party hostess Cobina Wright persuaded guest Wesley Ruggles to give Tucker a screen test, based on his photogenic good looks, thick wavy hair and height of six feet, five inches - putting him into the same league as John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness\\'s younger brother. Tucker was a sight reader who needed only one take. His film career started well despite a perception in most Hollywood studios that blond men were not photogenic. After twenty years spent mainly in Westerns and action roles, he returned to his roots, showing versatility as a comedic and stage musical actor. In the TV show \"F Troop\", he became identified with the character of Cavalry Sgt. Morgan O\\'Rourke (a manipulative character quite similar to Phil Silvers\\' role as MSgt Ernie Bilko). Tucker struggled with a drinking problem that began to affect his performances in the later years of his career.', 'Emergency Landing (1941 film) | Emergency Landing (a.k.a. Robot Pilot) is a 1941 American aviation spy-fi romantic screwball comedy film directed by William Beaudine. The film stars Forrest Tucker in his second film and in his first leading role with co-stars Carol Hughes and Evelyn Brent. \"Emergency Landing\" features lots of mismatched stock footage of various types of aircraft.', 'Forrest Tucker (criminal) | Forrest Silva \"Woody\" Tucker (June 23, 1920 – May 29, 2004) was an American career criminal first imprisoned at age 15 who spent the rest of his life in and out of jail. He is best known as an escape artist, having escaped from prison 18 times successfully and 12 times unsuccessfully.', 'Fess Parker | Fess Elisha Parker Jr. (August 16, 1924 – March 18, 2010) was an American film and television actor best known for his portrayals of Davy Crockett in the Walt Disney 1955–1956 TV miniseries and as Daniel Boone in a television series from 1964 to 1970. He was also known as a winemaker and resort owner-operator.', 'Davy Crockett, King of the Wild Frontier | Davy Crockett, King of the Wild Frontier is a 1955 live-action Walt Disney adventure film starring Fess Parker as Davy Crockett. This film is an edited compilation of the first three stories from the Disney television miniseries \"Davy Crockett\" :', 'Davy Crockett (miniseries) | Davy Crockett is a five-part serial which aired on ABC from 1954–1955 in one-hour episodes, on the \"Disneyland\" series. The series stars Fess Parker as real-life frontiersman Davy Crockett and Buddy Ebsen as his friend, George Russel.'], 'question': \"Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\", 'rationale': \"From the context, we know that Forrest Tucker was an American actor who appeared in nearly a hundred films. We also know that he was compared to John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness's younger brother. We need to find the actor that played a character in the 1955-1956 Walt Disney TV miniseries, which is Fess Parker.\", 'answer': 'Davy Crockett'}) (input_keys=None)], [Example({'augmented': True, 'context': ['Forrest Tucker | Forrest Meredith Tucker (February 12, 1919 – October 25, 1986) was an American actor in both movies and television who appeared in nearly a hundred films. Tucker worked as a vaudeville straight man, aged fifteen years old. A mentor provided funds and contacts for a trip to California, where party hostess Cobina Wright persuaded guest Wesley Ruggles to give Tucker a screen test, based on his photogenic good looks, thick wavy hair and height of six feet, five inches - putting him into the same league as John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness\\'s younger brother. Tucker was a sight reader who needed only one take. His film career started well despite a perception in most Hollywood studios that blond men were not photogenic. After twenty years spent mainly in Westerns and action roles, he returned to his roots, showing versatility as a comedic and stage musical actor. In the TV show \"F Troop\", he became identified with the character of Cavalry Sgt. Morgan O\\'Rourke (a manipulative character quite similar to Phil Silvers\\' role as MSgt Ernie Bilko). Tucker struggled with a drinking problem that began to affect his performances in the later years of his career.', 'Emergency Landing (1941 film) | Emergency Landing (a.k.a. Robot Pilot) is a 1941 American aviation spy-fi romantic screwball comedy film directed by William Beaudine. The film stars Forrest Tucker in his second film and in his first leading role with co-stars Carol Hughes and Evelyn Brent. \"Emergency Landing\" features lots of mismatched stock footage of various types of aircraft.', 'Forrest Tucker (criminal) | Forrest Silva \"Woody\" Tucker (June 23, 1920 – May 29, 2004) was an American career criminal first imprisoned at age 15 who spent the rest of his life in and out of jail. He is best known as an escape artist, having escaped from prison 18 times successfully and 12 times unsuccessfully.', 'Fess Parker | Fess Elisha Parker Jr. (August 16, 1924 – March 18, 2010) was an American film and television actor best known for his portrayals of Davy Crockett in the Walt Disney 1955–1956 TV miniseries and as Daniel Boone in a television series from 1964 to 1970. He was also known as a winemaker and resort owner-operator.', 'Davy Crockett, King of the Wild Frontier | Davy Crockett, King of the Wild Frontier is a 1955 live-action Walt Disney adventure film starring Fess Parker as Davy Crockett. This film is an edited compilation of the first three stories from the Disney television miniseries \"Davy Crockett\" :', 'Davy Crockett (miniseries) | Davy Crockett is a five-part serial which aired on ABC from 1954–1955 in one-hour episodes, on the \"Disneyland\" series. The series stars Fess Parker as real-life frontiersman Davy Crockett and Buddy Ebsen as his friend, George Russel.'], 'question': \"Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\", 'rationale': 'We need to find the actor who played the character in the 1955-1956 Walt Disney TV miniseries. From the context, we know that the miniseries was about Davy Crockett and starred Fess Parker as Davy Crockett.', 'answer': 'Davy Crockett'}) (input_keys=None), Example({'augmented': True, 'context': ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.', \"College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour\", 'Bass fishing | Bass fishing is the activity of angling for the North American gamefish known colloquially as the \"black bass\". There are numerous black bass species considered as gamefish in North America, including largemouth bass (\"Micropterus salmoides\"), smallmouth bass (\"Micropterus dolomieui\"), spotted bass or Kentucky bass (\"Micropterus punctulatus\"), and Guadalupe bass (order \"Perciformes\").', 'Cliff Crochet | Cliff Crochet (born August 1, 1983), also known as \"The Cajun Baby\", is a professional bass fisherman from Pierre Part, Louisiana. Crochet is a competitor on Bass Angler Sportsman Society\\'s (B.A.S.S.) Bassmaster Elite Series and has $368,837 in B.A.S.S. tournament winnings as of August 22, 2016. Crochet\\'s sponsors include: K2 Coolers, Huk, Skeeter, Yamaha Motor Company, Lowrance Electronics, Falcon Rods, Luck E Strike, Seaguar, Rat-L-Trap, Humdinger Spinnerbaits, Power-Pole. Since 2010, Crochet has competed in three Bassmaster Classic tournaments. Crochet\\'s boat for the 2016 season is a 2016 Skeeter FX 21 with a 250 hp Yamaha SHO outboard engine.'], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Context: [1] «Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.»\\n\\n[2] «College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour»\\n\\n[3] «G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.»\\n\\nReasoning: Let's think step by step in order to\", 'answer': 'Carhartt'}) (input_keys=None), Example({'question': 'How many years did Lenny Lyles spend with the organization dramatized in the filme \"The insder\"?', 'answer': '27'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Will Finn | Will Finn (born November 1, 1958) is an American animator, voice actor, and director. His work in animation includes characters from Disney and Don Bluth films such as \"The Secret of NIMH\", \"Oliver & Company\", \"The Little Mermaid\", \"The Rescuers Down Under\", and \"Pocahontas\". His characters includes Cogsworth in \"Beauty and the Beast\", Iago in \"Aladdin\", and Laverne in \"The Hunchback of Notre Dame\". Finn wrote and directed \"Home on the Range\" and did some voice acting the Hollywood Fish in \"Chicken Little\". In 2006, Finn directed the computer animated short Hammy\\'s Boomerang Adventure, a spin-off of \"Over the Hedge\".', 'Over the Hedge (film) | Over the Hedge is a 2006 American computer-animated comedy film, based on the characters from the United Media comic strip of the same name. Directed by Tim Johnson and Karey Kirkpatrick, and produced by Bonnie Arnold, it was released in the United States on May 19, 2006. The film was produced by DreamWorks Animation and distributed through Paramount Pictures. The film features the voices of Bruce Willis, Garry Shandling, Steve Carell, William Shatner, Wanda Sykes, and Nick Nolte. It is the first DreamWorks Animation film to be distributed by Paramount Pictures, which acquired the live-action DreamWorks studio in 2006. The film earned $336 million on an $80 million budget.', 'Tim Johnson (film director) | Tim Johnson (born August 27, 1961) is an American animation director. Johnson has directed in many films such as \"Antz\", \"\" and \"Over the Hedge\". Recently, he directed the DreamWorks 3D film \"Home.\"', 'Hammy\\'s Boomerang Adventure | Hammy\\'s Boomerang Adventure is a computer-animated short comedy film based on the film \"Over the Hedge\", which was based on a comic strip of the same name. It can be found on the movie\\'s DVD or the \"Madly Madagascar\" DVD as a bonus feature.', 'Boomerang (2015 film) | Boomerang is a 2015 French drama film directed by François Favrat and starring Laurent Lafitte, Mélanie Laurent, Audrey Dana and Wladimir Yordanoff. It is an adaptation of Tatiana de Rosnay\\'s novel \"A Secret Kept\" (alternative title:\"Boomerang\"). The film was released in theatres on 23 September 2015.', 'Over the Hedge: Hammy Goes Nuts! | Over the Hedge: Hammy Goes Nuts! is a video game published by Activision based on characters from the film \"Over the Hedge\". The Game Boy Advance (GBA) version, a miniature golf game, was developed by Vicarious Visions and released on October 10, 2006 in North America. The Nintendo DS and PlayStation Portable (PSP) versions, an action adventure game and a platform game respectively, were both developed by Amaze Entertainment and were released on October 24, 2006 and November 21, 2006. It is the second game to be based on the film.'], 'question': 'When was the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed?', 'rationale': 'Context: N/A\\n\\nQuestion: When was the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed?\\n\\nReasoning: Let\\'s think step by step in order to', 'answer': '2006'}) (input_keys=None), Example({'question': 'What is the capacity of the La Liga stadium in which The Madrid Derby team plays their home games?', 'answer': '68,000'}) (input_keys={'question'}), Example({'question': 'The Equestrian statue of Marshal Mannerheim was erected as a monument to the leader of the Whites during what event?', 'answer': 'Finnish Civil War'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.', 'Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.', \"Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world's third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.\"], 'question': 'The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?', 'rationale': 'Context: [1] «Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.»\\n\\n[2] «Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.»\\n\\n[3] «Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world\\'s third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.»\\n\\nReasoning: Let\\'s think step by step in order to', 'answer': 'India'}) (input_keys=None), Example({'question': 'Are Beth Gibbons and Ian Curtis the same nationality?', 'answer': 'yes'}) (input_keys={'question'}), Example({'question': \"When did Taylor Swift's satirical song get released to the radio?\", 'answer': 'February 9, 2015'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Wheat Field with Cypresses | A Wheatfield with Cypresses (occasionally called \"A Cornfield with Cypresses\") is any of three similar 1889 oil paintings by Vincent van Gogh, as part of his wheat field series. All were executed at the Saint-Paul-de-Mausole mental asylum at Saint-Rémy near Arles, France, where Van Gogh was voluntarily a patient from May 1889 to May 1890. The works were inspired by the view from the window at the asylum towards the Alpilles mountains.', 'Green Wheat Field with Cypress | Green Wheat Field with Cypress (French: Champ de blé vert avec cyprès) is an oil on canvas painting by a Dutch Post-Impressionist Vincent van Gogh. It was completed in 1889, while van Gogh was voluntarily incarcerated at the asylum of St. Paul near Saint-Rémy in Provence. Several paintings of wheat fields with cypresses were made when van Gogh was able to leave the asylum grounds and explore the local landscape. Besides a fondness for cypresses, van Gogh had a special affinty with wheat fields; he depicted them dozens of times over the years; to Vincent they symbolized the cycle of life and death, and he found in them both solace and inspiration.', 'Olive Trees (Van Gogh series) | Vincent van Gogh painted at least 18 paintings of olive trees, mostly in Saint-Rémy-de-Provence in 1889. At his own request, he lived at an asylum there from May 1889 through May 1890 painting the gardens of the asylum and, when he had permission to venture outside its walls, nearby olive trees, cypresses and wheat fields.', 'Wheat Fields (Van Gogh series) | The Wheat Fields is a series of dozens of paintings by Vincent van Gogh, borne out of his religious studies and sermons, connection to nature, appreciation of manual laborers and desire to provide a means of offering comfort to others. The wheat field works demonstrate his progression as an artist from the drab \"Wheat Sheaves\" made in 1885 in the Netherlands to the colorful, dramatic paintings from Arles, Saint-Rémy and Auvers-sur-Oise of rural France.'], 'question': 'What was the series inspired by the appreciation of manual laborers that Wheat Field with Cypresses is a part of?', 'rationale': \"Context: N/A\\n\\nQuestion: What was the series inspired by the appreciation of manual laborers that Wheat Field with Cypresses is a part of?\\n\\nReasoning: Let's think step by step in order to\", 'answer': 'Wheat Fields'}) (input_keys=None), Example({'question': 'What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?', 'answer': '2016'}) (input_keys={'question'}), Example({'question': 'How many shops does the larger of the Miami International Mall and Dolphin Mall have?', 'answer': '240'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Butch Reed | Bruce Franklin Reed (born July 11, 1954) is an American retired professional wrestler and football player, better known by the ring name Butch Reed. He is best known for his appearances with the World Wrestling Federation and World Championship Wrestling between 1986 and 1992.', 'Doug Reed (futsal player) | Doug Reed is a professional futsal player who is currently playing for Futsal Dinamo and England national futsal team. Previously he has played for Lanzarote Tias Yaiza (Spain), AEL Limassol, MNK Uspinjača, MNK Smederevo and Manchester Futsal Club. In 2011, he won the FA Futsal Cup with Manchester Futsal Club and was awarded Player of the Tournament in the same competition. In the FA National Futsal League, he won several FA National League Championships and FA Futsal Cup medals. Reed also works as a commentator for Sony SIX, Eurosport and BT Sport, and has worked with the emerging US Professional Futsal League as well as Indian Premier Futsal.', 'Doom (professional wrestling) | Doom was a professional wrestling tag team composed of Ron Simmons and Butch Reed. They competed from 1989 to 1991 in the National Wrestling Alliance (NWA) and World Championship Wrestling (WCW).', 'Robert Reed (gridiron football) | Robert E. Reed, Jr. (born February 23, 1943) is a former American football offensive lineman in the National Football League for the Washington Redskins. He played college football at Tennessee State University and was drafted in the sixteenth round of the 1965 NFL Draft. Reed was also selected in the 12th round of the 1965 AFL Draft by the Houston Oilers.', 'Reed Doughty | Reed Doughty (born November 4, 1982) is a former American football safety who played in the National Football League. He was drafted by the Washington Redskins in the sixth round, with the 173rd overall pick, of the 2006 NFL Draft. He played college football at the University of Northern Colorado.'], 'question': 'What other sport did Butch Reed, of the wrestling tag team Doom, play professionally?', 'rationale': \"Context: N/A\\n\\nQuestion: What other sport did Butch Reed, of the wrestling tag team Doom, play professionally?\\n\\nReasoning: Let's think step by step in order to\", 'answer': 'football'}) (input_keys=None), Example({'question': 'Which magazine is still in print American Heritage or High Times?', 'answer': 'High Times'}) (input_keys={'question'}), Example({'question': 'What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?', 'answer': '2016'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.', 'Ebenezer Howard | Sir Ebenezer Howard {\\'1\\': \", \\'2\\': \", \\'3\\': \", \\'4\\': \"} ( ; 29 January 1850 – 1 May 1928), the English founder of the garden city movement, is known for his publication \"\" (1898), the description of a utopian city in which people live harmoniously together with nature. The publication resulted in the founding of the garden city movement, and the building of the First Garden City, Letchworth Garden City, commenced in 1903.', 'Eugene Reynal | Eugene Reynal was an American Publisher and in 1933 Founder of Blue Ribbon Books of Garden City, New York and Reynal and Hitchcock of New York, New York.'], 'question': 'Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?', 'rationale': 'Context: [1] «Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.»\\n\\n[2] «A.T. Stewart | Alexander Turney Stewart (October 2, 1792 – April 10, 1876) was an American businessman, politician, and philanthropist. He was a prominent figure in New York City during the mid-19th century, and was known as the \"Merchant Prince\".»\\n\\n[3] «Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.»\\n\\nQuestion: Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?\\n\\nReasoning: Let\\'s think step by step in order to', 'answer': 'A.T. Stewart'}) (input_keys=None), Example({'augmented': True, 'context': ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.', \"College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour\", 'Bass fishing | Bass fishing is the activity of angling for the North American gamefish known colloquially as the \"black bass\". There are numerous black bass species considered as gamefish in North America, including largemouth bass (\"Micropterus salmoides\"), smallmouth bass (\"Micropterus dolomieui\"), spotted bass or Kentucky bass (\"Micropterus punctulatus\"), and Guadalupe bass (order \"Perciformes\").', 'Cliff Crochet | Cliff Crochet (born August 1, 1983), also known as \"The Cajun Baby\", is a professional bass fisherman from Pierre Part, Louisiana. Crochet is a competitor on Bass Angler Sportsman Society\\'s (B.A.S.S.) Bassmaster Elite Series and has $368,837 in B.A.S.S. tournament winnings as of August 22, 2016. Crochet\\'s sponsors include: K2 Coolers, Huk, Skeeter, Yamaha Motor Company, Lowrance Electronics, Falcon Rods, Luck E Strike, Seaguar, Rat-L-Trap, Humdinger Spinnerbaits, Power-Pole. Since 2010, Crochet has competed in three Bassmaster Classic tournaments. Crochet\\'s boat for the 2016 season is a 2016 Skeeter FX 21 with a 250 hp Yamaha SHO outboard engine.'], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Context: [1] «Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.»\\n\\n[2] «College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour»\\n\\n[3] «G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.»\\n\\nReasoning: Let's think step by step in order to\", 'answer': 'Carhartt'}) (input_keys=None), Example({'augmented': True, 'context': ['Stitch\\'s Great Escape! | Stitch\\'s Great Escape! is a Tomorrowland attraction at the Magic Kingdom theme park within the Walt Disney World Resort. It is a \"theater-in-the-round\" experience starring the title alien from Walt Disney Animation Studios\\' 2002 film \"Lilo & Stitch\". It opened November 16, 2004 and is the fourth attraction to occupy the site in Tomorrowland. Many of the animators who worked on \"Lilo & Stitch\" partnered with Walt Disney Imagineering for Stitch\\'s Great Escape! The attraction is a replacement of The ExtraTERRORestrial Alien Encounter, which formerly occupied the building the attraction is housed in. On September 21, 2016, it was announced that the attraction would be switching from a daily operated attraction to a seasonally operated one, depending on attendance, starting October 2, 2016.', 'Great Escape (amusement park) | Great Escape (sometimes referred to as Six Flags Great Escape) is an amusement and water park owned and operated by Six Flags Entertainment Corp. It is located approximately 60 mi north of Albany, in Queensbury, New York, but is advertised as being in Lake George, New York, a popular tourist and vacation spot nearby. It is one of two Six Flags parks not to be officially branded with the \"Six Flags\" name (La Ronde in Montreal, Quebec, Canada being the other).', 'Houdini\\'s Great Escape | Houdini\\'s Great Escape, also known as \"Boo-dini\" during Fright Fest and \"Houdini - The Great Escape\" at Six Flags New England, are indoor Vekoma Madhouse attractions at Six Flags Great Adventure and Six Flags New England. The ride is located in Great Adventure\\'s Fantasy Forest section and New England\\'s Crack Axle Canyon area. The attraction opened at both parks in 1999, and was Standing but not operating (SBNO) at Great Adventure from 2008 - 2009 but was opened again due to popular demand.', 'Magic Kingdom | Magic Kingdom is a theme park at the Walt Disney World Resort in Bay Lake, Florida, near Orlando. Owned and operated by The Walt Disney Company through its Parks and Resorts division, the park opened on October 1, 1971, as the first of four theme parks at the resort. Initialized by Walt Disney and designed by WED Enterprises, its layout and attractions are based on Disneyland Park in Anaheim, California, and is dedicated to fairy tales and Disney characters.', 'Magic Kingdom (disambiguation) | The Magic Kingdom is a theme park at the Walt Disney World Resort in Florida.', 'List of Magic Kingdom attractions | The Magic Kingdom is a theme park located at the Walt Disney World Resort in Florida. Below is a list of the current attractions found therein, arranged by \"land\" and with brief descriptions.'], 'question': 'Which theme park has Stitch’s Great Escape which replaced the attraction ExtraTERRORestial Alien Encounter?', 'rationale': \"We need to find the theme park that has Stitch's Great Escape, which replaced the attraction ExtraTERRORestial Alien Encounter. The context mentions that Stitch's Great Escape is located in the Magic Kingdom theme park.\", 'answer': 'Magic Kingdom'}) (input_keys=None)], [Example({'augmented': True, 'context': ['Simon Arora | Simon Arora (born November 1969) is a British billionaire businessman, CEO of the retail chain B & M.', 'Samir Arora | Samir Arora (born November 5, 1965) is an American businessman and former CEO of Mode Media (formerly Glam Media) from 2003 to April 2016. He was CEO and Chairman of the web design company NetObjects, Inc. from 1995 to 2001.', 'Dinesh Arora | Dinesh Arora (born 23 December 1976) is an Indian Administrative Service (IAS) officer, 2002 Kerala cadre, who previously worked as an Executive Director, Rural Electrification Corporation (REC) and CEO RECPDCL.', 'Bolliger & Mabillard | Bolliger & Mabillard, officially Bolliger & Mabillard Consulting Engineers Inc., and often abbreviated B&M, is a roller coaster design consultancy based in Monthey, Switzerland. The company was founded in 1988 by Walter Bolliger and Claude Mabillard, with Bolliger as president and Mabillard as vice-president. B&M has built over a hundred roller coasters around the world and pioneered several new ride technologies, most notably the inverted roller coaster. In North America, B&M coaster designs have been manufactured by Ohio company Clermont Steel Fabricators since 1990. B&M has grown significantly since its founding, originally employing four to as many as 37 in 2012, consisting primarily of engineers and draftsmen. In 2016, the company completed its 100th roller coaster.', \"Rakefet Russak-Aminoach | Rakefet Russak-Aminoach (Hebrew: רקפת רוסק עמינח\\u200e \\u200e ; born February 1, 1966) is the current President and CEO of Bank Leumi le-Israel B.M., a position she has held since May 1, 2012. In September 2015, she was listed in Fortune Magazine’s list of The World's 100 Most Powerful Women.\"], 'question': 'What company formed in 1978 has Simon Arora as its CEO?', 'rationale': 'There is no information about Simon Arora being the CEO of a company formed in 1978 in the context. However, we can find the information about Simon Arora being the CEO of B & M in the context.', 'answer': 'B & M'}) (input_keys=None), Example({'augmented': True, 'context': ['Electoral district of Bragg | Bragg is an electoral district of the House of Assembly in the Australian state of South Australia. The seat of Bragg is named after the eminent physicists Bragg – William Henry and his son, William Lawrence. The electorate is largely urban and encompasses a significant portion of the City of Burnside, stretching from the east parklands of Adelaide into the Adelaide Hills. After a redistribution following the 2006 election, the boundary moved eastwards to include suburbs that had formerly been in the seat of Heysen and now borders Kavel. Bragg currently includes the metropolitan suburbs of Beaumont, Burnside, Dulwich, Erindale, Hazelwood Park, Heathpool, Kensington Park, Leabrook, Linden Park, Rose Park, Marryatville, Skye, St Georges, Stonyfell, Toorak Gardens, Tusmore, Wattle Park and parts of Glen Osmond, and in the hills it includes Crafers, Cleland, Greenhill, Mount Osmond, Piccadilly, Summertown and parts of Ashton, Basket Range, Carey Gully, Horsnell Gully, Leawood Gardens and Uraidla.', 'Cleland, South Australia | Cleland is a suburb in South Australia located in the Adelaide metropolitan area about 10 km south-east of the Adelaide city centre. Its boundaries were created in October 2001, with additional land being added in 2010 from the adjoining suburb of Crafers. Its name is derived from the Cleland Conservation Park. The principal land use within the locality is conservation with the majority of its land area being occupied by the Cleland Conservation Park. Places within its extent include the summit of Mount Lofty. Cleland is located within the federal Division of Mayo, the state electoral district of Bragg and the local government area of the Adelaide Hills Council.', 'Electoral results for the district of Bragg | This is a list of electoral results for the Electoral district of Bragg in South Australian state elections.', 'Electoral district of Burnside | Burnside was an electoral district of the House of Assembly in the Australian state of South Australia from 1938 to 1970. Before 1938 the Burnside area was represented by the three-seat multi-member electorate of Sturt. It was abolished in the 1970 parliamentary reforms, and was replaced with Bragg.'], 'question': 'Bragg electoral district, incorporating the Kensington Park suburb, are associated with which South Australian city?', 'rationale': \"Context: N/A\\n\\nQuestion: Bragg electoral district, incorporating the Kensington Park suburb, are associated with which South Australian city?\\n\\nReasoning: Let's think step by step in order to\", 'answer': 'Adelaide'}) (input_keys=None), Example({'augmented': True, 'context': ['Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.', 'Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.', \"Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world's third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.\"], 'question': 'The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?', 'rationale': 'Context: [1] «Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.»\\n\\n[2] «Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.»\\n\\n[3] «Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world\\'s third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.»\\n\\nReasoning: Let\\'s think step by step in order to', 'answer': 'India'}) (input_keys=None)]]},6,{0: [[], [Example({'question': 'Oh Yeon-seo is one of the former members of the group LUV, which stands for Love yoUR Voice, is what kind of group?', 'answer': 'K-pop girls group'}) (input_keys={'question'}), Example({'question': 'Melbourne Central railway station feeds into which main metro station that serves the entire metropolitan rail network?', 'answer': 'Flinders Street railway station'}) (input_keys={'question'}), Example({'question': 'When was the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed?', 'answer': '2006'}) (input_keys={'question'})], [Example({'augmented': True, 'context': [], 'question': 'What race is the lieutenant governor of the state that held its primary for her election on May 19th, 2015?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'lieutenant governor race May 19 2015'}) (input_keys=None), Example({'augmented': True, 'context': [], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'clothing company founded 1889 college bass fishing'}) (input_keys=None), Example({'augmented': True, 'context': [], 'question': \"Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\", 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Forrest Tucker height Walt Disney TV miniseries'}) (input_keys=None)], [Example({'augmented': True, 'context': [], 'question': \"Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\", 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Forrest Tucker height Walt Disney TV miniseries'}) (input_keys=None), Example({'augmented': True, 'context': [], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'clothing company founded 1889 college bass fishing'}) (input_keys=None), Example({'question': 'How many years did Lenny Lyles spend with the organization dramatized in the filme \"The insder\"?', 'answer': '27'}) (input_keys={'question'})], [Example({'augmented': True, 'context': [], 'question': 'When was the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Will Finn Over the Hedge short film director'}) (input_keys=None), Example({'question': 'What is the capacity of the La Liga stadium in which The Madrid Derby team plays their home games?', 'answer': '68,000'}) (input_keys={'question'}), Example({'question': 'The Equestrian statue of Marshal Mannerheim was erected as a monument to the leader of the Whites during what event?', 'answer': 'Finnish Civil War'}) (input_keys={'question'})], [Example({'augmented': True, 'context': [], 'question': 'The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Kangchenjunga Singhi Kangri country'}) (input_keys=None), Example({'question': 'Are Beth Gibbons and Ian Curtis the same nationality?', 'answer': 'yes'}) (input_keys={'question'}), Example({'question': \"When did Taylor Swift's satirical song get released to the radio?\", 'answer': 'February 9, 2015'}) (input_keys={'question'})], [Example({'augmented': True, 'context': [], 'question': 'What was the series inspired by the appreciation of manual laborers that Wheat Field with Cypresses is a part of?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Wheat Field with Cypresses series'}) (input_keys=None), Example({'question': 'What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?', 'answer': '2016'}) (input_keys={'question'}), Example({'question': 'How many shops does the larger of the Miami International Mall and Dolphin Mall have?', 'answer': '240'}) (input_keys={'question'})], [Example({'augmented': True, 'context': [], 'question': 'What other sport did Butch Reed, of the wrestling tag team Doom, play professionally?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Butch Reed sports'}) (input_keys=None), Example({'question': 'Which magazine is still in print American Heritage or High Times?', 'answer': 'High Times'}) (input_keys={'question'}), Example({'question': 'What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?', 'answer': '2016'}) (input_keys={'question'})], [Example({'augmented': True, 'context': [], 'question': 'Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Garden City Hotel founder'}) (input_keys=None), Example({'augmented': True, 'context': [], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'clothing company founded 1889 college bass fishing'}) (input_keys=None), Example({'augmented': True, 'context': [], 'question': 'Which theme park has Stitch’s Great Escape which replaced the attraction ExtraTERRORestial Alien Encounter?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': \"Stitch's Great Escape theme park\"}) (input_keys=None)], [Example({'augmented': True, 'context': [], 'question': 'What company formed in 1978 has Simon Arora as its CEO?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Simon Arora CEO company 1978'}) (input_keys=None), Example({'augmented': True, 'context': [], 'question': 'Bragg electoral district, incorporating the Kensington Park suburb, are associated with which South Australian city?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Bragg electoral district Kensington Park suburb South Australian city'}) (input_keys=None), Example({'augmented': True, 'context': [], 'question': 'The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Kangchenjunga Singhi Kangri country'}) (input_keys=None)]], 1: [[], [Example({'question': 'Born Again starred the American film actor who played what role in The Best Years of Our Lives?', 'answer': 'Fred Derry'}) (input_keys={'question'}), Example({'question': 'What type of compilation does Chris Harris and Into the Labyrinth have in common?', 'answer': 'television series'}) (input_keys={'question'}), Example({'question': 'Are both Paul Schattel and Francis Ford Coppola American film directors?', 'answer': 'yes'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Alberta general election, 2015 | The 29th general election of Alberta, Canada, took place on May 5, 2015, following a request of Premier Jim Prentice to the Lieutenant Governor of Alberta Donald Ethell to dissolve the Legislative Assembly on April 7, 2015. This election elected members to the 29th Alberta Legislature. It was one of only four times that Alberta has changed governments.', \"Jenean Hampton | Jenean Michelle Hampton (born May 12, 1958) is the 57th and current Lieutenant Governor of Kentucky since December 8, 2015 after having won office as Republican gubernatorial nominee Matt Bevin's running-mate, in the 2015 gubernatorial election. She is the first African-American to hold any statewide office in Kentucky history, and only the third African-American woman to serve as lieutenant-governor of any U.S. state (after Jennette Bradley and Jennifer Carroll).\", 'Louisiana lieutenant gubernatorial election, 2015 | The Louisiana lieutenant gubernatorial election took place on October 24, 2015, to elect the Lieutenant Governor of Louisiana, with a runoff election held on November 21, 2015. Incumbent Republican Lieutenant Governor Jay Dardenne did not for re-election to a second full term in office. He instead ran for Governor. Billy Nungesser won the election defeating Kip Holden, despite a Democratic victory in the gubernatorial election, in which John Bel Edwards defeated David Vitter by a similar margin.'], 'question': 'What race is the lieutenant governor of the state that held its primary for her election on May 19th, 2015?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'lieutenant governor of the state that held its primary for her election on May 19th 2015'}) (input_keys=None), Example({'augmented': True, 'context': ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.'], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Carhartt bass fishing'}) (input_keys=None), Example({'augmented': True, 'context': ['Forrest Tucker | Forrest Meredith Tucker (February 12, 1919 – October 25, 1986) was an American actor in both movies and television who appeared in nearly a hundred films. Tucker worked as a vaudeville straight man, aged fifteen years old. A mentor provided funds and contacts for a trip to California, where party hostess Cobina Wright persuaded guest Wesley Ruggles to give Tucker a screen test, based on his photogenic good looks, thick wavy hair and height of six feet, five inches - putting him into the same league as John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness\\'s younger brother. Tucker was a sight reader who needed only one take. His film career started well despite a perception in most Hollywood studios that blond men were not photogenic. After twenty years spent mainly in Westerns and action roles, he returned to his roots, showing versatility as a comedic and stage musical actor. In the TV show \"F Troop\", he became identified with the character of Cavalry Sgt. Morgan O\\'Rourke (a manipulative character quite similar to Phil Silvers\\' role as MSgt Ernie Bilko). Tucker struggled with a drinking problem that began to affect his performances in the later years of his career.', 'Emergency Landing (1941 film) | Emergency Landing (a.k.a. Robot Pilot) is a 1941 American aviation spy-fi romantic screwball comedy film directed by William Beaudine. The film stars Forrest Tucker in his second film and in his first leading role with co-stars Carol Hughes and Evelyn Brent. \"Emergency Landing\" features lots of mismatched stock footage of various types of aircraft.', 'Forrest Tucker (criminal) | Forrest Silva \"Woody\" Tucker (June 23, 1920 – May 29, 2004) was an American career criminal first imprisoned at age 15 who spent the rest of his life in and out of jail. He is best known as an escape artist, having escaped from prison 18 times successfully and 12 times unsuccessfully.'], 'question': \"Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\", 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Walt Disney TV miniseries 1955-1956 character'}) (input_keys=None)], [Example({'augmented': True, 'context': ['Forrest Tucker | Forrest Meredith Tucker (February 12, 1919 – October 25, 1986) was an American actor in both movies and television who appeared in nearly a hundred films. Tucker worked as a vaudeville straight man, aged fifteen years old. A mentor provided funds and contacts for a trip to California, where party hostess Cobina Wright persuaded guest Wesley Ruggles to give Tucker a screen test, based on his photogenic good looks, thick wavy hair and height of six feet, five inches - putting him into the same league as John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness\\'s younger brother. Tucker was a sight reader who needed only one take. His film career started well despite a perception in most Hollywood studios that blond men were not photogenic. After twenty years spent mainly in Westerns and action roles, he returned to his roots, showing versatility as a comedic and stage musical actor. In the TV show \"F Troop\", he became identified with the character of Cavalry Sgt. Morgan O\\'Rourke (a manipulative character quite similar to Phil Silvers\\' role as MSgt Ernie Bilko). Tucker struggled with a drinking problem that began to affect his performances in the later years of his career.', 'Emergency Landing (1941 film) | Emergency Landing (a.k.a. Robot Pilot) is a 1941 American aviation spy-fi romantic screwball comedy film directed by William Beaudine. The film stars Forrest Tucker in his second film and in his first leading role with co-stars Carol Hughes and Evelyn Brent. \"Emergency Landing\" features lots of mismatched stock footage of various types of aircraft.', 'Forrest Tucker (criminal) | Forrest Silva \"Woody\" Tucker (June 23, 1920 – May 29, 2004) was an American career criminal first imprisoned at age 15 who spent the rest of his life in and out of jail. He is best known as an escape artist, having escaped from prison 18 times successfully and 12 times unsuccessfully.'], 'question': \"Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\", 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Walt Disney TV miniseries 1955-1956 character'}) (input_keys=None), Example({'augmented': True, 'context': ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.'], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Carhartt bass fishing'}) (input_keys=None), Example({'question': 'How many years did Lenny Lyles spend with the organization dramatized in the filme \"The insder\"?', 'answer': '27'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Will Finn | Will Finn (born November 1, 1958) is an American animator, voice actor, and director. His work in animation includes characters from Disney and Don Bluth films such as \"The Secret of NIMH\", \"Oliver & Company\", \"The Little Mermaid\", \"The Rescuers Down Under\", and \"Pocahontas\". His characters includes Cogsworth in \"Beauty and the Beast\", Iago in \"Aladdin\", and Laverne in \"The Hunchback of Notre Dame\". Finn wrote and directed \"Home on the Range\" and did some voice acting the Hollywood Fish in \"Chicken Little\". In 2006, Finn directed the computer animated short Hammy\\'s Boomerang Adventure, a spin-off of \"Over the Hedge\".', 'Over the Hedge (film) | Over the Hedge is a 2006 American computer-animated comedy film, based on the characters from the United Media comic strip of the same name. Directed by Tim Johnson and Karey Kirkpatrick, and produced by Bonnie Arnold, it was released in the United States on May 19, 2006. The film was produced by DreamWorks Animation and distributed through Paramount Pictures. The film features the voices of Bruce Willis, Garry Shandling, Steve Carell, William Shatner, Wanda Sykes, and Nick Nolte. It is the first DreamWorks Animation film to be distributed by Paramount Pictures, which acquired the live-action DreamWorks studio in 2006. The film earned $336 million on an $80 million budget.', 'Tim Johnson (film director) | Tim Johnson (born August 27, 1961) is an American animation director. Johnson has directed in many films such as \"Antz\", \"\" and \"Over the Hedge\". Recently, he directed the DreamWorks 3D film \"Home.\"'], 'question': 'When was the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed?', 'rationale': 'find the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed. The context mentions that Will Finn directed the computer animated short Hammy\\'s Boomerang Adventure, a spin-off of \"Over the Hedge\".', 'search_query': \"Hammy's Boomerang Adventure release date\"}) (input_keys=None), Example({'question': 'The Equestrian statue of Marshal Mannerheim was erected as a monument to the leader of the Whites during what event?', 'answer': 'Finnish Civil War'}) (input_keys={'question'}), Example({'question': 'What is the capacity of the La Liga stadium in which The Madrid Derby team plays their home games?', 'answer': '68,000'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.', 'Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.', \"Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world's third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.\"], 'question': 'The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Kangchenjunga Singhi Kangri country'}) (input_keys=None), Example({'question': \"When did Taylor Swift's satirical song get released to the radio?\", 'answer': 'February 9, 2015'}) (input_keys={'question'}), Example({'question': 'Are Beth Gibbons and Ian Curtis the same nationality?', 'answer': 'yes'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Wheat Field with Cypresses | A Wheatfield with Cypresses (occasionally called \"A Cornfield with Cypresses\") is any of three similar 1889 oil paintings by Vincent van Gogh, as part of his wheat field series. All were executed at the Saint-Paul-de-Mausole mental asylum at Saint-Rémy near Arles, France, where Van Gogh was voluntarily a patient from May 1889 to May 1890. The works were inspired by the view from the window at the asylum towards the Alpilles mountains.', 'Green Wheat Field with Cypress | Green Wheat Field with Cypress (French: Champ de blé vert avec cyprès) is an oil on canvas painting by a Dutch Post-Impressionist Vincent van Gogh. It was completed in 1889, while van Gogh was voluntarily incarcerated at the asylum of St. Paul near Saint-Rémy in Provence. Several paintings of wheat fields with cypresses were made when van Gogh was able to leave the asylum grounds and explore the local landscape. Besides a fondness for cypresses, van Gogh had a special affinty with wheat fields; he depicted them dozens of times over the years; to Vincent they symbolized the cycle of life and death, and he found in them both solace and inspiration.', 'Olive Trees (Van Gogh series) | Vincent van Gogh painted at least 18 paintings of olive trees, mostly in Saint-Rémy-de-Provence in 1889. At his own request, he lived at an asylum there from May 1889 through May 1890 painting the gardens of the asylum and, when he had permission to venture outside its walls, nearby olive trees, cypresses and wheat fields.'], 'question': 'What was the series inspired by the appreciation of manual laborers that Wheat Field with Cypresses is a part of?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Wheat Field with Cypresses series inspiration'}) (input_keys=None), Example({'question': 'How many shops does the larger of the Miami International Mall and Dolphin Mall have?', 'answer': '240'}) (input_keys={'question'}), Example({'question': 'What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?', 'answer': '2016'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Butch Reed | Bruce Franklin Reed (born July 11, 1954) is an American retired professional wrestler and football player, better known by the ring name Butch Reed. He is best known for his appearances with the World Wrestling Federation and World Championship Wrestling between 1986 and 1992.', 'Doug Reed (futsal player) | Doug Reed is a professional futsal player who is currently playing for Futsal Dinamo and England national futsal team. Previously he has played for Lanzarote Tias Yaiza (Spain), AEL Limassol, MNK Uspinjača, MNK Smederevo and Manchester Futsal Club. In 2011, he won the FA Futsal Cup with Manchester Futsal Club and was awarded Player of the Tournament in the same competition. In the FA National Futsal League, he won several FA National League Championships and FA Futsal Cup medals. Reed also works as a commentator for Sony SIX, Eurosport and BT Sport, and has worked with the emerging US Professional Futsal League as well as Indian Premier Futsal.', 'Doom (professional wrestling) | Doom was a professional wrestling tag team composed of Ron Simmons and Butch Reed. They competed from 1989 to 1991 in the National Wrestling Alliance (NWA) and World Championship Wrestling (WCW).'], 'question': 'What other sport did Butch Reed, of the wrestling tag team Doom, play professionally?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Butch Reed football'}) (input_keys=None), Example({'question': 'What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?', 'answer': '2016'}) (input_keys={'question'}), Example({'question': 'Which magazine is still in print American Heritage or High Times?', 'answer': 'High Times'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.', 'Ebenezer Howard | Sir Ebenezer Howard {\\'1\\': \", \\'2\\': \", \\'3\\': \", \\'4\\': \"} ( ; 29 January 1850 – 1 May 1928), the English founder of the garden city movement, is known for his publication \"\" (1898), the description of a utopian city in which people live harmoniously together with nature. The publication resulted in the founding of the garden city movement, and the building of the First Garden City, Letchworth Garden City, commenced in 1903.', 'Eugene Reynal | Eugene Reynal was an American Publisher and in 1933 Founder of Blue Ribbon Books of Garden City, New York and Reynal and Hitchcock of New York, New York.'], 'question': 'Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Garden City Hotel founder'}) (input_keys=None), Example({'augmented': True, 'context': ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.'], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Carhartt bass fishing'}) (input_keys=None), Example({'augmented': True, 'context': ['Stitch\\'s Great Escape! | Stitch\\'s Great Escape! is a Tomorrowland attraction at the Magic Kingdom theme park within the Walt Disney World Resort. It is a \"theater-in-the-round\" experience starring the title alien from Walt Disney Animation Studios\\' 2002 film \"Lilo & Stitch\". It opened November 16, 2004 and is the fourth attraction to occupy the site in Tomorrowland. Many of the animators who worked on \"Lilo & Stitch\" partnered with Walt Disney Imagineering for Stitch\\'s Great Escape! The attraction is a replacement of The ExtraTERRORestrial Alien Encounter, which formerly occupied the building the attraction is housed in. On September 21, 2016, it was announced that the attraction would be switching from a daily operated attraction to a seasonally operated one, depending on attendance, starting October 2, 2016.', 'Great Escape (amusement park) | Great Escape (sometimes referred to as Six Flags Great Escape) is an amusement and water park owned and operated by Six Flags Entertainment Corp. It is located approximately 60 mi north of Albany, in Queensbury, New York, but is advertised as being in Lake George, New York, a popular tourist and vacation spot nearby. It is one of two Six Flags parks not to be officially branded with the \"Six Flags\" name (La Ronde in Montreal, Quebec, Canada being the other).', 'Houdini\\'s Great Escape | Houdini\\'s Great Escape, also known as \"Boo-dini\" during Fright Fest and \"Houdini - The Great Escape\" at Six Flags New England, are indoor Vekoma Madhouse attractions at Six Flags Great Adventure and Six Flags New England. The ride is located in Great Adventure\\'s Fantasy Forest section and New England\\'s Crack Axle Canyon area. The attraction opened at both parks in 1999, and was Standing but not operating (SBNO) at Great Adventure from 2008 - 2009 but was opened again due to popular demand.'], 'question': 'Which theme park has Stitch’s Great Escape which replaced the attraction ExtraTERRORestial Alien Encounter?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Magic Kingdom theme park'}) (input_keys=None)], [Example({'augmented': True, 'context': ['Simon Arora | Simon Arora (born November 1969) is a British billionaire businessman, CEO of the retail chain B & M.', 'Samir Arora | Samir Arora (born November 5, 1965) is an American businessman and former CEO of Mode Media (formerly Glam Media) from 2003 to April 2016. He was CEO and Chairman of the web design company NetObjects, Inc. from 1995 to 2001.', 'Dinesh Arora | Dinesh Arora (born 23 December 1976) is an Indian Administrative Service (IAS) officer, 2002 Kerala cadre, who previously worked as an Executive Director, Rural Electrification Corporation (REC) and CEO RECPDCL.'], 'question': 'What company formed in 1978 has Simon Arora as its CEO?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'B & M CEO'}) (input_keys=None), Example({'augmented': True, 'context': ['Electoral district of Bragg | Bragg is an electoral district of the House of Assembly in the Australian state of South Australia. The seat of Bragg is named after the eminent physicists Bragg – William Henry and his son, William Lawrence. The electorate is largely urban and encompasses a significant portion of the City of Burnside, stretching from the east parklands of Adelaide into the Adelaide Hills. After a redistribution following the 2006 election, the boundary moved eastwards to include suburbs that had formerly been in the seat of Heysen and now borders Kavel. Bragg currently includes the metropolitan suburbs of Beaumont, Burnside, Dulwich, Erindale, Hazelwood Park, Heathpool, Kensington Park, Leabrook, Linden Park, Rose Park, Marryatville, Skye, St Georges, Stonyfell, Toorak Gardens, Tusmore, Wattle Park and parts of Glen Osmond, and in the hills it includes Crafers, Cleland, Greenhill, Mount Osmond, Piccadilly, Summertown and parts of Ashton, Basket Range, Carey Gully, Horsnell Gully, Leawood Gardens and Uraidla.', 'Cleland, South Australia | Cleland is a suburb in South Australia located in the Adelaide metropolitan area about 10 km south-east of the Adelaide city centre. Its boundaries were created in October 2001, with additional land being added in 2010 from the adjoining suburb of Crafers. Its name is derived from the Cleland Conservation Park. The principal land use within the locality is conservation with the majority of its land area being occupied by the Cleland Conservation Park. Places within its extent include the summit of Mount Lofty. Cleland is located within the federal Division of Mayo, the state electoral district of Bragg and the local government area of the Adelaide Hills Council.', 'Electoral results for the district of Bragg | This is a list of electoral results for the Electoral district of Bragg in South Australian state elections.'], 'question': 'Bragg electoral district, incorporating the Kensington Park suburb, are associated with which South Australian city?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'South Australian city associated with Bragg electoral district'}) (input_keys=None), Example({'augmented': True, 'context': ['Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.', 'Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.', \"Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world's third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.\"], 'question': 'The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Kangchenjunga Singhi Kangri country'}) (input_keys=None)]], 2: [[], [Example({'question': 'What character did this actor, who also played a supporting role in Sleepy Hollow, play in the final six Harry Potter films?', 'answer': 'Professor Albus Dumbledore'}) (input_keys={'question'}), Example({'question': 'The musical sitcom with an album by the same name has what special feature on the album cover?', 'answer': 'actual crossword puzzle'}) (input_keys={'question'}), Example({'question': 'What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?', 'answer': '2016'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Alberta general election, 2015 | The 29th general election of Alberta, Canada, took place on May 5, 2015, following a request of Premier Jim Prentice to the Lieutenant Governor of Alberta Donald Ethell to dissolve the Legislative Assembly on April 7, 2015. This election elected members to the 29th Alberta Legislature. It was one of only four times that Alberta has changed governments.', \"Jenean Hampton | Jenean Michelle Hampton (born May 12, 1958) is the 57th and current Lieutenant Governor of Kentucky since December 8, 2015 after having won office as Republican gubernatorial nominee Matt Bevin's running-mate, in the 2015 gubernatorial election. She is the first African-American to hold any statewide office in Kentucky history, and only the third African-American woman to serve as lieutenant-governor of any U.S. state (after Jennette Bradley and Jennifer Carroll).\", 'Louisiana lieutenant gubernatorial election, 2015 | The Louisiana lieutenant gubernatorial election took place on October 24, 2015, to elect the Lieutenant Governor of Louisiana, with a runoff election held on November 21, 2015. Incumbent Republican Lieutenant Governor Jay Dardenne did not for re-election to a second full term in office. He instead ran for Governor. Billy Nungesser won the election defeating Kip Holden, despite a Democratic victory in the gubernatorial election, in which John Bel Edwards defeated David Vitter by a similar margin.', 'Indiana gubernatorial election, 2016 | The 2016 Indiana gubernatorial election was held November 8, 2016, to elect the Governor and Lieutenant Governor of Indiana, concurrently with the 2016 U.S. presidential election as well as elections to the United States Senate and elections to the United States House of Representatives and various state and local elections. The primaries were held on May 3, 2016. Republican Lieutenant Governor Eric Holcomb won the race with 51.4% of the vote.', 'Virginia lieutenant gubernatorial election, 2017 | The Virginia lieutenant gubernatorial election of 2017 will take place on November 7, 2017. After the party primary elections were held, the major party nominees are Jill Vogel (Republican) and Justin Fairfax (Democrat). The incumbent Lieutenant Governor, Democrat Ralph Northam, is not running for re-election in order to run for Governor.'], 'question': 'What race is the lieutenant governor of the state that held its primary for her election on May 19th, 2015?', 'rationale': 'The lieutenant governor of the state that held its primary for her election on May 19th, 2015 is Jenean Hampton, who is the 57th and current Lieutenant Governor of Kentucky since December 8, 2015. The primary election was held in Kentucky, and Jenean Hampton is the lieutenant governor of Kentucky.', 'answer': 'African-American'}) (input_keys=None), Example({'augmented': True, 'context': ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.', \"College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour\", 'Bass fishing | Bass fishing is the activity of angling for the North American gamefish known colloquially as the \"black bass\". There are numerous black bass species considered as gamefish in North America, including largemouth bass (\"Micropterus salmoides\"), smallmouth bass (\"Micropterus dolomieui\"), spotted bass or Kentucky bass (\"Micropterus punctulatus\"), and Guadalupe bass (order \"Perciformes\").', 'Cliff Crochet | Cliff Crochet (born August 1, 1983), also known as \"The Cajun Baby\", is a professional bass fisherman from Pierre Part, Louisiana. Crochet is a competitor on Bass Angler Sportsman Society\\'s (B.A.S.S.) Bassmaster Elite Series and has $368,837 in B.A.S.S. tournament winnings as of August 22, 2016. Crochet\\'s sponsors include: K2 Coolers, Huk, Skeeter, Yamaha Motor Company, Lowrance Electronics, Falcon Rods, Luck E Strike, Seaguar, Rat-L-Trap, Humdinger Spinnerbaits, Power-Pole. Since 2010, Crochet has competed in three Bassmaster Classic tournaments. Crochet\\'s boat for the 2016 season is a 2016 Skeeter FX 21 with a 250 hp Yamaha SHO outboard engine.'], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': 'The context mentions Carhartt Bassmaster College Bass Fishing Series, which is a college bass fishing tournament series. This series is associated with the clothing company Carhartt, which was founded in 1889.', 'answer': 'Carhartt'}) (input_keys=None), Example({'augmented': True, 'context': ['Forrest Tucker | Forrest Meredith Tucker (February 12, 1919 – October 25, 1986) was an American actor in both movies and television who appeared in nearly a hundred films. Tucker worked as a vaudeville straight man, aged fifteen years old. A mentor provided funds and contacts for a trip to California, where party hostess Cobina Wright persuaded guest Wesley Ruggles to give Tucker a screen test, based on his photogenic good looks, thick wavy hair and height of six feet, five inches - putting him into the same league as John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness\\'s younger brother. Tucker was a sight reader who needed only one take. His film career started well despite a perception in most Hollywood studios that blond men were not photogenic. After twenty years spent mainly in Westerns and action roles, he returned to his roots, showing versatility as a comedic and stage musical actor. In the TV show \"F Troop\", he became identified with the character of Cavalry Sgt. Morgan O\\'Rourke (a manipulative character quite similar to Phil Silvers\\' role as MSgt Ernie Bilko). Tucker struggled with a drinking problem that began to affect his performances in the later years of his career.', 'Emergency Landing (1941 film) | Emergency Landing (a.k.a. Robot Pilot) is a 1941 American aviation spy-fi romantic screwball comedy film directed by William Beaudine. The film stars Forrest Tucker in his second film and in his first leading role with co-stars Carol Hughes and Evelyn Brent. \"Emergency Landing\" features lots of mismatched stock footage of various types of aircraft.', 'Forrest Tucker (criminal) | Forrest Silva \"Woody\" Tucker (June 23, 1920 – May 29, 2004) was an American career criminal first imprisoned at age 15 who spent the rest of his life in and out of jail. He is best known as an escape artist, having escaped from prison 18 times successfully and 12 times unsuccessfully.', 'Fess Parker | Fess Elisha Parker Jr. (August 16, 1924 – March 18, 2010) was an American film and television actor best known for his portrayals of Davy Crockett in the Walt Disney 1955–1956 TV miniseries and as Daniel Boone in a television series from 1964 to 1970. He was also known as a winemaker and resort owner-operator.', 'Davy Crockett, King of the Wild Frontier | Davy Crockett, King of the Wild Frontier is a 1955 live-action Walt Disney adventure film starring Fess Parker as Davy Crockett. This film is an edited compilation of the first three stories from the Disney television miniseries \"Davy Crockett\" :', 'Davy Crockett (miniseries) | Davy Crockett is a five-part serial which aired on ABC from 1954–1955 in one-hour episodes, on the \"Disneyland\" series. The series stars Fess Parker as real-life frontiersman Davy Crockett and Buddy Ebsen as his friend, George Russel.'], 'question': \"Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\", 'rationale': \"From the context, we know that Forrest Tucker was an American actor who appeared in nearly a hundred films. We also know that he was compared to John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness's younger brother. We need to find the actor that played a character in the 1955-1956 Walt Disney TV miniseries, which is Fess Parker.\", 'answer': 'Davy Crockett'}) (input_keys=None)], [Example({'augmented': True, 'context': ['Forrest Tucker | Forrest Meredith Tucker (February 12, 1919 – October 25, 1986) was an American actor in both movies and television who appeared in nearly a hundred films. Tucker worked as a vaudeville straight man, aged fifteen years old. A mentor provided funds and contacts for a trip to California, where party hostess Cobina Wright persuaded guest Wesley Ruggles to give Tucker a screen test, based on his photogenic good looks, thick wavy hair and height of six feet, five inches - putting him into the same league as John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness\\'s younger brother. Tucker was a sight reader who needed only one take. His film career started well despite a perception in most Hollywood studios that blond men were not photogenic. After twenty years spent mainly in Westerns and action roles, he returned to his roots, showing versatility as a comedic and stage musical actor. In the TV show \"F Troop\", he became identified with the character of Cavalry Sgt. Morgan O\\'Rourke (a manipulative character quite similar to Phil Silvers\\' role as MSgt Ernie Bilko). Tucker struggled with a drinking problem that began to affect his performances in the later years of his career.', 'Emergency Landing (1941 film) | Emergency Landing (a.k.a. Robot Pilot) is a 1941 American aviation spy-fi romantic screwball comedy film directed by William Beaudine. The film stars Forrest Tucker in his second film and in his first leading role with co-stars Carol Hughes and Evelyn Brent. \"Emergency Landing\" features lots of mismatched stock footage of various types of aircraft.', 'Forrest Tucker (criminal) | Forrest Silva \"Woody\" Tucker (June 23, 1920 – May 29, 2004) was an American career criminal first imprisoned at age 15 who spent the rest of his life in and out of jail. He is best known as an escape artist, having escaped from prison 18 times successfully and 12 times unsuccessfully.', 'Fess Parker | Fess Elisha Parker Jr. (August 16, 1924 – March 18, 2010) was an American film and television actor best known for his portrayals of Davy Crockett in the Walt Disney 1955–1956 TV miniseries and as Daniel Boone in a television series from 1964 to 1970. He was also known as a winemaker and resort owner-operator.', 'Davy Crockett, King of the Wild Frontier | Davy Crockett, King of the Wild Frontier is a 1955 live-action Walt Disney adventure film starring Fess Parker as Davy Crockett. This film is an edited compilation of the first three stories from the Disney television miniseries \"Davy Crockett\" :', 'Davy Crockett (miniseries) | Davy Crockett is a five-part serial which aired on ABC from 1954–1955 in one-hour episodes, on the \"Disneyland\" series. The series stars Fess Parker as real-life frontiersman Davy Crockett and Buddy Ebsen as his friend, George Russel.'], 'question': \"Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\", 'rationale': 'We need to find the actor who played the character in the 1955-1956 Walt Disney TV miniseries. From the context, we know that the miniseries was about Davy Crockett and starred Fess Parker as Davy Crockett.', 'answer': 'Davy Crockett'}) (input_keys=None), Example({'augmented': True, 'context': ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.', \"College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour\", 'Bass fishing | Bass fishing is the activity of angling for the North American gamefish known colloquially as the \"black bass\". There are numerous black bass species considered as gamefish in North America, including largemouth bass (\"Micropterus salmoides\"), smallmouth bass (\"Micropterus dolomieui\"), spotted bass or Kentucky bass (\"Micropterus punctulatus\"), and Guadalupe bass (order \"Perciformes\").', 'Cliff Crochet | Cliff Crochet (born August 1, 1983), also known as \"The Cajun Baby\", is a professional bass fisherman from Pierre Part, Louisiana. Crochet is a competitor on Bass Angler Sportsman Society\\'s (B.A.S.S.) Bassmaster Elite Series and has $368,837 in B.A.S.S. tournament winnings as of August 22, 2016. Crochet\\'s sponsors include: K2 Coolers, Huk, Skeeter, Yamaha Motor Company, Lowrance Electronics, Falcon Rods, Luck E Strike, Seaguar, Rat-L-Trap, Humdinger Spinnerbaits, Power-Pole. Since 2010, Crochet has competed in three Bassmaster Classic tournaments. Crochet\\'s boat for the 2016 season is a 2016 Skeeter FX 21 with a 250 hp Yamaha SHO outboard engine.'], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Context: [1] «Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.»\\n\\n[2] «College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour»\\n\\n[3] «G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.»\\n\\nReasoning: Let's think step by step in order to\", 'answer': 'Carhartt'}) (input_keys=None), Example({'question': 'How many years did Lenny Lyles spend with the organization dramatized in the filme \"The insder\"?', 'answer': '27'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Will Finn | Will Finn (born November 1, 1958) is an American animator, voice actor, and director. His work in animation includes characters from Disney and Don Bluth films such as \"The Secret of NIMH\", \"Oliver & Company\", \"The Little Mermaid\", \"The Rescuers Down Under\", and \"Pocahontas\". His characters includes Cogsworth in \"Beauty and the Beast\", Iago in \"Aladdin\", and Laverne in \"The Hunchback of Notre Dame\". Finn wrote and directed \"Home on the Range\" and did some voice acting the Hollywood Fish in \"Chicken Little\". In 2006, Finn directed the computer animated short Hammy\\'s Boomerang Adventure, a spin-off of \"Over the Hedge\".', 'Over the Hedge (film) | Over the Hedge is a 2006 American computer-animated comedy film, based on the characters from the United Media comic strip of the same name. Directed by Tim Johnson and Karey Kirkpatrick, and produced by Bonnie Arnold, it was released in the United States on May 19, 2006. The film was produced by DreamWorks Animation and distributed through Paramount Pictures. The film features the voices of Bruce Willis, Garry Shandling, Steve Carell, William Shatner, Wanda Sykes, and Nick Nolte. It is the first DreamWorks Animation film to be distributed by Paramount Pictures, which acquired the live-action DreamWorks studio in 2006. The film earned $336 million on an $80 million budget.', 'Tim Johnson (film director) | Tim Johnson (born August 27, 1961) is an American animation director. Johnson has directed in many films such as \"Antz\", \"\" and \"Over the Hedge\". Recently, he directed the DreamWorks 3D film \"Home.\"', 'Hammy\\'s Boomerang Adventure | Hammy\\'s Boomerang Adventure is a computer-animated short comedy film based on the film \"Over the Hedge\", which was based on a comic strip of the same name. It can be found on the movie\\'s DVD or the \"Madly Madagascar\" DVD as a bonus feature.', 'Boomerang (2015 film) | Boomerang is a 2015 French drama film directed by François Favrat and starring Laurent Lafitte, Mélanie Laurent, Audrey Dana and Wladimir Yordanoff. It is an adaptation of Tatiana de Rosnay\\'s novel \"A Secret Kept\" (alternative title:\"Boomerang\"). The film was released in theatres on 23 September 2015.', 'Over the Hedge: Hammy Goes Nuts! | Over the Hedge: Hammy Goes Nuts! is a video game published by Activision based on characters from the film \"Over the Hedge\". The Game Boy Advance (GBA) version, a miniature golf game, was developed by Vicarious Visions and released on October 10, 2006 in North America. The Nintendo DS and PlayStation Portable (PSP) versions, an action adventure game and a platform game respectively, were both developed by Amaze Entertainment and were released on October 24, 2006 and November 21, 2006. It is the second game to be based on the film.'], 'question': 'When was the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed?', 'rationale': 'Context: N/A\\n\\nQuestion: When was the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed?\\n\\nReasoning: Let\\'s think step by step in order to', 'answer': '2006'}) (input_keys=None), Example({'question': 'What is the capacity of the La Liga stadium in which The Madrid Derby team plays their home games?', 'answer': '68,000'}) (input_keys={'question'}), Example({'question': 'The Equestrian statue of Marshal Mannerheim was erected as a monument to the leader of the Whites during what event?', 'answer': 'Finnish Civil War'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.', 'Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.', \"Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world's third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.\"], 'question': 'The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?', 'rationale': 'Context: [1] «Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.»\\n\\n[2] «Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.»\\n\\n[3] «Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world\\'s third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.»\\n\\nReasoning: Let\\'s think step by step in order to', 'answer': 'India'}) (input_keys=None), Example({'question': 'Are Beth Gibbons and Ian Curtis the same nationality?', 'answer': 'yes'}) (input_keys={'question'}), Example({'question': \"When did Taylor Swift's satirical song get released to the radio?\", 'answer': 'February 9, 2015'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Wheat Field with Cypresses | A Wheatfield with Cypresses (occasionally called \"A Cornfield with Cypresses\") is any of three similar 1889 oil paintings by Vincent van Gogh, as part of his wheat field series. All were executed at the Saint-Paul-de-Mausole mental asylum at Saint-Rémy near Arles, France, where Van Gogh was voluntarily a patient from May 1889 to May 1890. The works were inspired by the view from the window at the asylum towards the Alpilles mountains.', 'Green Wheat Field with Cypress | Green Wheat Field with Cypress (French: Champ de blé vert avec cyprès) is an oil on canvas painting by a Dutch Post-Impressionist Vincent van Gogh. It was completed in 1889, while van Gogh was voluntarily incarcerated at the asylum of St. Paul near Saint-Rémy in Provence. Several paintings of wheat fields with cypresses were made when van Gogh was able to leave the asylum grounds and explore the local landscape. Besides a fondness for cypresses, van Gogh had a special affinty with wheat fields; he depicted them dozens of times over the years; to Vincent they symbolized the cycle of life and death, and he found in them both solace and inspiration.', 'Olive Trees (Van Gogh series) | Vincent van Gogh painted at least 18 paintings of olive trees, mostly in Saint-Rémy-de-Provence in 1889. At his own request, he lived at an asylum there from May 1889 through May 1890 painting the gardens of the asylum and, when he had permission to venture outside its walls, nearby olive trees, cypresses and wheat fields.', 'Wheat Fields (Van Gogh series) | The Wheat Fields is a series of dozens of paintings by Vincent van Gogh, borne out of his religious studies and sermons, connection to nature, appreciation of manual laborers and desire to provide a means of offering comfort to others. The wheat field works demonstrate his progression as an artist from the drab \"Wheat Sheaves\" made in 1885 in the Netherlands to the colorful, dramatic paintings from Arles, Saint-Rémy and Auvers-sur-Oise of rural France.'], 'question': 'What was the series inspired by the appreciation of manual laborers that Wheat Field with Cypresses is a part of?', 'rationale': \"Context: N/A\\n\\nQuestion: What was the series inspired by the appreciation of manual laborers that Wheat Field with Cypresses is a part of?\\n\\nReasoning: Let's think step by step in order to\", 'answer': 'Wheat Fields'}) (input_keys=None), Example({'question': 'What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?', 'answer': '2016'}) (input_keys={'question'}), Example({'question': 'How many shops does the larger of the Miami International Mall and Dolphin Mall have?', 'answer': '240'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Butch Reed | Bruce Franklin Reed (born July 11, 1954) is an American retired professional wrestler and football player, better known by the ring name Butch Reed. He is best known for his appearances with the World Wrestling Federation and World Championship Wrestling between 1986 and 1992.', 'Doug Reed (futsal player) | Doug Reed is a professional futsal player who is currently playing for Futsal Dinamo and England national futsal team. Previously he has played for Lanzarote Tias Yaiza (Spain), AEL Limassol, MNK Uspinjača, MNK Smederevo and Manchester Futsal Club. In 2011, he won the FA Futsal Cup with Manchester Futsal Club and was awarded Player of the Tournament in the same competition. In the FA National Futsal League, he won several FA National League Championships and FA Futsal Cup medals. Reed also works as a commentator for Sony SIX, Eurosport and BT Sport, and has worked with the emerging US Professional Futsal League as well as Indian Premier Futsal.', 'Doom (professional wrestling) | Doom was a professional wrestling tag team composed of Ron Simmons and Butch Reed. They competed from 1989 to 1991 in the National Wrestling Alliance (NWA) and World Championship Wrestling (WCW).', 'Robert Reed (gridiron football) | Robert E. Reed, Jr. (born February 23, 1943) is a former American football offensive lineman in the National Football League for the Washington Redskins. He played college football at Tennessee State University and was drafted in the sixteenth round of the 1965 NFL Draft. Reed was also selected in the 12th round of the 1965 AFL Draft by the Houston Oilers.', 'Reed Doughty | Reed Doughty (born November 4, 1982) is a former American football safety who played in the National Football League. He was drafted by the Washington Redskins in the sixth round, with the 173rd overall pick, of the 2006 NFL Draft. He played college football at the University of Northern Colorado.'], 'question': 'What other sport did Butch Reed, of the wrestling tag team Doom, play professionally?', 'rationale': \"Context: N/A\\n\\nQuestion: What other sport did Butch Reed, of the wrestling tag team Doom, play professionally?\\n\\nReasoning: Let's think step by step in order to\", 'answer': 'football'}) (input_keys=None), Example({'question': 'Which magazine is still in print American Heritage or High Times?', 'answer': 'High Times'}) (input_keys={'question'}), Example({'question': 'What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?', 'answer': '2016'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.', 'Ebenezer Howard | Sir Ebenezer Howard {\\'1\\': \", \\'2\\': \", \\'3\\': \", \\'4\\': \"} ( ; 29 January 1850 – 1 May 1928), the English founder of the garden city movement, is known for his publication \"\" (1898), the description of a utopian city in which people live harmoniously together with nature. The publication resulted in the founding of the garden city movement, and the building of the First Garden City, Letchworth Garden City, commenced in 1903.', 'Eugene Reynal | Eugene Reynal was an American Publisher and in 1933 Founder of Blue Ribbon Books of Garden City, New York and Reynal and Hitchcock of New York, New York.'], 'question': 'Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?', 'rationale': 'Context: [1] «Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.»\\n\\n[2] «A.T. Stewart | Alexander Turney Stewart (October 2, 1792 – April 10, 1876) was an American businessman, politician, and philanthropist. He was a prominent figure in New York City during the mid-19th century, and was known as the \"Merchant Prince\".»\\n\\n[3] «Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.»\\n\\nQuestion: Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?\\n\\nReasoning: Let\\'s think step by step in order to', 'answer': 'A.T. Stewart'}) (input_keys=None), Example({'augmented': True, 'context': ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.', \"College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour\", 'Bass fishing | Bass fishing is the activity of angling for the North American gamefish known colloquially as the \"black bass\". There are numerous black bass species considered as gamefish in North America, including largemouth bass (\"Micropterus salmoides\"), smallmouth bass (\"Micropterus dolomieui\"), spotted bass or Kentucky bass (\"Micropterus punctulatus\"), and Guadalupe bass (order \"Perciformes\").', 'Cliff Crochet | Cliff Crochet (born August 1, 1983), also known as \"The Cajun Baby\", is a professional bass fisherman from Pierre Part, Louisiana. Crochet is a competitor on Bass Angler Sportsman Society\\'s (B.A.S.S.) Bassmaster Elite Series and has $368,837 in B.A.S.S. tournament winnings as of August 22, 2016. Crochet\\'s sponsors include: K2 Coolers, Huk, Skeeter, Yamaha Motor Company, Lowrance Electronics, Falcon Rods, Luck E Strike, Seaguar, Rat-L-Trap, Humdinger Spinnerbaits, Power-Pole. Since 2010, Crochet has competed in three Bassmaster Classic tournaments. Crochet\\'s boat for the 2016 season is a 2016 Skeeter FX 21 with a 250 hp Yamaha SHO outboard engine.'], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Context: [1] «Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.»\\n\\n[2] «College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour»\\n\\n[3] «G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.»\\n\\nReasoning: Let's think step by step in order to\", 'answer': 'Carhartt'}) (input_keys=None), Example({'augmented': True, 'context': ['Stitch\\'s Great Escape! | Stitch\\'s Great Escape! is a Tomorrowland attraction at the Magic Kingdom theme park within the Walt Disney World Resort. It is a \"theater-in-the-round\" experience starring the title alien from Walt Disney Animation Studios\\' 2002 film \"Lilo & Stitch\". It opened November 16, 2004 and is the fourth attraction to occupy the site in Tomorrowland. Many of the animators who worked on \"Lilo & Stitch\" partnered with Walt Disney Imagineering for Stitch\\'s Great Escape! The attraction is a replacement of The ExtraTERRORestrial Alien Encounter, which formerly occupied the building the attraction is housed in. On September 21, 2016, it was announced that the attraction would be switching from a daily operated attraction to a seasonally operated one, depending on attendance, starting October 2, 2016.', 'Great Escape (amusement park) | Great Escape (sometimes referred to as Six Flags Great Escape) is an amusement and water park owned and operated by Six Flags Entertainment Corp. It is located approximately 60 mi north of Albany, in Queensbury, New York, but is advertised as being in Lake George, New York, a popular tourist and vacation spot nearby. It is one of two Six Flags parks not to be officially branded with the \"Six Flags\" name (La Ronde in Montreal, Quebec, Canada being the other).', 'Houdini\\'s Great Escape | Houdini\\'s Great Escape, also known as \"Boo-dini\" during Fright Fest and \"Houdini - The Great Escape\" at Six Flags New England, are indoor Vekoma Madhouse attractions at Six Flags Great Adventure and Six Flags New England. The ride is located in Great Adventure\\'s Fantasy Forest section and New England\\'s Crack Axle Canyon area. The attraction opened at both parks in 1999, and was Standing but not operating (SBNO) at Great Adventure from 2008 - 2009 but was opened again due to popular demand.', 'Magic Kingdom | Magic Kingdom is a theme park at the Walt Disney World Resort in Bay Lake, Florida, near Orlando. Owned and operated by The Walt Disney Company through its Parks and Resorts division, the park opened on October 1, 1971, as the first of four theme parks at the resort. Initialized by Walt Disney and designed by WED Enterprises, its layout and attractions are based on Disneyland Park in Anaheim, California, and is dedicated to fairy tales and Disney characters.', 'Magic Kingdom (disambiguation) | The Magic Kingdom is a theme park at the Walt Disney World Resort in Florida.', 'List of Magic Kingdom attractions | The Magic Kingdom is a theme park located at the Walt Disney World Resort in Florida. Below is a list of the current attractions found therein, arranged by \"land\" and with brief descriptions.'], 'question': 'Which theme park has Stitch’s Great Escape which replaced the attraction ExtraTERRORestial Alien Encounter?', 'rationale': \"We need to find the theme park that has Stitch's Great Escape, which replaced the attraction ExtraTERRORestial Alien Encounter. The context mentions that Stitch's Great Escape is located in the Magic Kingdom theme park.\", 'answer': 'Magic Kingdom'}) (input_keys=None)], [Example({'augmented': True, 'context': ['Simon Arora | Simon Arora (born November 1969) is a British billionaire businessman, CEO of the retail chain B & M.', 'Samir Arora | Samir Arora (born November 5, 1965) is an American businessman and former CEO of Mode Media (formerly Glam Media) from 2003 to April 2016. He was CEO and Chairman of the web design company NetObjects, Inc. from 1995 to 2001.', 'Dinesh Arora | Dinesh Arora (born 23 December 1976) is an Indian Administrative Service (IAS) officer, 2002 Kerala cadre, who previously worked as an Executive Director, Rural Electrification Corporation (REC) and CEO RECPDCL.', 'Bolliger & Mabillard | Bolliger & Mabillard, officially Bolliger & Mabillard Consulting Engineers Inc., and often abbreviated B&M, is a roller coaster design consultancy based in Monthey, Switzerland. The company was founded in 1988 by Walter Bolliger and Claude Mabillard, with Bolliger as president and Mabillard as vice-president. B&M has built over a hundred roller coasters around the world and pioneered several new ride technologies, most notably the inverted roller coaster. In North America, B&M coaster designs have been manufactured by Ohio company Clermont Steel Fabricators since 1990. B&M has grown significantly since its founding, originally employing four to as many as 37 in 2012, consisting primarily of engineers and draftsmen. In 2016, the company completed its 100th roller coaster.', \"Rakefet Russak-Aminoach | Rakefet Russak-Aminoach (Hebrew: רקפת רוסק עמינח\\u200e \\u200e ; born February 1, 1966) is the current President and CEO of Bank Leumi le-Israel B.M., a position she has held since May 1, 2012. In September 2015, she was listed in Fortune Magazine’s list of The World's 100 Most Powerful Women.\"], 'question': 'What company formed in 1978 has Simon Arora as its CEO?', 'rationale': 'There is no information about Simon Arora being the CEO of a company formed in 1978 in the context. However, we can find the information about Simon Arora being the CEO of B & M in the context.', 'answer': 'B & M'}) (input_keys=None), Example({'augmented': True, 'context': ['Electoral district of Bragg | Bragg is an electoral district of the House of Assembly in the Australian state of South Australia. The seat of Bragg is named after the eminent physicists Bragg – William Henry and his son, William Lawrence. The electorate is largely urban and encompasses a significant portion of the City of Burnside, stretching from the east parklands of Adelaide into the Adelaide Hills. After a redistribution following the 2006 election, the boundary moved eastwards to include suburbs that had formerly been in the seat of Heysen and now borders Kavel. Bragg currently includes the metropolitan suburbs of Beaumont, Burnside, Dulwich, Erindale, Hazelwood Park, Heathpool, Kensington Park, Leabrook, Linden Park, Rose Park, Marryatville, Skye, St Georges, Stonyfell, Toorak Gardens, Tusmore, Wattle Park and parts of Glen Osmond, and in the hills it includes Crafers, Cleland, Greenhill, Mount Osmond, Piccadilly, Summertown and parts of Ashton, Basket Range, Carey Gully, Horsnell Gully, Leawood Gardens and Uraidla.', 'Cleland, South Australia | Cleland is a suburb in South Australia located in the Adelaide metropolitan area about 10 km south-east of the Adelaide city centre. Its boundaries were created in October 2001, with additional land being added in 2010 from the adjoining suburb of Crafers. Its name is derived from the Cleland Conservation Park. The principal land use within the locality is conservation with the majority of its land area being occupied by the Cleland Conservation Park. Places within its extent include the summit of Mount Lofty. Cleland is located within the federal Division of Mayo, the state electoral district of Bragg and the local government area of the Adelaide Hills Council.', 'Electoral results for the district of Bragg | This is a list of electoral results for the Electoral district of Bragg in South Australian state elections.', 'Electoral district of Burnside | Burnside was an electoral district of the House of Assembly in the Australian state of South Australia from 1938 to 1970. Before 1938 the Burnside area was represented by the three-seat multi-member electorate of Sturt. It was abolished in the 1970 parliamentary reforms, and was replaced with Bragg.'], 'question': 'Bragg electoral district, incorporating the Kensington Park suburb, are associated with which South Australian city?', 'rationale': \"Context: N/A\\n\\nQuestion: Bragg electoral district, incorporating the Kensington Park suburb, are associated with which South Australian city?\\n\\nReasoning: Let's think step by step in order to\", 'answer': 'Adelaide'}) (input_keys=None), Example({'augmented': True, 'context': ['Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.', 'Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.', \"Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world's third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.\"], 'question': 'The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?', 'rationale': 'Context: [1] «Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.»\\n\\n[2] «Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.»\\n\\n[3] «Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world\\'s third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.»\\n\\nReasoning: Let\\'s think step by step in order to', 'answer': 'India'}) (input_keys=None)]]},8,{0: [[], [Example({'question': 'Oh Yeon-seo is one of the former members of the group LUV, which stands for Love yoUR Voice, is what kind of group?', 'answer': 'K-pop girls group'}) (input_keys={'question'}), Example({'question': 'Melbourne Central railway station feeds into which main metro station that serves the entire metropolitan rail network?', 'answer': 'Flinders Street railway station'}) (input_keys={'question'}), Example({'question': 'When was the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed?', 'answer': '2006'}) (input_keys={'question'})], [Example({'augmented': True, 'context': [], 'question': 'What race is the lieutenant governor of the state that held its primary for her election on May 19th, 2015?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'lieutenant governor race May 19 2015'}) (input_keys=None), Example({'augmented': True, 'context': [], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'clothing company founded 1889 college bass fishing'}) (input_keys=None), Example({'augmented': True, 'context': [], 'question': \"Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\", 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Forrest Tucker height Walt Disney TV miniseries'}) (input_keys=None)], [Example({'augmented': True, 'context': [], 'question': \"Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\", 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Forrest Tucker height Walt Disney TV miniseries'}) (input_keys=None), Example({'augmented': True, 'context': [], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'clothing company founded 1889 college bass fishing'}) (input_keys=None), Example({'question': 'How many years did Lenny Lyles spend with the organization dramatized in the filme \"The insder\"?', 'answer': '27'}) (input_keys={'question'})], [Example({'augmented': True, 'context': [], 'question': 'When was the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Will Finn Over the Hedge short film director'}) (input_keys=None), Example({'question': 'What is the capacity of the La Liga stadium in which The Madrid Derby team plays their home games?', 'answer': '68,000'}) (input_keys={'question'}), Example({'question': 'The Equestrian statue of Marshal Mannerheim was erected as a monument to the leader of the Whites during what event?', 'answer': 'Finnish Civil War'}) (input_keys={'question'})], [Example({'augmented': True, 'context': [], 'question': 'The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Kangchenjunga Singhi Kangri country'}) (input_keys=None), Example({'question': 'Are Beth Gibbons and Ian Curtis the same nationality?', 'answer': 'yes'}) (input_keys={'question'}), Example({'question': \"When did Taylor Swift's satirical song get released to the radio?\", 'answer': 'February 9, 2015'}) (input_keys={'question'})], [Example({'augmented': True, 'context': [], 'question': 'What was the series inspired by the appreciation of manual laborers that Wheat Field with Cypresses is a part of?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Wheat Field with Cypresses series'}) (input_keys=None), Example({'question': 'What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?', 'answer': '2016'}) (input_keys={'question'}), Example({'question': 'How many shops does the larger of the Miami International Mall and Dolphin Mall have?', 'answer': '240'}) (input_keys={'question'})], [Example({'augmented': True, 'context': [], 'question': 'What other sport did Butch Reed, of the wrestling tag team Doom, play professionally?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Butch Reed sports'}) (input_keys=None), Example({'question': 'Which magazine is still in print American Heritage or High Times?', 'answer': 'High Times'}) (input_keys={'question'}), Example({'question': 'What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?', 'answer': '2016'}) (input_keys={'question'})], [Example({'augmented': True, 'context': [], 'question': 'Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Garden City Hotel founder'}) (input_keys=None), Example({'augmented': True, 'context': [], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'clothing company founded 1889 college bass fishing'}) (input_keys=None), Example({'augmented': True, 'context': [], 'question': 'Which theme park has Stitch’s Great Escape which replaced the attraction ExtraTERRORestial Alien Encounter?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': \"Stitch's Great Escape theme park\"}) (input_keys=None)], [Example({'augmented': True, 'context': [], 'question': 'What company formed in 1978 has Simon Arora as its CEO?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Simon Arora CEO company 1978'}) (input_keys=None), Example({'augmented': True, 'context': [], 'question': 'Bragg electoral district, incorporating the Kensington Park suburb, are associated with which South Australian city?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Bragg electoral district Kensington Park suburb South Australian city'}) (input_keys=None), Example({'augmented': True, 'context': [], 'question': 'The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Kangchenjunga Singhi Kangri country'}) (input_keys=None)]], 1: [[], [Example({'question': 'Born Again starred the American film actor who played what role in The Best Years of Our Lives?', 'answer': 'Fred Derry'}) (input_keys={'question'}), Example({'question': 'What type of compilation does Chris Harris and Into the Labyrinth have in common?', 'answer': 'television series'}) (input_keys={'question'}), Example({'question': 'Are both Paul Schattel and Francis Ford Coppola American film directors?', 'answer': 'yes'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Alberta general election, 2015 | The 29th general election of Alberta, Canada, took place on May 5, 2015, following a request of Premier Jim Prentice to the Lieutenant Governor of Alberta Donald Ethell to dissolve the Legislative Assembly on April 7, 2015. This election elected members to the 29th Alberta Legislature. It was one of only four times that Alberta has changed governments.', \"Jenean Hampton | Jenean Michelle Hampton (born May 12, 1958) is the 57th and current Lieutenant Governor of Kentucky since December 8, 2015 after having won office as Republican gubernatorial nominee Matt Bevin's running-mate, in the 2015 gubernatorial election. She is the first African-American to hold any statewide office in Kentucky history, and only the third African-American woman to serve as lieutenant-governor of any U.S. state (after Jennette Bradley and Jennifer Carroll).\", 'Louisiana lieutenant gubernatorial election, 2015 | The Louisiana lieutenant gubernatorial election took place on October 24, 2015, to elect the Lieutenant Governor of Louisiana, with a runoff election held on November 21, 2015. Incumbent Republican Lieutenant Governor Jay Dardenne did not for re-election to a second full term in office. He instead ran for Governor. Billy Nungesser won the election defeating Kip Holden, despite a Democratic victory in the gubernatorial election, in which John Bel Edwards defeated David Vitter by a similar margin.'], 'question': 'What race is the lieutenant governor of the state that held its primary for her election on May 19th, 2015?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'lieutenant governor of the state that held its primary for her election on May 19th 2015'}) (input_keys=None), Example({'augmented': True, 'context': ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.'], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Carhartt bass fishing'}) (input_keys=None), Example({'augmented': True, 'context': ['Forrest Tucker | Forrest Meredith Tucker (February 12, 1919 – October 25, 1986) was an American actor in both movies and television who appeared in nearly a hundred films. Tucker worked as a vaudeville straight man, aged fifteen years old. A mentor provided funds and contacts for a trip to California, where party hostess Cobina Wright persuaded guest Wesley Ruggles to give Tucker a screen test, based on his photogenic good looks, thick wavy hair and height of six feet, five inches - putting him into the same league as John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness\\'s younger brother. Tucker was a sight reader who needed only one take. His film career started well despite a perception in most Hollywood studios that blond men were not photogenic. After twenty years spent mainly in Westerns and action roles, he returned to his roots, showing versatility as a comedic and stage musical actor. In the TV show \"F Troop\", he became identified with the character of Cavalry Sgt. Morgan O\\'Rourke (a manipulative character quite similar to Phil Silvers\\' role as MSgt Ernie Bilko). Tucker struggled with a drinking problem that began to affect his performances in the later years of his career.', 'Emergency Landing (1941 film) | Emergency Landing (a.k.a. Robot Pilot) is a 1941 American aviation spy-fi romantic screwball comedy film directed by William Beaudine. The film stars Forrest Tucker in his second film and in his first leading role with co-stars Carol Hughes and Evelyn Brent. \"Emergency Landing\" features lots of mismatched stock footage of various types of aircraft.', 'Forrest Tucker (criminal) | Forrest Silva \"Woody\" Tucker (June 23, 1920 – May 29, 2004) was an American career criminal first imprisoned at age 15 who spent the rest of his life in and out of jail. He is best known as an escape artist, having escaped from prison 18 times successfully and 12 times unsuccessfully.'], 'question': \"Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\", 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Walt Disney TV miniseries 1955-1956 character'}) (input_keys=None)], [Example({'augmented': True, 'context': ['Forrest Tucker | Forrest Meredith Tucker (February 12, 1919 – October 25, 1986) was an American actor in both movies and television who appeared in nearly a hundred films. Tucker worked as a vaudeville straight man, aged fifteen years old. A mentor provided funds and contacts for a trip to California, where party hostess Cobina Wright persuaded guest Wesley Ruggles to give Tucker a screen test, based on his photogenic good looks, thick wavy hair and height of six feet, five inches - putting him into the same league as John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness\\'s younger brother. Tucker was a sight reader who needed only one take. His film career started well despite a perception in most Hollywood studios that blond men were not photogenic. After twenty years spent mainly in Westerns and action roles, he returned to his roots, showing versatility as a comedic and stage musical actor. In the TV show \"F Troop\", he became identified with the character of Cavalry Sgt. Morgan O\\'Rourke (a manipulative character quite similar to Phil Silvers\\' role as MSgt Ernie Bilko). Tucker struggled with a drinking problem that began to affect his performances in the later years of his career.', 'Emergency Landing (1941 film) | Emergency Landing (a.k.a. Robot Pilot) is a 1941 American aviation spy-fi romantic screwball comedy film directed by William Beaudine. The film stars Forrest Tucker in his second film and in his first leading role with co-stars Carol Hughes and Evelyn Brent. \"Emergency Landing\" features lots of mismatched stock footage of various types of aircraft.', 'Forrest Tucker (criminal) | Forrest Silva \"Woody\" Tucker (June 23, 1920 – May 29, 2004) was an American career criminal first imprisoned at age 15 who spent the rest of his life in and out of jail. He is best known as an escape artist, having escaped from prison 18 times successfully and 12 times unsuccessfully.'], 'question': \"Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\", 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Walt Disney TV miniseries 1955-1956 character'}) (input_keys=None), Example({'augmented': True, 'context': ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.'], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Carhartt bass fishing'}) (input_keys=None), Example({'question': 'How many years did Lenny Lyles spend with the organization dramatized in the filme \"The insder\"?', 'answer': '27'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Will Finn | Will Finn (born November 1, 1958) is an American animator, voice actor, and director. His work in animation includes characters from Disney and Don Bluth films such as \"The Secret of NIMH\", \"Oliver & Company\", \"The Little Mermaid\", \"The Rescuers Down Under\", and \"Pocahontas\". His characters includes Cogsworth in \"Beauty and the Beast\", Iago in \"Aladdin\", and Laverne in \"The Hunchback of Notre Dame\". Finn wrote and directed \"Home on the Range\" and did some voice acting the Hollywood Fish in \"Chicken Little\". In 2006, Finn directed the computer animated short Hammy\\'s Boomerang Adventure, a spin-off of \"Over the Hedge\".', 'Over the Hedge (film) | Over the Hedge is a 2006 American computer-animated comedy film, based on the characters from the United Media comic strip of the same name. Directed by Tim Johnson and Karey Kirkpatrick, and produced by Bonnie Arnold, it was released in the United States on May 19, 2006. The film was produced by DreamWorks Animation and distributed through Paramount Pictures. The film features the voices of Bruce Willis, Garry Shandling, Steve Carell, William Shatner, Wanda Sykes, and Nick Nolte. It is the first DreamWorks Animation film to be distributed by Paramount Pictures, which acquired the live-action DreamWorks studio in 2006. The film earned $336 million on an $80 million budget.', 'Tim Johnson (film director) | Tim Johnson (born August 27, 1961) is an American animation director. Johnson has directed in many films such as \"Antz\", \"\" and \"Over the Hedge\". Recently, he directed the DreamWorks 3D film \"Home.\"'], 'question': 'When was the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed?', 'rationale': 'find the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed. The context mentions that Will Finn directed the computer animated short Hammy\\'s Boomerang Adventure, a spin-off of \"Over the Hedge\".', 'search_query': \"Hammy's Boomerang Adventure release date\"}) (input_keys=None), Example({'question': 'The Equestrian statue of Marshal Mannerheim was erected as a monument to the leader of the Whites during what event?', 'answer': 'Finnish Civil War'}) (input_keys={'question'}), Example({'question': 'What is the capacity of the La Liga stadium in which The Madrid Derby team plays their home games?', 'answer': '68,000'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.', 'Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.', \"Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world's third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.\"], 'question': 'The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Kangchenjunga Singhi Kangri country'}) (input_keys=None), Example({'question': \"When did Taylor Swift's satirical song get released to the radio?\", 'answer': 'February 9, 2015'}) (input_keys={'question'}), Example({'question': 'Are Beth Gibbons and Ian Curtis the same nationality?', 'answer': 'yes'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Wheat Field with Cypresses | A Wheatfield with Cypresses (occasionally called \"A Cornfield with Cypresses\") is any of three similar 1889 oil paintings by Vincent van Gogh, as part of his wheat field series. All were executed at the Saint-Paul-de-Mausole mental asylum at Saint-Rémy near Arles, France, where Van Gogh was voluntarily a patient from May 1889 to May 1890. The works were inspired by the view from the window at the asylum towards the Alpilles mountains.', 'Green Wheat Field with Cypress | Green Wheat Field with Cypress (French: Champ de blé vert avec cyprès) is an oil on canvas painting by a Dutch Post-Impressionist Vincent van Gogh. It was completed in 1889, while van Gogh was voluntarily incarcerated at the asylum of St. Paul near Saint-Rémy in Provence. Several paintings of wheat fields with cypresses were made when van Gogh was able to leave the asylum grounds and explore the local landscape. Besides a fondness for cypresses, van Gogh had a special affinty with wheat fields; he depicted them dozens of times over the years; to Vincent they symbolized the cycle of life and death, and he found in them both solace and inspiration.', 'Olive Trees (Van Gogh series) | Vincent van Gogh painted at least 18 paintings of olive trees, mostly in Saint-Rémy-de-Provence in 1889. At his own request, he lived at an asylum there from May 1889 through May 1890 painting the gardens of the asylum and, when he had permission to venture outside its walls, nearby olive trees, cypresses and wheat fields.'], 'question': 'What was the series inspired by the appreciation of manual laborers that Wheat Field with Cypresses is a part of?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Wheat Field with Cypresses series inspiration'}) (input_keys=None), Example({'question': 'How many shops does the larger of the Miami International Mall and Dolphin Mall have?', 'answer': '240'}) (input_keys={'question'}), Example({'question': 'What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?', 'answer': '2016'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Butch Reed | Bruce Franklin Reed (born July 11, 1954) is an American retired professional wrestler and football player, better known by the ring name Butch Reed. He is best known for his appearances with the World Wrestling Federation and World Championship Wrestling between 1986 and 1992.', 'Doug Reed (futsal player) | Doug Reed is a professional futsal player who is currently playing for Futsal Dinamo and England national futsal team. Previously he has played for Lanzarote Tias Yaiza (Spain), AEL Limassol, MNK Uspinjača, MNK Smederevo and Manchester Futsal Club. In 2011, he won the FA Futsal Cup with Manchester Futsal Club and was awarded Player of the Tournament in the same competition. In the FA National Futsal League, he won several FA National League Championships and FA Futsal Cup medals. Reed also works as a commentator for Sony SIX, Eurosport and BT Sport, and has worked with the emerging US Professional Futsal League as well as Indian Premier Futsal.', 'Doom (professional wrestling) | Doom was a professional wrestling tag team composed of Ron Simmons and Butch Reed. They competed from 1989 to 1991 in the National Wrestling Alliance (NWA) and World Championship Wrestling (WCW).'], 'question': 'What other sport did Butch Reed, of the wrestling tag team Doom, play professionally?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Butch Reed football'}) (input_keys=None), Example({'question': 'What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?', 'answer': '2016'}) (input_keys={'question'}), Example({'question': 'Which magazine is still in print American Heritage or High Times?', 'answer': 'High Times'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.', 'Ebenezer Howard | Sir Ebenezer Howard {\\'1\\': \", \\'2\\': \", \\'3\\': \", \\'4\\': \"} ( ; 29 January 1850 – 1 May 1928), the English founder of the garden city movement, is known for his publication \"\" (1898), the description of a utopian city in which people live harmoniously together with nature. The publication resulted in the founding of the garden city movement, and the building of the First Garden City, Letchworth Garden City, commenced in 1903.', 'Eugene Reynal | Eugene Reynal was an American Publisher and in 1933 Founder of Blue Ribbon Books of Garden City, New York and Reynal and Hitchcock of New York, New York.'], 'question': 'Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Garden City Hotel founder'}) (input_keys=None), Example({'augmented': True, 'context': ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.'], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Carhartt bass fishing'}) (input_keys=None), Example({'augmented': True, 'context': ['Stitch\\'s Great Escape! | Stitch\\'s Great Escape! is a Tomorrowland attraction at the Magic Kingdom theme park within the Walt Disney World Resort. It is a \"theater-in-the-round\" experience starring the title alien from Walt Disney Animation Studios\\' 2002 film \"Lilo & Stitch\". It opened November 16, 2004 and is the fourth attraction to occupy the site in Tomorrowland. Many of the animators who worked on \"Lilo & Stitch\" partnered with Walt Disney Imagineering for Stitch\\'s Great Escape! The attraction is a replacement of The ExtraTERRORestrial Alien Encounter, which formerly occupied the building the attraction is housed in. On September 21, 2016, it was announced that the attraction would be switching from a daily operated attraction to a seasonally operated one, depending on attendance, starting October 2, 2016.', 'Great Escape (amusement park) | Great Escape (sometimes referred to as Six Flags Great Escape) is an amusement and water park owned and operated by Six Flags Entertainment Corp. It is located approximately 60 mi north of Albany, in Queensbury, New York, but is advertised as being in Lake George, New York, a popular tourist and vacation spot nearby. It is one of two Six Flags parks not to be officially branded with the \"Six Flags\" name (La Ronde in Montreal, Quebec, Canada being the other).', 'Houdini\\'s Great Escape | Houdini\\'s Great Escape, also known as \"Boo-dini\" during Fright Fest and \"Houdini - The Great Escape\" at Six Flags New England, are indoor Vekoma Madhouse attractions at Six Flags Great Adventure and Six Flags New England. The ride is located in Great Adventure\\'s Fantasy Forest section and New England\\'s Crack Axle Canyon area. The attraction opened at both parks in 1999, and was Standing but not operating (SBNO) at Great Adventure from 2008 - 2009 but was opened again due to popular demand.'], 'question': 'Which theme park has Stitch’s Great Escape which replaced the attraction ExtraTERRORestial Alien Encounter?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Magic Kingdom theme park'}) (input_keys=None)], [Example({'augmented': True, 'context': ['Simon Arora | Simon Arora (born November 1969) is a British billionaire businessman, CEO of the retail chain B & M.', 'Samir Arora | Samir Arora (born November 5, 1965) is an American businessman and former CEO of Mode Media (formerly Glam Media) from 2003 to April 2016. He was CEO and Chairman of the web design company NetObjects, Inc. from 1995 to 2001.', 'Dinesh Arora | Dinesh Arora (born 23 December 1976) is an Indian Administrative Service (IAS) officer, 2002 Kerala cadre, who previously worked as an Executive Director, Rural Electrification Corporation (REC) and CEO RECPDCL.'], 'question': 'What company formed in 1978 has Simon Arora as its CEO?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'B & M CEO'}) (input_keys=None), Example({'augmented': True, 'context': ['Electoral district of Bragg | Bragg is an electoral district of the House of Assembly in the Australian state of South Australia. The seat of Bragg is named after the eminent physicists Bragg – William Henry and his son, William Lawrence. The electorate is largely urban and encompasses a significant portion of the City of Burnside, stretching from the east parklands of Adelaide into the Adelaide Hills. After a redistribution following the 2006 election, the boundary moved eastwards to include suburbs that had formerly been in the seat of Heysen and now borders Kavel. Bragg currently includes the metropolitan suburbs of Beaumont, Burnside, Dulwich, Erindale, Hazelwood Park, Heathpool, Kensington Park, Leabrook, Linden Park, Rose Park, Marryatville, Skye, St Georges, Stonyfell, Toorak Gardens, Tusmore, Wattle Park and parts of Glen Osmond, and in the hills it includes Crafers, Cleland, Greenhill, Mount Osmond, Piccadilly, Summertown and parts of Ashton, Basket Range, Carey Gully, Horsnell Gully, Leawood Gardens and Uraidla.', 'Cleland, South Australia | Cleland is a suburb in South Australia located in the Adelaide metropolitan area about 10 km south-east of the Adelaide city centre. Its boundaries were created in October 2001, with additional land being added in 2010 from the adjoining suburb of Crafers. Its name is derived from the Cleland Conservation Park. The principal land use within the locality is conservation with the majority of its land area being occupied by the Cleland Conservation Park. Places within its extent include the summit of Mount Lofty. Cleland is located within the federal Division of Mayo, the state electoral district of Bragg and the local government area of the Adelaide Hills Council.', 'Electoral results for the district of Bragg | This is a list of electoral results for the Electoral district of Bragg in South Australian state elections.'], 'question': 'Bragg electoral district, incorporating the Kensington Park suburb, are associated with which South Australian city?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'South Australian city associated with Bragg electoral district'}) (input_keys=None), Example({'augmented': True, 'context': ['Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.', 'Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.', \"Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world's third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.\"], 'question': 'The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Kangchenjunga Singhi Kangri country'}) (input_keys=None)]], 2: [[], [Example({'question': 'What character did this actor, who also played a supporting role in Sleepy Hollow, play in the final six Harry Potter films?', 'answer': 'Professor Albus Dumbledore'}) (input_keys={'question'}), Example({'question': 'The musical sitcom with an album by the same name has what special feature on the album cover?', 'answer': 'actual crossword puzzle'}) (input_keys={'question'}), Example({'question': 'What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?', 'answer': '2016'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Alberta general election, 2015 | The 29th general election of Alberta, Canada, took place on May 5, 2015, following a request of Premier Jim Prentice to the Lieutenant Governor of Alberta Donald Ethell to dissolve the Legislative Assembly on April 7, 2015. This election elected members to the 29th Alberta Legislature. It was one of only four times that Alberta has changed governments.', \"Jenean Hampton | Jenean Michelle Hampton (born May 12, 1958) is the 57th and current Lieutenant Governor of Kentucky since December 8, 2015 after having won office as Republican gubernatorial nominee Matt Bevin's running-mate, in the 2015 gubernatorial election. She is the first African-American to hold any statewide office in Kentucky history, and only the third African-American woman to serve as lieutenant-governor of any U.S. state (after Jennette Bradley and Jennifer Carroll).\", 'Louisiana lieutenant gubernatorial election, 2015 | The Louisiana lieutenant gubernatorial election took place on October 24, 2015, to elect the Lieutenant Governor of Louisiana, with a runoff election held on November 21, 2015. Incumbent Republican Lieutenant Governor Jay Dardenne did not for re-election to a second full term in office. He instead ran for Governor. Billy Nungesser won the election defeating Kip Holden, despite a Democratic victory in the gubernatorial election, in which John Bel Edwards defeated David Vitter by a similar margin.', 'Indiana gubernatorial election, 2016 | The 2016 Indiana gubernatorial election was held November 8, 2016, to elect the Governor and Lieutenant Governor of Indiana, concurrently with the 2016 U.S. presidential election as well as elections to the United States Senate and elections to the United States House of Representatives and various state and local elections. The primaries were held on May 3, 2016. Republican Lieutenant Governor Eric Holcomb won the race with 51.4% of the vote.', 'Virginia lieutenant gubernatorial election, 2017 | The Virginia lieutenant gubernatorial election of 2017 will take place on November 7, 2017. After the party primary elections were held, the major party nominees are Jill Vogel (Republican) and Justin Fairfax (Democrat). The incumbent Lieutenant Governor, Democrat Ralph Northam, is not running for re-election in order to run for Governor.'], 'question': 'What race is the lieutenant governor of the state that held its primary for her election on May 19th, 2015?', 'rationale': 'The lieutenant governor of the state that held its primary for her election on May 19th, 2015 is Jenean Hampton, who is the 57th and current Lieutenant Governor of Kentucky since December 8, 2015. The primary election was held in Kentucky, and Jenean Hampton is the lieutenant governor of Kentucky.', 'answer': 'African-American'}) (input_keys=None), Example({'augmented': True, 'context': ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.', \"College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour\", 'Bass fishing | Bass fishing is the activity of angling for the North American gamefish known colloquially as the \"black bass\". There are numerous black bass species considered as gamefish in North America, including largemouth bass (\"Micropterus salmoides\"), smallmouth bass (\"Micropterus dolomieui\"), spotted bass or Kentucky bass (\"Micropterus punctulatus\"), and Guadalupe bass (order \"Perciformes\").', 'Cliff Crochet | Cliff Crochet (born August 1, 1983), also known as \"The Cajun Baby\", is a professional bass fisherman from Pierre Part, Louisiana. Crochet is a competitor on Bass Angler Sportsman Society\\'s (B.A.S.S.) Bassmaster Elite Series and has $368,837 in B.A.S.S. tournament winnings as of August 22, 2016. Crochet\\'s sponsors include: K2 Coolers, Huk, Skeeter, Yamaha Motor Company, Lowrance Electronics, Falcon Rods, Luck E Strike, Seaguar, Rat-L-Trap, Humdinger Spinnerbaits, Power-Pole. Since 2010, Crochet has competed in three Bassmaster Classic tournaments. Crochet\\'s boat for the 2016 season is a 2016 Skeeter FX 21 with a 250 hp Yamaha SHO outboard engine.'], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': 'The context mentions Carhartt Bassmaster College Bass Fishing Series, which is a college bass fishing tournament series. This series is associated with the clothing company Carhartt, which was founded in 1889.', 'answer': 'Carhartt'}) (input_keys=None), Example({'augmented': True, 'context': ['Forrest Tucker | Forrest Meredith Tucker (February 12, 1919 – October 25, 1986) was an American actor in both movies and television who appeared in nearly a hundred films. Tucker worked as a vaudeville straight man, aged fifteen years old. A mentor provided funds and contacts for a trip to California, where party hostess Cobina Wright persuaded guest Wesley Ruggles to give Tucker a screen test, based on his photogenic good looks, thick wavy hair and height of six feet, five inches - putting him into the same league as John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness\\'s younger brother. Tucker was a sight reader who needed only one take. His film career started well despite a perception in most Hollywood studios that blond men were not photogenic. After twenty years spent mainly in Westerns and action roles, he returned to his roots, showing versatility as a comedic and stage musical actor. In the TV show \"F Troop\", he became identified with the character of Cavalry Sgt. Morgan O\\'Rourke (a manipulative character quite similar to Phil Silvers\\' role as MSgt Ernie Bilko). Tucker struggled with a drinking problem that began to affect his performances in the later years of his career.', 'Emergency Landing (1941 film) | Emergency Landing (a.k.a. Robot Pilot) is a 1941 American aviation spy-fi romantic screwball comedy film directed by William Beaudine. The film stars Forrest Tucker in his second film and in his first leading role with co-stars Carol Hughes and Evelyn Brent. \"Emergency Landing\" features lots of mismatched stock footage of various types of aircraft.', 'Forrest Tucker (criminal) | Forrest Silva \"Woody\" Tucker (June 23, 1920 – May 29, 2004) was an American career criminal first imprisoned at age 15 who spent the rest of his life in and out of jail. He is best known as an escape artist, having escaped from prison 18 times successfully and 12 times unsuccessfully.', 'Fess Parker | Fess Elisha Parker Jr. (August 16, 1924 – March 18, 2010) was an American film and television actor best known for his portrayals of Davy Crockett in the Walt Disney 1955–1956 TV miniseries and as Daniel Boone in a television series from 1964 to 1970. He was also known as a winemaker and resort owner-operator.', 'Davy Crockett, King of the Wild Frontier | Davy Crockett, King of the Wild Frontier is a 1955 live-action Walt Disney adventure film starring Fess Parker as Davy Crockett. This film is an edited compilation of the first three stories from the Disney television miniseries \"Davy Crockett\" :', 'Davy Crockett (miniseries) | Davy Crockett is a five-part serial which aired on ABC from 1954–1955 in one-hour episodes, on the \"Disneyland\" series. The series stars Fess Parker as real-life frontiersman Davy Crockett and Buddy Ebsen as his friend, George Russel.'], 'question': \"Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\", 'rationale': \"From the context, we know that Forrest Tucker was an American actor who appeared in nearly a hundred films. We also know that he was compared to John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness's younger brother. We need to find the actor that played a character in the 1955-1956 Walt Disney TV miniseries, which is Fess Parker.\", 'answer': 'Davy Crockett'}) (input_keys=None)], [Example({'augmented': True, 'context': ['Forrest Tucker | Forrest Meredith Tucker (February 12, 1919 – October 25, 1986) was an American actor in both movies and television who appeared in nearly a hundred films. Tucker worked as a vaudeville straight man, aged fifteen years old. A mentor provided funds and contacts for a trip to California, where party hostess Cobina Wright persuaded guest Wesley Ruggles to give Tucker a screen test, based on his photogenic good looks, thick wavy hair and height of six feet, five inches - putting him into the same league as John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness\\'s younger brother. Tucker was a sight reader who needed only one take. His film career started well despite a perception in most Hollywood studios that blond men were not photogenic. After twenty years spent mainly in Westerns and action roles, he returned to his roots, showing versatility as a comedic and stage musical actor. In the TV show \"F Troop\", he became identified with the character of Cavalry Sgt. Morgan O\\'Rourke (a manipulative character quite similar to Phil Silvers\\' role as MSgt Ernie Bilko). Tucker struggled with a drinking problem that began to affect his performances in the later years of his career.', 'Emergency Landing (1941 film) | Emergency Landing (a.k.a. Robot Pilot) is a 1941 American aviation spy-fi romantic screwball comedy film directed by William Beaudine. The film stars Forrest Tucker in his second film and in his first leading role with co-stars Carol Hughes and Evelyn Brent. \"Emergency Landing\" features lots of mismatched stock footage of various types of aircraft.', 'Forrest Tucker (criminal) | Forrest Silva \"Woody\" Tucker (June 23, 1920 – May 29, 2004) was an American career criminal first imprisoned at age 15 who spent the rest of his life in and out of jail. He is best known as an escape artist, having escaped from prison 18 times successfully and 12 times unsuccessfully.', 'Fess Parker | Fess Elisha Parker Jr. (August 16, 1924 – March 18, 2010) was an American film and television actor best known for his portrayals of Davy Crockett in the Walt Disney 1955–1956 TV miniseries and as Daniel Boone in a television series from 1964 to 1970. He was also known as a winemaker and resort owner-operator.', 'Davy Crockett, King of the Wild Frontier | Davy Crockett, King of the Wild Frontier is a 1955 live-action Walt Disney adventure film starring Fess Parker as Davy Crockett. This film is an edited compilation of the first three stories from the Disney television miniseries \"Davy Crockett\" :', 'Davy Crockett (miniseries) | Davy Crockett is a five-part serial which aired on ABC from 1954–1955 in one-hour episodes, on the \"Disneyland\" series. The series stars Fess Parker as real-life frontiersman Davy Crockett and Buddy Ebsen as his friend, George Russel.'], 'question': \"Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\", 'rationale': 'We need to find the actor who played the character in the 1955-1956 Walt Disney TV miniseries. From the context, we know that the miniseries was about Davy Crockett and starred Fess Parker as Davy Crockett.', 'answer': 'Davy Crockett'}) (input_keys=None), Example({'augmented': True, 'context': ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.', \"College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour\", 'Bass fishing | Bass fishing is the activity of angling for the North American gamefish known colloquially as the \"black bass\". There are numerous black bass species considered as gamefish in North America, including largemouth bass (\"Micropterus salmoides\"), smallmouth bass (\"Micropterus dolomieui\"), spotted bass or Kentucky bass (\"Micropterus punctulatus\"), and Guadalupe bass (order \"Perciformes\").', 'Cliff Crochet | Cliff Crochet (born August 1, 1983), also known as \"The Cajun Baby\", is a professional bass fisherman from Pierre Part, Louisiana. Crochet is a competitor on Bass Angler Sportsman Society\\'s (B.A.S.S.) Bassmaster Elite Series and has $368,837 in B.A.S.S. tournament winnings as of August 22, 2016. Crochet\\'s sponsors include: K2 Coolers, Huk, Skeeter, Yamaha Motor Company, Lowrance Electronics, Falcon Rods, Luck E Strike, Seaguar, Rat-L-Trap, Humdinger Spinnerbaits, Power-Pole. Since 2010, Crochet has competed in three Bassmaster Classic tournaments. Crochet\\'s boat for the 2016 season is a 2016 Skeeter FX 21 with a 250 hp Yamaha SHO outboard engine.'], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Context: [1] «Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.»\\n\\n[2] «College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour»\\n\\n[3] «G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.»\\n\\nReasoning: Let's think step by step in order to\", 'answer': 'Carhartt'}) (input_keys=None), Example({'question': 'How many years did Lenny Lyles spend with the organization dramatized in the filme \"The insder\"?', 'answer': '27'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Will Finn | Will Finn (born November 1, 1958) is an American animator, voice actor, and director. His work in animation includes characters from Disney and Don Bluth films such as \"The Secret of NIMH\", \"Oliver & Company\", \"The Little Mermaid\", \"The Rescuers Down Under\", and \"Pocahontas\". His characters includes Cogsworth in \"Beauty and the Beast\", Iago in \"Aladdin\", and Laverne in \"The Hunchback of Notre Dame\". Finn wrote and directed \"Home on the Range\" and did some voice acting the Hollywood Fish in \"Chicken Little\". In 2006, Finn directed the computer animated short Hammy\\'s Boomerang Adventure, a spin-off of \"Over the Hedge\".', 'Over the Hedge (film) | Over the Hedge is a 2006 American computer-animated comedy film, based on the characters from the United Media comic strip of the same name. Directed by Tim Johnson and Karey Kirkpatrick, and produced by Bonnie Arnold, it was released in the United States on May 19, 2006. The film was produced by DreamWorks Animation and distributed through Paramount Pictures. The film features the voices of Bruce Willis, Garry Shandling, Steve Carell, William Shatner, Wanda Sykes, and Nick Nolte. It is the first DreamWorks Animation film to be distributed by Paramount Pictures, which acquired the live-action DreamWorks studio in 2006. The film earned $336 million on an $80 million budget.', 'Tim Johnson (film director) | Tim Johnson (born August 27, 1961) is an American animation director. Johnson has directed in many films such as \"Antz\", \"\" and \"Over the Hedge\". Recently, he directed the DreamWorks 3D film \"Home.\"', 'Hammy\\'s Boomerang Adventure | Hammy\\'s Boomerang Adventure is a computer-animated short comedy film based on the film \"Over the Hedge\", which was based on a comic strip of the same name. It can be found on the movie\\'s DVD or the \"Madly Madagascar\" DVD as a bonus feature.', 'Boomerang (2015 film) | Boomerang is a 2015 French drama film directed by François Favrat and starring Laurent Lafitte, Mélanie Laurent, Audrey Dana and Wladimir Yordanoff. It is an adaptation of Tatiana de Rosnay\\'s novel \"A Secret Kept\" (alternative title:\"Boomerang\"). The film was released in theatres on 23 September 2015.', 'Over the Hedge: Hammy Goes Nuts! | Over the Hedge: Hammy Goes Nuts! is a video game published by Activision based on characters from the film \"Over the Hedge\". The Game Boy Advance (GBA) version, a miniature golf game, was developed by Vicarious Visions and released on October 10, 2006 in North America. The Nintendo DS and PlayStation Portable (PSP) versions, an action adventure game and a platform game respectively, were both developed by Amaze Entertainment and were released on October 24, 2006 and November 21, 2006. It is the second game to be based on the film.'], 'question': 'When was the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed?', 'rationale': 'Context: N/A\\n\\nQuestion: When was the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed?\\n\\nReasoning: Let\\'s think step by step in order to', 'answer': '2006'}) (input_keys=None), Example({'question': 'What is the capacity of the La Liga stadium in which The Madrid Derby team plays their home games?', 'answer': '68,000'}) (input_keys={'question'}), Example({'question': 'The Equestrian statue of Marshal Mannerheim was erected as a monument to the leader of the Whites during what event?', 'answer': 'Finnish Civil War'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.', 'Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.', \"Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world's third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.\"], 'question': 'The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?', 'rationale': 'Context: [1] «Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.»\\n\\n[2] «Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.»\\n\\n[3] «Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world\\'s third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.»\\n\\nReasoning: Let\\'s think step by step in order to', 'answer': 'India'}) (input_keys=None), Example({'question': 'Are Beth Gibbons and Ian Curtis the same nationality?', 'answer': 'yes'}) (input_keys={'question'}), Example({'question': \"When did Taylor Swift's satirical song get released to the radio?\", 'answer': 'February 9, 2015'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Wheat Field with Cypresses | A Wheatfield with Cypresses (occasionally called \"A Cornfield with Cypresses\") is any of three similar 1889 oil paintings by Vincent van Gogh, as part of his wheat field series. All were executed at the Saint-Paul-de-Mausole mental asylum at Saint-Rémy near Arles, France, where Van Gogh was voluntarily a patient from May 1889 to May 1890. The works were inspired by the view from the window at the asylum towards the Alpilles mountains.', 'Green Wheat Field with Cypress | Green Wheat Field with Cypress (French: Champ de blé vert avec cyprès) is an oil on canvas painting by a Dutch Post-Impressionist Vincent van Gogh. It was completed in 1889, while van Gogh was voluntarily incarcerated at the asylum of St. Paul near Saint-Rémy in Provence. Several paintings of wheat fields with cypresses were made when van Gogh was able to leave the asylum grounds and explore the local landscape. Besides a fondness for cypresses, van Gogh had a special affinty with wheat fields; he depicted them dozens of times over the years; to Vincent they symbolized the cycle of life and death, and he found in them both solace and inspiration.', 'Olive Trees (Van Gogh series) | Vincent van Gogh painted at least 18 paintings of olive trees, mostly in Saint-Rémy-de-Provence in 1889. At his own request, he lived at an asylum there from May 1889 through May 1890 painting the gardens of the asylum and, when he had permission to venture outside its walls, nearby olive trees, cypresses and wheat fields.', 'Wheat Fields (Van Gogh series) | The Wheat Fields is a series of dozens of paintings by Vincent van Gogh, borne out of his religious studies and sermons, connection to nature, appreciation of manual laborers and desire to provide a means of offering comfort to others. The wheat field works demonstrate his progression as an artist from the drab \"Wheat Sheaves\" made in 1885 in the Netherlands to the colorful, dramatic paintings from Arles, Saint-Rémy and Auvers-sur-Oise of rural France.'], 'question': 'What was the series inspired by the appreciation of manual laborers that Wheat Field with Cypresses is a part of?', 'rationale': \"Context: N/A\\n\\nQuestion: What was the series inspired by the appreciation of manual laborers that Wheat Field with Cypresses is a part of?\\n\\nReasoning: Let's think step by step in order to\", 'answer': 'Wheat Fields'}) (input_keys=None), Example({'question': 'What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?', 'answer': '2016'}) (input_keys={'question'}), Example({'question': 'How many shops does the larger of the Miami International Mall and Dolphin Mall have?', 'answer': '240'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Butch Reed | Bruce Franklin Reed (born July 11, 1954) is an American retired professional wrestler and football player, better known by the ring name Butch Reed. He is best known for his appearances with the World Wrestling Federation and World Championship Wrestling between 1986 and 1992.', 'Doug Reed (futsal player) | Doug Reed is a professional futsal player who is currently playing for Futsal Dinamo and England national futsal team. Previously he has played for Lanzarote Tias Yaiza (Spain), AEL Limassol, MNK Uspinjača, MNK Smederevo and Manchester Futsal Club. In 2011, he won the FA Futsal Cup with Manchester Futsal Club and was awarded Player of the Tournament in the same competition. In the FA National Futsal League, he won several FA National League Championships and FA Futsal Cup medals. Reed also works as a commentator for Sony SIX, Eurosport and BT Sport, and has worked with the emerging US Professional Futsal League as well as Indian Premier Futsal.', 'Doom (professional wrestling) | Doom was a professional wrestling tag team composed of Ron Simmons and Butch Reed. They competed from 1989 to 1991 in the National Wrestling Alliance (NWA) and World Championship Wrestling (WCW).', 'Robert Reed (gridiron football) | Robert E. Reed, Jr. (born February 23, 1943) is a former American football offensive lineman in the National Football League for the Washington Redskins. He played college football at Tennessee State University and was drafted in the sixteenth round of the 1965 NFL Draft. Reed was also selected in the 12th round of the 1965 AFL Draft by the Houston Oilers.', 'Reed Doughty | Reed Doughty (born November 4, 1982) is a former American football safety who played in the National Football League. He was drafted by the Washington Redskins in the sixth round, with the 173rd overall pick, of the 2006 NFL Draft. He played college football at the University of Northern Colorado.'], 'question': 'What other sport did Butch Reed, of the wrestling tag team Doom, play professionally?', 'rationale': \"Context: N/A\\n\\nQuestion: What other sport did Butch Reed, of the wrestling tag team Doom, play professionally?\\n\\nReasoning: Let's think step by step in order to\", 'answer': 'football'}) (input_keys=None), Example({'question': 'Which magazine is still in print American Heritage or High Times?', 'answer': 'High Times'}) (input_keys={'question'}), Example({'question': 'What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?', 'answer': '2016'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.', 'Ebenezer Howard | Sir Ebenezer Howard {\\'1\\': \", \\'2\\': \", \\'3\\': \", \\'4\\': \"} ( ; 29 January 1850 – 1 May 1928), the English founder of the garden city movement, is known for his publication \"\" (1898), the description of a utopian city in which people live harmoniously together with nature. The publication resulted in the founding of the garden city movement, and the building of the First Garden City, Letchworth Garden City, commenced in 1903.', 'Eugene Reynal | Eugene Reynal was an American Publisher and in 1933 Founder of Blue Ribbon Books of Garden City, New York and Reynal and Hitchcock of New York, New York.'], 'question': 'Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?', 'rationale': 'Context: [1] «Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.»\\n\\n[2] «A.T. Stewart | Alexander Turney Stewart (October 2, 1792 – April 10, 1876) was an American businessman, politician, and philanthropist. He was a prominent figure in New York City during the mid-19th century, and was known as the \"Merchant Prince\".»\\n\\n[3] «Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.»\\n\\nQuestion: Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?\\n\\nReasoning: Let\\'s think step by step in order to', 'answer': 'A.T. Stewart'}) (input_keys=None), Example({'augmented': True, 'context': ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.', \"College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour\", 'Bass fishing | Bass fishing is the activity of angling for the North American gamefish known colloquially as the \"black bass\". There are numerous black bass species considered as gamefish in North America, including largemouth bass (\"Micropterus salmoides\"), smallmouth bass (\"Micropterus dolomieui\"), spotted bass or Kentucky bass (\"Micropterus punctulatus\"), and Guadalupe bass (order \"Perciformes\").', 'Cliff Crochet | Cliff Crochet (born August 1, 1983), also known as \"The Cajun Baby\", is a professional bass fisherman from Pierre Part, Louisiana. Crochet is a competitor on Bass Angler Sportsman Society\\'s (B.A.S.S.) Bassmaster Elite Series and has $368,837 in B.A.S.S. tournament winnings as of August 22, 2016. Crochet\\'s sponsors include: K2 Coolers, Huk, Skeeter, Yamaha Motor Company, Lowrance Electronics, Falcon Rods, Luck E Strike, Seaguar, Rat-L-Trap, Humdinger Spinnerbaits, Power-Pole. Since 2010, Crochet has competed in three Bassmaster Classic tournaments. Crochet\\'s boat for the 2016 season is a 2016 Skeeter FX 21 with a 250 hp Yamaha SHO outboard engine.'], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Context: [1] «Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.»\\n\\n[2] «College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour»\\n\\n[3] «G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.»\\n\\nReasoning: Let's think step by step in order to\", 'answer': 'Carhartt'}) (input_keys=None), Example({'augmented': True, 'context': ['Stitch\\'s Great Escape! | Stitch\\'s Great Escape! is a Tomorrowland attraction at the Magic Kingdom theme park within the Walt Disney World Resort. It is a \"theater-in-the-round\" experience starring the title alien from Walt Disney Animation Studios\\' 2002 film \"Lilo & Stitch\". It opened November 16, 2004 and is the fourth attraction to occupy the site in Tomorrowland. Many of the animators who worked on \"Lilo & Stitch\" partnered with Walt Disney Imagineering for Stitch\\'s Great Escape! The attraction is a replacement of The ExtraTERRORestrial Alien Encounter, which formerly occupied the building the attraction is housed in. On September 21, 2016, it was announced that the attraction would be switching from a daily operated attraction to a seasonally operated one, depending on attendance, starting October 2, 2016.', 'Great Escape (amusement park) | Great Escape (sometimes referred to as Six Flags Great Escape) is an amusement and water park owned and operated by Six Flags Entertainment Corp. It is located approximately 60 mi north of Albany, in Queensbury, New York, but is advertised as being in Lake George, New York, a popular tourist and vacation spot nearby. It is one of two Six Flags parks not to be officially branded with the \"Six Flags\" name (La Ronde in Montreal, Quebec, Canada being the other).', 'Houdini\\'s Great Escape | Houdini\\'s Great Escape, also known as \"Boo-dini\" during Fright Fest and \"Houdini - The Great Escape\" at Six Flags New England, are indoor Vekoma Madhouse attractions at Six Flags Great Adventure and Six Flags New England. The ride is located in Great Adventure\\'s Fantasy Forest section and New England\\'s Crack Axle Canyon area. The attraction opened at both parks in 1999, and was Standing but not operating (SBNO) at Great Adventure from 2008 - 2009 but was opened again due to popular demand.', 'Magic Kingdom | Magic Kingdom is a theme park at the Walt Disney World Resort in Bay Lake, Florida, near Orlando. Owned and operated by The Walt Disney Company through its Parks and Resorts division, the park opened on October 1, 1971, as the first of four theme parks at the resort. Initialized by Walt Disney and designed by WED Enterprises, its layout and attractions are based on Disneyland Park in Anaheim, California, and is dedicated to fairy tales and Disney characters.', 'Magic Kingdom (disambiguation) | The Magic Kingdom is a theme park at the Walt Disney World Resort in Florida.', 'List of Magic Kingdom attractions | The Magic Kingdom is a theme park located at the Walt Disney World Resort in Florida. Below is a list of the current attractions found therein, arranged by \"land\" and with brief descriptions.'], 'question': 'Which theme park has Stitch’s Great Escape which replaced the attraction ExtraTERRORestial Alien Encounter?', 'rationale': \"We need to find the theme park that has Stitch's Great Escape, which replaced the attraction ExtraTERRORestial Alien Encounter. The context mentions that Stitch's Great Escape is located in the Magic Kingdom theme park.\", 'answer': 'Magic Kingdom'}) (input_keys=None)], [Example({'augmented': True, 'context': ['Simon Arora | Simon Arora (born November 1969) is a British billionaire businessman, CEO of the retail chain B & M.', 'Samir Arora | Samir Arora (born November 5, 1965) is an American businessman and former CEO of Mode Media (formerly Glam Media) from 2003 to April 2016. He was CEO and Chairman of the web design company NetObjects, Inc. from 1995 to 2001.', 'Dinesh Arora | Dinesh Arora (born 23 December 1976) is an Indian Administrative Service (IAS) officer, 2002 Kerala cadre, who previously worked as an Executive Director, Rural Electrification Corporation (REC) and CEO RECPDCL.', 'Bolliger & Mabillard | Bolliger & Mabillard, officially Bolliger & Mabillard Consulting Engineers Inc., and often abbreviated B&M, is a roller coaster design consultancy based in Monthey, Switzerland. The company was founded in 1988 by Walter Bolliger and Claude Mabillard, with Bolliger as president and Mabillard as vice-president. B&M has built over a hundred roller coasters around the world and pioneered several new ride technologies, most notably the inverted roller coaster. In North America, B&M coaster designs have been manufactured by Ohio company Clermont Steel Fabricators since 1990. B&M has grown significantly since its founding, originally employing four to as many as 37 in 2012, consisting primarily of engineers and draftsmen. In 2016, the company completed its 100th roller coaster.', \"Rakefet Russak-Aminoach | Rakefet Russak-Aminoach (Hebrew: רקפת רוסק עמינח\\u200e \\u200e ; born February 1, 1966) is the current President and CEO of Bank Leumi le-Israel B.M., a position she has held since May 1, 2012. In September 2015, she was listed in Fortune Magazine’s list of The World's 100 Most Powerful Women.\"], 'question': 'What company formed in 1978 has Simon Arora as its CEO?', 'rationale': 'There is no information about Simon Arora being the CEO of a company formed in 1978 in the context. However, we can find the information about Simon Arora being the CEO of B & M in the context.', 'answer': 'B & M'}) (input_keys=None), Example({'augmented': True, 'context': ['Electoral district of Bragg | Bragg is an electoral district of the House of Assembly in the Australian state of South Australia. The seat of Bragg is named after the eminent physicists Bragg – William Henry and his son, William Lawrence. The electorate is largely urban and encompasses a significant portion of the City of Burnside, stretching from the east parklands of Adelaide into the Adelaide Hills. After a redistribution following the 2006 election, the boundary moved eastwards to include suburbs that had formerly been in the seat of Heysen and now borders Kavel. Bragg currently includes the metropolitan suburbs of Beaumont, Burnside, Dulwich, Erindale, Hazelwood Park, Heathpool, Kensington Park, Leabrook, Linden Park, Rose Park, Marryatville, Skye, St Georges, Stonyfell, Toorak Gardens, Tusmore, Wattle Park and parts of Glen Osmond, and in the hills it includes Crafers, Cleland, Greenhill, Mount Osmond, Piccadilly, Summertown and parts of Ashton, Basket Range, Carey Gully, Horsnell Gully, Leawood Gardens and Uraidla.', 'Cleland, South Australia | Cleland is a suburb in South Australia located in the Adelaide metropolitan area about 10 km south-east of the Adelaide city centre. Its boundaries were created in October 2001, with additional land being added in 2010 from the adjoining suburb of Crafers. Its name is derived from the Cleland Conservation Park. The principal land use within the locality is conservation with the majority of its land area being occupied by the Cleland Conservation Park. Places within its extent include the summit of Mount Lofty. Cleland is located within the federal Division of Mayo, the state electoral district of Bragg and the local government area of the Adelaide Hills Council.', 'Electoral results for the district of Bragg | This is a list of electoral results for the Electoral district of Bragg in South Australian state elections.', 'Electoral district of Burnside | Burnside was an electoral district of the House of Assembly in the Australian state of South Australia from 1938 to 1970. Before 1938 the Burnside area was represented by the three-seat multi-member electorate of Sturt. It was abolished in the 1970 parliamentary reforms, and was replaced with Bragg.'], 'question': 'Bragg electoral district, incorporating the Kensington Park suburb, are associated with which South Australian city?', 'rationale': \"Context: N/A\\n\\nQuestion: Bragg electoral district, incorporating the Kensington Park suburb, are associated with which South Australian city?\\n\\nReasoning: Let's think step by step in order to\", 'answer': 'Adelaide'}) (input_keys=None), Example({'augmented': True, 'context': ['Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.', 'Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.', \"Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world's third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.\"], 'question': 'The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?', 'rationale': 'Context: [1] «Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.»\\n\\n[2] «Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.»\\n\\n[3] «Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world\\'s third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.»\\n\\nReasoning: Let\\'s think step by step in order to', 'answer': 'India'}) (input_keys=None)]]} with Mean = 52.0\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 42 / 100 (42.0): 100%|██████████| 100/100 [00:27<00:00, 3.69it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "UPDATING BEST SCORE WITH 42.0\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 64 / 150 (42.7): 100%|██████████| 150/150 [01:01<00:00, 2.44it/s]\n",+ "[I 2024-09-23 19:09:24,681] Trial 0 finished with value: 52.0 and parameters: {'0_predictor_instruction': 1, '0_predictor_demos': 2, '1_predictor_instruction': 6, '1_predictor_demos': 2, '2_predictor_instruction': 8, '2_predictor_demos': 6}. Best is trial 0 with value: 52.0.\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "CANDIDATE PROGRAM:\n",+ "Predictor 0\n",+ "i: Given the context and question, generate a concise search query that accurately captures the essence of the query and facilitates the retrieval of relevant information.\n",+ "p: Search Query:\n",+ "\n",+ "\n",+ "Predictor 1\n",+ "i: Generate a search query that accurately captures the essence of the question and enables the retrieval of relevant information from the dataset, taking into account the context provided, the question being asked, and the relationship between the two.\n",+ "p: Search Query:\n",+ "\n",+ "\n",+ "Predictor 2\n",+ "i: Given a context and a question, generate a logical and coherent answer that accurately addresses the query by meticulously analyzing the provided context, identifying relevant information, and applying critical thinking to provide a precise and informative response.\n",+ "p: Answer:\n",+ "\n",+ "\n",+ "...\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 11 / 25 (44.0): 100%|██████████| 25/25 [00:15<00:00, 1.66it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "FULL TRACE\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 1 / 1 (100.0): 100%|██████████| 1/1 [00:03<00:00, 3.00s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "\n",+ "\n",+ "\n",+ "Generate a search query that accurately captures the essence of the question and enables the retrieval of relevant information from the dataset, taking into account the context provided, the question being asked, and the relationship between the two.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Search Query: ${search_query}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.»\n",+ "[2] «Ebenezer Howard | Sir Ebenezer Howard {'1': \", '2': \", '3': \", '4': \"} ( ; 29 January 1850 – 1 May 1928), the English founder of the garden city movement, is known for his publication \"\" (1898), the description of a utopian city in which people live harmoniously together with nature. The publication resulted in the founding of the garden city movement, and the building of the First Garden City, Letchworth Garden City, commenced in 1903.»\n",+ "[3] «Eugene Reynal | Eugene Reynal was an American Publisher and in 1933 Founder of Blue Ribbon Books of Garden City, New York and Reynal and Hitchcock of New York, New York.»\n",+ "\n",+ "Question: Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Garden City Hotel founder\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.»\n",+ "[2] «Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford's second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.»\n",+ "[3] «G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.»\n",+ "\n",+ "Question: Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Carhartt bass fishing\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Stitch's Great Escape! | Stitch's Great Escape! is a Tomorrowland attraction at the Magic Kingdom theme park within the Walt Disney World Resort. It is a \"theater-in-the-round\" experience starring the title alien from Walt Disney Animation Studios' 2002 film \"Lilo & Stitch\". It opened November 16, 2004 and is the fourth attraction to occupy the site in Tomorrowland. Many of the animators who worked on \"Lilo & Stitch\" partnered with Walt Disney Imagineering for Stitch's Great Escape! The attraction is a replacement of The ExtraTERRORestrial Alien Encounter, which formerly occupied the building the attraction is housed in. On September 21, 2016, it was announced that the attraction would be switching from a daily operated attraction to a seasonally operated one, depending on attendance, starting October 2, 2016.»\n",+ "[2] «Great Escape (amusement park) | Great Escape (sometimes referred to as Six Flags Great Escape) is an amusement and water park owned and operated by Six Flags Entertainment Corp. It is located approximately 60 mi north of Albany, in Queensbury, New York, but is advertised as being in Lake George, New York, a popular tourist and vacation spot nearby. It is one of two Six Flags parks not to be officially branded with the \"Six Flags\" name (La Ronde in Montreal, Quebec, Canada being the other).»\n",+ "[3] «Houdini's Great Escape | Houdini's Great Escape, also known as \"Boo-dini\" during Fright Fest and \"Houdini - The Great Escape\" at Six Flags New England, are indoor Vekoma Madhouse attractions at Six Flags Great Adventure and Six Flags New England. The ride is located in Great Adventure's Fantasy Forest section and New England's Crack Axle Canyon area. The attraction opened at both parks in 1999, and was Standing but not operating (SBNO) at Great Adventure from 2008 - 2009 but was opened again due to popular demand.»\n",+ "\n",+ "Question: Which theme park has Stitch’s Great Escape which replaced the attraction ExtraTERRORestial Alien Encounter?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Magic Kingdom theme park\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Archibald Wavell, 1st Earl Wavell | Field Marshal Archibald Percival Wavell, 1st Earl Wavell, {'1': \", '2': \", '3': \", '4': \"} (5 May 1883 – 24 May 1950) was a senior officer of the British Army. He served in the Second Boer War, the Bazar Valley Campaign and the Great War, during which he was wounded in the Second Battle of Ypres. He served in the Second World War, initially as Commander-in-Chief Middle East, in which role he led British forces to victory over the Italians in western Egypt and eastern Libya during Operation \"Compass\" in December 1940, only to be defeated by the German Army in the Western Desert in April 1941. He served as Commander-in-Chief, India, from July 1941 until June 1943 (apart from a brief tour as Commander of ABDACOM) and then served as Viceroy of India until his retirement in February 1947.»\n",+ "[2] «John French, 1st Earl of Ypres | Field Marshal John Denton Pinkstone French, 1st Earl of Ypres (28 September 1852 – 22 May 1925), known as Sir John French from 1901 to 1916, and as The Viscount French between 1916 and 1922, was a senior British Army officer. Born in Kent to an Anglo-Irish family, he saw brief service as a midshipman in the Royal Navy, before becoming a cavalry officer. He achieved rapid promotion and distinguished himself on the Gordon Relief Expedition. French had a considerable reputation as a womaniser throughout his life and his career nearly ended when he was cited in the divorce of a brother officer whilst in India in the early 1890s.»\n",+ "[3] «John French, 2nd Earl of Ypres | John Richard Lowndes French, 2nd Earl of Ypres (6 July 1881 – 5 April 1958) was the son of the British field marshal and the first commander of the British Expeditionary Force (BEF) in World War I Sir John French.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Search Query. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Generate a search query that accurately captures the essence of the question and enables the retrieval of relevant information from the dataset, taking into account the context provided, the question being asked, and the relationship between the two.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Search Query: ${search_query}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.»\n",+ "[2] «Ebenezer Howard | Sir Ebenezer Howard {'1': \", '2': \", '3': \", '4': \"} ( ; 29 January 1850 – 1 May 1928), the English founder of the garden city movement, is known for his publication \"\" (1898), the description of a utopian city in which people live harmoniously together with nature. The publication resulted in the founding of the garden city movement, and the building of the First Garden City, Letchworth Garden City, commenced in 1903.»\n",+ "[3] «Eugene Reynal | Eugene Reynal was an American Publisher and in 1933 Founder of Blue Ribbon Books of Garden City, New York and Reynal and Hitchcock of New York, New York.»\n",+ "\n",+ "Question: Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Garden City Hotel founder\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.»\n",+ "[2] «Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford's second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.»\n",+ "[3] «G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.»\n",+ "\n",+ "Question: Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Carhartt bass fishing\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Stitch's Great Escape! | Stitch's Great Escape! is a Tomorrowland attraction at the Magic Kingdom theme park within the Walt Disney World Resort. It is a \"theater-in-the-round\" experience starring the title alien from Walt Disney Animation Studios' 2002 film \"Lilo & Stitch\". It opened November 16, 2004 and is the fourth attraction to occupy the site in Tomorrowland. Many of the animators who worked on \"Lilo & Stitch\" partnered with Walt Disney Imagineering for Stitch's Great Escape! The attraction is a replacement of The ExtraTERRORestrial Alien Encounter, which formerly occupied the building the attraction is housed in. On September 21, 2016, it was announced that the attraction would be switching from a daily operated attraction to a seasonally operated one, depending on attendance, starting October 2, 2016.»\n",+ "[2] «Great Escape (amusement park) | Great Escape (sometimes referred to as Six Flags Great Escape) is an amusement and water park owned and operated by Six Flags Entertainment Corp. It is located approximately 60 mi north of Albany, in Queensbury, New York, but is advertised as being in Lake George, New York, a popular tourist and vacation spot nearby. It is one of two Six Flags parks not to be officially branded with the \"Six Flags\" name (La Ronde in Montreal, Quebec, Canada being the other).»\n",+ "[3] «Houdini's Great Escape | Houdini's Great Escape, also known as \"Boo-dini\" during Fright Fest and \"Houdini - The Great Escape\" at Six Flags New England, are indoor Vekoma Madhouse attractions at Six Flags Great Adventure and Six Flags New England. The ride is located in Great Adventure's Fantasy Forest section and New England's Crack Axle Canyon area. The attraction opened at both parks in 1999, and was Standing but not operating (SBNO) at Great Adventure from 2008 - 2009 but was opened again due to popular demand.»\n",+ "\n",+ "Question: Which theme park has Stitch’s Great Escape which replaced the attraction ExtraTERRORestial Alien Encounter?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Magic Kingdom theme park\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Archibald Wavell, 1st Earl Wavell | Field Marshal Archibald Percival Wavell, 1st Earl Wavell, {'1': \", '2': \", '3': \", '4': \"} (5 May 1883 – 24 May 1950) was a senior officer of the British Army. He served in the Second Boer War, the Bazar Valley Campaign and the Great War, during which he was wounded in the Second Battle of Ypres. He served in the Second World War, initially as Commander-in-Chief Middle East, in which role he led British forces to victory over the Italians in western Egypt and eastern Libya during Operation \"Compass\" in December 1940, only to be defeated by the German Army in the Western Desert in April 1941. He served as Commander-in-Chief, India, from July 1941 until June 1943 (apart from a brief tour as Commander of ABDACOM) and then served as Viceroy of India until his retirement in February 1947.»\n",+ "[2] «John French, 1st Earl of Ypres | Field Marshal John Denton Pinkstone French, 1st Earl of Ypres (28 September 1852 – 22 May 1925), known as Sir John French from 1901 to 1916, and as The Viscount French between 1916 and 1922, was a senior British Army officer. Born in Kent to an Anglo-Irish family, he saw brief service as a midshipman in the Royal Navy, before becoming a cavalry officer. He achieved rapid promotion and distinguished himself on the Gordon Relief Expedition. French had a considerable reputation as a womaniser throughout his life and his career nearly ended when he was cited in the divorce of a brother officer whilst in India in the early 1890s.»\n",+ "[3] «John French, 2nd Earl of Ypres | John Richard Lowndes French, 2nd Earl of Ypres (6 July 1881 – 5 April 1958) was the son of the British field marshal and the first commander of the British Expeditionary Force (BEF) in World War I Sir John French.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Please provide the output field Search Query. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field Search Query.\n",+ "\n",+ "Search Query: Archibald Wavell birthdate\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Given a context and a question, generate a logical and coherent answer that accurately addresses the query by meticulously analyzing the provided context, identifying relevant information, and applying critical thinking to provide a precise and informative response.\n",+ "\n",+ "---\n",+ "\n",+ "Question: How many years did Lenny Lyles spend with the organization dramatized in the filme \"The insder\"?\n",+ "\n",+ "Answer: 27\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Answer: ${answer}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Forrest Tucker | Forrest Meredith Tucker (February 12, 1919 – October 25, 1986) was an American actor in both movies and television who appeared in nearly a hundred films. Tucker worked as a vaudeville straight man, aged fifteen years old. A mentor provided funds and contacts for a trip to California, where party hostess Cobina Wright persuaded guest Wesley Ruggles to give Tucker a screen test, based on his photogenic good looks, thick wavy hair and height of six feet, five inches - putting him into the same league as John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness's younger brother. Tucker was a sight reader who needed only one take. His film career started well despite a perception in most Hollywood studios that blond men were not photogenic. After twenty years spent mainly in Westerns and action roles, he returned to his roots, showing versatility as a comedic and stage musical actor. In the TV show \"F Troop\", he became identified with the character of Cavalry Sgt. Morgan O'Rourke (a manipulative character quite similar to Phil Silvers' role as MSgt Ernie Bilko). Tucker struggled with a drinking problem that began to affect his performances in the later years of his career.»\n",+ "[2] «Emergency Landing (1941 film) | Emergency Landing (a.k.a. Robot Pilot) is a 1941 American aviation spy-fi romantic screwball comedy film directed by William Beaudine. The film stars Forrest Tucker in his second film and in his first leading role with co-stars Carol Hughes and Evelyn Brent. \"Emergency Landing\" features lots of mismatched stock footage of various types of aircraft.»\n",+ "[3] «Forrest Tucker (criminal) | Forrest Silva \"Woody\" Tucker (June 23, 1920 – May 29, 2004) was an American career criminal first imprisoned at age 15 who spent the rest of his life in and out of jail. He is best known as an escape artist, having escaped from prison 18 times successfully and 12 times unsuccessfully.»\n",+ "[4] «Fess Parker | Fess Elisha Parker Jr. (August 16, 1924 – March 18, 2010) was an American film and television actor best known for his portrayals of Davy Crockett in the Walt Disney 1955–1956 TV miniseries and as Daniel Boone in a television series from 1964 to 1970. He was also known as a winemaker and resort owner-operator.»\n",+ "[5] «Davy Crockett, King of the Wild Frontier | Davy Crockett, King of the Wild Frontier is a 1955 live-action Walt Disney adventure film starring Fess Parker as Davy Crockett. This film is an edited compilation of the first three stories from the Disney television miniseries \"Davy Crockett\" :»\n",+ "[6] «Davy Crockett (miniseries) | Davy Crockett is a five-part serial which aired on ABC from 1954–1955 in one-hour episodes, on the \"Disneyland\" series. The series stars Fess Parker as real-life frontiersman Davy Crockett and Buddy Ebsen as his friend, George Russel.»\n",+ "\n",+ "Question: Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\n",+ "\n",+ "Reasoning: Let's think step by step in order to We need to find the actor who played the character in the 1955-1956 Walt Disney TV miniseries. From the context, we know that the miniseries was about Davy Crockett and starred Fess Parker as Davy Crockett.\n",+ "\n",+ "Answer: Davy Crockett\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.»\n",+ "[2] «Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford's second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.»\n",+ "[3] «G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.»\n",+ "[4] «College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour»\n",+ "[5] «Bass fishing | Bass fishing is the activity of angling for the North American gamefish known colloquially as the \"black bass\". There are numerous black bass species considered as gamefish in North America, including largemouth bass (\"Micropterus salmoides\"), smallmouth bass (\"Micropterus dolomieui\"), spotted bass or Kentucky bass (\"Micropterus punctulatus\"), and Guadalupe bass (order \"Perciformes\").»\n",+ "[6] «Cliff Crochet | Cliff Crochet (born August 1, 1983), also known as \"The Cajun Baby\", is a professional bass fisherman from Pierre Part, Louisiana. Crochet is a competitor on Bass Angler Sportsman Society's (B.A.S.S.) Bassmaster Elite Series and has $368,837 in B.A.S.S. tournament winnings as of August 22, 2016. Crochet's sponsors include: K2 Coolers, Huk, Skeeter, Yamaha Motor Company, Lowrance Electronics, Falcon Rods, Luck E Strike, Seaguar, Rat-L-Trap, Humdinger Spinnerbaits, Power-Pole. Since 2010, Crochet has competed in three Bassmaster Classic tournaments. Crochet's boat for the 2016 season is a 2016 Skeeter FX 21 with a 250 hp Yamaha SHO outboard engine.»\n",+ "\n",+ "Question: Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "Context: [1] «Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.»\n",+ "\n",+ "[2] «College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour»\n",+ "\n",+ "[3] «G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.»\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "\n",+ "Answer: Carhartt\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Archibald Wavell, 1st Earl Wavell | Field Marshal Archibald Percival Wavell, 1st Earl Wavell, {'1': \", '2': \", '3': \", '4': \"} (5 May 1883 – 24 May 1950) was a senior officer of the British Army. He served in the Second Boer War, the Bazar Valley Campaign and the Great War, during which he was wounded in the Second Battle of Ypres. He served in the Second World War, initially as Commander-in-Chief Middle East, in which role he led British forces to victory over the Italians in western Egypt and eastern Libya during Operation \"Compass\" in December 1940, only to be defeated by the German Army in the Western Desert in April 1941. He served as Commander-in-Chief, India, from July 1941 until June 1943 (apart from a brief tour as Commander of ABDACOM) and then served as Viceroy of India until his retirement in February 1947.»\n",+ "[2] «John French, 1st Earl of Ypres | Field Marshal John Denton Pinkstone French, 1st Earl of Ypres (28 September 1852 – 22 May 1925), known as Sir John French from 1901 to 1916, and as The Viscount French between 1916 and 1922, was a senior British Army officer. Born in Kent to an Anglo-Irish family, he saw brief service as a midshipman in the Royal Navy, before becoming a cavalry officer. He achieved rapid promotion and distinguished himself on the Gordon Relief Expedition. French had a considerable reputation as a womaniser throughout his life and his career nearly ended when he was cited in the divorce of a brother officer whilst in India in the early 1890s.»\n",+ "[3] «John French, 2nd Earl of Ypres | John Richard Lowndes French, 2nd Earl of Ypres (6 July 1881 – 5 April 1958) was the son of the British field marshal and the first commander of the British Expeditionary Force (BEF) in World War I Sir John French.»\n",+ "[4] «Archibald Wavell, 2nd Earl Wavell | Major Archibald John Arthur Wavell, 2nd Earl Wavell MC (11 May 1916 – 24 December 1953) was the second and last Earl Wavell and Viscount Keren of Eritrea. He was educated at Winchester College and succeeded his father in 1950.»\n",+ "[5] «Archibald MacKinnon | Archibald (\"Archie\") MacKinnon (born January 13, 1937 in Cranbrook, British Columbia) is a Canadian competition rower and Olympic champion.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Answer. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to We need to find the 1st Earl of Ypres who attacked the Boer force from the rear while mounted infantry. From the context, we know that the 1st Earl of Ypres was John French, 1st Earl of Ypres.\n",+ "\n",+ "Answer: 28 September 1852\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Generate a search query that accurately captures the essence of the question and enables the retrieval of relevant information from the dataset, taking into account the context provided, the question being asked, and the relationship between the two.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Search Query: ${search_query}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.»\n",+ "[2] «Ebenezer Howard | Sir Ebenezer Howard {'1': \", '2': \", '3': \", '4': \"} ( ; 29 January 1850 – 1 May 1928), the English founder of the garden city movement, is known for his publication \"\" (1898), the description of a utopian city in which people live harmoniously together with nature. The publication resulted in the founding of the garden city movement, and the building of the First Garden City, Letchworth Garden City, commenced in 1903.»\n",+ "[3] «Eugene Reynal | Eugene Reynal was an American Publisher and in 1933 Founder of Blue Ribbon Books of Garden City, New York and Reynal and Hitchcock of New York, New York.»\n",+ "\n",+ "Question: Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Garden City Hotel founder\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.»\n",+ "[2] «Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford's second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.»\n",+ "[3] «G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.»\n",+ "\n",+ "Question: Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Carhartt bass fishing\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Stitch's Great Escape! | Stitch's Great Escape! is a Tomorrowland attraction at the Magic Kingdom theme park within the Walt Disney World Resort. It is a \"theater-in-the-round\" experience starring the title alien from Walt Disney Animation Studios' 2002 film \"Lilo & Stitch\". It opened November 16, 2004 and is the fourth attraction to occupy the site in Tomorrowland. Many of the animators who worked on \"Lilo & Stitch\" partnered with Walt Disney Imagineering for Stitch's Great Escape! The attraction is a replacement of The ExtraTERRORestrial Alien Encounter, which formerly occupied the building the attraction is housed in. On September 21, 2016, it was announced that the attraction would be switching from a daily operated attraction to a seasonally operated one, depending on attendance, starting October 2, 2016.»\n",+ "[2] «Great Escape (amusement park) | Great Escape (sometimes referred to as Six Flags Great Escape) is an amusement and water park owned and operated by Six Flags Entertainment Corp. It is located approximately 60 mi north of Albany, in Queensbury, New York, but is advertised as being in Lake George, New York, a popular tourist and vacation spot nearby. It is one of two Six Flags parks not to be officially branded with the \"Six Flags\" name (La Ronde in Montreal, Quebec, Canada being the other).»\n",+ "[3] «Houdini's Great Escape | Houdini's Great Escape, also known as \"Boo-dini\" during Fright Fest and \"Houdini - The Great Escape\" at Six Flags New England, are indoor Vekoma Madhouse attractions at Six Flags Great Adventure and Six Flags New England. The ride is located in Great Adventure's Fantasy Forest section and New England's Crack Axle Canyon area. The attraction opened at both parks in 1999, and was Standing but not operating (SBNO) at Great Adventure from 2008 - 2009 but was opened again due to popular demand.»\n",+ "\n",+ "Question: Which theme park has Stitch’s Great Escape which replaced the attraction ExtraTERRORestial Alien Encounter?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Magic Kingdom theme park\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Archibald Wavell, 1st Earl Wavell | Field Marshal Archibald Percival Wavell, 1st Earl Wavell, {'1': \", '2': \", '3': \", '4': \"} (5 May 1883 – 24 May 1950) was a senior officer of the British Army. He served in the Second Boer War, the Bazar Valley Campaign and the Great War, during which he was wounded in the Second Battle of Ypres. He served in the Second World War, initially as Commander-in-Chief Middle East, in which role he led British forces to victory over the Italians in western Egypt and eastern Libya during Operation \"Compass\" in December 1940, only to be defeated by the German Army in the Western Desert in April 1941. He served as Commander-in-Chief, India, from July 1941 until June 1943 (apart from a brief tour as Commander of ABDACOM) and then served as Viceroy of India until his retirement in February 1947.»\n",+ "[2] «John French, 1st Earl of Ypres | Field Marshal John Denton Pinkstone French, 1st Earl of Ypres (28 September 1852 – 22 May 1925), known as Sir John French from 1901 to 1916, and as The Viscount French between 1916 and 1922, was a senior British Army officer. Born in Kent to an Anglo-Irish family, he saw brief service as a midshipman in the Royal Navy, before becoming a cavalry officer. He achieved rapid promotion and distinguished himself on the Gordon Relief Expedition. French had a considerable reputation as a womaniser throughout his life and his career nearly ended when he was cited in the divorce of a brother officer whilst in India in the early 1890s.»\n",+ "[3] «John French, 2nd Earl of Ypres | John Richard Lowndes French, 2nd Earl of Ypres (6 July 1881 – 5 April 1958) was the son of the British field marshal and the first commander of the British Expeditionary Force (BEF) in World War I Sir John French.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Search Query. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Generate a search query that accurately captures the essence of the question and enables the retrieval of relevant information from the dataset, taking into account the context provided, the question being asked, and the relationship between the two.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Search Query: ${search_query}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.»\n",+ "[2] «Ebenezer Howard | Sir Ebenezer Howard {'1': \", '2': \", '3': \", '4': \"} ( ; 29 January 1850 – 1 May 1928), the English founder of the garden city movement, is known for his publication \"\" (1898), the description of a utopian city in which people live harmoniously together with nature. The publication resulted in the founding of the garden city movement, and the building of the First Garden City, Letchworth Garden City, commenced in 1903.»\n",+ "[3] «Eugene Reynal | Eugene Reynal was an American Publisher and in 1933 Founder of Blue Ribbon Books of Garden City, New York and Reynal and Hitchcock of New York, New York.»\n",+ "\n",+ "Question: Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Garden City Hotel founder\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.»\n",+ "[2] «Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford's second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.»\n",+ "[3] «G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.»\n",+ "\n",+ "Question: Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Carhartt bass fishing\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Stitch's Great Escape! | Stitch's Great Escape! is a Tomorrowland attraction at the Magic Kingdom theme park within the Walt Disney World Resort. It is a \"theater-in-the-round\" experience starring the title alien from Walt Disney Animation Studios' 2002 film \"Lilo & Stitch\". It opened November 16, 2004 and is the fourth attraction to occupy the site in Tomorrowland. Many of the animators who worked on \"Lilo & Stitch\" partnered with Walt Disney Imagineering for Stitch's Great Escape! The attraction is a replacement of The ExtraTERRORestrial Alien Encounter, which formerly occupied the building the attraction is housed in. On September 21, 2016, it was announced that the attraction would be switching from a daily operated attraction to a seasonally operated one, depending on attendance, starting October 2, 2016.»\n",+ "[2] «Great Escape (amusement park) | Great Escape (sometimes referred to as Six Flags Great Escape) is an amusement and water park owned and operated by Six Flags Entertainment Corp. It is located approximately 60 mi north of Albany, in Queensbury, New York, but is advertised as being in Lake George, New York, a popular tourist and vacation spot nearby. It is one of two Six Flags parks not to be officially branded with the \"Six Flags\" name (La Ronde in Montreal, Quebec, Canada being the other).»\n",+ "[3] «Houdini's Great Escape | Houdini's Great Escape, also known as \"Boo-dini\" during Fright Fest and \"Houdini - The Great Escape\" at Six Flags New England, are indoor Vekoma Madhouse attractions at Six Flags Great Adventure and Six Flags New England. The ride is located in Great Adventure's Fantasy Forest section and New England's Crack Axle Canyon area. The attraction opened at both parks in 1999, and was Standing but not operating (SBNO) at Great Adventure from 2008 - 2009 but was opened again due to popular demand.»\n",+ "\n",+ "Question: Which theme park has Stitch’s Great Escape which replaced the attraction ExtraTERRORestial Alien Encounter?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Magic Kingdom theme park\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Archibald Wavell, 1st Earl Wavell | Field Marshal Archibald Percival Wavell, 1st Earl Wavell, {'1': \", '2': \", '3': \", '4': \"} (5 May 1883 – 24 May 1950) was a senior officer of the British Army. He served in the Second Boer War, the Bazar Valley Campaign and the Great War, during which he was wounded in the Second Battle of Ypres. He served in the Second World War, initially as Commander-in-Chief Middle East, in which role he led British forces to victory over the Italians in western Egypt and eastern Libya during Operation \"Compass\" in December 1940, only to be defeated by the German Army in the Western Desert in April 1941. He served as Commander-in-Chief, India, from July 1941 until June 1943 (apart from a brief tour as Commander of ABDACOM) and then served as Viceroy of India until his retirement in February 1947.»\n",+ "[2] «John French, 1st Earl of Ypres | Field Marshal John Denton Pinkstone French, 1st Earl of Ypres (28 September 1852 – 22 May 1925), known as Sir John French from 1901 to 1916, and as The Viscount French between 1916 and 1922, was a senior British Army officer. Born in Kent to an Anglo-Irish family, he saw brief service as a midshipman in the Royal Navy, before becoming a cavalry officer. He achieved rapid promotion and distinguished himself on the Gordon Relief Expedition. French had a considerable reputation as a womaniser throughout his life and his career nearly ended when he was cited in the divorce of a brother officer whilst in India in the early 1890s.»\n",+ "[3] «John French, 2nd Earl of Ypres | John Richard Lowndes French, 2nd Earl of Ypres (6 July 1881 – 5 April 1958) was the son of the British field marshal and the first commander of the British Expeditionary Force (BEF) in World War I Sir John French.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Please provide the output field Search Query. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field Search Query.\n",+ "\n",+ "Search Query: Archibald Wavell birthdate\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Given a context and a question, generate a logical and coherent answer that accurately addresses the query by meticulously analyzing the provided context, identifying relevant information, and applying critical thinking to provide a precise and informative response.\n",+ "\n",+ "---\n",+ "\n",+ "Question: How many years did Lenny Lyles spend with the organization dramatized in the filme \"The insder\"?\n",+ "\n",+ "Answer: 27\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Answer: ${answer}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Forrest Tucker | Forrest Meredith Tucker (February 12, 1919 – October 25, 1986) was an American actor in both movies and television who appeared in nearly a hundred films. Tucker worked as a vaudeville straight man, aged fifteen years old. A mentor provided funds and contacts for a trip to California, where party hostess Cobina Wright persuaded guest Wesley Ruggles to give Tucker a screen test, based on his photogenic good looks, thick wavy hair and height of six feet, five inches - putting him into the same league as John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness's younger brother. Tucker was a sight reader who needed only one take. His film career started well despite a perception in most Hollywood studios that blond men were not photogenic. After twenty years spent mainly in Westerns and action roles, he returned to his roots, showing versatility as a comedic and stage musical actor. In the TV show \"F Troop\", he became identified with the character of Cavalry Sgt. Morgan O'Rourke (a manipulative character quite similar to Phil Silvers' role as MSgt Ernie Bilko). Tucker struggled with a drinking problem that began to affect his performances in the later years of his career.»\n",+ "[2] «Emergency Landing (1941 film) | Emergency Landing (a.k.a. Robot Pilot) is a 1941 American aviation spy-fi romantic screwball comedy film directed by William Beaudine. The film stars Forrest Tucker in his second film and in his first leading role with co-stars Carol Hughes and Evelyn Brent. \"Emergency Landing\" features lots of mismatched stock footage of various types of aircraft.»\n",+ "[3] «Forrest Tucker (criminal) | Forrest Silva \"Woody\" Tucker (June 23, 1920 – May 29, 2004) was an American career criminal first imprisoned at age 15 who spent the rest of his life in and out of jail. He is best known as an escape artist, having escaped from prison 18 times successfully and 12 times unsuccessfully.»\n",+ "[4] «Fess Parker | Fess Elisha Parker Jr. (August 16, 1924 – March 18, 2010) was an American film and television actor best known for his portrayals of Davy Crockett in the Walt Disney 1955–1956 TV miniseries and as Daniel Boone in a television series from 1964 to 1970. He was also known as a winemaker and resort owner-operator.»\n",+ "[5] «Davy Crockett, King of the Wild Frontier | Davy Crockett, King of the Wild Frontier is a 1955 live-action Walt Disney adventure film starring Fess Parker as Davy Crockett. This film is an edited compilation of the first three stories from the Disney television miniseries \"Davy Crockett\" :»\n",+ "[6] «Davy Crockett (miniseries) | Davy Crockett is a five-part serial which aired on ABC from 1954–1955 in one-hour episodes, on the \"Disneyland\" series. The series stars Fess Parker as real-life frontiersman Davy Crockett and Buddy Ebsen as his friend, George Russel.»\n",+ "\n",+ "Question: Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\n",+ "\n",+ "Reasoning: Let's think step by step in order to We need to find the actor who played the character in the 1955-1956 Walt Disney TV miniseries. From the context, we know that the miniseries was about Davy Crockett and starred Fess Parker as Davy Crockett.\n",+ "\n",+ "Answer: Davy Crockett\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.»\n",+ "[2] «Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford's second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.»\n",+ "[3] «G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.»\n",+ "[4] «College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour»\n",+ "[5] «Bass fishing | Bass fishing is the activity of angling for the North American gamefish known colloquially as the \"black bass\". There are numerous black bass species considered as gamefish in North America, including largemouth bass (\"Micropterus salmoides\"), smallmouth bass (\"Micropterus dolomieui\"), spotted bass or Kentucky bass (\"Micropterus punctulatus\"), and Guadalupe bass (order \"Perciformes\").»\n",+ "[6] «Cliff Crochet | Cliff Crochet (born August 1, 1983), also known as \"The Cajun Baby\", is a professional bass fisherman from Pierre Part, Louisiana. Crochet is a competitor on Bass Angler Sportsman Society's (B.A.S.S.) Bassmaster Elite Series and has $368,837 in B.A.S.S. tournament winnings as of August 22, 2016. Crochet's sponsors include: K2 Coolers, Huk, Skeeter, Yamaha Motor Company, Lowrance Electronics, Falcon Rods, Luck E Strike, Seaguar, Rat-L-Trap, Humdinger Spinnerbaits, Power-Pole. Since 2010, Crochet has competed in three Bassmaster Classic tournaments. Crochet's boat for the 2016 season is a 2016 Skeeter FX 21 with a 250 hp Yamaha SHO outboard engine.»\n",+ "\n",+ "Question: Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "Context: [1] «Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.»\n",+ "\n",+ "[2] «College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour»\n",+ "\n",+ "[3] «G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.»\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "\n",+ "Answer: Carhartt\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Archibald Wavell, 1st Earl Wavell | Field Marshal Archibald Percival Wavell, 1st Earl Wavell, {'1': \", '2': \", '3': \", '4': \"} (5 May 1883 – 24 May 1950) was a senior officer of the British Army. He served in the Second Boer War, the Bazar Valley Campaign and the Great War, during which he was wounded in the Second Battle of Ypres. He served in the Second World War, initially as Commander-in-Chief Middle East, in which role he led British forces to victory over the Italians in western Egypt and eastern Libya during Operation \"Compass\" in December 1940, only to be defeated by the German Army in the Western Desert in April 1941. He served as Commander-in-Chief, India, from July 1941 until June 1943 (apart from a brief tour as Commander of ABDACOM) and then served as Viceroy of India until his retirement in February 1947.»\n",+ "[2] «John French, 1st Earl of Ypres | Field Marshal John Denton Pinkstone French, 1st Earl of Ypres (28 September 1852 – 22 May 1925), known as Sir John French from 1901 to 1916, and as The Viscount French between 1916 and 1922, was a senior British Army officer. Born in Kent to an Anglo-Irish family, he saw brief service as a midshipman in the Royal Navy, before becoming a cavalry officer. He achieved rapid promotion and distinguished himself on the Gordon Relief Expedition. French had a considerable reputation as a womaniser throughout his life and his career nearly ended when he was cited in the divorce of a brother officer whilst in India in the early 1890s.»\n",+ "[3] «John French, 2nd Earl of Ypres | John Richard Lowndes French, 2nd Earl of Ypres (6 July 1881 – 5 April 1958) was the son of the British field marshal and the first commander of the British Expeditionary Force (BEF) in World War I Sir John French.»\n",+ "[4] «Archibald Wavell, 2nd Earl Wavell | Major Archibald John Arthur Wavell, 2nd Earl Wavell MC (11 May 1916 – 24 December 1953) was the second and last Earl Wavell and Viscount Keren of Eritrea. He was educated at Winchester College and succeeded his father in 1950.»\n",+ "[5] «Archibald MacKinnon | Archibald (\"Archie\") MacKinnon (born January 13, 1937 in Cranbrook, British Columbia) is a Canadian competition rower and Olympic champion.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Answer. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to We need to find the 1st Earl of Ypres who attacked the Boer force from the rear while mounted infantry. From the context, we know that the 1st Earl of Ypres was John French, 1st Earl of Ypres.\n",+ "\n",+ "Answer: 28 September 1852\n",+ "\n",+ "\n",+ "\n",+ "...\n",+ "Score 44.0\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "[I 2024-09-23 19:09:42,889] Trial 1 finished with value: 44.0 and parameters: {'0_predictor_instruction': 4, '0_predictor_demos': 5, '1_predictor_instruction': 3, '1_predictor_demos': 8, '2_predictor_instruction': 2, '2_predictor_demos': 3}. Best is trial 0 with value: 52.0.\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "CANDIDATE PROGRAM:\n",+ "Predictor 0\n",+ "i: Generate a search query that accurately captures the essence of the question within the given context, leveraging specific keywords and phrases to guide the retrieval of relevant passages.\n",+ "p: Search Query:\n",+ "\n",+ "\n",+ "Predictor 1\n",+ "i: Generate a concise and informative search query that leverages the context information and question description to retrieve relevant passages from the database, effectively guiding the subsequent reasoning and answer generation process, taking into account the nuances and specificity of the provided context and question.\n",+ "p: Search Query:\n",+ "\n",+ "\n",+ "Predictor 2\n",+ "i: Given the fields `context`, `question`, produce the fields `answer`.\n",+ "p: Answer:\n",+ "\n",+ "\n",+ "...\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 8 / 25 (32.0): 100%|██████████| 25/25 [00:14<00:00, 1.76it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "FULL TRACE\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 0 / 1 (0.0): 100%|██████████| 1/1 [00:03<00:00, 3.43s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "\n",+ "\n",+ "\n",+ "Generate a concise and informative search query that leverages the context information and question description to retrieve relevant passages from the database, effectively guiding the subsequent reasoning and answer generation process, taking into account the nuances and specificity of the provided context and question.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Search Query: ${search_query}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Will Finn | Will Finn (born November 1, 1958) is an American animator, voice actor, and director. His work in animation includes characters from Disney and Don Bluth films such as \"The Secret of NIMH\", \"Oliver & Company\", \"The Little Mermaid\", \"The Rescuers Down Under\", and \"Pocahontas\". His characters includes Cogsworth in \"Beauty and the Beast\", Iago in \"Aladdin\", and Laverne in \"The Hunchback of Notre Dame\". Finn wrote and directed \"Home on the Range\" and did some voice acting the Hollywood Fish in \"Chicken Little\". In 2006, Finn directed the computer animated short Hammy's Boomerang Adventure, a spin-off of \"Over the Hedge\".»\n",+ "[2] «Over the Hedge (film) | Over the Hedge is a 2006 American computer-animated comedy film, based on the characters from the United Media comic strip of the same name. Directed by Tim Johnson and Karey Kirkpatrick, and produced by Bonnie Arnold, it was released in the United States on May 19, 2006. The film was produced by DreamWorks Animation and distributed through Paramount Pictures. The film features the voices of Bruce Willis, Garry Shandling, Steve Carell, William Shatner, Wanda Sykes, and Nick Nolte. It is the first DreamWorks Animation film to be distributed by Paramount Pictures, which acquired the live-action DreamWorks studio in 2006. The film earned $336 million on an $80 million budget.»\n",+ "[3] «Tim Johnson (film director) | Tim Johnson (born August 27, 1961) is an American animation director. Johnson has directed in many films such as \"Antz\", \"\" and \"Over the Hedge\". Recently, he directed the DreamWorks 3D film \"Home.\"»\n",+ "\n",+ "Question: When was the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed?\n",+ "\n",+ "Reasoning: Let's think step by step in order to find the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed. The context mentions that Will Finn directed the computer animated short Hammy's Boomerang Adventure, a spin-off of \"Over the Hedge\".\n",+ "\n",+ "Search Query: Hammy's Boomerang Adventure release date\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Archibald Wavell, 1st Earl Wavell | Field Marshal Archibald Percival Wavell, 1st Earl Wavell, {'1': \", '2': \", '3': \", '4': \"} (5 May 1883 – 24 May 1950) was a senior officer of the British Army. He served in the Second Boer War, the Bazar Valley Campaign and the Great War, during which he was wounded in the Second Battle of Ypres. He served in the Second World War, initially as Commander-in-Chief Middle East, in which role he led British forces to victory over the Italians in western Egypt and eastern Libya during Operation \"Compass\" in December 1940, only to be defeated by the German Army in the Western Desert in April 1941. He served as Commander-in-Chief, India, from July 1941 until June 1943 (apart from a brief tour as Commander of ABDACOM) and then served as Viceroy of India until his retirement in February 1947.»\n",+ "[2] «John French, 1st Earl of Ypres | Field Marshal John Denton Pinkstone French, 1st Earl of Ypres (28 September 1852 – 22 May 1925), known as Sir John French from 1901 to 1916, and as The Viscount French between 1916 and 1922, was a senior British Army officer. Born in Kent to an Anglo-Irish family, he saw brief service as a midshipman in the Royal Navy, before becoming a cavalry officer. He achieved rapid promotion and distinguished himself on the Gordon Relief Expedition. French had a considerable reputation as a womaniser throughout his life and his career nearly ended when he was cited in the divorce of a brother officer whilst in India in the early 1890s.»\n",+ "[3] «John French, 2nd Earl of Ypres | John Richard Lowndes French, 2nd Earl of Ypres (6 July 1881 – 5 April 1958) was the son of the British field marshal and the first commander of the British Expeditionary Force (BEF) in World War I Sir John French.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Search Query. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to find the 1st Earl of Ypres born who attacked the Boer force from the rear while mounted infantry. The context mentions that Archibald Wavell, 1st Earl Wavell, was a senior officer of the British Army and served in the Second Boer War. The question asks about the 1st Earl of Ypres, who is John French, 1st Earl of Ypres.\n",+ "Search Query: John French 1st Earl of Ypres birthdate\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Given the fields `context`, `question`, produce the fields `answer`.\n",+ "\n",+ "---\n",+ "\n",+ "Question: Which magazine is still in print American Heritage or High Times?\n",+ "\n",+ "Answer: High Times\n",+ "\n",+ "---\n",+ "\n",+ "Question: What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?\n",+ "\n",+ "Answer: 2016\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Answer: ${answer}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Butch Reed | Bruce Franklin Reed (born July 11, 1954) is an American retired professional wrestler and football player, better known by the ring name Butch Reed. He is best known for his appearances with the World Wrestling Federation and World Championship Wrestling between 1986 and 1992.»\n",+ "[2] «Doug Reed (futsal player) | Doug Reed is a professional futsal player who is currently playing for Futsal Dinamo and England national futsal team. Previously he has played for Lanzarote Tias Yaiza (Spain), AEL Limassol, MNK Uspinjača, MNK Smederevo and Manchester Futsal Club. In 2011, he won the FA Futsal Cup with Manchester Futsal Club and was awarded Player of the Tournament in the same competition. In the FA National Futsal League, he won several FA National League Championships and FA Futsal Cup medals. Reed also works as a commentator for Sony SIX, Eurosport and BT Sport, and has worked with the emerging US Professional Futsal League as well as Indian Premier Futsal.»\n",+ "[3] «Doom (professional wrestling) | Doom was a professional wrestling tag team composed of Ron Simmons and Butch Reed. They competed from 1989 to 1991 in the National Wrestling Alliance (NWA) and World Championship Wrestling (WCW).»\n",+ "[4] «Robert Reed (gridiron football) | Robert E. Reed, Jr. (born February 23, 1943) is a former American football offensive lineman in the National Football League for the Washington Redskins. He played college football at Tennessee State University and was drafted in the sixteenth round of the 1965 NFL Draft. Reed was also selected in the 12th round of the 1965 AFL Draft by the Houston Oilers.»\n",+ "[5] «Reed Doughty | Reed Doughty (born November 4, 1982) is a former American football safety who played in the National Football League. He was drafted by the Washington Redskins in the sixth round, with the 173rd overall pick, of the 2006 NFL Draft. He played college football at the University of Northern Colorado.»\n",+ "\n",+ "Question: What other sport did Butch Reed, of the wrestling tag team Doom, play professionally?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "Context: N/A\n",+ "\n",+ "Question: What other sport did Butch Reed, of the wrestling tag team Doom, play professionally?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "\n",+ "Answer: football\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Archibald Wavell, 1st Earl Wavell | Field Marshal Archibald Percival Wavell, 1st Earl Wavell, {'1': \", '2': \", '3': \", '4': \"} (5 May 1883 – 24 May 1950) was a senior officer of the British Army. He served in the Second Boer War, the Bazar Valley Campaign and the Great War, during which he was wounded in the Second Battle of Ypres. He served in the Second World War, initially as Commander-in-Chief Middle East, in which role he led British forces to victory over the Italians in western Egypt and eastern Libya during Operation \"Compass\" in December 1940, only to be defeated by the German Army in the Western Desert in April 1941. He served as Commander-in-Chief, India, from July 1941 until June 1943 (apart from a brief tour as Commander of ABDACOM) and then served as Viceroy of India until his retirement in February 1947.»\n",+ "[2] «John French, 1st Earl of Ypres | Field Marshal John Denton Pinkstone French, 1st Earl of Ypres (28 September 1852 – 22 May 1925), known as Sir John French from 1901 to 1916, and as The Viscount French between 1916 and 1922, was a senior British Army officer. Born in Kent to an Anglo-Irish family, he saw brief service as a midshipman in the Royal Navy, before becoming a cavalry officer. He achieved rapid promotion and distinguished himself on the Gordon Relief Expedition. French had a considerable reputation as a womaniser throughout his life and his career nearly ended when he was cited in the divorce of a brother officer whilst in India in the early 1890s.»\n",+ "[3] «John French, 2nd Earl of Ypres | John Richard Lowndes French, 2nd Earl of Ypres (6 July 1881 – 5 April 1958) was the son of the British field marshal and the first commander of the British Expeditionary Force (BEF) in World War I Sir John French.»\n",+ "[4] «Ripple, Kent | John French, 1st Earl of Ypres, the commander of the first British Expeditionary Force was born there in 1852, and is buried at the village church. His sister Charlotte Despard, the suffragist, novelist and Sinn Féin activist was also born in Ripple in 1844.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Answer. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Given the fields `context`, `question`, produce the fields `answer`.\n",+ "\n",+ "---\n",+ "\n",+ "Question: Which magazine is still in print American Heritage or High Times?\n",+ "\n",+ "Answer: High Times\n",+ "\n",+ "---\n",+ "\n",+ "Question: What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?\n",+ "\n",+ "Answer: 2016\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Answer: ${answer}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Butch Reed | Bruce Franklin Reed (born July 11, 1954) is an American retired professional wrestler and football player, better known by the ring name Butch Reed. He is best known for his appearances with the World Wrestling Federation and World Championship Wrestling between 1986 and 1992.»\n",+ "[2] «Doug Reed (futsal player) | Doug Reed is a professional futsal player who is currently playing for Futsal Dinamo and England national futsal team. Previously he has played for Lanzarote Tias Yaiza (Spain), AEL Limassol, MNK Uspinjača, MNK Smederevo and Manchester Futsal Club. In 2011, he won the FA Futsal Cup with Manchester Futsal Club and was awarded Player of the Tournament in the same competition. In the FA National Futsal League, he won several FA National League Championships and FA Futsal Cup medals. Reed also works as a commentator for Sony SIX, Eurosport and BT Sport, and has worked with the emerging US Professional Futsal League as well as Indian Premier Futsal.»\n",+ "[3] «Doom (professional wrestling) | Doom was a professional wrestling tag team composed of Ron Simmons and Butch Reed. They competed from 1989 to 1991 in the National Wrestling Alliance (NWA) and World Championship Wrestling (WCW).»\n",+ "[4] «Robert Reed (gridiron football) | Robert E. Reed, Jr. (born February 23, 1943) is a former American football offensive lineman in the National Football League for the Washington Redskins. He played college football at Tennessee State University and was drafted in the sixteenth round of the 1965 NFL Draft. Reed was also selected in the 12th round of the 1965 AFL Draft by the Houston Oilers.»\n",+ "[5] «Reed Doughty | Reed Doughty (born November 4, 1982) is a former American football safety who played in the National Football League. He was drafted by the Washington Redskins in the sixth round, with the 173rd overall pick, of the 2006 NFL Draft. He played college football at the University of Northern Colorado.»\n",+ "\n",+ "Question: What other sport did Butch Reed, of the wrestling tag team Doom, play professionally?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "Context: N/A\n",+ "\n",+ "Question: What other sport did Butch Reed, of the wrestling tag team Doom, play professionally?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "\n",+ "Answer: football\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Archibald Wavell, 1st Earl Wavell | Field Marshal Archibald Percival Wavell, 1st Earl Wavell, {'1': \", '2': \", '3': \", '4': \"} (5 May 1883 – 24 May 1950) was a senior officer of the British Army. He served in the Second Boer War, the Bazar Valley Campaign and the Great War, during which he was wounded in the Second Battle of Ypres. He served in the Second World War, initially as Commander-in-Chief Middle East, in which role he led British forces to victory over the Italians in western Egypt and eastern Libya during Operation \"Compass\" in December 1940, only to be defeated by the German Army in the Western Desert in April 1941. He served as Commander-in-Chief, India, from July 1941 until June 1943 (apart from a brief tour as Commander of ABDACOM) and then served as Viceroy of India until his retirement in February 1947.»\n",+ "[2] «John French, 1st Earl of Ypres | Field Marshal John Denton Pinkstone French, 1st Earl of Ypres (28 September 1852 – 22 May 1925), known as Sir John French from 1901 to 1916, and as The Viscount French between 1916 and 1922, was a senior British Army officer. Born in Kent to an Anglo-Irish family, he saw brief service as a midshipman in the Royal Navy, before becoming a cavalry officer. He achieved rapid promotion and distinguished himself on the Gordon Relief Expedition. French had a considerable reputation as a womaniser throughout his life and his career nearly ended when he was cited in the divorce of a brother officer whilst in India in the early 1890s.»\n",+ "[3] «John French, 2nd Earl of Ypres | John Richard Lowndes French, 2nd Earl of Ypres (6 July 1881 – 5 April 1958) was the son of the British field marshal and the first commander of the British Expeditionary Force (BEF) in World War I Sir John French.»\n",+ "[4] «Ripple, Kent | John French, 1st Earl of Ypres, the commander of the first British Expeditionary Force was born there in 1852, and is buried at the village church. His sister Charlotte Despard, the suffragist, novelist and Sinn Féin activist was also born in Ripple in 1844.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Please provide the output field Answer. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field Answer.\n",+ "\n",+ "1883\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Generate a concise and informative search query that leverages the context information and question description to retrieve relevant passages from the database, effectively guiding the subsequent reasoning and answer generation process, taking into account the nuances and specificity of the provided context and question.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Search Query: ${search_query}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Will Finn | Will Finn (born November 1, 1958) is an American animator, voice actor, and director. His work in animation includes characters from Disney and Don Bluth films such as \"The Secret of NIMH\", \"Oliver & Company\", \"The Little Mermaid\", \"The Rescuers Down Under\", and \"Pocahontas\". His characters includes Cogsworth in \"Beauty and the Beast\", Iago in \"Aladdin\", and Laverne in \"The Hunchback of Notre Dame\". Finn wrote and directed \"Home on the Range\" and did some voice acting the Hollywood Fish in \"Chicken Little\". In 2006, Finn directed the computer animated short Hammy's Boomerang Adventure, a spin-off of \"Over the Hedge\".»\n",+ "[2] «Over the Hedge (film) | Over the Hedge is a 2006 American computer-animated comedy film, based on the characters from the United Media comic strip of the same name. Directed by Tim Johnson and Karey Kirkpatrick, and produced by Bonnie Arnold, it was released in the United States on May 19, 2006. The film was produced by DreamWorks Animation and distributed through Paramount Pictures. The film features the voices of Bruce Willis, Garry Shandling, Steve Carell, William Shatner, Wanda Sykes, and Nick Nolte. It is the first DreamWorks Animation film to be distributed by Paramount Pictures, which acquired the live-action DreamWorks studio in 2006. The film earned $336 million on an $80 million budget.»\n",+ "[3] «Tim Johnson (film director) | Tim Johnson (born August 27, 1961) is an American animation director. Johnson has directed in many films such as \"Antz\", \"\" and \"Over the Hedge\". Recently, he directed the DreamWorks 3D film \"Home.\"»\n",+ "\n",+ "Question: When was the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed?\n",+ "\n",+ "Reasoning: Let's think step by step in order to find the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed. The context mentions that Will Finn directed the computer animated short Hammy's Boomerang Adventure, a spin-off of \"Over the Hedge\".\n",+ "\n",+ "Search Query: Hammy's Boomerang Adventure release date\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Archibald Wavell, 1st Earl Wavell | Field Marshal Archibald Percival Wavell, 1st Earl Wavell, {'1': \", '2': \", '3': \", '4': \"} (5 May 1883 – 24 May 1950) was a senior officer of the British Army. He served in the Second Boer War, the Bazar Valley Campaign and the Great War, during which he was wounded in the Second Battle of Ypres. He served in the Second World War, initially as Commander-in-Chief Middle East, in which role he led British forces to victory over the Italians in western Egypt and eastern Libya during Operation \"Compass\" in December 1940, only to be defeated by the German Army in the Western Desert in April 1941. He served as Commander-in-Chief, India, from July 1941 until June 1943 (apart from a brief tour as Commander of ABDACOM) and then served as Viceroy of India until his retirement in February 1947.»\n",+ "[2] «John French, 1st Earl of Ypres | Field Marshal John Denton Pinkstone French, 1st Earl of Ypres (28 September 1852 – 22 May 1925), known as Sir John French from 1901 to 1916, and as The Viscount French between 1916 and 1922, was a senior British Army officer. Born in Kent to an Anglo-Irish family, he saw brief service as a midshipman in the Royal Navy, before becoming a cavalry officer. He achieved rapid promotion and distinguished himself on the Gordon Relief Expedition. French had a considerable reputation as a womaniser throughout his life and his career nearly ended when he was cited in the divorce of a brother officer whilst in India in the early 1890s.»\n",+ "[3] «John French, 2nd Earl of Ypres | John Richard Lowndes French, 2nd Earl of Ypres (6 July 1881 – 5 April 1958) was the son of the British field marshal and the first commander of the British Expeditionary Force (BEF) in World War I Sir John French.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Search Query. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to find the 1st Earl of Ypres born who attacked the Boer force from the rear while mounted infantry. The context mentions that Archibald Wavell, 1st Earl Wavell, was a senior officer of the British Army and served in the Second Boer War. The question asks about the 1st Earl of Ypres, who is John French, 1st Earl of Ypres.\n",+ "Search Query: John French 1st Earl of Ypres birthdate\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Given the fields `context`, `question`, produce the fields `answer`.\n",+ "\n",+ "---\n",+ "\n",+ "Question: Which magazine is still in print American Heritage or High Times?\n",+ "\n",+ "Answer: High Times\n",+ "\n",+ "---\n",+ "\n",+ "Question: What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?\n",+ "\n",+ "Answer: 2016\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Answer: ${answer}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Butch Reed | Bruce Franklin Reed (born July 11, 1954) is an American retired professional wrestler and football player, better known by the ring name Butch Reed. He is best known for his appearances with the World Wrestling Federation and World Championship Wrestling between 1986 and 1992.»\n",+ "[2] «Doug Reed (futsal player) | Doug Reed is a professional futsal player who is currently playing for Futsal Dinamo and England national futsal team. Previously he has played for Lanzarote Tias Yaiza (Spain), AEL Limassol, MNK Uspinjača, MNK Smederevo and Manchester Futsal Club. In 2011, he won the FA Futsal Cup with Manchester Futsal Club and was awarded Player of the Tournament in the same competition. In the FA National Futsal League, he won several FA National League Championships and FA Futsal Cup medals. Reed also works as a commentator for Sony SIX, Eurosport and BT Sport, and has worked with the emerging US Professional Futsal League as well as Indian Premier Futsal.»\n",+ "[3] «Doom (professional wrestling) | Doom was a professional wrestling tag team composed of Ron Simmons and Butch Reed. They competed from 1989 to 1991 in the National Wrestling Alliance (NWA) and World Championship Wrestling (WCW).»\n",+ "[4] «Robert Reed (gridiron football) | Robert E. Reed, Jr. (born February 23, 1943) is a former American football offensive lineman in the National Football League for the Washington Redskins. He played college football at Tennessee State University and was drafted in the sixteenth round of the 1965 NFL Draft. Reed was also selected in the 12th round of the 1965 AFL Draft by the Houston Oilers.»\n",+ "[5] «Reed Doughty | Reed Doughty (born November 4, 1982) is a former American football safety who played in the National Football League. He was drafted by the Washington Redskins in the sixth round, with the 173rd overall pick, of the 2006 NFL Draft. He played college football at the University of Northern Colorado.»\n",+ "\n",+ "Question: What other sport did Butch Reed, of the wrestling tag team Doom, play professionally?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "Context: N/A\n",+ "\n",+ "Question: What other sport did Butch Reed, of the wrestling tag team Doom, play professionally?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "\n",+ "Answer: football\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Archibald Wavell, 1st Earl Wavell | Field Marshal Archibald Percival Wavell, 1st Earl Wavell, {'1': \", '2': \", '3': \", '4': \"} (5 May 1883 – 24 May 1950) was a senior officer of the British Army. He served in the Second Boer War, the Bazar Valley Campaign and the Great War, during which he was wounded in the Second Battle of Ypres. He served in the Second World War, initially as Commander-in-Chief Middle East, in which role he led British forces to victory over the Italians in western Egypt and eastern Libya during Operation \"Compass\" in December 1940, only to be defeated by the German Army in the Western Desert in April 1941. He served as Commander-in-Chief, India, from July 1941 until June 1943 (apart from a brief tour as Commander of ABDACOM) and then served as Viceroy of India until his retirement in February 1947.»\n",+ "[2] «John French, 1st Earl of Ypres | Field Marshal John Denton Pinkstone French, 1st Earl of Ypres (28 September 1852 – 22 May 1925), known as Sir John French from 1901 to 1916, and as The Viscount French between 1916 and 1922, was a senior British Army officer. Born in Kent to an Anglo-Irish family, he saw brief service as a midshipman in the Royal Navy, before becoming a cavalry officer. He achieved rapid promotion and distinguished himself on the Gordon Relief Expedition. French had a considerable reputation as a womaniser throughout his life and his career nearly ended when he was cited in the divorce of a brother officer whilst in India in the early 1890s.»\n",+ "[3] «John French, 2nd Earl of Ypres | John Richard Lowndes French, 2nd Earl of Ypres (6 July 1881 – 5 April 1958) was the son of the British field marshal and the first commander of the British Expeditionary Force (BEF) in World War I Sir John French.»\n",+ "[4] «Ripple, Kent | John French, 1st Earl of Ypres, the commander of the first British Expeditionary Force was born there in 1852, and is buried at the village church. His sister Charlotte Despard, the suffragist, novelist and Sinn Féin activist was also born in Ripple in 1844.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Answer. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Given the fields `context`, `question`, produce the fields `answer`.\n",+ "\n",+ "---\n",+ "\n",+ "Question: Which magazine is still in print American Heritage or High Times?\n",+ "\n",+ "Answer: High Times\n",+ "\n",+ "---\n",+ "\n",+ "Question: What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?\n",+ "\n",+ "Answer: 2016\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Answer: ${answer}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Butch Reed | Bruce Franklin Reed (born July 11, 1954) is an American retired professional wrestler and football player, better known by the ring name Butch Reed. He is best known for his appearances with the World Wrestling Federation and World Championship Wrestling between 1986 and 1992.»\n",+ "[2] «Doug Reed (futsal player) | Doug Reed is a professional futsal player who is currently playing for Futsal Dinamo and England national futsal team. Previously he has played for Lanzarote Tias Yaiza (Spain), AEL Limassol, MNK Uspinjača, MNK Smederevo and Manchester Futsal Club. In 2011, he won the FA Futsal Cup with Manchester Futsal Club and was awarded Player of the Tournament in the same competition. In the FA National Futsal League, he won several FA National League Championships and FA Futsal Cup medals. Reed also works as a commentator for Sony SIX, Eurosport and BT Sport, and has worked with the emerging US Professional Futsal League as well as Indian Premier Futsal.»\n",+ "[3] «Doom (professional wrestling) | Doom was a professional wrestling tag team composed of Ron Simmons and Butch Reed. They competed from 1989 to 1991 in the National Wrestling Alliance (NWA) and World Championship Wrestling (WCW).»\n",+ "[4] «Robert Reed (gridiron football) | Robert E. Reed, Jr. (born February 23, 1943) is a former American football offensive lineman in the National Football League for the Washington Redskins. He played college football at Tennessee State University and was drafted in the sixteenth round of the 1965 NFL Draft. Reed was also selected in the 12th round of the 1965 AFL Draft by the Houston Oilers.»\n",+ "[5] «Reed Doughty | Reed Doughty (born November 4, 1982) is a former American football safety who played in the National Football League. He was drafted by the Washington Redskins in the sixth round, with the 173rd overall pick, of the 2006 NFL Draft. He played college football at the University of Northern Colorado.»\n",+ "\n",+ "Question: What other sport did Butch Reed, of the wrestling tag team Doom, play professionally?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "Context: N/A\n",+ "\n",+ "Question: What other sport did Butch Reed, of the wrestling tag team Doom, play professionally?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "\n",+ "Answer: football\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Archibald Wavell, 1st Earl Wavell | Field Marshal Archibald Percival Wavell, 1st Earl Wavell, {'1': \", '2': \", '3': \", '4': \"} (5 May 1883 – 24 May 1950) was a senior officer of the British Army. He served in the Second Boer War, the Bazar Valley Campaign and the Great War, during which he was wounded in the Second Battle of Ypres. He served in the Second World War, initially as Commander-in-Chief Middle East, in which role he led British forces to victory over the Italians in western Egypt and eastern Libya during Operation \"Compass\" in December 1940, only to be defeated by the German Army in the Western Desert in April 1941. He served as Commander-in-Chief, India, from July 1941 until June 1943 (apart from a brief tour as Commander of ABDACOM) and then served as Viceroy of India until his retirement in February 1947.»\n",+ "[2] «John French, 1st Earl of Ypres | Field Marshal John Denton Pinkstone French, 1st Earl of Ypres (28 September 1852 – 22 May 1925), known as Sir John French from 1901 to 1916, and as The Viscount French between 1916 and 1922, was a senior British Army officer. Born in Kent to an Anglo-Irish family, he saw brief service as a midshipman in the Royal Navy, before becoming a cavalry officer. He achieved rapid promotion and distinguished himself on the Gordon Relief Expedition. French had a considerable reputation as a womaniser throughout his life and his career nearly ended when he was cited in the divorce of a brother officer whilst in India in the early 1890s.»\n",+ "[3] «John French, 2nd Earl of Ypres | John Richard Lowndes French, 2nd Earl of Ypres (6 July 1881 – 5 April 1958) was the son of the British field marshal and the first commander of the British Expeditionary Force (BEF) in World War I Sir John French.»\n",+ "[4] «Ripple, Kent | John French, 1st Earl of Ypres, the commander of the first British Expeditionary Force was born there in 1852, and is buried at the village church. His sister Charlotte Despard, the suffragist, novelist and Sinn Féin activist was also born in Ripple in 1844.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Please provide the output field Answer. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field Answer.\n",+ "\n",+ "1883\n",+ "\n",+ "\n",+ "\n",+ "...\n",+ "Score 32.0\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "[I 2024-09-23 19:10:00,687] Trial 2 finished with value: 32.0 and parameters: {'0_predictor_instruction': 9, '0_predictor_demos': 5, '1_predictor_instruction': 7, '1_predictor_demos': 4, '2_predictor_instruction': 0, '2_predictor_demos': 7}. Best is trial 0 with value: 52.0.\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "CANDIDATE PROGRAM:\n",+ "Predictor 0\n",+ "i: Generate a search query that accurately captures the essence of the question within the given context, leveraging specific keywords and phrases to guide the retrieval of relevant passages.\n",+ "p: Search Query:\n",+ "\n",+ "\n",+ "Predictor 1\n",+ "i: Given a context that provides information about specific locations and landmarks, and a question that requires identifying the country where certain high-peaked mountains are partially located, utilize the provided context to generate a search query that will yield relevant information to answer the question.\n",+ "p: Search Query:\n",+ "\n",+ "\n",+ "Predictor 2\n",+ "i: Given the context text and question to extract information, produce the answer and provide a step-by-step explanation of the reasoning and thought process used to arrive at the answer.\n",+ "p: Answer:\n",+ "\n",+ "\n",+ "...\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 10 / 25 (40.0): 100%|██████████| 25/25 [00:19<00:00, 1.28it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "FULL TRACE\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 0 / 1 (0.0): 100%|██████████| 1/1 [00:04<00:00, 4.25s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "\n",+ "\n",+ "\n",+ "Generate a search query that accurately captures the essence of the question within the given context, leveraging specific keywords and phrases to guide the retrieval of relevant passages.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Search Query: ${search_query}\n",+ "\n",+ "---\n",+ "\n",+ "Context: N/A\n",+ "\n",+ "Question: What other sport did Butch Reed, of the wrestling tag team Doom, play professionally?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Butch Reed sports\n",+ "\n",+ "---\n",+ "\n",+ "Context: N/A\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Please provide the output field Search Query. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field Search Query.\n",+ "\n",+ "Search Query: 1st Earl of Ypres Boer force\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Given a context that provides information about specific locations and landmarks, and a question that requires identifying the country where certain high-peaked mountains are partially located, utilize the provided context to generate a search query that will yield relevant information to answer the question.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Search Query: ${search_query}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Archibald Wavell, 1st Earl Wavell | Field Marshal Archibald Percival Wavell, 1st Earl Wavell, {'1': \", '2': \", '3': \", '4': \"} (5 May 1883 – 24 May 1950) was a senior officer of the British Army. He served in the Second Boer War, the Bazar Valley Campaign and the Great War, during which he was wounded in the Second Battle of Ypres. He served in the Second World War, initially as Commander-in-Chief Middle East, in which role he led British forces to victory over the Italians in western Egypt and eastern Libya during Operation \"Compass\" in December 1940, only to be defeated by the German Army in the Western Desert in April 1941. He served as Commander-in-Chief, India, from July 1941 until June 1943 (apart from a brief tour as Commander of ABDACOM) and then served as Viceroy of India until his retirement in February 1947.»\n",+ "[2] «John French, 2nd Earl of Ypres | John Richard Lowndes French, 2nd Earl of Ypres (6 July 1881 – 5 April 1958) was the son of the British field marshal and the first commander of the British Expeditionary Force (BEF) in World War I Sir John French.»\n",+ "[3] «Ripple, Kent | John French, 1st Earl of Ypres, the commander of the first British Expeditionary Force was born there in 1852, and is buried at the village church. His sister Charlotte Despard, the suffragist, novelist and Sinn Féin activist was also born in Ripple in 1844.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Search Query. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to find the country where certain high-peaked mountains are partially located. We need to find the 1st Earl of Ypres, who is mentioned in the context as the commander of the first British Expeditionary Force in World War I. We need to find the mountains that he attacked while mounted infantry.\n",+ "Search Query: 1st Earl of Ypres mountains\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Given the context text and question to extract information, produce the answer and provide a step-by-step explanation of the reasoning and thought process used to arrive at the answer.\n",+ "\n",+ "---\n",+ "\n",+ "Question: Are Beth Gibbons and Ian Curtis the same nationality?\n",+ "\n",+ "Answer: yes\n",+ "\n",+ "---\n",+ "\n",+ "Question: When did Taylor Swift's satirical song get released to the radio?\n",+ "\n",+ "Answer: February 9, 2015\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Answer: ${answer}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.»\n",+ "[2] «Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.»\n",+ "[3] «Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world's third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.»\n",+ "\n",+ "Question: The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "Context: [1] «Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.»\n",+ "\n",+ "[2] «Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.»\n",+ "\n",+ "[3] «Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world's third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.»\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "\n",+ "Answer: India\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Archibald Wavell, 1st Earl Wavell | Field Marshal Archibald Percival Wavell, 1st Earl Wavell, {'1': \", '2': \", '3': \", '4': \"} (5 May 1883 – 24 May 1950) was a senior officer of the British Army. He served in the Second Boer War, the Bazar Valley Campaign and the Great War, during which he was wounded in the Second Battle of Ypres. He served in the Second World War, initially as Commander-in-Chief Middle East, in which role he led British forces to victory over the Italians in western Egypt and eastern Libya during Operation \"Compass\" in December 1940, only to be defeated by the German Army in the Western Desert in April 1941. He served as Commander-in-Chief, India, from July 1941 until June 1943 (apart from a brief tour as Commander of ABDACOM) and then served as Viceroy of India until his retirement in February 1947.»\n",+ "[2] «John French, 2nd Earl of Ypres | John Richard Lowndes French, 2nd Earl of Ypres (6 July 1881 – 5 April 1958) was the son of the British field marshal and the first commander of the British Expeditionary Force (BEF) in World War I Sir John French.»\n",+ "[3] «Ripple, Kent | John French, 1st Earl of Ypres, the commander of the first British Expeditionary Force was born there in 1852, and is buried at the village church. His sister Charlotte Despard, the suffragist, novelist and Sinn Féin activist was also born in Ripple in 1844.»\n",+ "[4] «John French, 1st Earl of Ypres | Field Marshal John Denton Pinkstone French, 1st Earl of Ypres (28 September 1852 – 22 May 1925), known as Sir John French from 1901 to 1916, and as The Viscount French between 1916 and 1922, was a senior British Army officer. Born in Kent to an Anglo-Irish family, he saw brief service as a midshipman in the Royal Navy, before becoming a cavalry officer. He achieved rapid promotion and distinguished himself on the Gordon Relief Expedition. French had a considerable reputation as a womaniser throughout his life and his career nearly ended when he was cited in the divorce of a brother officer whilst in India in the early 1890s.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Answer. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to find the 1st Earl of Ypres who attacked the Boer force from the rear while mounted infantry. We need to find the 1st Earl of Ypres who was born in 1852, as mentioned in the context.\n",+ "Answer: John French, 1st Earl of Ypres\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Generate a search query that accurately captures the essence of the question within the given context, leveraging specific keywords and phrases to guide the retrieval of relevant passages.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Search Query: ${search_query}\n",+ "\n",+ "---\n",+ "\n",+ "Context: N/A\n",+ "\n",+ "Question: What other sport did Butch Reed, of the wrestling tag team Doom, play professionally?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Butch Reed sports\n",+ "\n",+ "---\n",+ "\n",+ "Context: N/A\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Please provide the output field Search Query. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field Search Query.\n",+ "\n",+ "Search Query: 1st Earl of Ypres Boer force\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Given a context that provides information about specific locations and landmarks, and a question that requires identifying the country where certain high-peaked mountains are partially located, utilize the provided context to generate a search query that will yield relevant information to answer the question.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Search Query: ${search_query}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Archibald Wavell, 1st Earl Wavell | Field Marshal Archibald Percival Wavell, 1st Earl Wavell, {'1': \", '2': \", '3': \", '4': \"} (5 May 1883 – 24 May 1950) was a senior officer of the British Army. He served in the Second Boer War, the Bazar Valley Campaign and the Great War, during which he was wounded in the Second Battle of Ypres. He served in the Second World War, initially as Commander-in-Chief Middle East, in which role he led British forces to victory over the Italians in western Egypt and eastern Libya during Operation \"Compass\" in December 1940, only to be defeated by the German Army in the Western Desert in April 1941. He served as Commander-in-Chief, India, from July 1941 until June 1943 (apart from a brief tour as Commander of ABDACOM) and then served as Viceroy of India until his retirement in February 1947.»\n",+ "[2] «John French, 2nd Earl of Ypres | John Richard Lowndes French, 2nd Earl of Ypres (6 July 1881 – 5 April 1958) was the son of the British field marshal and the first commander of the British Expeditionary Force (BEF) in World War I Sir John French.»\n",+ "[3] «Ripple, Kent | John French, 1st Earl of Ypres, the commander of the first British Expeditionary Force was born there in 1852, and is buried at the village church. His sister Charlotte Despard, the suffragist, novelist and Sinn Féin activist was also born in Ripple in 1844.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Search Query. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to find the country where certain high-peaked mountains are partially located. We need to find the 1st Earl of Ypres, who is mentioned in the context as the commander of the first British Expeditionary Force in World War I. We need to find the mountains that he attacked while mounted infantry.\n",+ "Search Query: 1st Earl of Ypres mountains\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Given the context text and question to extract information, produce the answer and provide a step-by-step explanation of the reasoning and thought process used to arrive at the answer.\n",+ "\n",+ "---\n",+ "\n",+ "Question: Are Beth Gibbons and Ian Curtis the same nationality?\n",+ "\n",+ "Answer: yes\n",+ "\n",+ "---\n",+ "\n",+ "Question: When did Taylor Swift's satirical song get released to the radio?\n",+ "\n",+ "Answer: February 9, 2015\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Answer: ${answer}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.»\n",+ "[2] «Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.»\n",+ "[3] «Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world's third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.»\n",+ "\n",+ "Question: The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "Context: [1] «Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.»\n",+ "\n",+ "[2] «Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.»\n",+ "\n",+ "[3] «Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world's third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.»\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "\n",+ "Answer: India\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Archibald Wavell, 1st Earl Wavell | Field Marshal Archibald Percival Wavell, 1st Earl Wavell, {'1': \", '2': \", '3': \", '4': \"} (5 May 1883 – 24 May 1950) was a senior officer of the British Army. He served in the Second Boer War, the Bazar Valley Campaign and the Great War, during which he was wounded in the Second Battle of Ypres. He served in the Second World War, initially as Commander-in-Chief Middle East, in which role he led British forces to victory over the Italians in western Egypt and eastern Libya during Operation \"Compass\" in December 1940, only to be defeated by the German Army in the Western Desert in April 1941. He served as Commander-in-Chief, India, from July 1941 until June 1943 (apart from a brief tour as Commander of ABDACOM) and then served as Viceroy of India until his retirement in February 1947.»\n",+ "[2] «John French, 2nd Earl of Ypres | John Richard Lowndes French, 2nd Earl of Ypres (6 July 1881 – 5 April 1958) was the son of the British field marshal and the first commander of the British Expeditionary Force (BEF) in World War I Sir John French.»\n",+ "[3] «Ripple, Kent | John French, 1st Earl of Ypres, the commander of the first British Expeditionary Force was born there in 1852, and is buried at the village church. His sister Charlotte Despard, the suffragist, novelist and Sinn Féin activist was also born in Ripple in 1844.»\n",+ "[4] «John French, 1st Earl of Ypres | Field Marshal John Denton Pinkstone French, 1st Earl of Ypres (28 September 1852 – 22 May 1925), known as Sir John French from 1901 to 1916, and as The Viscount French between 1916 and 1922, was a senior British Army officer. Born in Kent to an Anglo-Irish family, he saw brief service as a midshipman in the Royal Navy, before becoming a cavalry officer. He achieved rapid promotion and distinguished himself on the Gordon Relief Expedition. French had a considerable reputation as a womaniser throughout his life and his career nearly ended when he was cited in the divorce of a brother officer whilst in India in the early 1890s.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Answer. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to find the 1st Earl of Ypres who attacked the Boer force from the rear while mounted infantry. We need to find the 1st Earl of Ypres who was born in 1852, as mentioned in the context.\n",+ "Answer: John French, 1st Earl of Ypres\n",+ "\n",+ "\n",+ "\n",+ "...\n",+ "Score 40.0\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "[I 2024-09-23 19:10:24,572] Trial 3 finished with value: 40.0 and parameters: {'0_predictor_instruction': 9, '0_predictor_demos': 7, '1_predictor_instruction': 5, '1_predictor_demos': 0, '2_predictor_instruction': 9, '2_predictor_demos': 5}. Best is trial 0 with value: 52.0.\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "CANDIDATE PROGRAM:\n",+ "Predictor 0\n",+ "i: Given the context and question, generate a search query that accurately captures the essence of the query and retrieves relevant information to answer the question, ensuring the query is concise, descriptive, and specific to the context and question.\n",+ "p: Search Query:\n",+ "\n",+ "\n",+ "Predictor 1\n",+ "i: Provide a descriptive search query that entails the critical information from the context and question to find a precise answer to the query asked, such as finding the release date of the computer-animated short comedy film Hammy's Boomerang Adventure, which is a spin-off of \"Over the Hedge\".\n",+ "p: Search Query:\n",+ "\n",+ "\n",+ "Predictor 2\n",+ "i: Given the context and question, produce a specific and concise answer that accurately responds to the query, by iteratively generating relevant search queries and reasoning step by step.\n",+ "p: Answer:\n",+ "\n",+ "\n",+ "...\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 13 / 25 (52.0): 100%|██████████| 25/25 [00:20<00:00, 1.24it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "FULL TRACE\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 0 / 1 (0.0): 100%|██████████| 1/1 [00:04<00:00, 4.51s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "\n",+ "\n",+ "\n",+ "Given the context and question, generate a search query that accurately captures the essence of the query and retrieves relevant information to answer the question, ensuring the query is concise, descriptive, and specific to the context and question.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Search Query: ${search_query}\n",+ "\n",+ "---\n",+ "\n",+ "Context: N/A\n",+ "\n",+ "Question: What company formed in 1978 has Simon Arora as its CEO?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Simon Arora CEO company 1978\n",+ "\n",+ "---\n",+ "\n",+ "Context: N/A\n",+ "\n",+ "Question: Bragg electoral district, incorporating the Kensington Park suburb, are associated with which South Australian city?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Bragg electoral district Kensington Park suburb South Australian city\n",+ "\n",+ "---\n",+ "\n",+ "Context: N/A\n",+ "\n",+ "Question: The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Kangchenjunga Singhi Kangri country\n",+ "\n",+ "---\n",+ "\n",+ "Context: N/A\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Search Query. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: 1st Earl of Ypres birthdate Boer force mounted infantry\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Provide a descriptive search query that entails the critical information from the context and question to find a precise answer to the query asked, such as finding the release date of the computer-animated short comedy film Hammy's Boomerang Adventure, which is a spin-off of \"Over the Hedge\".\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Search Query: ${search_query}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Butch Reed | Bruce Franklin Reed (born July 11, 1954) is an American retired professional wrestler and football player, better known by the ring name Butch Reed. He is best known for his appearances with the World Wrestling Federation and World Championship Wrestling between 1986 and 1992.»\n",+ "[2] «Doug Reed (futsal player) | Doug Reed is a professional futsal player who is currently playing for Futsal Dinamo and England national futsal team. Previously he has played for Lanzarote Tias Yaiza (Spain), AEL Limassol, MNK Uspinjača, MNK Smederevo and Manchester Futsal Club. In 2011, he won the FA Futsal Cup with Manchester Futsal Club and was awarded Player of the Tournament in the same competition. In the FA National Futsal League, he won several FA National League Championships and FA Futsal Cup medals. Reed also works as a commentator for Sony SIX, Eurosport and BT Sport, and has worked with the emerging US Professional Futsal League as well as Indian Premier Futsal.»\n",+ "[3] «Doom (professional wrestling) | Doom was a professional wrestling tag team composed of Ron Simmons and Butch Reed. They competed from 1989 to 1991 in the National Wrestling Alliance (NWA) and World Championship Wrestling (WCW).»\n",+ "\n",+ "Question: What other sport did Butch Reed, of the wrestling tag team Doom, play professionally?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Butch Reed football\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «John French, 1st Earl of Ypres | Field Marshal John Denton Pinkstone French, 1st Earl of Ypres (28 September 1852 – 22 May 1925), known as Sir John French from 1901 to 1916, and as The Viscount French between 1916 and 1922, was a senior British Army officer. Born in Kent to an Anglo-Irish family, he saw brief service as a midshipman in the Royal Navy, before becoming a cavalry officer. He achieved rapid promotion and distinguished himself on the Gordon Relief Expedition. French had a considerable reputation as a womaniser throughout his life and his career nearly ended when he was cited in the divorce of a brother officer whilst in India in the early 1890s.»\n",+ "[2] «Archibald Wavell, 1st Earl Wavell | Field Marshal Archibald Percival Wavell, 1st Earl Wavell, {'1': \", '2': \", '3': \", '4': \"} (5 May 1883 – 24 May 1950) was a senior officer of the British Army. He served in the Second Boer War, the Bazar Valley Campaign and the Great War, during which he was wounded in the Second Battle of Ypres. He served in the Second World War, initially as Commander-in-Chief Middle East, in which role he led British forces to victory over the Italians in western Egypt and eastern Libya during Operation \"Compass\" in December 1940, only to be defeated by the German Army in the Western Desert in April 1941. He served as Commander-in-Chief, India, from July 1941 until June 1943 (apart from a brief tour as Commander of ABDACOM) and then served as Viceroy of India until his retirement in February 1947.»\n",+ "[3] «Ripple, Kent | John French, 1st Earl of Ypres, the commander of the first British Expeditionary Force was born there in 1852, and is buried at the village church. His sister Charlotte Despard, the suffragist, novelist and Sinn Féin activist was also born in Ripple in 1844.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Search Query. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to find the release date of the computer-animated short comedy film Hammy's Boomerang Adventure, which is a spin-off of \"Over the Hedge\". We need to find the release date of the film, but the context does not provide the information. We need to find the release date of the film, which is a spin-off of \"Over the Hedge\". We can find the release date of \"Over the Hedge\" and then find the release date of the spin-off film.\n",+ "Search Query: Over the Hedge release date\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Given the context and question, produce a specific and concise answer that accurately responds to the query, by iteratively generating relevant search queries and reasoning step by step.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Answer: ${answer}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.»\n",+ "[2] «Ebenezer Howard | Sir Ebenezer Howard {'1': \", '2': \", '3': \", '4': \"} ( ; 29 January 1850 – 1 May 1928), the English founder of the garden city movement, is known for his publication \"\" (1898), the description of a utopian city in which people live harmoniously together with nature. The publication resulted in the founding of the garden city movement, and the building of the First Garden City, Letchworth Garden City, commenced in 1903.»\n",+ "[3] «Eugene Reynal | Eugene Reynal was an American Publisher and in 1933 Founder of Blue Ribbon Books of Garden City, New York and Reynal and Hitchcock of New York, New York.»\n",+ "\n",+ "Question: Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "Context: [1] «Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.»\n",+ "\n",+ "[2] «A.T. Stewart | Alexander Turney Stewart (October 2, 1792 – April 10, 1876) was an American businessman, politician, and philanthropist. He was a prominent figure in New York City during the mid-19th century, and was known as the \"Merchant Prince\".»\n",+ "\n",+ "[3] «Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.»\n",+ "\n",+ "Question: Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "\n",+ "Answer: A.T. Stewart\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.»\n",+ "[2] «Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford's second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.»\n",+ "[3] «G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.»\n",+ "[4] «College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour»\n",+ "[5] «Bass fishing | Bass fishing is the activity of angling for the North American gamefish known colloquially as the \"black bass\". There are numerous black bass species considered as gamefish in North America, including largemouth bass (\"Micropterus salmoides\"), smallmouth bass (\"Micropterus dolomieui\"), spotted bass or Kentucky bass (\"Micropterus punctulatus\"), and Guadalupe bass (order \"Perciformes\").»\n",+ "[6] «Cliff Crochet | Cliff Crochet (born August 1, 1983), also known as \"The Cajun Baby\", is a professional bass fisherman from Pierre Part, Louisiana. Crochet is a competitor on Bass Angler Sportsman Society's (B.A.S.S.) Bassmaster Elite Series and has $368,837 in B.A.S.S. tournament winnings as of August 22, 2016. Crochet's sponsors include: K2 Coolers, Huk, Skeeter, Yamaha Motor Company, Lowrance Electronics, Falcon Rods, Luck E Strike, Seaguar, Rat-L-Trap, Humdinger Spinnerbaits, Power-Pole. Since 2010, Crochet has competed in three Bassmaster Classic tournaments. Crochet's boat for the 2016 season is a 2016 Skeeter FX 21 with a 250 hp Yamaha SHO outboard engine.»\n",+ "\n",+ "Question: Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "Context: [1] «Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.»\n",+ "\n",+ "[2] «College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour»\n",+ "\n",+ "[3] «G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.»\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "\n",+ "Answer: Carhartt\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Stitch's Great Escape! | Stitch's Great Escape! is a Tomorrowland attraction at the Magic Kingdom theme park within the Walt Disney World Resort. It is a \"theater-in-the-round\" experience starring the title alien from Walt Disney Animation Studios' 2002 film \"Lilo & Stitch\". It opened November 16, 2004 and is the fourth attraction to occupy the site in Tomorrowland. Many of the animators who worked on \"Lilo & Stitch\" partnered with Walt Disney Imagineering for Stitch's Great Escape! The attraction is a replacement of The ExtraTERRORestrial Alien Encounter, which formerly occupied the building the attraction is housed in. On September 21, 2016, it was announced that the attraction would be switching from a daily operated attraction to a seasonally operated one, depending on attendance, starting October 2, 2016.»\n",+ "[2] «Great Escape (amusement park) | Great Escape (sometimes referred to as Six Flags Great Escape) is an amusement and water park owned and operated by Six Flags Entertainment Corp. It is located approximately 60 mi north of Albany, in Queensbury, New York, but is advertised as being in Lake George, New York, a popular tourist and vacation spot nearby. It is one of two Six Flags parks not to be officially branded with the \"Six Flags\" name (La Ronde in Montreal, Quebec, Canada being the other).»\n",+ "[3] «Houdini's Great Escape | Houdini's Great Escape, also known as \"Boo-dini\" during Fright Fest and \"Houdini - The Great Escape\" at Six Flags New England, are indoor Vekoma Madhouse attractions at Six Flags Great Adventure and Six Flags New England. The ride is located in Great Adventure's Fantasy Forest section and New England's Crack Axle Canyon area. The attraction opened at both parks in 1999, and was Standing but not operating (SBNO) at Great Adventure from 2008 - 2009 but was opened again due to popular demand.»\n",+ "[4] «Magic Kingdom | Magic Kingdom is a theme park at the Walt Disney World Resort in Bay Lake, Florida, near Orlando. Owned and operated by The Walt Disney Company through its Parks and Resorts division, the park opened on October 1, 1971, as the first of four theme parks at the resort. Initialized by Walt Disney and designed by WED Enterprises, its layout and attractions are based on Disneyland Park in Anaheim, California, and is dedicated to fairy tales and Disney characters.»\n",+ "[5] «Magic Kingdom (disambiguation) | The Magic Kingdom is a theme park at the Walt Disney World Resort in Florida.»\n",+ "[6] «List of Magic Kingdom attractions | The Magic Kingdom is a theme park located at the Walt Disney World Resort in Florida. Below is a list of the current attractions found therein, arranged by \"land\" and with brief descriptions.»\n",+ "\n",+ "Question: Which theme park has Stitch’s Great Escape which replaced the attraction ExtraTERRORestial Alien Encounter?\n",+ "\n",+ "Reasoning: Let's think step by step in order to We need to find the theme park that has Stitch's Great Escape, which replaced the attraction ExtraTERRORestial Alien Encounter. The context mentions that Stitch's Great Escape is located in the Magic Kingdom theme park.\n",+ "\n",+ "Answer: Magic Kingdom\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «John French, 1st Earl of Ypres | Field Marshal John Denton Pinkstone French, 1st Earl of Ypres (28 September 1852 – 22 May 1925), known as Sir John French from 1901 to 1916, and as The Viscount French between 1916 and 1922, was a senior British Army officer. Born in Kent to an Anglo-Irish family, he saw brief service as a midshipman in the Royal Navy, before becoming a cavalry officer. He achieved rapid promotion and distinguished himself on the Gordon Relief Expedition. French had a considerable reputation as a womaniser throughout his life and his career nearly ended when he was cited in the divorce of a brother officer whilst in India in the early 1890s.»\n",+ "[2] «Archibald Wavell, 1st Earl Wavell | Field Marshal Archibald Percival Wavell, 1st Earl Wavell, {'1': \", '2': \", '3': \", '4': \"} (5 May 1883 – 24 May 1950) was a senior officer of the British Army. He served in the Second Boer War, the Bazar Valley Campaign and the Great War, during which he was wounded in the Second Battle of Ypres. He served in the Second World War, initially as Commander-in-Chief Middle East, in which role he led British forces to victory over the Italians in western Egypt and eastern Libya during Operation \"Compass\" in December 1940, only to be defeated by the German Army in the Western Desert in April 1941. He served as Commander-in-Chief, India, from July 1941 until June 1943 (apart from a brief tour as Commander of ABDACOM) and then served as Viceroy of India until his retirement in February 1947.»\n",+ "[3] «Ripple, Kent | John French, 1st Earl of Ypres, the commander of the first British Expeditionary Force was born there in 1852, and is buried at the village church. His sister Charlotte Despard, the suffragist, novelist and Sinn Féin activist was also born in Ripple in 1844.»\n",+ "[4] «Over the Hedge (film) | Over the Hedge is a 2006 American computer-animated comedy film, based on the characters from the United Media comic strip of the same name. Directed by Tim Johnson and Karey Kirkpatrick, and produced by Bonnie Arnold, it was released in the United States on May 19, 2006. The film was produced by DreamWorks Animation and distributed through Paramount Pictures. The film features the voices of Bruce Willis, Garry Shandling, Steve Carell, William Shatner, Wanda Sykes, and Nick Nolte. It is the first DreamWorks Animation film to be distributed by Paramount Pictures, which acquired the live-action DreamWorks studio in 2006. The film earned $336 million on an $80 million budget.»\n",+ "[5] «Over the Hedge: Hammy Goes Nuts! | Over the Hedge: Hammy Goes Nuts! is a video game published by Activision based on characters from the film \"Over the Hedge\". The Game Boy Advance (GBA) version, a miniature golf game, was developed by Vicarious Visions and released on October 10, 2006 in North America. The Nintendo DS and PlayStation Portable (PSP) versions, an action adventure game and a platform game respectively, were both developed by Amaze Entertainment and were released on October 24, 2006 and November 21, 2006. It is the second game to be based on the film.»\n",+ "[6] «Over the Hedge | Over the Hedge is a syndicated comic strip written and drawn by Michael Fry and T. Lewis. It tells the story of a raccoon, turtle, a squirrel, and their friends who come to terms with their woodlands being taken over by suburbia, trying to survive the increasing flow of humanity and technology while becoming enticed by it at the same time. The strip debuted in June 1995.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Answer. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to find the birth year of the 1st Earl of Ypres who attacked the Boer force from the rear while mounted infantry. We need to find the context that mentions the 1st Earl of Ypres and his actions during the Second Boer War.\n",+ "Answer: 1852\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Given the context and question, generate a search query that accurately captures the essence of the query and retrieves relevant information to answer the question, ensuring the query is concise, descriptive, and specific to the context and question.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Search Query: ${search_query}\n",+ "\n",+ "---\n",+ "\n",+ "Context: N/A\n",+ "\n",+ "Question: What company formed in 1978 has Simon Arora as its CEO?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Simon Arora CEO company 1978\n",+ "\n",+ "---\n",+ "\n",+ "Context: N/A\n",+ "\n",+ "Question: Bragg electoral district, incorporating the Kensington Park suburb, are associated with which South Australian city?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Bragg electoral district Kensington Park suburb South Australian city\n",+ "\n",+ "---\n",+ "\n",+ "Context: N/A\n",+ "\n",+ "Question: The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Kangchenjunga Singhi Kangri country\n",+ "\n",+ "---\n",+ "\n",+ "Context: N/A\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Search Query. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: 1st Earl of Ypres birthdate Boer force mounted infantry\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Provide a descriptive search query that entails the critical information from the context and question to find a precise answer to the query asked, such as finding the release date of the computer-animated short comedy film Hammy's Boomerang Adventure, which is a spin-off of \"Over the Hedge\".\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Search Query: ${search_query}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Butch Reed | Bruce Franklin Reed (born July 11, 1954) is an American retired professional wrestler and football player, better known by the ring name Butch Reed. He is best known for his appearances with the World Wrestling Federation and World Championship Wrestling between 1986 and 1992.»\n",+ "[2] «Doug Reed (futsal player) | Doug Reed is a professional futsal player who is currently playing for Futsal Dinamo and England national futsal team. Previously he has played for Lanzarote Tias Yaiza (Spain), AEL Limassol, MNK Uspinjača, MNK Smederevo and Manchester Futsal Club. In 2011, he won the FA Futsal Cup with Manchester Futsal Club and was awarded Player of the Tournament in the same competition. In the FA National Futsal League, he won several FA National League Championships and FA Futsal Cup medals. Reed also works as a commentator for Sony SIX, Eurosport and BT Sport, and has worked with the emerging US Professional Futsal League as well as Indian Premier Futsal.»\n",+ "[3] «Doom (professional wrestling) | Doom was a professional wrestling tag team composed of Ron Simmons and Butch Reed. They competed from 1989 to 1991 in the National Wrestling Alliance (NWA) and World Championship Wrestling (WCW).»\n",+ "\n",+ "Question: What other sport did Butch Reed, of the wrestling tag team Doom, play professionally?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Butch Reed football\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «John French, 1st Earl of Ypres | Field Marshal John Denton Pinkstone French, 1st Earl of Ypres (28 September 1852 – 22 May 1925), known as Sir John French from 1901 to 1916, and as The Viscount French between 1916 and 1922, was a senior British Army officer. Born in Kent to an Anglo-Irish family, he saw brief service as a midshipman in the Royal Navy, before becoming a cavalry officer. He achieved rapid promotion and distinguished himself on the Gordon Relief Expedition. French had a considerable reputation as a womaniser throughout his life and his career nearly ended when he was cited in the divorce of a brother officer whilst in India in the early 1890s.»\n",+ "[2] «Archibald Wavell, 1st Earl Wavell | Field Marshal Archibald Percival Wavell, 1st Earl Wavell, {'1': \", '2': \", '3': \", '4': \"} (5 May 1883 – 24 May 1950) was a senior officer of the British Army. He served in the Second Boer War, the Bazar Valley Campaign and the Great War, during which he was wounded in the Second Battle of Ypres. He served in the Second World War, initially as Commander-in-Chief Middle East, in which role he led British forces to victory over the Italians in western Egypt and eastern Libya during Operation \"Compass\" in December 1940, only to be defeated by the German Army in the Western Desert in April 1941. He served as Commander-in-Chief, India, from July 1941 until June 1943 (apart from a brief tour as Commander of ABDACOM) and then served as Viceroy of India until his retirement in February 1947.»\n",+ "[3] «Ripple, Kent | John French, 1st Earl of Ypres, the commander of the first British Expeditionary Force was born there in 1852, and is buried at the village church. His sister Charlotte Despard, the suffragist, novelist and Sinn Féin activist was also born in Ripple in 1844.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Search Query. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to find the release date of the computer-animated short comedy film Hammy's Boomerang Adventure, which is a spin-off of \"Over the Hedge\". We need to find the release date of the film, but the context does not provide the information. We need to find the release date of the film, which is a spin-off of \"Over the Hedge\". We can find the release date of \"Over the Hedge\" and then find the release date of the spin-off film.\n",+ "Search Query: Over the Hedge release date\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Given the context and question, produce a specific and concise answer that accurately responds to the query, by iteratively generating relevant search queries and reasoning step by step.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Answer: ${answer}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.»\n",+ "[2] «Ebenezer Howard | Sir Ebenezer Howard {'1': \", '2': \", '3': \", '4': \"} ( ; 29 January 1850 – 1 May 1928), the English founder of the garden city movement, is known for his publication \"\" (1898), the description of a utopian city in which people live harmoniously together with nature. The publication resulted in the founding of the garden city movement, and the building of the First Garden City, Letchworth Garden City, commenced in 1903.»\n",+ "[3] «Eugene Reynal | Eugene Reynal was an American Publisher and in 1933 Founder of Blue Ribbon Books of Garden City, New York and Reynal and Hitchcock of New York, New York.»\n",+ "\n",+ "Question: Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "Context: [1] «Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.»\n",+ "\n",+ "[2] «A.T. Stewart | Alexander Turney Stewart (October 2, 1792 – April 10, 1876) was an American businessman, politician, and philanthropist. He was a prominent figure in New York City during the mid-19th century, and was known as the \"Merchant Prince\".»\n",+ "\n",+ "[3] «Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.»\n",+ "\n",+ "Question: Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "\n",+ "Answer: A.T. Stewart\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.»\n",+ "[2] «Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford's second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.»\n",+ "[3] «G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.»\n",+ "[4] «College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour»\n",+ "[5] «Bass fishing | Bass fishing is the activity of angling for the North American gamefish known colloquially as the \"black bass\". There are numerous black bass species considered as gamefish in North America, including largemouth bass (\"Micropterus salmoides\"), smallmouth bass (\"Micropterus dolomieui\"), spotted bass or Kentucky bass (\"Micropterus punctulatus\"), and Guadalupe bass (order \"Perciformes\").»\n",+ "[6] «Cliff Crochet | Cliff Crochet (born August 1, 1983), also known as \"The Cajun Baby\", is a professional bass fisherman from Pierre Part, Louisiana. Crochet is a competitor on Bass Angler Sportsman Society's (B.A.S.S.) Bassmaster Elite Series and has $368,837 in B.A.S.S. tournament winnings as of August 22, 2016. Crochet's sponsors include: K2 Coolers, Huk, Skeeter, Yamaha Motor Company, Lowrance Electronics, Falcon Rods, Luck E Strike, Seaguar, Rat-L-Trap, Humdinger Spinnerbaits, Power-Pole. Since 2010, Crochet has competed in three Bassmaster Classic tournaments. Crochet's boat for the 2016 season is a 2016 Skeeter FX 21 with a 250 hp Yamaha SHO outboard engine.»\n",+ "\n",+ "Question: Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "Context: [1] «Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.»\n",+ "\n",+ "[2] «College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour»\n",+ "\n",+ "[3] «G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.»\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "\n",+ "Answer: Carhartt\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Stitch's Great Escape! | Stitch's Great Escape! is a Tomorrowland attraction at the Magic Kingdom theme park within the Walt Disney World Resort. It is a \"theater-in-the-round\" experience starring the title alien from Walt Disney Animation Studios' 2002 film \"Lilo & Stitch\". It opened November 16, 2004 and is the fourth attraction to occupy the site in Tomorrowland. Many of the animators who worked on \"Lilo & Stitch\" partnered with Walt Disney Imagineering for Stitch's Great Escape! The attraction is a replacement of The ExtraTERRORestrial Alien Encounter, which formerly occupied the building the attraction is housed in. On September 21, 2016, it was announced that the attraction would be switching from a daily operated attraction to a seasonally operated one, depending on attendance, starting October 2, 2016.»\n",+ "[2] «Great Escape (amusement park) | Great Escape (sometimes referred to as Six Flags Great Escape) is an amusement and water park owned and operated by Six Flags Entertainment Corp. It is located approximately 60 mi north of Albany, in Queensbury, New York, but is advertised as being in Lake George, New York, a popular tourist and vacation spot nearby. It is one of two Six Flags parks not to be officially branded with the \"Six Flags\" name (La Ronde in Montreal, Quebec, Canada being the other).»\n",+ "[3] «Houdini's Great Escape | Houdini's Great Escape, also known as \"Boo-dini\" during Fright Fest and \"Houdini - The Great Escape\" at Six Flags New England, are indoor Vekoma Madhouse attractions at Six Flags Great Adventure and Six Flags New England. The ride is located in Great Adventure's Fantasy Forest section and New England's Crack Axle Canyon area. The attraction opened at both parks in 1999, and was Standing but not operating (SBNO) at Great Adventure from 2008 - 2009 but was opened again due to popular demand.»\n",+ "[4] «Magic Kingdom | Magic Kingdom is a theme park at the Walt Disney World Resort in Bay Lake, Florida, near Orlando. Owned and operated by The Walt Disney Company through its Parks and Resorts division, the park opened on October 1, 1971, as the first of four theme parks at the resort. Initialized by Walt Disney and designed by WED Enterprises, its layout and attractions are based on Disneyland Park in Anaheim, California, and is dedicated to fairy tales and Disney characters.»\n",+ "[5] «Magic Kingdom (disambiguation) | The Magic Kingdom is a theme park at the Walt Disney World Resort in Florida.»\n",+ "[6] «List of Magic Kingdom attractions | The Magic Kingdom is a theme park located at the Walt Disney World Resort in Florida. Below is a list of the current attractions found therein, arranged by \"land\" and with brief descriptions.»\n",+ "\n",+ "Question: Which theme park has Stitch’s Great Escape which replaced the attraction ExtraTERRORestial Alien Encounter?\n",+ "\n",+ "Reasoning: Let's think step by step in order to We need to find the theme park that has Stitch's Great Escape, which replaced the attraction ExtraTERRORestial Alien Encounter. The context mentions that Stitch's Great Escape is located in the Magic Kingdom theme park.\n",+ "\n",+ "Answer: Magic Kingdom\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «John French, 1st Earl of Ypres | Field Marshal John Denton Pinkstone French, 1st Earl of Ypres (28 September 1852 – 22 May 1925), known as Sir John French from 1901 to 1916, and as The Viscount French between 1916 and 1922, was a senior British Army officer. Born in Kent to an Anglo-Irish family, he saw brief service as a midshipman in the Royal Navy, before becoming a cavalry officer. He achieved rapid promotion and distinguished himself on the Gordon Relief Expedition. French had a considerable reputation as a womaniser throughout his life and his career nearly ended when he was cited in the divorce of a brother officer whilst in India in the early 1890s.»\n",+ "[2] «Archibald Wavell, 1st Earl Wavell | Field Marshal Archibald Percival Wavell, 1st Earl Wavell, {'1': \", '2': \", '3': \", '4': \"} (5 May 1883 – 24 May 1950) was a senior officer of the British Army. He served in the Second Boer War, the Bazar Valley Campaign and the Great War, during which he was wounded in the Second Battle of Ypres. He served in the Second World War, initially as Commander-in-Chief Middle East, in which role he led British forces to victory over the Italians in western Egypt and eastern Libya during Operation \"Compass\" in December 1940, only to be defeated by the German Army in the Western Desert in April 1941. He served as Commander-in-Chief, India, from July 1941 until June 1943 (apart from a brief tour as Commander of ABDACOM) and then served as Viceroy of India until his retirement in February 1947.»\n",+ "[3] «Ripple, Kent | John French, 1st Earl of Ypres, the commander of the first British Expeditionary Force was born there in 1852, and is buried at the village church. His sister Charlotte Despard, the suffragist, novelist and Sinn Féin activist was also born in Ripple in 1844.»\n",+ "[4] «Over the Hedge (film) | Over the Hedge is a 2006 American computer-animated comedy film, based on the characters from the United Media comic strip of the same name. Directed by Tim Johnson and Karey Kirkpatrick, and produced by Bonnie Arnold, it was released in the United States on May 19, 2006. The film was produced by DreamWorks Animation and distributed through Paramount Pictures. The film features the voices of Bruce Willis, Garry Shandling, Steve Carell, William Shatner, Wanda Sykes, and Nick Nolte. It is the first DreamWorks Animation film to be distributed by Paramount Pictures, which acquired the live-action DreamWorks studio in 2006. The film earned $336 million on an $80 million budget.»\n",+ "[5] «Over the Hedge: Hammy Goes Nuts! | Over the Hedge: Hammy Goes Nuts! is a video game published by Activision based on characters from the film \"Over the Hedge\". The Game Boy Advance (GBA) version, a miniature golf game, was developed by Vicarious Visions and released on October 10, 2006 in North America. The Nintendo DS and PlayStation Portable (PSP) versions, an action adventure game and a platform game respectively, were both developed by Amaze Entertainment and were released on October 24, 2006 and November 21, 2006. It is the second game to be based on the film.»\n",+ "[6] «Over the Hedge | Over the Hedge is a syndicated comic strip written and drawn by Michael Fry and T. Lewis. It tells the story of a raccoon, turtle, a squirrel, and their friends who come to terms with their woodlands being taken over by suburbia, trying to survive the increasing flow of humanity and technology while becoming enticed by it at the same time. The strip debuted in June 1995.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Answer. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to find the birth year of the 1st Earl of Ypres who attacked the Boer force from the rear while mounted infantry. We need to find the context that mentions the 1st Earl of Ypres and his actions during the Second Boer War.\n",+ "Answer: 1852\n",+ "\n",+ "\n",+ "\n",+ "...\n",+ "Score 52.0\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "[I 2024-09-23 19:10:49,348] Trial 4 finished with value: 52.0 and parameters: {'0_predictor_instruction': 6, '0_predictor_demos': 9, '1_predictor_instruction': 4, '1_predictor_demos': 7, '2_predictor_instruction': 1, '2_predictor_demos': 8}. Best is trial 0 with value: 52.0.\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "CANDIDATE PROGRAM:\n",+ "Predictor 0\n",+ "i: Generate a search query that accurately captures the essence of the question within the given context, leveraging specific keywords and phrases to guide the retrieval of relevant passages.\n",+ "p: Search Query:\n",+ "\n",+ "\n",+ "Predictor 1\n",+ "i: Given the context and question, formulate a search query that accurately captures the key information required to answer the question, incorporating relevant entities and concepts from the context, and ensuring that the query is specific, concise, and effectively retrieves relevant passages to support the question's answer.\n",+ "p: Search Query:\n",+ "\n",+ "\n",+ "Predictor 2\n",+ "i: Given the context, question, and relevant background information, generate the most accurate and informative answer by carefully analyzing the context and question, identifying the key details and relationships, and providing a clear and concise response that incorporates the relevant facts and justifications.\n",+ "p: Answer:\n",+ "\n",+ "\n",+ "...\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 10 / 25 (40.0): 100%|██████████| 25/25 [00:17<00:00, 1.44it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "FULL TRACE\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 0 / 1 (0.0): 100%|██████████| 1/1 [00:04<00:00, 4.41s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "\n",+ "\n",+ "\n",+ "Generate a search query that accurately captures the essence of the question within the given context, leveraging specific keywords and phrases to guide the retrieval of relevant passages.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Search Query: ${search_query}\n",+ "\n",+ "---\n",+ "\n",+ "Context: N/A\n",+ "\n",+ "Question: Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Garden City Hotel founder\n",+ "\n",+ "---\n",+ "\n",+ "Context: N/A\n",+ "\n",+ "Question: Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: clothing company founded 1889 college bass fishing\n",+ "\n",+ "---\n",+ "\n",+ "Context: N/A\n",+ "\n",+ "Question: Which theme park has Stitch’s Great Escape which replaced the attraction ExtraTERRORestial Alien Encounter?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Stitch's Great Escape theme park\n",+ "\n",+ "---\n",+ "\n",+ "Context: N/A\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Please provide the output field Search Query. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field Search Query.\n",+ "\n",+ "Search Query: 1st Earl of Ypres birthdate\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Given the context and question, formulate a search query that accurately captures the key information required to answer the question, incorporating relevant entities and concepts from the context, and ensuring that the query is specific, concise, and effectively retrieves relevant passages to support the question's answer.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Search Query: ${search_query}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Will Finn | Will Finn (born November 1, 1958) is an American animator, voice actor, and director. His work in animation includes characters from Disney and Don Bluth films such as \"The Secret of NIMH\", \"Oliver & Company\", \"The Little Mermaid\", \"The Rescuers Down Under\", and \"Pocahontas\". His characters includes Cogsworth in \"Beauty and the Beast\", Iago in \"Aladdin\", and Laverne in \"The Hunchback of Notre Dame\". Finn wrote and directed \"Home on the Range\" and did some voice acting the Hollywood Fish in \"Chicken Little\". In 2006, Finn directed the computer animated short Hammy's Boomerang Adventure, a spin-off of \"Over the Hedge\".»\n",+ "[2] «Over the Hedge (film) | Over the Hedge is a 2006 American computer-animated comedy film, based on the characters from the United Media comic strip of the same name. Directed by Tim Johnson and Karey Kirkpatrick, and produced by Bonnie Arnold, it was released in the United States on May 19, 2006. The film was produced by DreamWorks Animation and distributed through Paramount Pictures. The film features the voices of Bruce Willis, Garry Shandling, Steve Carell, William Shatner, Wanda Sykes, and Nick Nolte. It is the first DreamWorks Animation film to be distributed by Paramount Pictures, which acquired the live-action DreamWorks studio in 2006. The film earned $336 million on an $80 million budget.»\n",+ "[3] «Tim Johnson (film director) | Tim Johnson (born August 27, 1961) is an American animation director. Johnson has directed in many films such as \"Antz\", \"\" and \"Over the Hedge\". Recently, he directed the DreamWorks 3D film \"Home.\"»\n",+ "\n",+ "Question: When was the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed?\n",+ "\n",+ "Reasoning: Let's think step by step in order to find the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed. The context mentions that Will Finn directed the computer animated short Hammy's Boomerang Adventure, a spin-off of \"Over the Hedge\".\n",+ "\n",+ "Search Query: Hammy's Boomerang Adventure release date\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Ripple, Kent | John French, 1st Earl of Ypres, the commander of the first British Expeditionary Force was born there in 1852, and is buried at the village church. His sister Charlotte Despard, the suffragist, novelist and Sinn Féin activist was also born in Ripple in 1844.»\n",+ "[2] «John French, 1st Earl of Ypres | Field Marshal John Denton Pinkstone French, 1st Earl of Ypres (28 September 1852 – 22 May 1925), known as Sir John French from 1901 to 1916, and as The Viscount French between 1916 and 1922, was a senior British Army officer. Born in Kent to an Anglo-Irish family, he saw brief service as a midshipman in the Royal Navy, before becoming a cavalry officer. He achieved rapid promotion and distinguished himself on the Gordon Relief Expedition. French had a considerable reputation as a womaniser throughout his life and his career nearly ended when he was cited in the divorce of a brother officer whilst in India in the early 1890s.»\n",+ "[3] «John French, 2nd Earl of Ypres | John Richard Lowndes French, 2nd Earl of Ypres (6 July 1881 – 5 April 1958) was the son of the British field marshal and the first commander of the British Expeditionary Force (BEF) in World War I Sir John French.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Search Query. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to find the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed. The context mentions that Will Finn directed the computer animated short Hammy's Boomerang Adventure, a spin-off of \"Over the Hedge\". However, the question is asking about the 1st Earl of Ypres, which is not related to the context.\n",+ "Search Query: John French 1st Earl of Ypres birthdate\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Given the context, question, and relevant background information, generate the most accurate and informative answer by carefully analyzing the context and question, identifying the key details and relationships, and providing a clear and concise response that incorporates the relevant facts and justifications.\n",+ "\n",+ "---\n",+ "\n",+ "Question: What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?\n",+ "\n",+ "Answer: 2016\n",+ "\n",+ "---\n",+ "\n",+ "Question: How many shops does the larger of the Miami International Mall and Dolphin Mall have?\n",+ "\n",+ "Answer: 240\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Answer: ${answer}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Wheat Field with Cypresses | A Wheatfield with Cypresses (occasionally called \"A Cornfield with Cypresses\") is any of three similar 1889 oil paintings by Vincent van Gogh, as part of his wheat field series. All were executed at the Saint-Paul-de-Mausole mental asylum at Saint-Rémy near Arles, France, where Van Gogh was voluntarily a patient from May 1889 to May 1890. The works were inspired by the view from the window at the asylum towards the Alpilles mountains.»\n",+ "[2] «Green Wheat Field with Cypress | Green Wheat Field with Cypress (French: Champ de blé vert avec cyprès) is an oil on canvas painting by a Dutch Post-Impressionist Vincent van Gogh. It was completed in 1889, while van Gogh was voluntarily incarcerated at the asylum of St. Paul near Saint-Rémy in Provence. Several paintings of wheat fields with cypresses were made when van Gogh was able to leave the asylum grounds and explore the local landscape. Besides a fondness for cypresses, van Gogh had a special affinty with wheat fields; he depicted them dozens of times over the years; to Vincent they symbolized the cycle of life and death, and he found in them both solace and inspiration.»\n",+ "[3] «Olive Trees (Van Gogh series) | Vincent van Gogh painted at least 18 paintings of olive trees, mostly in Saint-Rémy-de-Provence in 1889. At his own request, he lived at an asylum there from May 1889 through May 1890 painting the gardens of the asylum and, when he had permission to venture outside its walls, nearby olive trees, cypresses and wheat fields.»\n",+ "[4] «Wheat Fields (Van Gogh series) | The Wheat Fields is a series of dozens of paintings by Vincent van Gogh, borne out of his religious studies and sermons, connection to nature, appreciation of manual laborers and desire to provide a means of offering comfort to others. The wheat field works demonstrate his progression as an artist from the drab \"Wheat Sheaves\" made in 1885 in the Netherlands to the colorful, dramatic paintings from Arles, Saint-Rémy and Auvers-sur-Oise of rural France.»\n",+ "\n",+ "Question: What was the series inspired by the appreciation of manual laborers that Wheat Field with Cypresses is a part of?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "Context: N/A\n",+ "\n",+ "Question: What was the series inspired by the appreciation of manual laborers that Wheat Field with Cypresses is a part of?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "\n",+ "Answer: Wheat Fields\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Ripple, Kent | John French, 1st Earl of Ypres, the commander of the first British Expeditionary Force was born there in 1852, and is buried at the village church. His sister Charlotte Despard, the suffragist, novelist and Sinn Féin activist was also born in Ripple in 1844.»\n",+ "[2] «John French, 1st Earl of Ypres | Field Marshal John Denton Pinkstone French, 1st Earl of Ypres (28 September 1852 – 22 May 1925), known as Sir John French from 1901 to 1916, and as The Viscount French between 1916 and 1922, was a senior British Army officer. Born in Kent to an Anglo-Irish family, he saw brief service as a midshipman in the Royal Navy, before becoming a cavalry officer. He achieved rapid promotion and distinguished himself on the Gordon Relief Expedition. French had a considerable reputation as a womaniser throughout his life and his career nearly ended when he was cited in the divorce of a brother officer whilst in India in the early 1890s.»\n",+ "[3] «John French, 2nd Earl of Ypres | John Richard Lowndes French, 2nd Earl of Ypres (6 July 1881 – 5 April 1958) was the son of the British field marshal and the first commander of the British Expeditionary Force (BEF) in World War I Sir John French.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Answer. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to find the 1st Earl of Ypres who attacked the Boer force from the rear while mounted infantry. We need to find the 1st Earl of Ypres' military career, which is mentioned in the context as the commander of the first British Expeditionary Force. This is John French, 1st Earl of Ypres.\n",+ "Answer: 1852\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Generate a search query that accurately captures the essence of the question within the given context, leveraging specific keywords and phrases to guide the retrieval of relevant passages.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Search Query: ${search_query}\n",+ "\n",+ "---\n",+ "\n",+ "Context: N/A\n",+ "\n",+ "Question: Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Garden City Hotel founder\n",+ "\n",+ "---\n",+ "\n",+ "Context: N/A\n",+ "\n",+ "Question: Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: clothing company founded 1889 college bass fishing\n",+ "\n",+ "---\n",+ "\n",+ "Context: N/A\n",+ "\n",+ "Question: Which theme park has Stitch’s Great Escape which replaced the attraction ExtraTERRORestial Alien Encounter?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Stitch's Great Escape theme park\n",+ "\n",+ "---\n",+ "\n",+ "Context: N/A\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Please provide the output field Search Query. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field Search Query.\n",+ "\n",+ "Search Query: 1st Earl of Ypres birthdate\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Given the context and question, formulate a search query that accurately captures the key information required to answer the question, incorporating relevant entities and concepts from the context, and ensuring that the query is specific, concise, and effectively retrieves relevant passages to support the question's answer.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Search Query: ${search_query}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Will Finn | Will Finn (born November 1, 1958) is an American animator, voice actor, and director. His work in animation includes characters from Disney and Don Bluth films such as \"The Secret of NIMH\", \"Oliver & Company\", \"The Little Mermaid\", \"The Rescuers Down Under\", and \"Pocahontas\". His characters includes Cogsworth in \"Beauty and the Beast\", Iago in \"Aladdin\", and Laverne in \"The Hunchback of Notre Dame\". Finn wrote and directed \"Home on the Range\" and did some voice acting the Hollywood Fish in \"Chicken Little\". In 2006, Finn directed the computer animated short Hammy's Boomerang Adventure, a spin-off of \"Over the Hedge\".»\n",+ "[2] «Over the Hedge (film) | Over the Hedge is a 2006 American computer-animated comedy film, based on the characters from the United Media comic strip of the same name. Directed by Tim Johnson and Karey Kirkpatrick, and produced by Bonnie Arnold, it was released in the United States on May 19, 2006. The film was produced by DreamWorks Animation and distributed through Paramount Pictures. The film features the voices of Bruce Willis, Garry Shandling, Steve Carell, William Shatner, Wanda Sykes, and Nick Nolte. It is the first DreamWorks Animation film to be distributed by Paramount Pictures, which acquired the live-action DreamWorks studio in 2006. The film earned $336 million on an $80 million budget.»\n",+ "[3] «Tim Johnson (film director) | Tim Johnson (born August 27, 1961) is an American animation director. Johnson has directed in many films such as \"Antz\", \"\" and \"Over the Hedge\". Recently, he directed the DreamWorks 3D film \"Home.\"»\n",+ "\n",+ "Question: When was the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed?\n",+ "\n",+ "Reasoning: Let's think step by step in order to find the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed. The context mentions that Will Finn directed the computer animated short Hammy's Boomerang Adventure, a spin-off of \"Over the Hedge\".\n",+ "\n",+ "Search Query: Hammy's Boomerang Adventure release date\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Ripple, Kent | John French, 1st Earl of Ypres, the commander of the first British Expeditionary Force was born there in 1852, and is buried at the village church. His sister Charlotte Despard, the suffragist, novelist and Sinn Féin activist was also born in Ripple in 1844.»\n",+ "[2] «John French, 1st Earl of Ypres | Field Marshal John Denton Pinkstone French, 1st Earl of Ypres (28 September 1852 – 22 May 1925), known as Sir John French from 1901 to 1916, and as The Viscount French between 1916 and 1922, was a senior British Army officer. Born in Kent to an Anglo-Irish family, he saw brief service as a midshipman in the Royal Navy, before becoming a cavalry officer. He achieved rapid promotion and distinguished himself on the Gordon Relief Expedition. French had a considerable reputation as a womaniser throughout his life and his career nearly ended when he was cited in the divorce of a brother officer whilst in India in the early 1890s.»\n",+ "[3] «John French, 2nd Earl of Ypres | John Richard Lowndes French, 2nd Earl of Ypres (6 July 1881 – 5 April 1958) was the son of the British field marshal and the first commander of the British Expeditionary Force (BEF) in World War I Sir John French.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Search Query. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to find the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed. The context mentions that Will Finn directed the computer animated short Hammy's Boomerang Adventure, a spin-off of \"Over the Hedge\". However, the question is asking about the 1st Earl of Ypres, which is not related to the context.\n",+ "Search Query: John French 1st Earl of Ypres birthdate\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Given the context, question, and relevant background information, generate the most accurate and informative answer by carefully analyzing the context and question, identifying the key details and relationships, and providing a clear and concise response that incorporates the relevant facts and justifications.\n",+ "\n",+ "---\n",+ "\n",+ "Question: What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?\n",+ "\n",+ "Answer: 2016\n",+ "\n",+ "---\n",+ "\n",+ "Question: How many shops does the larger of the Miami International Mall and Dolphin Mall have?\n",+ "\n",+ "Answer: 240\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Answer: ${answer}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Wheat Field with Cypresses | A Wheatfield with Cypresses (occasionally called \"A Cornfield with Cypresses\") is any of three similar 1889 oil paintings by Vincent van Gogh, as part of his wheat field series. All were executed at the Saint-Paul-de-Mausole mental asylum at Saint-Rémy near Arles, France, where Van Gogh was voluntarily a patient from May 1889 to May 1890. The works were inspired by the view from the window at the asylum towards the Alpilles mountains.»\n",+ "[2] «Green Wheat Field with Cypress | Green Wheat Field with Cypress (French: Champ de blé vert avec cyprès) is an oil on canvas painting by a Dutch Post-Impressionist Vincent van Gogh. It was completed in 1889, while van Gogh was voluntarily incarcerated at the asylum of St. Paul near Saint-Rémy in Provence. Several paintings of wheat fields with cypresses were made when van Gogh was able to leave the asylum grounds and explore the local landscape. Besides a fondness for cypresses, van Gogh had a special affinty with wheat fields; he depicted them dozens of times over the years; to Vincent they symbolized the cycle of life and death, and he found in them both solace and inspiration.»\n",+ "[3] «Olive Trees (Van Gogh series) | Vincent van Gogh painted at least 18 paintings of olive trees, mostly in Saint-Rémy-de-Provence in 1889. At his own request, he lived at an asylum there from May 1889 through May 1890 painting the gardens of the asylum and, when he had permission to venture outside its walls, nearby olive trees, cypresses and wheat fields.»\n",+ "[4] «Wheat Fields (Van Gogh series) | The Wheat Fields is a series of dozens of paintings by Vincent van Gogh, borne out of his religious studies and sermons, connection to nature, appreciation of manual laborers and desire to provide a means of offering comfort to others. The wheat field works demonstrate his progression as an artist from the drab \"Wheat Sheaves\" made in 1885 in the Netherlands to the colorful, dramatic paintings from Arles, Saint-Rémy and Auvers-sur-Oise of rural France.»\n",+ "\n",+ "Question: What was the series inspired by the appreciation of manual laborers that Wheat Field with Cypresses is a part of?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "Context: N/A\n",+ "\n",+ "Question: What was the series inspired by the appreciation of manual laborers that Wheat Field with Cypresses is a part of?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "\n",+ "Answer: Wheat Fields\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Ripple, Kent | John French, 1st Earl of Ypres, the commander of the first British Expeditionary Force was born there in 1852, and is buried at the village church. His sister Charlotte Despard, the suffragist, novelist and Sinn Féin activist was also born in Ripple in 1844.»\n",+ "[2] «John French, 1st Earl of Ypres | Field Marshal John Denton Pinkstone French, 1st Earl of Ypres (28 September 1852 – 22 May 1925), known as Sir John French from 1901 to 1916, and as The Viscount French between 1916 and 1922, was a senior British Army officer. Born in Kent to an Anglo-Irish family, he saw brief service as a midshipman in the Royal Navy, before becoming a cavalry officer. He achieved rapid promotion and distinguished himself on the Gordon Relief Expedition. French had a considerable reputation as a womaniser throughout his life and his career nearly ended when he was cited in the divorce of a brother officer whilst in India in the early 1890s.»\n",+ "[3] «John French, 2nd Earl of Ypres | John Richard Lowndes French, 2nd Earl of Ypres (6 July 1881 – 5 April 1958) was the son of the British field marshal and the first commander of the British Expeditionary Force (BEF) in World War I Sir John French.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Answer. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to find the 1st Earl of Ypres who attacked the Boer force from the rear while mounted infantry. We need to find the 1st Earl of Ypres' military career, which is mentioned in the context as the commander of the first British Expeditionary Force. This is John French, 1st Earl of Ypres.\n",+ "Answer: 1852\n",+ "\n",+ "\n",+ "\n",+ "...\n",+ "Score 40.0\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "[I 2024-09-23 19:11:11,304] Trial 5 finished with value: 40.0 and parameters: {'0_predictor_instruction': 9, '0_predictor_demos': 8, '1_predictor_instruction': 2, '1_predictor_demos': 4, '2_predictor_instruction': 7, '2_predictor_demos': 6}. Best is trial 0 with value: 52.0.\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "CANDIDATE PROGRAM:\n",+ "Predictor 0\n",+ "i: Given the context and question, generate an informative search query that accurately captures the essence of the question, incorporating relevant keywords and phrases to retrieve relevant passages that can aid in answering the question.\n",+ "p: Search Query:\n",+ "\n",+ "\n",+ "Predictor 1\n",+ "i: Given the fields `context` and `question`, produce a search query that is a concise and specific phrase that accurately captures the main idea of the question and is likely to retrieve relevant passages from the context, by carefully considering the relationships between the context, question, and topic, and generating a query that will allow the model to execute a precise and effective search in the context.\n",+ "p: Search Query:\n",+ "\n",+ "\n",+ "Predictor 2\n",+ "i: Given the context and question, produce a specific and concise answer that accurately responds to the query, by iteratively generating relevant search queries and reasoning step by step.\n",+ "p: Answer:\n",+ "\n",+ "\n",+ "...\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 7 / 25 (28.0): 100%|██████████| 25/25 [00:22<00:00, 1.11it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "FULL TRACE\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 0 / 1 (0.0): 100%|██████████| 1/1 [00:03<00:00, 3.67s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "\n",+ "\n",+ "\n",+ "Given the fields `context` and `question`, produce a search query that is a concise and specific phrase that accurately captures the main idea of the question and is likely to retrieve relevant passages from the context, by carefully considering the relationships between the context, question, and topic, and generating a query that will allow the model to execute a precise and effective search in the context.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Search Query: ${search_query}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.»\n",+ "[2] «Ebenezer Howard | Sir Ebenezer Howard {'1': \", '2': \", '3': \", '4': \"} ( ; 29 January 1850 – 1 May 1928), the English founder of the garden city movement, is known for his publication \"\" (1898), the description of a utopian city in which people live harmoniously together with nature. The publication resulted in the founding of the garden city movement, and the building of the First Garden City, Letchworth Garden City, commenced in 1903.»\n",+ "[3] «Eugene Reynal | Eugene Reynal was an American Publisher and in 1933 Founder of Blue Ribbon Books of Garden City, New York and Reynal and Hitchcock of New York, New York.»\n",+ "\n",+ "Question: Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Garden City Hotel founder\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.»\n",+ "[2] «Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford's second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.»\n",+ "[3] «G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.»\n",+ "\n",+ "Question: Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Carhartt bass fishing\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Stitch's Great Escape! | Stitch's Great Escape! is a Tomorrowland attraction at the Magic Kingdom theme park within the Walt Disney World Resort. It is a \"theater-in-the-round\" experience starring the title alien from Walt Disney Animation Studios' 2002 film \"Lilo & Stitch\". It opened November 16, 2004 and is the fourth attraction to occupy the site in Tomorrowland. Many of the animators who worked on \"Lilo & Stitch\" partnered with Walt Disney Imagineering for Stitch's Great Escape! The attraction is a replacement of The ExtraTERRORestrial Alien Encounter, which formerly occupied the building the attraction is housed in. On September 21, 2016, it was announced that the attraction would be switching from a daily operated attraction to a seasonally operated one, depending on attendance, starting October 2, 2016.»\n",+ "[2] «Great Escape (amusement park) | Great Escape (sometimes referred to as Six Flags Great Escape) is an amusement and water park owned and operated by Six Flags Entertainment Corp. It is located approximately 60 mi north of Albany, in Queensbury, New York, but is advertised as being in Lake George, New York, a popular tourist and vacation spot nearby. It is one of two Six Flags parks not to be officially branded with the \"Six Flags\" name (La Ronde in Montreal, Quebec, Canada being the other).»\n",+ "[3] «Houdini's Great Escape | Houdini's Great Escape, also known as \"Boo-dini\" during Fright Fest and \"Houdini - The Great Escape\" at Six Flags New England, are indoor Vekoma Madhouse attractions at Six Flags Great Adventure and Six Flags New England. The ride is located in Great Adventure's Fantasy Forest section and New England's Crack Axle Canyon area. The attraction opened at both parks in 1999, and was Standing but not operating (SBNO) at Great Adventure from 2008 - 2009 but was opened again due to popular demand.»\n",+ "\n",+ "Question: Which theme park has Stitch’s Great Escape which replaced the attraction ExtraTERRORestial Alien Encounter?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Magic Kingdom theme park\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Archibald Wavell, 1st Earl Wavell | Field Marshal Archibald Percival Wavell, 1st Earl Wavell, {'1': \", '2': \", '3': \", '4': \"} (5 May 1883 – 24 May 1950) was a senior officer of the British Army. He served in the Second Boer War, the Bazar Valley Campaign and the Great War, during which he was wounded in the Second Battle of Ypres. He served in the Second World War, initially as Commander-in-Chief Middle East, in which role he led British forces to victory over the Italians in western Egypt and eastern Libya during Operation \"Compass\" in December 1940, only to be defeated by the German Army in the Western Desert in April 1941. He served as Commander-in-Chief, India, from July 1941 until June 1943 (apart from a brief tour as Commander of ABDACOM) and then served as Viceroy of India until his retirement in February 1947.»\n",+ "[2] «John French, 1st Earl of Ypres | Field Marshal John Denton Pinkstone French, 1st Earl of Ypres (28 September 1852 – 22 May 1925), known as Sir John French from 1901 to 1916, and as The Viscount French between 1916 and 1922, was a senior British Army officer. Born in Kent to an Anglo-Irish family, he saw brief service as a midshipman in the Royal Navy, before becoming a cavalry officer. He achieved rapid promotion and distinguished himself on the Gordon Relief Expedition. French had a considerable reputation as a womaniser throughout his life and his career nearly ended when he was cited in the divorce of a brother officer whilst in India in the early 1890s.»\n",+ "[3] «John French, 2nd Earl of Ypres | John Richard Lowndes French, 2nd Earl of Ypres (6 July 1881 – 5 April 1958) was the son of the British field marshal and the first commander of the British Expeditionary Force (BEF) in World War I Sir John French.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Search Query. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Given the fields `context` and `question`, produce a search query that is a concise and specific phrase that accurately captures the main idea of the question and is likely to retrieve relevant passages from the context, by carefully considering the relationships between the context, question, and topic, and generating a query that will allow the model to execute a precise and effective search in the context.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Search Query: ${search_query}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.»\n",+ "[2] «Ebenezer Howard | Sir Ebenezer Howard {'1': \", '2': \", '3': \", '4': \"} ( ; 29 January 1850 – 1 May 1928), the English founder of the garden city movement, is known for his publication \"\" (1898), the description of a utopian city in which people live harmoniously together with nature. The publication resulted in the founding of the garden city movement, and the building of the First Garden City, Letchworth Garden City, commenced in 1903.»\n",+ "[3] «Eugene Reynal | Eugene Reynal was an American Publisher and in 1933 Founder of Blue Ribbon Books of Garden City, New York and Reynal and Hitchcock of New York, New York.»\n",+ "\n",+ "Question: Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Garden City Hotel founder\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.»\n",+ "[2] «Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford's second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.»\n",+ "[3] «G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.»\n",+ "\n",+ "Question: Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Carhartt bass fishing\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Stitch's Great Escape! | Stitch's Great Escape! is a Tomorrowland attraction at the Magic Kingdom theme park within the Walt Disney World Resort. It is a \"theater-in-the-round\" experience starring the title alien from Walt Disney Animation Studios' 2002 film \"Lilo & Stitch\". It opened November 16, 2004 and is the fourth attraction to occupy the site in Tomorrowland. Many of the animators who worked on \"Lilo & Stitch\" partnered with Walt Disney Imagineering for Stitch's Great Escape! The attraction is a replacement of The ExtraTERRORestrial Alien Encounter, which formerly occupied the building the attraction is housed in. On September 21, 2016, it was announced that the attraction would be switching from a daily operated attraction to a seasonally operated one, depending on attendance, starting October 2, 2016.»\n",+ "[2] «Great Escape (amusement park) | Great Escape (sometimes referred to as Six Flags Great Escape) is an amusement and water park owned and operated by Six Flags Entertainment Corp. It is located approximately 60 mi north of Albany, in Queensbury, New York, but is advertised as being in Lake George, New York, a popular tourist and vacation spot nearby. It is one of two Six Flags parks not to be officially branded with the \"Six Flags\" name (La Ronde in Montreal, Quebec, Canada being the other).»\n",+ "[3] «Houdini's Great Escape | Houdini's Great Escape, also known as \"Boo-dini\" during Fright Fest and \"Houdini - The Great Escape\" at Six Flags New England, are indoor Vekoma Madhouse attractions at Six Flags Great Adventure and Six Flags New England. The ride is located in Great Adventure's Fantasy Forest section and New England's Crack Axle Canyon area. The attraction opened at both parks in 1999, and was Standing but not operating (SBNO) at Great Adventure from 2008 - 2009 but was opened again due to popular demand.»\n",+ "\n",+ "Question: Which theme park has Stitch’s Great Escape which replaced the attraction ExtraTERRORestial Alien Encounter?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Magic Kingdom theme park\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Archibald Wavell, 1st Earl Wavell | Field Marshal Archibald Percival Wavell, 1st Earl Wavell, {'1': \", '2': \", '3': \", '4': \"} (5 May 1883 – 24 May 1950) was a senior officer of the British Army. He served in the Second Boer War, the Bazar Valley Campaign and the Great War, during which he was wounded in the Second Battle of Ypres. He served in the Second World War, initially as Commander-in-Chief Middle East, in which role he led British forces to victory over the Italians in western Egypt and eastern Libya during Operation \"Compass\" in December 1940, only to be defeated by the German Army in the Western Desert in April 1941. He served as Commander-in-Chief, India, from July 1941 until June 1943 (apart from a brief tour as Commander of ABDACOM) and then served as Viceroy of India until his retirement in February 1947.»\n",+ "[2] «John French, 1st Earl of Ypres | Field Marshal John Denton Pinkstone French, 1st Earl of Ypres (28 September 1852 – 22 May 1925), known as Sir John French from 1901 to 1916, and as The Viscount French between 1916 and 1922, was a senior British Army officer. Born in Kent to an Anglo-Irish family, he saw brief service as a midshipman in the Royal Navy, before becoming a cavalry officer. He achieved rapid promotion and distinguished himself on the Gordon Relief Expedition. French had a considerable reputation as a womaniser throughout his life and his career nearly ended when he was cited in the divorce of a brother officer whilst in India in the early 1890s.»\n",+ "[3] «John French, 2nd Earl of Ypres | John Richard Lowndes French, 2nd Earl of Ypres (6 July 1881 – 5 April 1958) was the son of the British field marshal and the first commander of the British Expeditionary Force (BEF) in World War I Sir John French.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Please provide the output field Search Query. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field Search Query.\n",+ "\n",+ "Search Query: Archibald Wavell birthdate\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Given the context and question, produce a specific and concise answer that accurately responds to the query, by iteratively generating relevant search queries and reasoning step by step.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Answer: ${answer}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.»\n",+ "[2] «Ebenezer Howard | Sir Ebenezer Howard {'1': \", '2': \", '3': \", '4': \"} ( ; 29 January 1850 – 1 May 1928), the English founder of the garden city movement, is known for his publication \"\" (1898), the description of a utopian city in which people live harmoniously together with nature. The publication resulted in the founding of the garden city movement, and the building of the First Garden City, Letchworth Garden City, commenced in 1903.»\n",+ "[3] «Eugene Reynal | Eugene Reynal was an American Publisher and in 1933 Founder of Blue Ribbon Books of Garden City, New York and Reynal and Hitchcock of New York, New York.»\n",+ "\n",+ "Question: Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "Context: [1] «Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.»\n",+ "\n",+ "[2] «A.T. Stewart | Alexander Turney Stewart (October 2, 1792 – April 10, 1876) was an American businessman, politician, and philanthropist. He was a prominent figure in New York City during the mid-19th century, and was known as the \"Merchant Prince\".»\n",+ "\n",+ "[3] «Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.»\n",+ "\n",+ "Question: Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "\n",+ "Answer: A.T. Stewart\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.»\n",+ "[2] «Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford's second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.»\n",+ "[3] «G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.»\n",+ "[4] «College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour»\n",+ "[5] «Bass fishing | Bass fishing is the activity of angling for the North American gamefish known colloquially as the \"black bass\". There are numerous black bass species considered as gamefish in North America, including largemouth bass (\"Micropterus salmoides\"), smallmouth bass (\"Micropterus dolomieui\"), spotted bass or Kentucky bass (\"Micropterus punctulatus\"), and Guadalupe bass (order \"Perciformes\").»\n",+ "[6] «Cliff Crochet | Cliff Crochet (born August 1, 1983), also known as \"The Cajun Baby\", is a professional bass fisherman from Pierre Part, Louisiana. Crochet is a competitor on Bass Angler Sportsman Society's (B.A.S.S.) Bassmaster Elite Series and has $368,837 in B.A.S.S. tournament winnings as of August 22, 2016. Crochet's sponsors include: K2 Coolers, Huk, Skeeter, Yamaha Motor Company, Lowrance Electronics, Falcon Rods, Luck E Strike, Seaguar, Rat-L-Trap, Humdinger Spinnerbaits, Power-Pole. Since 2010, Crochet has competed in three Bassmaster Classic tournaments. Crochet's boat for the 2016 season is a 2016 Skeeter FX 21 with a 250 hp Yamaha SHO outboard engine.»\n",+ "\n",+ "Question: Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "Context: [1] «Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.»\n",+ "\n",+ "[2] «College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour»\n",+ "\n",+ "[3] «G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.»\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "\n",+ "Answer: Carhartt\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Stitch's Great Escape! | Stitch's Great Escape! is a Tomorrowland attraction at the Magic Kingdom theme park within the Walt Disney World Resort. It is a \"theater-in-the-round\" experience starring the title alien from Walt Disney Animation Studios' 2002 film \"Lilo & Stitch\". It opened November 16, 2004 and is the fourth attraction to occupy the site in Tomorrowland. Many of the animators who worked on \"Lilo & Stitch\" partnered with Walt Disney Imagineering for Stitch's Great Escape! The attraction is a replacement of The ExtraTERRORestrial Alien Encounter, which formerly occupied the building the attraction is housed in. On September 21, 2016, it was announced that the attraction would be switching from a daily operated attraction to a seasonally operated one, depending on attendance, starting October 2, 2016.»\n",+ "[2] «Great Escape (amusement park) | Great Escape (sometimes referred to as Six Flags Great Escape) is an amusement and water park owned and operated by Six Flags Entertainment Corp. It is located approximately 60 mi north of Albany, in Queensbury, New York, but is advertised as being in Lake George, New York, a popular tourist and vacation spot nearby. It is one of two Six Flags parks not to be officially branded with the \"Six Flags\" name (La Ronde in Montreal, Quebec, Canada being the other).»\n",+ "[3] «Houdini's Great Escape | Houdini's Great Escape, also known as \"Boo-dini\" during Fright Fest and \"Houdini - The Great Escape\" at Six Flags New England, are indoor Vekoma Madhouse attractions at Six Flags Great Adventure and Six Flags New England. The ride is located in Great Adventure's Fantasy Forest section and New England's Crack Axle Canyon area. The attraction opened at both parks in 1999, and was Standing but not operating (SBNO) at Great Adventure from 2008 - 2009 but was opened again due to popular demand.»\n",+ "[4] «Magic Kingdom | Magic Kingdom is a theme park at the Walt Disney World Resort in Bay Lake, Florida, near Orlando. Owned and operated by The Walt Disney Company through its Parks and Resorts division, the park opened on October 1, 1971, as the first of four theme parks at the resort. Initialized by Walt Disney and designed by WED Enterprises, its layout and attractions are based on Disneyland Park in Anaheim, California, and is dedicated to fairy tales and Disney characters.»\n",+ "[5] «Magic Kingdom (disambiguation) | The Magic Kingdom is a theme park at the Walt Disney World Resort in Florida.»\n",+ "[6] «List of Magic Kingdom attractions | The Magic Kingdom is a theme park located at the Walt Disney World Resort in Florida. Below is a list of the current attractions found therein, arranged by \"land\" and with brief descriptions.»\n",+ "\n",+ "Question: Which theme park has Stitch’s Great Escape which replaced the attraction ExtraTERRORestial Alien Encounter?\n",+ "\n",+ "Reasoning: Let's think step by step in order to We need to find the theme park that has Stitch's Great Escape, which replaced the attraction ExtraTERRORestial Alien Encounter. The context mentions that Stitch's Great Escape is located in the Magic Kingdom theme park.\n",+ "\n",+ "Answer: Magic Kingdom\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Archibald Wavell, 1st Earl Wavell | Field Marshal Archibald Percival Wavell, 1st Earl Wavell, {'1': \", '2': \", '3': \", '4': \"} (5 May 1883 – 24 May 1950) was a senior officer of the British Army. He served in the Second Boer War, the Bazar Valley Campaign and the Great War, during which he was wounded in the Second Battle of Ypres. He served in the Second World War, initially as Commander-in-Chief Middle East, in which role he led British forces to victory over the Italians in western Egypt and eastern Libya during Operation \"Compass\" in December 1940, only to be defeated by the German Army in the Western Desert in April 1941. He served as Commander-in-Chief, India, from July 1941 until June 1943 (apart from a brief tour as Commander of ABDACOM) and then served as Viceroy of India until his retirement in February 1947.»\n",+ "[2] «John French, 1st Earl of Ypres | Field Marshal John Denton Pinkstone French, 1st Earl of Ypres (28 September 1852 – 22 May 1925), known as Sir John French from 1901 to 1916, and as The Viscount French between 1916 and 1922, was a senior British Army officer. Born in Kent to an Anglo-Irish family, he saw brief service as a midshipman in the Royal Navy, before becoming a cavalry officer. He achieved rapid promotion and distinguished himself on the Gordon Relief Expedition. French had a considerable reputation as a womaniser throughout his life and his career nearly ended when he was cited in the divorce of a brother officer whilst in India in the early 1890s.»\n",+ "[3] «John French, 2nd Earl of Ypres | John Richard Lowndes French, 2nd Earl of Ypres (6 July 1881 – 5 April 1958) was the son of the British field marshal and the first commander of the British Expeditionary Force (BEF) in World War I Sir John French.»\n",+ "[4] «Archibald Wavell, 2nd Earl Wavell | Major Archibald John Arthur Wavell, 2nd Earl Wavell MC (11 May 1916 – 24 December 1953) was the second and last Earl Wavell and Viscount Keren of Eritrea. He was educated at Winchester College and succeeded his father in 1950.»\n",+ "[5] «Archibald MacKinnon | Archibald (\"Archie\") MacKinnon (born January 13, 1937 in Cranbrook, British Columbia) is a Canadian competition rower and Olympic champion.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Answer. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to find the 1st Earl of Ypres who attacked the Boer force from the rear while mounted infantry. We need to find the context that mentions the 1st Earl of Ypres and the event where he attacked the Boer force from the rear while mounted infantry.\n",+ "\n",+ "Answer: John French, 1st Earl of Ypres\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Given the fields `context` and `question`, produce a search query that is a concise and specific phrase that accurately captures the main idea of the question and is likely to retrieve relevant passages from the context, by carefully considering the relationships between the context, question, and topic, and generating a query that will allow the model to execute a precise and effective search in the context.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Search Query: ${search_query}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.»\n",+ "[2] «Ebenezer Howard | Sir Ebenezer Howard {'1': \", '2': \", '3': \", '4': \"} ( ; 29 January 1850 – 1 May 1928), the English founder of the garden city movement, is known for his publication \"\" (1898), the description of a utopian city in which people live harmoniously together with nature. The publication resulted in the founding of the garden city movement, and the building of the First Garden City, Letchworth Garden City, commenced in 1903.»\n",+ "[3] «Eugene Reynal | Eugene Reynal was an American Publisher and in 1933 Founder of Blue Ribbon Books of Garden City, New York and Reynal and Hitchcock of New York, New York.»\n",+ "\n",+ "Question: Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Garden City Hotel founder\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.»\n",+ "[2] «Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford's second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.»\n",+ "[3] «G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.»\n",+ "\n",+ "Question: Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Carhartt bass fishing\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Stitch's Great Escape! | Stitch's Great Escape! is a Tomorrowland attraction at the Magic Kingdom theme park within the Walt Disney World Resort. It is a \"theater-in-the-round\" experience starring the title alien from Walt Disney Animation Studios' 2002 film \"Lilo & Stitch\". It opened November 16, 2004 and is the fourth attraction to occupy the site in Tomorrowland. Many of the animators who worked on \"Lilo & Stitch\" partnered with Walt Disney Imagineering for Stitch's Great Escape! The attraction is a replacement of The ExtraTERRORestrial Alien Encounter, which formerly occupied the building the attraction is housed in. On September 21, 2016, it was announced that the attraction would be switching from a daily operated attraction to a seasonally operated one, depending on attendance, starting October 2, 2016.»\n",+ "[2] «Great Escape (amusement park) | Great Escape (sometimes referred to as Six Flags Great Escape) is an amusement and water park owned and operated by Six Flags Entertainment Corp. It is located approximately 60 mi north of Albany, in Queensbury, New York, but is advertised as being in Lake George, New York, a popular tourist and vacation spot nearby. It is one of two Six Flags parks not to be officially branded with the \"Six Flags\" name (La Ronde in Montreal, Quebec, Canada being the other).»\n",+ "[3] «Houdini's Great Escape | Houdini's Great Escape, also known as \"Boo-dini\" during Fright Fest and \"Houdini - The Great Escape\" at Six Flags New England, are indoor Vekoma Madhouse attractions at Six Flags Great Adventure and Six Flags New England. The ride is located in Great Adventure's Fantasy Forest section and New England's Crack Axle Canyon area. The attraction opened at both parks in 1999, and was Standing but not operating (SBNO) at Great Adventure from 2008 - 2009 but was opened again due to popular demand.»\n",+ "\n",+ "Question: Which theme park has Stitch’s Great Escape which replaced the attraction ExtraTERRORestial Alien Encounter?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Magic Kingdom theme park\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Archibald Wavell, 1st Earl Wavell | Field Marshal Archibald Percival Wavell, 1st Earl Wavell, {'1': \", '2': \", '3': \", '4': \"} (5 May 1883 – 24 May 1950) was a senior officer of the British Army. He served in the Second Boer War, the Bazar Valley Campaign and the Great War, during which he was wounded in the Second Battle of Ypres. He served in the Second World War, initially as Commander-in-Chief Middle East, in which role he led British forces to victory over the Italians in western Egypt and eastern Libya during Operation \"Compass\" in December 1940, only to be defeated by the German Army in the Western Desert in April 1941. He served as Commander-in-Chief, India, from July 1941 until June 1943 (apart from a brief tour as Commander of ABDACOM) and then served as Viceroy of India until his retirement in February 1947.»\n",+ "[2] «John French, 1st Earl of Ypres | Field Marshal John Denton Pinkstone French, 1st Earl of Ypres (28 September 1852 – 22 May 1925), known as Sir John French from 1901 to 1916, and as The Viscount French between 1916 and 1922, was a senior British Army officer. Born in Kent to an Anglo-Irish family, he saw brief service as a midshipman in the Royal Navy, before becoming a cavalry officer. He achieved rapid promotion and distinguished himself on the Gordon Relief Expedition. French had a considerable reputation as a womaniser throughout his life and his career nearly ended when he was cited in the divorce of a brother officer whilst in India in the early 1890s.»\n",+ "[3] «John French, 2nd Earl of Ypres | John Richard Lowndes French, 2nd Earl of Ypres (6 July 1881 – 5 April 1958) was the son of the British field marshal and the first commander of the British Expeditionary Force (BEF) in World War I Sir John French.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Search Query. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Given the fields `context` and `question`, produce a search query that is a concise and specific phrase that accurately captures the main idea of the question and is likely to retrieve relevant passages from the context, by carefully considering the relationships between the context, question, and topic, and generating a query that will allow the model to execute a precise and effective search in the context.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Search Query: ${search_query}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.»\n",+ "[2] «Ebenezer Howard | Sir Ebenezer Howard {'1': \", '2': \", '3': \", '4': \"} ( ; 29 January 1850 – 1 May 1928), the English founder of the garden city movement, is known for his publication \"\" (1898), the description of a utopian city in which people live harmoniously together with nature. The publication resulted in the founding of the garden city movement, and the building of the First Garden City, Letchworth Garden City, commenced in 1903.»\n",+ "[3] «Eugene Reynal | Eugene Reynal was an American Publisher and in 1933 Founder of Blue Ribbon Books of Garden City, New York and Reynal and Hitchcock of New York, New York.»\n",+ "\n",+ "Question: Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Garden City Hotel founder\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.»\n",+ "[2] «Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford's second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.»\n",+ "[3] «G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.»\n",+ "\n",+ "Question: Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Carhartt bass fishing\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Stitch's Great Escape! | Stitch's Great Escape! is a Tomorrowland attraction at the Magic Kingdom theme park within the Walt Disney World Resort. It is a \"theater-in-the-round\" experience starring the title alien from Walt Disney Animation Studios' 2002 film \"Lilo & Stitch\". It opened November 16, 2004 and is the fourth attraction to occupy the site in Tomorrowland. Many of the animators who worked on \"Lilo & Stitch\" partnered with Walt Disney Imagineering for Stitch's Great Escape! The attraction is a replacement of The ExtraTERRORestrial Alien Encounter, which formerly occupied the building the attraction is housed in. On September 21, 2016, it was announced that the attraction would be switching from a daily operated attraction to a seasonally operated one, depending on attendance, starting October 2, 2016.»\n",+ "[2] «Great Escape (amusement park) | Great Escape (sometimes referred to as Six Flags Great Escape) is an amusement and water park owned and operated by Six Flags Entertainment Corp. It is located approximately 60 mi north of Albany, in Queensbury, New York, but is advertised as being in Lake George, New York, a popular tourist and vacation spot nearby. It is one of two Six Flags parks not to be officially branded with the \"Six Flags\" name (La Ronde in Montreal, Quebec, Canada being the other).»\n",+ "[3] «Houdini's Great Escape | Houdini's Great Escape, also known as \"Boo-dini\" during Fright Fest and \"Houdini - The Great Escape\" at Six Flags New England, are indoor Vekoma Madhouse attractions at Six Flags Great Adventure and Six Flags New England. The ride is located in Great Adventure's Fantasy Forest section and New England's Crack Axle Canyon area. The attraction opened at both parks in 1999, and was Standing but not operating (SBNO) at Great Adventure from 2008 - 2009 but was opened again due to popular demand.»\n",+ "\n",+ "Question: Which theme park has Stitch’s Great Escape which replaced the attraction ExtraTERRORestial Alien Encounter?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Magic Kingdom theme park\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Archibald Wavell, 1st Earl Wavell | Field Marshal Archibald Percival Wavell, 1st Earl Wavell, {'1': \", '2': \", '3': \", '4': \"} (5 May 1883 – 24 May 1950) was a senior officer of the British Army. He served in the Second Boer War, the Bazar Valley Campaign and the Great War, during which he was wounded in the Second Battle of Ypres. He served in the Second World War, initially as Commander-in-Chief Middle East, in which role he led British forces to victory over the Italians in western Egypt and eastern Libya during Operation \"Compass\" in December 1940, only to be defeated by the German Army in the Western Desert in April 1941. He served as Commander-in-Chief, India, from July 1941 until June 1943 (apart from a brief tour as Commander of ABDACOM) and then served as Viceroy of India until his retirement in February 1947.»\n",+ "[2] «John French, 1st Earl of Ypres | Field Marshal John Denton Pinkstone French, 1st Earl of Ypres (28 September 1852 – 22 May 1925), known as Sir John French from 1901 to 1916, and as The Viscount French between 1916 and 1922, was a senior British Army officer. Born in Kent to an Anglo-Irish family, he saw brief service as a midshipman in the Royal Navy, before becoming a cavalry officer. He achieved rapid promotion and distinguished himself on the Gordon Relief Expedition. French had a considerable reputation as a womaniser throughout his life and his career nearly ended when he was cited in the divorce of a brother officer whilst in India in the early 1890s.»\n",+ "[3] «John French, 2nd Earl of Ypres | John Richard Lowndes French, 2nd Earl of Ypres (6 July 1881 – 5 April 1958) was the son of the British field marshal and the first commander of the British Expeditionary Force (BEF) in World War I Sir John French.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Please provide the output field Search Query. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field Search Query.\n",+ "\n",+ "Search Query: Archibald Wavell birthdate\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Given the context and question, produce a specific and concise answer that accurately responds to the query, by iteratively generating relevant search queries and reasoning step by step.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Answer: ${answer}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.»\n",+ "[2] «Ebenezer Howard | Sir Ebenezer Howard {'1': \", '2': \", '3': \", '4': \"} ( ; 29 January 1850 – 1 May 1928), the English founder of the garden city movement, is known for his publication \"\" (1898), the description of a utopian city in which people live harmoniously together with nature. The publication resulted in the founding of the garden city movement, and the building of the First Garden City, Letchworth Garden City, commenced in 1903.»\n",+ "[3] «Eugene Reynal | Eugene Reynal was an American Publisher and in 1933 Founder of Blue Ribbon Books of Garden City, New York and Reynal and Hitchcock of New York, New York.»\n",+ "\n",+ "Question: Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "Context: [1] «Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.»\n",+ "\n",+ "[2] «A.T. Stewart | Alexander Turney Stewart (October 2, 1792 – April 10, 1876) was an American businessman, politician, and philanthropist. He was a prominent figure in New York City during the mid-19th century, and was known as the \"Merchant Prince\".»\n",+ "\n",+ "[3] «Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.»\n",+ "\n",+ "Question: Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "\n",+ "Answer: A.T. Stewart\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.»\n",+ "[2] «Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford's second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.»\n",+ "[3] «G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.»\n",+ "[4] «College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour»\n",+ "[5] «Bass fishing | Bass fishing is the activity of angling for the North American gamefish known colloquially as the \"black bass\". There are numerous black bass species considered as gamefish in North America, including largemouth bass (\"Micropterus salmoides\"), smallmouth bass (\"Micropterus dolomieui\"), spotted bass or Kentucky bass (\"Micropterus punctulatus\"), and Guadalupe bass (order \"Perciformes\").»\n",+ "[6] «Cliff Crochet | Cliff Crochet (born August 1, 1983), also known as \"The Cajun Baby\", is a professional bass fisherman from Pierre Part, Louisiana. Crochet is a competitor on Bass Angler Sportsman Society's (B.A.S.S.) Bassmaster Elite Series and has $368,837 in B.A.S.S. tournament winnings as of August 22, 2016. Crochet's sponsors include: K2 Coolers, Huk, Skeeter, Yamaha Motor Company, Lowrance Electronics, Falcon Rods, Luck E Strike, Seaguar, Rat-L-Trap, Humdinger Spinnerbaits, Power-Pole. Since 2010, Crochet has competed in three Bassmaster Classic tournaments. Crochet's boat for the 2016 season is a 2016 Skeeter FX 21 with a 250 hp Yamaha SHO outboard engine.»\n",+ "\n",+ "Question: Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "Context: [1] «Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.»\n",+ "\n",+ "[2] «College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour»\n",+ "\n",+ "[3] «G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.»\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "\n",+ "Answer: Carhartt\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Stitch's Great Escape! | Stitch's Great Escape! is a Tomorrowland attraction at the Magic Kingdom theme park within the Walt Disney World Resort. It is a \"theater-in-the-round\" experience starring the title alien from Walt Disney Animation Studios' 2002 film \"Lilo & Stitch\". It opened November 16, 2004 and is the fourth attraction to occupy the site in Tomorrowland. Many of the animators who worked on \"Lilo & Stitch\" partnered with Walt Disney Imagineering for Stitch's Great Escape! The attraction is a replacement of The ExtraTERRORestrial Alien Encounter, which formerly occupied the building the attraction is housed in. On September 21, 2016, it was announced that the attraction would be switching from a daily operated attraction to a seasonally operated one, depending on attendance, starting October 2, 2016.»\n",+ "[2] «Great Escape (amusement park) | Great Escape (sometimes referred to as Six Flags Great Escape) is an amusement and water park owned and operated by Six Flags Entertainment Corp. It is located approximately 60 mi north of Albany, in Queensbury, New York, but is advertised as being in Lake George, New York, a popular tourist and vacation spot nearby. It is one of two Six Flags parks not to be officially branded with the \"Six Flags\" name (La Ronde in Montreal, Quebec, Canada being the other).»\n",+ "[3] «Houdini's Great Escape | Houdini's Great Escape, also known as \"Boo-dini\" during Fright Fest and \"Houdini - The Great Escape\" at Six Flags New England, are indoor Vekoma Madhouse attractions at Six Flags Great Adventure and Six Flags New England. The ride is located in Great Adventure's Fantasy Forest section and New England's Crack Axle Canyon area. The attraction opened at both parks in 1999, and was Standing but not operating (SBNO) at Great Adventure from 2008 - 2009 but was opened again due to popular demand.»\n",+ "[4] «Magic Kingdom | Magic Kingdom is a theme park at the Walt Disney World Resort in Bay Lake, Florida, near Orlando. Owned and operated by The Walt Disney Company through its Parks and Resorts division, the park opened on October 1, 1971, as the first of four theme parks at the resort. Initialized by Walt Disney and designed by WED Enterprises, its layout and attractions are based on Disneyland Park in Anaheim, California, and is dedicated to fairy tales and Disney characters.»\n",+ "[5] «Magic Kingdom (disambiguation) | The Magic Kingdom is a theme park at the Walt Disney World Resort in Florida.»\n",+ "[6] «List of Magic Kingdom attractions | The Magic Kingdom is a theme park located at the Walt Disney World Resort in Florida. Below is a list of the current attractions found therein, arranged by \"land\" and with brief descriptions.»\n",+ "\n",+ "Question: Which theme park has Stitch’s Great Escape which replaced the attraction ExtraTERRORestial Alien Encounter?\n",+ "\n",+ "Reasoning: Let's think step by step in order to We need to find the theme park that has Stitch's Great Escape, which replaced the attraction ExtraTERRORestial Alien Encounter. The context mentions that Stitch's Great Escape is located in the Magic Kingdom theme park.\n",+ "\n",+ "Answer: Magic Kingdom\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Archibald Wavell, 1st Earl Wavell | Field Marshal Archibald Percival Wavell, 1st Earl Wavell, {'1': \", '2': \", '3': \", '4': \"} (5 May 1883 – 24 May 1950) was a senior officer of the British Army. He served in the Second Boer War, the Bazar Valley Campaign and the Great War, during which he was wounded in the Second Battle of Ypres. He served in the Second World War, initially as Commander-in-Chief Middle East, in which role he led British forces to victory over the Italians in western Egypt and eastern Libya during Operation \"Compass\" in December 1940, only to be defeated by the German Army in the Western Desert in April 1941. He served as Commander-in-Chief, India, from July 1941 until June 1943 (apart from a brief tour as Commander of ABDACOM) and then served as Viceroy of India until his retirement in February 1947.»\n",+ "[2] «John French, 1st Earl of Ypres | Field Marshal John Denton Pinkstone French, 1st Earl of Ypres (28 September 1852 – 22 May 1925), known as Sir John French from 1901 to 1916, and as The Viscount French between 1916 and 1922, was a senior British Army officer. Born in Kent to an Anglo-Irish family, he saw brief service as a midshipman in the Royal Navy, before becoming a cavalry officer. He achieved rapid promotion and distinguished himself on the Gordon Relief Expedition. French had a considerable reputation as a womaniser throughout his life and his career nearly ended when he was cited in the divorce of a brother officer whilst in India in the early 1890s.»\n",+ "[3] «John French, 2nd Earl of Ypres | John Richard Lowndes French, 2nd Earl of Ypres (6 July 1881 – 5 April 1958) was the son of the British field marshal and the first commander of the British Expeditionary Force (BEF) in World War I Sir John French.»\n",+ "[4] «Archibald Wavell, 2nd Earl Wavell | Major Archibald John Arthur Wavell, 2nd Earl Wavell MC (11 May 1916 – 24 December 1953) was the second and last Earl Wavell and Viscount Keren of Eritrea. He was educated at Winchester College and succeeded his father in 1950.»\n",+ "[5] «Archibald MacKinnon | Archibald (\"Archie\") MacKinnon (born January 13, 1937 in Cranbrook, British Columbia) is a Canadian competition rower and Olympic champion.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Answer. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to find the 1st Earl of Ypres who attacked the Boer force from the rear while mounted infantry. We need to find the context that mentions the 1st Earl of Ypres and the event where he attacked the Boer force from the rear while mounted infantry.\n",+ "\n",+ "Answer: John French, 1st Earl of Ypres\n",+ "\n",+ "\n",+ "\n",+ "...\n",+ "Score 28.0\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "[I 2024-09-23 19:11:37,745] Trial 6 finished with value: 28.0 and parameters: {'0_predictor_instruction': 2, '0_predictor_demos': 6, '1_predictor_instruction': 8, '1_predictor_demos': 8, '2_predictor_instruction': 1, '2_predictor_demos': 8}. Best is trial 0 with value: 52.0.\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "CANDIDATE PROGRAM:\n",+ "Predictor 0\n",+ "i: Produce a search query that effectively captures the essence of the provided context and question, allowing me to retrieve relevant information that can aid in answering the question with precision and accuracy.\n",+ "p: Search Query:\n",+ "\n",+ "\n",+ "Predictor 1\n",+ "i: Generate a subsequent search query based on the provided context and question, aiming to narrow down the scope of relevant information and guide the retrieval of precise passages to support the forthcoming answer.\n",+ "p: Search Query:\n",+ "\n",+ "\n",+ "Predictor 2\n",+ "i: Provide a concise and accurate answer to the question, citing relevant information from the provided context, ensuring credibility and clarity.\n",+ "p: Answer:\n",+ "\n",+ "\n",+ "...\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 10 / 25 (40.0): 100%|██████████| 25/25 [00:16<00:00, 1.53it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "FULL TRACE\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 0 / 1 (0.0): 100%|██████████| 1/1 [00:03<00:00, 3.77s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "\n",+ "\n",+ "\n",+ "Produce a search query that effectively captures the essence of the provided context and question, allowing me to retrieve relevant information that can aid in answering the question with precision and accuracy.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Search Query: ${search_query}\n",+ "\n",+ "---\n",+ "\n",+ "Context: N/A\n",+ "\n",+ "Question: Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Forrest Tucker height Walt Disney TV miniseries\n",+ "\n",+ "---\n",+ "\n",+ "Context: N/A\n",+ "\n",+ "Question: Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: clothing company founded 1889 college bass fishing\n",+ "\n",+ "---\n",+ "\n",+ "Context: N/A\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Please provide the output field Search Query. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field Search Query.\n",+ "\n",+ "Search Query: 1st Earl of Ypres Boer force mounted infantry\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Generate a subsequent search query based on the provided context and question, aiming to narrow down the scope of relevant information and guide the retrieval of precise passages to support the forthcoming answer.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Search Query: ${search_query}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Butch Reed | Bruce Franklin Reed (born July 11, 1954) is an American retired professional wrestler and football player, better known by the ring name Butch Reed. He is best known for his appearances with the World Wrestling Federation and World Championship Wrestling between 1986 and 1992.»\n",+ "[2] «Doug Reed (futsal player) | Doug Reed is a professional futsal player who is currently playing for Futsal Dinamo and England national futsal team. Previously he has played for Lanzarote Tias Yaiza (Spain), AEL Limassol, MNK Uspinjača, MNK Smederevo and Manchester Futsal Club. In 2011, he won the FA Futsal Cup with Manchester Futsal Club and was awarded Player of the Tournament in the same competition. In the FA National Futsal League, he won several FA National League Championships and FA Futsal Cup medals. Reed also works as a commentator for Sony SIX, Eurosport and BT Sport, and has worked with the emerging US Professional Futsal League as well as Indian Premier Futsal.»\n",+ "[3] «Doom (professional wrestling) | Doom was a professional wrestling tag team composed of Ron Simmons and Butch Reed. They competed from 1989 to 1991 in the National Wrestling Alliance (NWA) and World Championship Wrestling (WCW).»\n",+ "\n",+ "Question: What other sport did Butch Reed, of the wrestling tag team Doom, play professionally?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Butch Reed football\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Archibald Wavell, 1st Earl Wavell | Field Marshal Archibald Percival Wavell, 1st Earl Wavell, {'1': \", '2': \", '3': \", '4': \"} (5 May 1883 – 24 May 1950) was a senior officer of the British Army. He served in the Second Boer War, the Bazar Valley Campaign and the Great War, during which he was wounded in the Second Battle of Ypres. He served in the Second World War, initially as Commander-in-Chief Middle East, in which role he led British forces to victory over the Italians in western Egypt and eastern Libya during Operation \"Compass\" in December 1940, only to be defeated by the German Army in the Western Desert in April 1941. He served as Commander-in-Chief, India, from July 1941 until June 1943 (apart from a brief tour as Commander of ABDACOM) and then served as Viceroy of India until his retirement in February 1947.»\n",+ "[2] «John French, 1st Earl of Ypres | Field Marshal John Denton Pinkstone French, 1st Earl of Ypres (28 September 1852 – 22 May 1925), known as Sir John French from 1901 to 1916, and as The Viscount French between 1916 and 1922, was a senior British Army officer. Born in Kent to an Anglo-Irish family, he saw brief service as a midshipman in the Royal Navy, before becoming a cavalry officer. He achieved rapid promotion and distinguished himself on the Gordon Relief Expedition. French had a considerable reputation as a womaniser throughout his life and his career nearly ended when he was cited in the divorce of a brother officer whilst in India in the early 1890s.»\n",+ "[3] «John French, 2nd Earl of Ypres | John Richard Lowndes French, 2nd Earl of Ypres (6 July 1881 – 5 April 1958) was the son of the British field marshal and the first commander of the British Expeditionary Force (BEF) in World War I Sir John French.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Search Query. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to find the birth year of the 1st Earl of Ypres who attacked the Boer force from the rear while mounted infantry. We need to find the 1st Earl of Ypres who was involved in the Second Boer War, and then find his birth year.\n",+ "Search Query: Archibald Wavell birth year\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Provide a concise and accurate answer to the question, citing relevant information from the provided context, ensuring credibility and clarity.\n",+ "\n",+ "---\n",+ "\n",+ "Question: What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?\n",+ "\n",+ "Answer: 2016\n",+ "\n",+ "---\n",+ "\n",+ "Question: How many shops does the larger of the Miami International Mall and Dolphin Mall have?\n",+ "\n",+ "Answer: 240\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Answer: ${answer}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Wheat Field with Cypresses | A Wheatfield with Cypresses (occasionally called \"A Cornfield with Cypresses\") is any of three similar 1889 oil paintings by Vincent van Gogh, as part of his wheat field series. All were executed at the Saint-Paul-de-Mausole mental asylum at Saint-Rémy near Arles, France, where Van Gogh was voluntarily a patient from May 1889 to May 1890. The works were inspired by the view from the window at the asylum towards the Alpilles mountains.»\n",+ "[2] «Green Wheat Field with Cypress | Green Wheat Field with Cypress (French: Champ de blé vert avec cyprès) is an oil on canvas painting by a Dutch Post-Impressionist Vincent van Gogh. It was completed in 1889, while van Gogh was voluntarily incarcerated at the asylum of St. Paul near Saint-Rémy in Provence. Several paintings of wheat fields with cypresses were made when van Gogh was able to leave the asylum grounds and explore the local landscape. Besides a fondness for cypresses, van Gogh had a special affinty with wheat fields; he depicted them dozens of times over the years; to Vincent they symbolized the cycle of life and death, and he found in them both solace and inspiration.»\n",+ "[3] «Olive Trees (Van Gogh series) | Vincent van Gogh painted at least 18 paintings of olive trees, mostly in Saint-Rémy-de-Provence in 1889. At his own request, he lived at an asylum there from May 1889 through May 1890 painting the gardens of the asylum and, when he had permission to venture outside its walls, nearby olive trees, cypresses and wheat fields.»\n",+ "[4] «Wheat Fields (Van Gogh series) | The Wheat Fields is a series of dozens of paintings by Vincent van Gogh, borne out of his religious studies and sermons, connection to nature, appreciation of manual laborers and desire to provide a means of offering comfort to others. The wheat field works demonstrate his progression as an artist from the drab \"Wheat Sheaves\" made in 1885 in the Netherlands to the colorful, dramatic paintings from Arles, Saint-Rémy and Auvers-sur-Oise of rural France.»\n",+ "\n",+ "Question: What was the series inspired by the appreciation of manual laborers that Wheat Field with Cypresses is a part of?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "Context: N/A\n",+ "\n",+ "Question: What was the series inspired by the appreciation of manual laborers that Wheat Field with Cypresses is a part of?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "\n",+ "Answer: Wheat Fields\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Archibald Wavell, 1st Earl Wavell | Field Marshal Archibald Percival Wavell, 1st Earl Wavell, {'1': \", '2': \", '3': \", '4': \"} (5 May 1883 – 24 May 1950) was a senior officer of the British Army. He served in the Second Boer War, the Bazar Valley Campaign and the Great War, during which he was wounded in the Second Battle of Ypres. He served in the Second World War, initially as Commander-in-Chief Middle East, in which role he led British forces to victory over the Italians in western Egypt and eastern Libya during Operation \"Compass\" in December 1940, only to be defeated by the German Army in the Western Desert in April 1941. He served as Commander-in-Chief, India, from July 1941 until June 1943 (apart from a brief tour as Commander of ABDACOM) and then served as Viceroy of India until his retirement in February 1947.»\n",+ "[2] «John French, 1st Earl of Ypres | Field Marshal John Denton Pinkstone French, 1st Earl of Ypres (28 September 1852 – 22 May 1925), known as Sir John French from 1901 to 1916, and as The Viscount French between 1916 and 1922, was a senior British Army officer. Born in Kent to an Anglo-Irish family, he saw brief service as a midshipman in the Royal Navy, before becoming a cavalry officer. He achieved rapid promotion and distinguished himself on the Gordon Relief Expedition. French had a considerable reputation as a womaniser throughout his life and his career nearly ended when he was cited in the divorce of a brother officer whilst in India in the early 1890s.»\n",+ "[3] «John French, 2nd Earl of Ypres | John Richard Lowndes French, 2nd Earl of Ypres (6 July 1881 – 5 April 1958) was the son of the British field marshal and the first commander of the British Expeditionary Force (BEF) in World War I Sir John French.»\n",+ "[4] «Archibald Wavell, 2nd Earl Wavell | Major Archibald John Arthur Wavell, 2nd Earl Wavell MC (11 May 1916 – 24 December 1953) was the second and last Earl Wavell and Viscount Keren of Eritrea. He was educated at Winchester College and succeeded his father in 1950.»\n",+ "[5] «Archibald Frank Nicoll | Archibald Frank Nicoll {'1': \", '2': \", '3': \", '4': \"} (14 June 1886 – 1 February 1953) was a New Zealand artist and art teacher.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Answer. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to find the 1st Earl of Ypres who attacked the Boer force from the rear while mounted infantry. We need to find the 1st Earl of Ypres who served in the Second Boer War.\n",+ "Answer: John French\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Produce a search query that effectively captures the essence of the provided context and question, allowing me to retrieve relevant information that can aid in answering the question with precision and accuracy.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Search Query: ${search_query}\n",+ "\n",+ "---\n",+ "\n",+ "Context: N/A\n",+ "\n",+ "Question: Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Forrest Tucker height Walt Disney TV miniseries\n",+ "\n",+ "---\n",+ "\n",+ "Context: N/A\n",+ "\n",+ "Question: Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: clothing company founded 1889 college bass fishing\n",+ "\n",+ "---\n",+ "\n",+ "Context: N/A\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Please provide the output field Search Query. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field Search Query.\n",+ "\n",+ "Search Query: 1st Earl of Ypres Boer force mounted infantry\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Generate a subsequent search query based on the provided context and question, aiming to narrow down the scope of relevant information and guide the retrieval of precise passages to support the forthcoming answer.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Search Query: ${search_query}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Butch Reed | Bruce Franklin Reed (born July 11, 1954) is an American retired professional wrestler and football player, better known by the ring name Butch Reed. He is best known for his appearances with the World Wrestling Federation and World Championship Wrestling between 1986 and 1992.»\n",+ "[2] «Doug Reed (futsal player) | Doug Reed is a professional futsal player who is currently playing for Futsal Dinamo and England national futsal team. Previously he has played for Lanzarote Tias Yaiza (Spain), AEL Limassol, MNK Uspinjača, MNK Smederevo and Manchester Futsal Club. In 2011, he won the FA Futsal Cup with Manchester Futsal Club and was awarded Player of the Tournament in the same competition. In the FA National Futsal League, he won several FA National League Championships and FA Futsal Cup medals. Reed also works as a commentator for Sony SIX, Eurosport and BT Sport, and has worked with the emerging US Professional Futsal League as well as Indian Premier Futsal.»\n",+ "[3] «Doom (professional wrestling) | Doom was a professional wrestling tag team composed of Ron Simmons and Butch Reed. They competed from 1989 to 1991 in the National Wrestling Alliance (NWA) and World Championship Wrestling (WCW).»\n",+ "\n",+ "Question: What other sport did Butch Reed, of the wrestling tag team Doom, play professionally?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Butch Reed football\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Archibald Wavell, 1st Earl Wavell | Field Marshal Archibald Percival Wavell, 1st Earl Wavell, {'1': \", '2': \", '3': \", '4': \"} (5 May 1883 – 24 May 1950) was a senior officer of the British Army. He served in the Second Boer War, the Bazar Valley Campaign and the Great War, during which he was wounded in the Second Battle of Ypres. He served in the Second World War, initially as Commander-in-Chief Middle East, in which role he led British forces to victory over the Italians in western Egypt and eastern Libya during Operation \"Compass\" in December 1940, only to be defeated by the German Army in the Western Desert in April 1941. He served as Commander-in-Chief, India, from July 1941 until June 1943 (apart from a brief tour as Commander of ABDACOM) and then served as Viceroy of India until his retirement in February 1947.»\n",+ "[2] «John French, 1st Earl of Ypres | Field Marshal John Denton Pinkstone French, 1st Earl of Ypres (28 September 1852 – 22 May 1925), known as Sir John French from 1901 to 1916, and as The Viscount French between 1916 and 1922, was a senior British Army officer. Born in Kent to an Anglo-Irish family, he saw brief service as a midshipman in the Royal Navy, before becoming a cavalry officer. He achieved rapid promotion and distinguished himself on the Gordon Relief Expedition. French had a considerable reputation as a womaniser throughout his life and his career nearly ended when he was cited in the divorce of a brother officer whilst in India in the early 1890s.»\n",+ "[3] «John French, 2nd Earl of Ypres | John Richard Lowndes French, 2nd Earl of Ypres (6 July 1881 – 5 April 1958) was the son of the British field marshal and the first commander of the British Expeditionary Force (BEF) in World War I Sir John French.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Search Query. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to find the birth year of the 1st Earl of Ypres who attacked the Boer force from the rear while mounted infantry. We need to find the 1st Earl of Ypres who was involved in the Second Boer War, and then find his birth year.\n",+ "Search Query: Archibald Wavell birth year\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Provide a concise and accurate answer to the question, citing relevant information from the provided context, ensuring credibility and clarity.\n",+ "\n",+ "---\n",+ "\n",+ "Question: What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?\n",+ "\n",+ "Answer: 2016\n",+ "\n",+ "---\n",+ "\n",+ "Question: How many shops does the larger of the Miami International Mall and Dolphin Mall have?\n",+ "\n",+ "Answer: 240\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Answer: ${answer}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Wheat Field with Cypresses | A Wheatfield with Cypresses (occasionally called \"A Cornfield with Cypresses\") is any of three similar 1889 oil paintings by Vincent van Gogh, as part of his wheat field series. All were executed at the Saint-Paul-de-Mausole mental asylum at Saint-Rémy near Arles, France, where Van Gogh was voluntarily a patient from May 1889 to May 1890. The works were inspired by the view from the window at the asylum towards the Alpilles mountains.»\n",+ "[2] «Green Wheat Field with Cypress | Green Wheat Field with Cypress (French: Champ de blé vert avec cyprès) is an oil on canvas painting by a Dutch Post-Impressionist Vincent van Gogh. It was completed in 1889, while van Gogh was voluntarily incarcerated at the asylum of St. Paul near Saint-Rémy in Provence. Several paintings of wheat fields with cypresses were made when van Gogh was able to leave the asylum grounds and explore the local landscape. Besides a fondness for cypresses, van Gogh had a special affinty with wheat fields; he depicted them dozens of times over the years; to Vincent they symbolized the cycle of life and death, and he found in them both solace and inspiration.»\n",+ "[3] «Olive Trees (Van Gogh series) | Vincent van Gogh painted at least 18 paintings of olive trees, mostly in Saint-Rémy-de-Provence in 1889. At his own request, he lived at an asylum there from May 1889 through May 1890 painting the gardens of the asylum and, when he had permission to venture outside its walls, nearby olive trees, cypresses and wheat fields.»\n",+ "[4] «Wheat Fields (Van Gogh series) | The Wheat Fields is a series of dozens of paintings by Vincent van Gogh, borne out of his religious studies and sermons, connection to nature, appreciation of manual laborers and desire to provide a means of offering comfort to others. The wheat field works demonstrate his progression as an artist from the drab \"Wheat Sheaves\" made in 1885 in the Netherlands to the colorful, dramatic paintings from Arles, Saint-Rémy and Auvers-sur-Oise of rural France.»\n",+ "\n",+ "Question: What was the series inspired by the appreciation of manual laborers that Wheat Field with Cypresses is a part of?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "Context: N/A\n",+ "\n",+ "Question: What was the series inspired by the appreciation of manual laborers that Wheat Field with Cypresses is a part of?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "\n",+ "Answer: Wheat Fields\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Archibald Wavell, 1st Earl Wavell | Field Marshal Archibald Percival Wavell, 1st Earl Wavell, {'1': \", '2': \", '3': \", '4': \"} (5 May 1883 – 24 May 1950) was a senior officer of the British Army. He served in the Second Boer War, the Bazar Valley Campaign and the Great War, during which he was wounded in the Second Battle of Ypres. He served in the Second World War, initially as Commander-in-Chief Middle East, in which role he led British forces to victory over the Italians in western Egypt and eastern Libya during Operation \"Compass\" in December 1940, only to be defeated by the German Army in the Western Desert in April 1941. He served as Commander-in-Chief, India, from July 1941 until June 1943 (apart from a brief tour as Commander of ABDACOM) and then served as Viceroy of India until his retirement in February 1947.»\n",+ "[2] «John French, 1st Earl of Ypres | Field Marshal John Denton Pinkstone French, 1st Earl of Ypres (28 September 1852 – 22 May 1925), known as Sir John French from 1901 to 1916, and as The Viscount French between 1916 and 1922, was a senior British Army officer. Born in Kent to an Anglo-Irish family, he saw brief service as a midshipman in the Royal Navy, before becoming a cavalry officer. He achieved rapid promotion and distinguished himself on the Gordon Relief Expedition. French had a considerable reputation as a womaniser throughout his life and his career nearly ended when he was cited in the divorce of a brother officer whilst in India in the early 1890s.»\n",+ "[3] «John French, 2nd Earl of Ypres | John Richard Lowndes French, 2nd Earl of Ypres (6 July 1881 – 5 April 1958) was the son of the British field marshal and the first commander of the British Expeditionary Force (BEF) in World War I Sir John French.»\n",+ "[4] «Archibald Wavell, 2nd Earl Wavell | Major Archibald John Arthur Wavell, 2nd Earl Wavell MC (11 May 1916 – 24 December 1953) was the second and last Earl Wavell and Viscount Keren of Eritrea. He was educated at Winchester College and succeeded his father in 1950.»\n",+ "[5] «Archibald Frank Nicoll | Archibald Frank Nicoll {'1': \", '2': \", '3': \", '4': \"} (14 June 1886 – 1 February 1953) was a New Zealand artist and art teacher.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Answer. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to find the 1st Earl of Ypres who attacked the Boer force from the rear while mounted infantry. We need to find the 1st Earl of Ypres who served in the Second Boer War.\n",+ "Answer: John French\n",+ "\n",+ "\n",+ "\n",+ "...\n",+ "Score 40.0\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "[I 2024-09-23 19:11:57,963] Trial 7 finished with value: 40.0 and parameters: {'0_predictor_instruction': 8, '0_predictor_demos': 3, '1_predictor_instruction': 1, '1_predictor_demos': 7, '2_predictor_instruction': 3, '2_predictor_demos': 6}. Best is trial 0 with value: 52.0.\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "CANDIDATE PROGRAM:\n",+ "Predictor 0\n",+ "i: Generate a search query that accurately captures the essence of the question within the given context, leveraging specific keywords and phrases to guide the retrieval of relevant passages.\n",+ "p: Search Query:\n",+ "\n",+ "\n",+ "Predictor 1\n",+ "i: Generate a search query that accurately captures the essence of the question and enables the retrieval of relevant information from the dataset, taking into account the context provided, the question being asked, and the relationship between the two.\n",+ "p: Search Query:\n",+ "\n",+ "\n",+ "Predictor 2\n",+ "i: Given a context and a question, generate a logical and coherent answer that accurately addresses the query by meticulously analyzing the provided context, identifying relevant information, and applying critical thinking to provide a precise and informative response.\n",+ "p: Answer:\n",+ "\n",+ "\n",+ "...\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 15 / 25 (60.0): 100%|██████████| 25/25 [00:28<00:00, 1.16s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "FULL TRACE\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 0 / 1 (0.0): 100%|██████████| 1/1 [00:00<00:00, 250.05it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "\n",+ "\n",+ "\n",+ "Generate a search query that accurately captures the essence of the question within the given context, leveraging specific keywords and phrases to guide the retrieval of relevant passages.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Search Query: ${search_query}\n",+ "\n",+ "---\n",+ "\n",+ "Context: N/A\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Search Query. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to produce the output fields. We need to extract the key information from the question, which is the title of the person, the event, and the action. This will help us formulate a search query that can provide the answer to the question.\n",+ "Search Query: 1st Earl of Ypres birth Boer force mounted infantry\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Generate a search query that accurately captures the essence of the question and enables the retrieval of relevant information from the dataset, taking into account the context provided, the question being asked, and the relationship between the two.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Search Query: ${search_query}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Will Finn | Will Finn (born November 1, 1958) is an American animator, voice actor, and director. His work in animation includes characters from Disney and Don Bluth films such as \"The Secret of NIMH\", \"Oliver & Company\", \"The Little Mermaid\", \"The Rescuers Down Under\", and \"Pocahontas\". His characters includes Cogsworth in \"Beauty and the Beast\", Iago in \"Aladdin\", and Laverne in \"The Hunchback of Notre Dame\". Finn wrote and directed \"Home on the Range\" and did some voice acting the Hollywood Fish in \"Chicken Little\". In 2006, Finn directed the computer animated short Hammy's Boomerang Adventure, a spin-off of \"Over the Hedge\".»\n",+ "[2] «Over the Hedge (film) | Over the Hedge is a 2006 American computer-animated comedy film, based on the characters from the United Media comic strip of the same name. Directed by Tim Johnson and Karey Kirkpatrick, and produced by Bonnie Arnold, it was released in the United States on May 19, 2006. The film was produced by DreamWorks Animation and distributed through Paramount Pictures. The film features the voices of Bruce Willis, Garry Shandling, Steve Carell, William Shatner, Wanda Sykes, and Nick Nolte. It is the first DreamWorks Animation film to be distributed by Paramount Pictures, which acquired the live-action DreamWorks studio in 2006. The film earned $336 million on an $80 million budget.»\n",+ "[3] «Tim Johnson (film director) | Tim Johnson (born August 27, 1961) is an American animation director. Johnson has directed in many films such as \"Antz\", \"\" and \"Over the Hedge\". Recently, he directed the DreamWorks 3D film \"Home.\"»\n",+ "\n",+ "Question: When was the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed?\n",+ "\n",+ "Reasoning: Let's think step by step in order to find the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed. The context mentions that Will Finn directed the computer animated short Hammy's Boomerang Adventure, a spin-off of \"Over the Hedge\".\n",+ "\n",+ "Search Query: Hammy's Boomerang Adventure release date\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Archibald Wavell, 1st Earl Wavell | Field Marshal Archibald Percival Wavell, 1st Earl Wavell, {'1': \", '2': \", '3': \", '4': \"} (5 May 1883 – 24 May 1950) was a senior officer of the British Army. He served in the Second Boer War, the Bazar Valley Campaign and the Great War, during which he was wounded in the Second Battle of Ypres. He served in the Second World War, initially as Commander-in-Chief Middle East, in which role he led British forces to victory over the Italians in western Egypt and eastern Libya during Operation \"Compass\" in December 1940, only to be defeated by the German Army in the Western Desert in April 1941. He served as Commander-in-Chief, India, from July 1941 until June 1943 (apart from a brief tour as Commander of ABDACOM) and then served as Viceroy of India until his retirement in February 1947.»\n",+ "[2] «John French, 1st Earl of Ypres | Field Marshal John Denton Pinkstone French, 1st Earl of Ypres (28 September 1852 – 22 May 1925), known as Sir John French from 1901 to 1916, and as The Viscount French between 1916 and 1922, was a senior British Army officer. Born in Kent to an Anglo-Irish family, he saw brief service as a midshipman in the Royal Navy, before becoming a cavalry officer. He achieved rapid promotion and distinguished himself on the Gordon Relief Expedition. French had a considerable reputation as a womaniser throughout his life and his career nearly ended when he was cited in the divorce of a brother officer whilst in India in the early 1890s.»\n",+ "[3] «Ripple, Kent | John French, 1st Earl of Ypres, the commander of the first British Expeditionary Force was born there in 1852, and is buried at the village church. His sister Charlotte Despard, the suffragist, novelist and Sinn Féin activist was also born in Ripple in 1844.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Search Query. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to find the 1st Earl of Ypres born who attacked the Boer force from the rear while mounted infantry. The context mentions that Archibald Wavell, 1st Earl Wavell, was a senior officer of the British Army who served in the Second Boer War. He was the one who attacked the Boer force from the rear while mounted infantry.\n",+ "Search Query: Archibald Wavell birthdate\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Given a context and a question, generate a logical and coherent answer that accurately addresses the query by meticulously analyzing the provided context, identifying relevant information, and applying critical thinking to provide a precise and informative response.\n",+ "\n",+ "---\n",+ "\n",+ "Question: How many years did Lenny Lyles spend with the organization dramatized in the filme \"The insder\"?\n",+ "\n",+ "Answer: 27\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Answer: ${answer}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Forrest Tucker | Forrest Meredith Tucker (February 12, 1919 – October 25, 1986) was an American actor in both movies and television who appeared in nearly a hundred films. Tucker worked as a vaudeville straight man, aged fifteen years old. A mentor provided funds and contacts for a trip to California, where party hostess Cobina Wright persuaded guest Wesley Ruggles to give Tucker a screen test, based on his photogenic good looks, thick wavy hair and height of six feet, five inches - putting him into the same league as John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness's younger brother. Tucker was a sight reader who needed only one take. His film career started well despite a perception in most Hollywood studios that blond men were not photogenic. After twenty years spent mainly in Westerns and action roles, he returned to his roots, showing versatility as a comedic and stage musical actor. In the TV show \"F Troop\", he became identified with the character of Cavalry Sgt. Morgan O'Rourke (a manipulative character quite similar to Phil Silvers' role as MSgt Ernie Bilko). Tucker struggled with a drinking problem that began to affect his performances in the later years of his career.»\n",+ "[2] «Emergency Landing (1941 film) | Emergency Landing (a.k.a. Robot Pilot) is a 1941 American aviation spy-fi romantic screwball comedy film directed by William Beaudine. The film stars Forrest Tucker in his second film and in his first leading role with co-stars Carol Hughes and Evelyn Brent. \"Emergency Landing\" features lots of mismatched stock footage of various types of aircraft.»\n",+ "[3] «Forrest Tucker (criminal) | Forrest Silva \"Woody\" Tucker (June 23, 1920 – May 29, 2004) was an American career criminal first imprisoned at age 15 who spent the rest of his life in and out of jail. He is best known as an escape artist, having escaped from prison 18 times successfully and 12 times unsuccessfully.»\n",+ "[4] «Fess Parker | Fess Elisha Parker Jr. (August 16, 1924 – March 18, 2010) was an American film and television actor best known for his portrayals of Davy Crockett in the Walt Disney 1955–1956 TV miniseries and as Daniel Boone in a television series from 1964 to 1970. He was also known as a winemaker and resort owner-operator.»\n",+ "[5] «Davy Crockett, King of the Wild Frontier | Davy Crockett, King of the Wild Frontier is a 1955 live-action Walt Disney adventure film starring Fess Parker as Davy Crockett. This film is an edited compilation of the first three stories from the Disney television miniseries \"Davy Crockett\" :»\n",+ "[6] «Davy Crockett (miniseries) | Davy Crockett is a five-part serial which aired on ABC from 1954–1955 in one-hour episodes, on the \"Disneyland\" series. The series stars Fess Parker as real-life frontiersman Davy Crockett and Buddy Ebsen as his friend, George Russel.»\n",+ "\n",+ "Question: Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\n",+ "\n",+ "Reasoning: Let's think step by step in order to We need to find the actor who played the character in the 1955-1956 Walt Disney TV miniseries. From the context, we know that the miniseries was about Davy Crockett and starred Fess Parker as Davy Crockett.\n",+ "\n",+ "Answer: Davy Crockett\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.»\n",+ "[2] «Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford's second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.»\n",+ "[3] «G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.»\n",+ "[4] «College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour»\n",+ "[5] «Bass fishing | Bass fishing is the activity of angling for the North American gamefish known colloquially as the \"black bass\". There are numerous black bass species considered as gamefish in North America, including largemouth bass (\"Micropterus salmoides\"), smallmouth bass (\"Micropterus dolomieui\"), spotted bass or Kentucky bass (\"Micropterus punctulatus\"), and Guadalupe bass (order \"Perciformes\").»\n",+ "[6] «Cliff Crochet | Cliff Crochet (born August 1, 1983), also known as \"The Cajun Baby\", is a professional bass fisherman from Pierre Part, Louisiana. Crochet is a competitor on Bass Angler Sportsman Society's (B.A.S.S.) Bassmaster Elite Series and has $368,837 in B.A.S.S. tournament winnings as of August 22, 2016. Crochet's sponsors include: K2 Coolers, Huk, Skeeter, Yamaha Motor Company, Lowrance Electronics, Falcon Rods, Luck E Strike, Seaguar, Rat-L-Trap, Humdinger Spinnerbaits, Power-Pole. Since 2010, Crochet has competed in three Bassmaster Classic tournaments. Crochet's boat for the 2016 season is a 2016 Skeeter FX 21 with a 250 hp Yamaha SHO outboard engine.»\n",+ "\n",+ "Question: Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "Context: [1] «Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.»\n",+ "\n",+ "[2] «College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour»\n",+ "\n",+ "[3] «G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.»\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "\n",+ "Answer: Carhartt\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Archibald Wavell, 1st Earl Wavell | Field Marshal Archibald Percival Wavell, 1st Earl Wavell, {'1': \", '2': \", '3': \", '4': \"} (5 May 1883 – 24 May 1950) was a senior officer of the British Army. He served in the Second Boer War, the Bazar Valley Campaign and the Great War, during which he was wounded in the Second Battle of Ypres. He served in the Second World War, initially as Commander-in-Chief Middle East, in which role he led British forces to victory over the Italians in western Egypt and eastern Libya during Operation \"Compass\" in December 1940, only to be defeated by the German Army in the Western Desert in April 1941. He served as Commander-in-Chief, India, from July 1941 until June 1943 (apart from a brief tour as Commander of ABDACOM) and then served as Viceroy of India until his retirement in February 1947.»\n",+ "[2] «John French, 1st Earl of Ypres | Field Marshal John Denton Pinkstone French, 1st Earl of Ypres (28 September 1852 – 22 May 1925), known as Sir John French from 1901 to 1916, and as The Viscount French between 1916 and 1922, was a senior British Army officer. Born in Kent to an Anglo-Irish family, he saw brief service as a midshipman in the Royal Navy, before becoming a cavalry officer. He achieved rapid promotion and distinguished himself on the Gordon Relief Expedition. French had a considerable reputation as a womaniser throughout his life and his career nearly ended when he was cited in the divorce of a brother officer whilst in India in the early 1890s.»\n",+ "[3] «Ripple, Kent | John French, 1st Earl of Ypres, the commander of the first British Expeditionary Force was born there in 1852, and is buried at the village church. His sister Charlotte Despard, the suffragist, novelist and Sinn Féin activist was also born in Ripple in 1844.»\n",+ "[4] «Archibald Wavell, 2nd Earl Wavell | Major Archibald John Arthur Wavell, 2nd Earl Wavell MC (11 May 1916 – 24 December 1953) was the second and last Earl Wavell and Viscount Keren of Eritrea. He was educated at Winchester College and succeeded his father in 1950.»\n",+ "[5] «Archibald MacKinnon | Archibald (\"Archie\") MacKinnon (born January 13, 1937 in Cranbrook, British Columbia) is a Canadian competition rower and Olympic champion.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Answer. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to We need to find the 1st Earl of Ypres who attacked the Boer force from the rear while mounted infantry. From the context, we know that the 1st Earl of Ypres was John French, 1st Earl of Ypres.\n",+ "\n",+ "Answer: 1852\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Generate a search query that accurately captures the essence of the question within the given context, leveraging specific keywords and phrases to guide the retrieval of relevant passages.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Search Query: ${search_query}\n",+ "\n",+ "---\n",+ "\n",+ "Context: N/A\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Search Query. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to produce the output fields. We need to extract the key information from the question, which is the title of the person, the event, and the action. This will help us formulate a search query that can provide the answer to the question.\n",+ "Search Query: 1st Earl of Ypres birth Boer force mounted infantry\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Generate a search query that accurately captures the essence of the question and enables the retrieval of relevant information from the dataset, taking into account the context provided, the question being asked, and the relationship between the two.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Search Query: ${search_query}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Will Finn | Will Finn (born November 1, 1958) is an American animator, voice actor, and director. His work in animation includes characters from Disney and Don Bluth films such as \"The Secret of NIMH\", \"Oliver & Company\", \"The Little Mermaid\", \"The Rescuers Down Under\", and \"Pocahontas\". His characters includes Cogsworth in \"Beauty and the Beast\", Iago in \"Aladdin\", and Laverne in \"The Hunchback of Notre Dame\". Finn wrote and directed \"Home on the Range\" and did some voice acting the Hollywood Fish in \"Chicken Little\". In 2006, Finn directed the computer animated short Hammy's Boomerang Adventure, a spin-off of \"Over the Hedge\".»\n",+ "[2] «Over the Hedge (film) | Over the Hedge is a 2006 American computer-animated comedy film, based on the characters from the United Media comic strip of the same name. Directed by Tim Johnson and Karey Kirkpatrick, and produced by Bonnie Arnold, it was released in the United States on May 19, 2006. The film was produced by DreamWorks Animation and distributed through Paramount Pictures. The film features the voices of Bruce Willis, Garry Shandling, Steve Carell, William Shatner, Wanda Sykes, and Nick Nolte. It is the first DreamWorks Animation film to be distributed by Paramount Pictures, which acquired the live-action DreamWorks studio in 2006. The film earned $336 million on an $80 million budget.»\n",+ "[3] «Tim Johnson (film director) | Tim Johnson (born August 27, 1961) is an American animation director. Johnson has directed in many films such as \"Antz\", \"\" and \"Over the Hedge\". Recently, he directed the DreamWorks 3D film \"Home.\"»\n",+ "\n",+ "Question: When was the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed?\n",+ "\n",+ "Reasoning: Let's think step by step in order to find the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed. The context mentions that Will Finn directed the computer animated short Hammy's Boomerang Adventure, a spin-off of \"Over the Hedge\".\n",+ "\n",+ "Search Query: Hammy's Boomerang Adventure release date\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Archibald Wavell, 1st Earl Wavell | Field Marshal Archibald Percival Wavell, 1st Earl Wavell, {'1': \", '2': \", '3': \", '4': \"} (5 May 1883 – 24 May 1950) was a senior officer of the British Army. He served in the Second Boer War, the Bazar Valley Campaign and the Great War, during which he was wounded in the Second Battle of Ypres. He served in the Second World War, initially as Commander-in-Chief Middle East, in which role he led British forces to victory over the Italians in western Egypt and eastern Libya during Operation \"Compass\" in December 1940, only to be defeated by the German Army in the Western Desert in April 1941. He served as Commander-in-Chief, India, from July 1941 until June 1943 (apart from a brief tour as Commander of ABDACOM) and then served as Viceroy of India until his retirement in February 1947.»\n",+ "[2] «John French, 1st Earl of Ypres | Field Marshal John Denton Pinkstone French, 1st Earl of Ypres (28 September 1852 – 22 May 1925), known as Sir John French from 1901 to 1916, and as The Viscount French between 1916 and 1922, was a senior British Army officer. Born in Kent to an Anglo-Irish family, he saw brief service as a midshipman in the Royal Navy, before becoming a cavalry officer. He achieved rapid promotion and distinguished himself on the Gordon Relief Expedition. French had a considerable reputation as a womaniser throughout his life and his career nearly ended when he was cited in the divorce of a brother officer whilst in India in the early 1890s.»\n",+ "[3] «Ripple, Kent | John French, 1st Earl of Ypres, the commander of the first British Expeditionary Force was born there in 1852, and is buried at the village church. His sister Charlotte Despard, the suffragist, novelist and Sinn Féin activist was also born in Ripple in 1844.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Search Query. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to find the 1st Earl of Ypres born who attacked the Boer force from the rear while mounted infantry. The context mentions that Archibald Wavell, 1st Earl Wavell, was a senior officer of the British Army who served in the Second Boer War. He was the one who attacked the Boer force from the rear while mounted infantry.\n",+ "Search Query: Archibald Wavell birthdate\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Given a context and a question, generate a logical and coherent answer that accurately addresses the query by meticulously analyzing the provided context, identifying relevant information, and applying critical thinking to provide a precise and informative response.\n",+ "\n",+ "---\n",+ "\n",+ "Question: How many years did Lenny Lyles spend with the organization dramatized in the filme \"The insder\"?\n",+ "\n",+ "Answer: 27\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Answer: ${answer}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Forrest Tucker | Forrest Meredith Tucker (February 12, 1919 – October 25, 1986) was an American actor in both movies and television who appeared in nearly a hundred films. Tucker worked as a vaudeville straight man, aged fifteen years old. A mentor provided funds and contacts for a trip to California, where party hostess Cobina Wright persuaded guest Wesley Ruggles to give Tucker a screen test, based on his photogenic good looks, thick wavy hair and height of six feet, five inches - putting him into the same league as John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness's younger brother. Tucker was a sight reader who needed only one take. His film career started well despite a perception in most Hollywood studios that blond men were not photogenic. After twenty years spent mainly in Westerns and action roles, he returned to his roots, showing versatility as a comedic and stage musical actor. In the TV show \"F Troop\", he became identified with the character of Cavalry Sgt. Morgan O'Rourke (a manipulative character quite similar to Phil Silvers' role as MSgt Ernie Bilko). Tucker struggled with a drinking problem that began to affect his performances in the later years of his career.»\n",+ "[2] «Emergency Landing (1941 film) | Emergency Landing (a.k.a. Robot Pilot) is a 1941 American aviation spy-fi romantic screwball comedy film directed by William Beaudine. The film stars Forrest Tucker in his second film and in his first leading role with co-stars Carol Hughes and Evelyn Brent. \"Emergency Landing\" features lots of mismatched stock footage of various types of aircraft.»\n",+ "[3] «Forrest Tucker (criminal) | Forrest Silva \"Woody\" Tucker (June 23, 1920 – May 29, 2004) was an American career criminal first imprisoned at age 15 who spent the rest of his life in and out of jail. He is best known as an escape artist, having escaped from prison 18 times successfully and 12 times unsuccessfully.»\n",+ "[4] «Fess Parker | Fess Elisha Parker Jr. (August 16, 1924 – March 18, 2010) was an American film and television actor best known for his portrayals of Davy Crockett in the Walt Disney 1955–1956 TV miniseries and as Daniel Boone in a television series from 1964 to 1970. He was also known as a winemaker and resort owner-operator.»\n",+ "[5] «Davy Crockett, King of the Wild Frontier | Davy Crockett, King of the Wild Frontier is a 1955 live-action Walt Disney adventure film starring Fess Parker as Davy Crockett. This film is an edited compilation of the first three stories from the Disney television miniseries \"Davy Crockett\" :»\n",+ "[6] «Davy Crockett (miniseries) | Davy Crockett is a five-part serial which aired on ABC from 1954–1955 in one-hour episodes, on the \"Disneyland\" series. The series stars Fess Parker as real-life frontiersman Davy Crockett and Buddy Ebsen as his friend, George Russel.»\n",+ "\n",+ "Question: Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\n",+ "\n",+ "Reasoning: Let's think step by step in order to We need to find the actor who played the character in the 1955-1956 Walt Disney TV miniseries. From the context, we know that the miniseries was about Davy Crockett and starred Fess Parker as Davy Crockett.\n",+ "\n",+ "Answer: Davy Crockett\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.»\n",+ "[2] «Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford's second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.»\n",+ "[3] «G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.»\n",+ "[4] «College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour»\n",+ "[5] «Bass fishing | Bass fishing is the activity of angling for the North American gamefish known colloquially as the \"black bass\". There are numerous black bass species considered as gamefish in North America, including largemouth bass (\"Micropterus salmoides\"), smallmouth bass (\"Micropterus dolomieui\"), spotted bass or Kentucky bass (\"Micropterus punctulatus\"), and Guadalupe bass (order \"Perciformes\").»\n",+ "[6] «Cliff Crochet | Cliff Crochet (born August 1, 1983), also known as \"The Cajun Baby\", is a professional bass fisherman from Pierre Part, Louisiana. Crochet is a competitor on Bass Angler Sportsman Society's (B.A.S.S.) Bassmaster Elite Series and has $368,837 in B.A.S.S. tournament winnings as of August 22, 2016. Crochet's sponsors include: K2 Coolers, Huk, Skeeter, Yamaha Motor Company, Lowrance Electronics, Falcon Rods, Luck E Strike, Seaguar, Rat-L-Trap, Humdinger Spinnerbaits, Power-Pole. Since 2010, Crochet has competed in three Bassmaster Classic tournaments. Crochet's boat for the 2016 season is a 2016 Skeeter FX 21 with a 250 hp Yamaha SHO outboard engine.»\n",+ "\n",+ "Question: Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "Context: [1] «Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.»\n",+ "\n",+ "[2] «College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour»\n",+ "\n",+ "[3] «G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.»\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "\n",+ "Answer: Carhartt\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Archibald Wavell, 1st Earl Wavell | Field Marshal Archibald Percival Wavell, 1st Earl Wavell, {'1': \", '2': \", '3': \", '4': \"} (5 May 1883 – 24 May 1950) was a senior officer of the British Army. He served in the Second Boer War, the Bazar Valley Campaign and the Great War, during which he was wounded in the Second Battle of Ypres. He served in the Second World War, initially as Commander-in-Chief Middle East, in which role he led British forces to victory over the Italians in western Egypt and eastern Libya during Operation \"Compass\" in December 1940, only to be defeated by the German Army in the Western Desert in April 1941. He served as Commander-in-Chief, India, from July 1941 until June 1943 (apart from a brief tour as Commander of ABDACOM) and then served as Viceroy of India until his retirement in February 1947.»\n",+ "[2] «John French, 1st Earl of Ypres | Field Marshal John Denton Pinkstone French, 1st Earl of Ypres (28 September 1852 – 22 May 1925), known as Sir John French from 1901 to 1916, and as The Viscount French between 1916 and 1922, was a senior British Army officer. Born in Kent to an Anglo-Irish family, he saw brief service as a midshipman in the Royal Navy, before becoming a cavalry officer. He achieved rapid promotion and distinguished himself on the Gordon Relief Expedition. French had a considerable reputation as a womaniser throughout his life and his career nearly ended when he was cited in the divorce of a brother officer whilst in India in the early 1890s.»\n",+ "[3] «Ripple, Kent | John French, 1st Earl of Ypres, the commander of the first British Expeditionary Force was born there in 1852, and is buried at the village church. His sister Charlotte Despard, the suffragist, novelist and Sinn Féin activist was also born in Ripple in 1844.»\n",+ "[4] «Archibald Wavell, 2nd Earl Wavell | Major Archibald John Arthur Wavell, 2nd Earl Wavell MC (11 May 1916 – 24 December 1953) was the second and last Earl Wavell and Viscount Keren of Eritrea. He was educated at Winchester College and succeeded his father in 1950.»\n",+ "[5] «Archibald MacKinnon | Archibald (\"Archie\") MacKinnon (born January 13, 1937 in Cranbrook, British Columbia) is a Canadian competition rower and Olympic champion.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Answer. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to We need to find the 1st Earl of Ypres who attacked the Boer force from the rear while mounted infantry. From the context, we know that the 1st Earl of Ypres was John French, 1st Earl of Ypres.\n",+ "\n",+ "Answer: 1852\n",+ "\n",+ "\n",+ "\n",+ "...\n",+ "Score 60.0\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "[I 2024-09-23 19:12:27,004] Trial 8 finished with value: 60.0 and parameters: {'0_predictor_instruction': 9, '0_predictor_demos': 0, '1_predictor_instruction': 3, '1_predictor_demos': 4, '2_predictor_instruction': 2, '2_predictor_demos': 3}. Best is trial 8 with value: 60.0.\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "CANDIDATE PROGRAM:\n",+ "Predictor 0\n",+ "i: Given the context and question, generate an informative search query that accurately captures the essence of the question, incorporating relevant keywords and phrases to retrieve relevant passages that can aid in answering the question.\n",+ "p: Search Query:\n",+ "\n",+ "\n",+ "Predictor 1\n",+ "i: Provide a descriptive search query that entails the critical information from the context and question to find a precise answer to the query asked, such as finding the release date of the computer-animated short comedy film Hammy's Boomerang Adventure, which is a spin-off of \"Over the Hedge\".\n",+ "p: Search Query:\n",+ "\n",+ "\n",+ "Predictor 2\n",+ "i: Given a context and a question that mentions two high-peaked mountains, generate the answer that indicates the country where the mountains are partially located, by considering the geographical information provided in the context and the reasoning required to answer the question.\n",+ "p: Answer:\n",+ "\n",+ "\n",+ "...\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 14 / 25 (56.0): 100%|██████████| 25/25 [00:34<00:00, 1.40s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "FULL TRACE\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 0 / 1 (0.0): 100%|██████████| 1/1 [00:00<00:00, 252.15it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "\n",+ "\n",+ "\n",+ "Given the context and question, generate an informative search query that accurately captures the essence of the question, incorporating relevant keywords and phrases to retrieve relevant passages that can aid in answering the question.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Search Query: ${search_query}\n",+ "\n",+ "---\n",+ "\n",+ "Context: N/A\n",+ "\n",+ "Question: What race is the lieutenant governor of the state that held its primary for her election on May 19th, 2015?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: lieutenant governor race May 19 2015\n",+ "\n",+ "---\n",+ "\n",+ "Context: N/A\n",+ "\n",+ "Question: Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: clothing company founded 1889 college bass fishing\n",+ "\n",+ "---\n",+ "\n",+ "Context: N/A\n",+ "\n",+ "Question: Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Forrest Tucker height Walt Disney TV miniseries\n",+ "\n",+ "---\n",+ "\n",+ "Context: N/A\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Search Query. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: 1st Earl of Ypres Boer force mounted infantry\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Provide a descriptive search query that entails the critical information from the context and question to find a precise answer to the query asked, such as finding the release date of the computer-animated short comedy film Hammy's Boomerang Adventure, which is a spin-off of \"Over the Hedge\".\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Search Query: ${search_query}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Forrest Tucker | Forrest Meredith Tucker (February 12, 1919 – October 25, 1986) was an American actor in both movies and television who appeared in nearly a hundred films. Tucker worked as a vaudeville straight man, aged fifteen years old. A mentor provided funds and contacts for a trip to California, where party hostess Cobina Wright persuaded guest Wesley Ruggles to give Tucker a screen test, based on his photogenic good looks, thick wavy hair and height of six feet, five inches - putting him into the same league as John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness's younger brother. Tucker was a sight reader who needed only one take. His film career started well despite a perception in most Hollywood studios that blond men were not photogenic. After twenty years spent mainly in Westerns and action roles, he returned to his roots, showing versatility as a comedic and stage musical actor. In the TV show \"F Troop\", he became identified with the character of Cavalry Sgt. Morgan O'Rourke (a manipulative character quite similar to Phil Silvers' role as MSgt Ernie Bilko). Tucker struggled with a drinking problem that began to affect his performances in the later years of his career.»\n",+ "[2] «Emergency Landing (1941 film) | Emergency Landing (a.k.a. Robot Pilot) is a 1941 American aviation spy-fi romantic screwball comedy film directed by William Beaudine. The film stars Forrest Tucker in his second film and in his first leading role with co-stars Carol Hughes and Evelyn Brent. \"Emergency Landing\" features lots of mismatched stock footage of various types of aircraft.»\n",+ "[3] «Forrest Tucker (criminal) | Forrest Silva \"Woody\" Tucker (June 23, 1920 – May 29, 2004) was an American career criminal first imprisoned at age 15 who spent the rest of his life in and out of jail. He is best known as an escape artist, having escaped from prison 18 times successfully and 12 times unsuccessfully.»\n",+ "\n",+ "Question: Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Walt Disney TV miniseries 1955-1956 character\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.»\n",+ "[2] «Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford's second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.»\n",+ "[3] «G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.»\n",+ "\n",+ "Question: Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Carhartt bass fishing\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Archibald Wavell, 1st Earl Wavell | Field Marshal Archibald Percival Wavell, 1st Earl Wavell, {'1': \", '2': \", '3': \", '4': \"} (5 May 1883 – 24 May 1950) was a senior officer of the British Army. He served in the Second Boer War, the Bazar Valley Campaign and the Great War, during which he was wounded in the Second Battle of Ypres. He served in the Second World War, initially as Commander-in-Chief Middle East, in which role he led British forces to victory over the Italians in western Egypt and eastern Libya during Operation \"Compass\" in December 1940, only to be defeated by the German Army in the Western Desert in April 1941. He served as Commander-in-Chief, India, from July 1941 until June 1943 (apart from a brief tour as Commander of ABDACOM) and then served as Viceroy of India until his retirement in February 1947.»\n",+ "[2] «John French, 1st Earl of Ypres | Field Marshal John Denton Pinkstone French, 1st Earl of Ypres (28 September 1852 – 22 May 1925), known as Sir John French from 1901 to 1916, and as The Viscount French between 1916 and 1922, was a senior British Army officer. Born in Kent to an Anglo-Irish family, he saw brief service as a midshipman in the Royal Navy, before becoming a cavalry officer. He achieved rapid promotion and distinguished himself on the Gordon Relief Expedition. French had a considerable reputation as a womaniser throughout his life and his career nearly ended when he was cited in the divorce of a brother officer whilst in India in the early 1890s.»\n",+ "[3] «John French, 2nd Earl of Ypres | John Richard Lowndes French, 2nd Earl of Ypres (6 July 1881 – 5 April 1958) was the son of the British field marshal and the first commander of the British Expeditionary Force (BEF) in World War I Sir John French.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Search Query. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to find the birth year of the 1st Earl of Ypres who attacked the Boer force from the rear while mounted infantry. We need to find the 1st Earl of Ypres who was a senior officer of the British Army and served in the Second Boer War.\n",+ "Search Query: Archibald Wavell birth year\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Given a context and a question that mentions two high-peaked mountains, generate the answer that indicates the country where the mountains are partially located, by considering the geographical information provided in the context and the reasoning required to answer the question.\n",+ "\n",+ "---\n",+ "\n",+ "Question: What character did this actor, who also played a supporting role in Sleepy Hollow, play in the final six Harry Potter films?\n",+ "\n",+ "Answer: Professor Albus Dumbledore\n",+ "\n",+ "---\n",+ "\n",+ "Question: The musical sitcom with an album by the same name has what special feature on the album cover?\n",+ "\n",+ "Answer: actual crossword puzzle\n",+ "\n",+ "---\n",+ "\n",+ "Question: What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?\n",+ "\n",+ "Answer: 2016\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Answer: ${answer}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Archibald Wavell, 1st Earl Wavell | Field Marshal Archibald Percival Wavell, 1st Earl Wavell, {'1': \", '2': \", '3': \", '4': \"} (5 May 1883 – 24 May 1950) was a senior officer of the British Army. He served in the Second Boer War, the Bazar Valley Campaign and the Great War, during which he was wounded in the Second Battle of Ypres. He served in the Second World War, initially as Commander-in-Chief Middle East, in which role he led British forces to victory over the Italians in western Egypt and eastern Libya during Operation \"Compass\" in December 1940, only to be defeated by the German Army in the Western Desert in April 1941. He served as Commander-in-Chief, India, from July 1941 until June 1943 (apart from a brief tour as Commander of ABDACOM) and then served as Viceroy of India until his retirement in February 1947.»\n",+ "[2] «John French, 1st Earl of Ypres | Field Marshal John Denton Pinkstone French, 1st Earl of Ypres (28 September 1852 – 22 May 1925), known as Sir John French from 1901 to 1916, and as The Viscount French between 1916 and 1922, was a senior British Army officer. Born in Kent to an Anglo-Irish family, he saw brief service as a midshipman in the Royal Navy, before becoming a cavalry officer. He achieved rapid promotion and distinguished himself on the Gordon Relief Expedition. French had a considerable reputation as a womaniser throughout his life and his career nearly ended when he was cited in the divorce of a brother officer whilst in India in the early 1890s.»\n",+ "[3] «John French, 2nd Earl of Ypres | John Richard Lowndes French, 2nd Earl of Ypres (6 July 1881 – 5 April 1958) was the son of the British field marshal and the first commander of the British Expeditionary Force (BEF) in World War I Sir John French.»\n",+ "[4] «Archibald Wavell, 2nd Earl Wavell | Major Archibald John Arthur Wavell, 2nd Earl Wavell MC (11 May 1916 – 24 December 1953) was the second and last Earl Wavell and Viscount Keren of Eritrea. He was educated at Winchester College and succeeded his father in 1950.»\n",+ "[5] «Archibald Frank Nicoll | Archibald Frank Nicoll {'1': \", '2': \", '3': \", '4': \"} (14 June 1886 – 1 February 1953) was a New Zealand artist and art teacher.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Answer. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Context:\n",+ "[1] «Archibald Wavell, 1st Earl Wavell | Field Marshal Archibald Percival Wavell, 1st Earl Wavell, {'1': \", '2': \", '3': \", '4': \"} (5 May 1883 – 24 May 1950) was a senior officer of the British Army. He served in the Second Boer War, the Bazar Valley Campaign and the Great War, during which he was wounded in the Second Battle of Ypres. He served in the Second World War, initially as Commander-in-Chief Middle East, in which role he led British forces to victory over the Italians in western Egypt and eastern Libya during Operation \"Compass\" in December 1940, only to be defeated by the German Army in the Western Desert in April 1941. He served as Commander-in-Chief, India, from July 1941 until June 1943 (apart from a brief tour as Commander of ABDACOM) and then served as Viceroy of India until his retirement in February 1947.»\n",+ "[2] «John French, 1st Earl of Ypres | Field Marshal John Denton Pinkstone French, 1st Earl of Ypres (28 September 1852 – 22 May 1925), known as Sir John French from 1901 to 1916, and as The Viscount French between 1916 and 1922, was a senior British Army officer. Born in Kent to an Anglo-Irish family, he saw brief service as a midshipman in the Royal Navy, before becoming a cavalry officer. He achieved rapid promotion and distinguished himself on the Gordon Relief Expedition. French had a considerable reputation as a womaniser throughout his life and his career nearly ended when he was cited in the divorce of a brother officer whilst in India in the early 1890s.»\n",+ "[3] «John French, 2nd Earl of Ypres | John Richard Lowndes French, 2nd Earl of Ypres (6 July 1881 – 5 April 1958) was the son of the British field marshal and the first commander of the British Expeditionary Force (BEF) in World War I Sir John French.»\n",+ "[4] «Archibald Wavell, 2nd Earl Wavell | Major Archibald John Arthur Wavell, 2nd Earl Wavell MC (11 May 1916 – 24 December 1953) was the second and last Earl Wavell and Viscount Keren of Eritrea. He was educated at Winchester College and succeeded his father in 1950.»\n",+ "[5] «Archibald Frank Nicoll | Archibald Frank Nicoll {'1': \", '2': \", '3': \", '4': \"} (14 June 1886 – 1 February 1953) was a New Zealand artist and art teacher.»\n",+ "\n",+ "Reasoning: We need to find the 1st Earl of Ypres who attacked the Boer force from the rear while mounted infantry. The context does not provide the information about the 1st Earl of Ypres attacking the Boer force from the rear while mounted infantry. However, we can find the information about John French, 1st Earl of Ypres, who was a senior British Army officer and achieved rapid promotion and distinguished himself on the Gordon Relief Expedition.\n",+ "Answer: John French\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Given the context and question, generate an informative search query that accurately captures the essence of the question, incorporating relevant keywords and phrases to retrieve relevant passages that can aid in answering the question.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Search Query: ${search_query}\n",+ "\n",+ "---\n",+ "\n",+ "Context: N/A\n",+ "\n",+ "Question: What race is the lieutenant governor of the state that held its primary for her election on May 19th, 2015?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: lieutenant governor race May 19 2015\n",+ "\n",+ "---\n",+ "\n",+ "Context: N/A\n",+ "\n",+ "Question: Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: clothing company founded 1889 college bass fishing\n",+ "\n",+ "---\n",+ "\n",+ "Context: N/A\n",+ "\n",+ "Question: Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Forrest Tucker height Walt Disney TV miniseries\n",+ "\n",+ "---\n",+ "\n",+ "Context: N/A\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Search Query. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: 1st Earl of Ypres Boer force mounted infantry\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Provide a descriptive search query that entails the critical information from the context and question to find a precise answer to the query asked, such as finding the release date of the computer-animated short comedy film Hammy's Boomerang Adventure, which is a spin-off of \"Over the Hedge\".\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Search Query: ${search_query}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Forrest Tucker | Forrest Meredith Tucker (February 12, 1919 – October 25, 1986) was an American actor in both movies and television who appeared in nearly a hundred films. Tucker worked as a vaudeville straight man, aged fifteen years old. A mentor provided funds and contacts for a trip to California, where party hostess Cobina Wright persuaded guest Wesley Ruggles to give Tucker a screen test, based on his photogenic good looks, thick wavy hair and height of six feet, five inches - putting him into the same league as John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness's younger brother. Tucker was a sight reader who needed only one take. His film career started well despite a perception in most Hollywood studios that blond men were not photogenic. After twenty years spent mainly in Westerns and action roles, he returned to his roots, showing versatility as a comedic and stage musical actor. In the TV show \"F Troop\", he became identified with the character of Cavalry Sgt. Morgan O'Rourke (a manipulative character quite similar to Phil Silvers' role as MSgt Ernie Bilko). Tucker struggled with a drinking problem that began to affect his performances in the later years of his career.»\n",+ "[2] «Emergency Landing (1941 film) | Emergency Landing (a.k.a. Robot Pilot) is a 1941 American aviation spy-fi romantic screwball comedy film directed by William Beaudine. The film stars Forrest Tucker in his second film and in his first leading role with co-stars Carol Hughes and Evelyn Brent. \"Emergency Landing\" features lots of mismatched stock footage of various types of aircraft.»\n",+ "[3] «Forrest Tucker (criminal) | Forrest Silva \"Woody\" Tucker (June 23, 1920 – May 29, 2004) was an American career criminal first imprisoned at age 15 who spent the rest of his life in and out of jail. He is best known as an escape artist, having escaped from prison 18 times successfully and 12 times unsuccessfully.»\n",+ "\n",+ "Question: Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Walt Disney TV miniseries 1955-1956 character\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.»\n",+ "[2] «Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford's second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.»\n",+ "[3] «G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.»\n",+ "\n",+ "Question: Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?\n",+ "\n",+ "Reasoning: Let's think step by step in order to Reasoning: Let's think step by step in order to\n",+ "\n",+ "Search Query: Carhartt bass fishing\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Archibald Wavell, 1st Earl Wavell | Field Marshal Archibald Percival Wavell, 1st Earl Wavell, {'1': \", '2': \", '3': \", '4': \"} (5 May 1883 – 24 May 1950) was a senior officer of the British Army. He served in the Second Boer War, the Bazar Valley Campaign and the Great War, during which he was wounded in the Second Battle of Ypres. He served in the Second World War, initially as Commander-in-Chief Middle East, in which role he led British forces to victory over the Italians in western Egypt and eastern Libya during Operation \"Compass\" in December 1940, only to be defeated by the German Army in the Western Desert in April 1941. He served as Commander-in-Chief, India, from July 1941 until June 1943 (apart from a brief tour as Commander of ABDACOM) and then served as Viceroy of India until his retirement in February 1947.»\n",+ "[2] «John French, 1st Earl of Ypres | Field Marshal John Denton Pinkstone French, 1st Earl of Ypres (28 September 1852 – 22 May 1925), known as Sir John French from 1901 to 1916, and as The Viscount French between 1916 and 1922, was a senior British Army officer. Born in Kent to an Anglo-Irish family, he saw brief service as a midshipman in the Royal Navy, before becoming a cavalry officer. He achieved rapid promotion and distinguished himself on the Gordon Relief Expedition. French had a considerable reputation as a womaniser throughout his life and his career nearly ended when he was cited in the divorce of a brother officer whilst in India in the early 1890s.»\n",+ "[3] «John French, 2nd Earl of Ypres | John Richard Lowndes French, 2nd Earl of Ypres (6 July 1881 – 5 April 1958) was the son of the British field marshal and the first commander of the British Expeditionary Force (BEF) in World War I Sir John French.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Search Query. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to find the birth year of the 1st Earl of Ypres who attacked the Boer force from the rear while mounted infantry. We need to find the 1st Earl of Ypres who was a senior officer of the British Army and served in the Second Boer War.\n",+ "Search Query: Archibald Wavell birth year\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Given a context and a question that mentions two high-peaked mountains, generate the answer that indicates the country where the mountains are partially located, by considering the geographical information provided in the context and the reasoning required to answer the question.\n",+ "\n",+ "---\n",+ "\n",+ "Question: What character did this actor, who also played a supporting role in Sleepy Hollow, play in the final six Harry Potter films?\n",+ "\n",+ "Answer: Professor Albus Dumbledore\n",+ "\n",+ "---\n",+ "\n",+ "Question: The musical sitcom with an album by the same name has what special feature on the album cover?\n",+ "\n",+ "Answer: actual crossword puzzle\n",+ "\n",+ "---\n",+ "\n",+ "Question: What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?\n",+ "\n",+ "Answer: 2016\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Answer: ${answer}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Archibald Wavell, 1st Earl Wavell | Field Marshal Archibald Percival Wavell, 1st Earl Wavell, {'1': \", '2': \", '3': \", '4': \"} (5 May 1883 – 24 May 1950) was a senior officer of the British Army. He served in the Second Boer War, the Bazar Valley Campaign and the Great War, during which he was wounded in the Second Battle of Ypres. He served in the Second World War, initially as Commander-in-Chief Middle East, in which role he led British forces to victory over the Italians in western Egypt and eastern Libya during Operation \"Compass\" in December 1940, only to be defeated by the German Army in the Western Desert in April 1941. He served as Commander-in-Chief, India, from July 1941 until June 1943 (apart from a brief tour as Commander of ABDACOM) and then served as Viceroy of India until his retirement in February 1947.»\n",+ "[2] «John French, 1st Earl of Ypres | Field Marshal John Denton Pinkstone French, 1st Earl of Ypres (28 September 1852 – 22 May 1925), known as Sir John French from 1901 to 1916, and as The Viscount French between 1916 and 1922, was a senior British Army officer. Born in Kent to an Anglo-Irish family, he saw brief service as a midshipman in the Royal Navy, before becoming a cavalry officer. He achieved rapid promotion and distinguished himself on the Gordon Relief Expedition. French had a considerable reputation as a womaniser throughout his life and his career nearly ended when he was cited in the divorce of a brother officer whilst in India in the early 1890s.»\n",+ "[3] «John French, 2nd Earl of Ypres | John Richard Lowndes French, 2nd Earl of Ypres (6 July 1881 – 5 April 1958) was the son of the British field marshal and the first commander of the British Expeditionary Force (BEF) in World War I Sir John French.»\n",+ "[4] «Archibald Wavell, 2nd Earl Wavell | Major Archibald John Arthur Wavell, 2nd Earl Wavell MC (11 May 1916 – 24 December 1953) was the second and last Earl Wavell and Viscount Keren of Eritrea. He was educated at Winchester College and succeeded his father in 1950.»\n",+ "[5] «Archibald Frank Nicoll | Archibald Frank Nicoll {'1': \", '2': \", '3': \", '4': \"} (14 June 1886 – 1 February 1953) was a New Zealand artist and art teacher.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Answer. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Context:\n",+ "[1] «Archibald Wavell, 1st Earl Wavell | Field Marshal Archibald Percival Wavell, 1st Earl Wavell, {'1': \", '2': \", '3': \", '4': \"} (5 May 1883 – 24 May 1950) was a senior officer of the British Army. He served in the Second Boer War, the Bazar Valley Campaign and the Great War, during which he was wounded in the Second Battle of Ypres. He served in the Second World War, initially as Commander-in-Chief Middle East, in which role he led British forces to victory over the Italians in western Egypt and eastern Libya during Operation \"Compass\" in December 1940, only to be defeated by the German Army in the Western Desert in April 1941. He served as Commander-in-Chief, India, from July 1941 until June 1943 (apart from a brief tour as Commander of ABDACOM) and then served as Viceroy of India until his retirement in February 1947.»\n",+ "[2] «John French, 1st Earl of Ypres | Field Marshal John Denton Pinkstone French, 1st Earl of Ypres (28 September 1852 – 22 May 1925), known as Sir John French from 1901 to 1916, and as The Viscount French between 1916 and 1922, was a senior British Army officer. Born in Kent to an Anglo-Irish family, he saw brief service as a midshipman in the Royal Navy, before becoming a cavalry officer. He achieved rapid promotion and distinguished himself on the Gordon Relief Expedition. French had a considerable reputation as a womaniser throughout his life and his career nearly ended when he was cited in the divorce of a brother officer whilst in India in the early 1890s.»\n",+ "[3] «John French, 2nd Earl of Ypres | John Richard Lowndes French, 2nd Earl of Ypres (6 July 1881 – 5 April 1958) was the son of the British field marshal and the first commander of the British Expeditionary Force (BEF) in World War I Sir John French.»\n",+ "[4] «Archibald Wavell, 2nd Earl Wavell | Major Archibald John Arthur Wavell, 2nd Earl Wavell MC (11 May 1916 – 24 December 1953) was the second and last Earl Wavell and Viscount Keren of Eritrea. He was educated at Winchester College and succeeded his father in 1950.»\n",+ "[5] «Archibald Frank Nicoll | Archibald Frank Nicoll {'1': \", '2': \", '3': \", '4': \"} (14 June 1886 – 1 February 1953) was a New Zealand artist and art teacher.»\n",+ "\n",+ "Reasoning: We need to find the 1st Earl of Ypres who attacked the Boer force from the rear while mounted infantry. The context does not provide the information about the 1st Earl of Ypres attacking the Boer force from the rear while mounted infantry. However, we can find the information about John French, 1st Earl of Ypres, who was a senior British Army officer and achieved rapid promotion and distinguished himself on the Gordon Relief Expedition.\n",+ "Answer: John French\n",+ "\n",+ "\n",+ "\n",+ "...\n",+ "Score 56.0\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "[I 2024-09-23 19:13:02,080] Trial 9 finished with value: 56.0 and parameters: {'0_predictor_instruction': 2, '0_predictor_demos': 2, '1_predictor_instruction': 4, '1_predictor_demos': 3, '2_predictor_instruction': 5, '2_predictor_demos': 1}. Best is trial 8 with value: 60.0.\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "CANDIDATE PROGRAM:\n",+ "Predictor 0\n",+ "i: Given the context and question, generate a specific search query that would lead to relevant passages for answering the question, and provide a detailed reasoning for this search query.\n",+ "p: Search Query:\n",+ "\n",+ "\n",+ "Predictor 1\n",+ "i: Generate a subsequent search query based on the provided context and question, aiming to narrow down the scope of relevant information and guide the retrieval of precise passages to support the forthcoming answer.\n",+ "p: Search Query:\n",+ "\n",+ "\n",+ "Predictor 2\n",+ "i: Given a context and a question, generate a logical and coherent answer that accurately addresses the query by meticulously analyzing the provided context, identifying relevant information, and applying critical thinking to provide a precise and informative response.\n",+ "p: Answer:\n",+ "\n",+ "\n",+ "...\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 12 / 25 (48.0): 100%|██████████| 25/25 [00:17<00:00, 1.42it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "FULL TRACE\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 0 / 1 (0.0): 100%|██████████| 1/1 [00:05<00:00, 5.32s/it]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "\n",+ "\n",+ "\n",+ "Given the context and question, generate a specific search query that would lead to relevant passages for answering the question, and provide a detailed reasoning for this search query.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Search Query: ${search_query}\n",+ "\n",+ "---\n",+ "\n",+ "Context: N/A\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Search Query. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to produce the output fields. We need to extract the key information from the question, which is the title of the person, the event, and the action. This will help us formulate a search query that can provide the answer to the question.\n",+ "Search Query: 1st Earl of Ypres birth Boer force mounted infantry\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Generate a subsequent search query based on the provided context and question, aiming to narrow down the scope of relevant information and guide the retrieval of precise passages to support the forthcoming answer.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Search Query: ${search_query}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Will Finn | Will Finn (born November 1, 1958) is an American animator, voice actor, and director. His work in animation includes characters from Disney and Don Bluth films such as \"The Secret of NIMH\", \"Oliver & Company\", \"The Little Mermaid\", \"The Rescuers Down Under\", and \"Pocahontas\". His characters includes Cogsworth in \"Beauty and the Beast\", Iago in \"Aladdin\", and Laverne in \"The Hunchback of Notre Dame\". Finn wrote and directed \"Home on the Range\" and did some voice acting the Hollywood Fish in \"Chicken Little\". In 2006, Finn directed the computer animated short Hammy's Boomerang Adventure, a spin-off of \"Over the Hedge\".»\n",+ "[2] «Over the Hedge (film) | Over the Hedge is a 2006 American computer-animated comedy film, based on the characters from the United Media comic strip of the same name. Directed by Tim Johnson and Karey Kirkpatrick, and produced by Bonnie Arnold, it was released in the United States on May 19, 2006. The film was produced by DreamWorks Animation and distributed through Paramount Pictures. The film features the voices of Bruce Willis, Garry Shandling, Steve Carell, William Shatner, Wanda Sykes, and Nick Nolte. It is the first DreamWorks Animation film to be distributed by Paramount Pictures, which acquired the live-action DreamWorks studio in 2006. The film earned $336 million on an $80 million budget.»\n",+ "[3] «Tim Johnson (film director) | Tim Johnson (born August 27, 1961) is an American animation director. Johnson has directed in many films such as \"Antz\", \"\" and \"Over the Hedge\". Recently, he directed the DreamWorks 3D film \"Home.\"»\n",+ "\n",+ "Question: When was the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed?\n",+ "\n",+ "Reasoning: Let's think step by step in order to find the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed. The context mentions that Will Finn directed the computer animated short Hammy's Boomerang Adventure, a spin-off of \"Over the Hedge\".\n",+ "\n",+ "Search Query: Hammy's Boomerang Adventure release date\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Archibald Wavell, 1st Earl Wavell | Field Marshal Archibald Percival Wavell, 1st Earl Wavell, {'1': \", '2': \", '3': \", '4': \"} (5 May 1883 – 24 May 1950) was a senior officer of the British Army. He served in the Second Boer War, the Bazar Valley Campaign and the Great War, during which he was wounded in the Second Battle of Ypres. He served in the Second World War, initially as Commander-in-Chief Middle East, in which role he led British forces to victory over the Italians in western Egypt and eastern Libya during Operation \"Compass\" in December 1940, only to be defeated by the German Army in the Western Desert in April 1941. He served as Commander-in-Chief, India, from July 1941 until June 1943 (apart from a brief tour as Commander of ABDACOM) and then served as Viceroy of India until his retirement in February 1947.»\n",+ "[2] «John French, 1st Earl of Ypres | Field Marshal John Denton Pinkstone French, 1st Earl of Ypres (28 September 1852 – 22 May 1925), known as Sir John French from 1901 to 1916, and as The Viscount French between 1916 and 1922, was a senior British Army officer. Born in Kent to an Anglo-Irish family, he saw brief service as a midshipman in the Royal Navy, before becoming a cavalry officer. He achieved rapid promotion and distinguished himself on the Gordon Relief Expedition. French had a considerable reputation as a womaniser throughout his life and his career nearly ended when he was cited in the divorce of a brother officer whilst in India in the early 1890s.»\n",+ "[3] «Ripple, Kent | John French, 1st Earl of Ypres, the commander of the first British Expeditionary Force was born there in 1852, and is buried at the village church. His sister Charlotte Despard, the suffragist, novelist and Sinn Féin activist was also born in Ripple in 1844.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Search Query. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to find the 1st Earl of Ypres born who attacked the Boer force from the rear while mounted infantry. The context mentions that Archibald Wavell, 1st Earl Wavell, was a senior officer of the British Army who served in the Second Boer War. We need to find the 1st Earl of Ypres who was born in the same war.\n",+ "Search Query: John French, 1st Earl of Ypres birthdate\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Given a context and a question, generate a logical and coherent answer that accurately addresses the query by meticulously analyzing the provided context, identifying relevant information, and applying critical thinking to provide a precise and informative response.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Answer: ${answer}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Simon Arora | Simon Arora (born November 1969) is a British billionaire businessman, CEO of the retail chain B & M.»\n",+ "[2] «Samir Arora | Samir Arora (born November 5, 1965) is an American businessman and former CEO of Mode Media (formerly Glam Media) from 2003 to April 2016. He was CEO and Chairman of the web design company NetObjects, Inc. from 1995 to 2001.»\n",+ "[3] «Dinesh Arora | Dinesh Arora (born 23 December 1976) is an Indian Administrative Service (IAS) officer, 2002 Kerala cadre, who previously worked as an Executive Director, Rural Electrification Corporation (REC) and CEO RECPDCL.»\n",+ "[4] «Bolliger & Mabillard | Bolliger & Mabillard, officially Bolliger & Mabillard Consulting Engineers Inc., and often abbreviated B&M, is a roller coaster design consultancy based in Monthey, Switzerland. The company was founded in 1988 by Walter Bolliger and Claude Mabillard, with Bolliger as president and Mabillard as vice-president. B&M has built over a hundred roller coasters around the world and pioneered several new ride technologies, most notably the inverted roller coaster. In North America, B&M coaster designs have been manufactured by Ohio company Clermont Steel Fabricators since 1990. B&M has grown significantly since its founding, originally employing four to as many as 37 in 2012, consisting primarily of engineers and draftsmen. In 2016, the company completed its 100th roller coaster.»\n",+ "[5] «Rakefet Russak-Aminoach | Rakefet Russak-Aminoach (Hebrew: רקפת רוסק עמינח ; born February 1, 1966) is the current President and CEO of Bank Leumi le-Israel B.M., a position she has held since May 1, 2012. In September 2015, she was listed in Fortune Magazine’s list of The World's 100 Most Powerful Women.»\n",+ "\n",+ "Question: What company formed in 1978 has Simon Arora as its CEO?\n",+ "\n",+ "Reasoning: Let's think step by step in order to There is no information about Simon Arora being the CEO of a company formed in 1978 in the context. However, we can find the information about Simon Arora being the CEO of B & M in the context.\n",+ "\n",+ "Answer: B & M\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Electoral district of Bragg | Bragg is an electoral district of the House of Assembly in the Australian state of South Australia. The seat of Bragg is named after the eminent physicists Bragg – William Henry and his son, William Lawrence. The electorate is largely urban and encompasses a significant portion of the City of Burnside, stretching from the east parklands of Adelaide into the Adelaide Hills. After a redistribution following the 2006 election, the boundary moved eastwards to include suburbs that had formerly been in the seat of Heysen and now borders Kavel. Bragg currently includes the metropolitan suburbs of Beaumont, Burnside, Dulwich, Erindale, Hazelwood Park, Heathpool, Kensington Park, Leabrook, Linden Park, Rose Park, Marryatville, Skye, St Georges, Stonyfell, Toorak Gardens, Tusmore, Wattle Park and parts of Glen Osmond, and in the hills it includes Crafers, Cleland, Greenhill, Mount Osmond, Piccadilly, Summertown and parts of Ashton, Basket Range, Carey Gully, Horsnell Gully, Leawood Gardens and Uraidla.»\n",+ "[2] «Cleland, South Australia | Cleland is a suburb in South Australia located in the Adelaide metropolitan area about 10 km south-east of the Adelaide city centre. Its boundaries were created in October 2001, with additional land being added in 2010 from the adjoining suburb of Crafers. Its name is derived from the Cleland Conservation Park. The principal land use within the locality is conservation with the majority of its land area being occupied by the Cleland Conservation Park. Places within its extent include the summit of Mount Lofty. Cleland is located within the federal Division of Mayo, the state electoral district of Bragg and the local government area of the Adelaide Hills Council.»\n",+ "[3] «Electoral results for the district of Bragg | This is a list of electoral results for the Electoral district of Bragg in South Australian state elections.»\n",+ "[4] «Electoral district of Burnside | Burnside was an electoral district of the House of Assembly in the Australian state of South Australia from 1938 to 1970. Before 1938 the Burnside area was represented by the three-seat multi-member electorate of Sturt. It was abolished in the 1970 parliamentary reforms, and was replaced with Bragg.»\n",+ "\n",+ "Question: Bragg electoral district, incorporating the Kensington Park suburb, are associated with which South Australian city?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "Context: N/A\n",+ "\n",+ "Question: Bragg electoral district, incorporating the Kensington Park suburb, are associated with which South Australian city?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "\n",+ "Answer: Adelaide\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.»\n",+ "[2] «Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.»\n",+ "[3] «Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world's third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.»\n",+ "\n",+ "Question: The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "Context: [1] «Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.»\n",+ "\n",+ "[2] «Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.»\n",+ "\n",+ "[3] «Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world's third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.»\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "\n",+ "Answer: India\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Archibald Wavell, 1st Earl Wavell | Field Marshal Archibald Percival Wavell, 1st Earl Wavell, {'1': \", '2': \", '3': \", '4': \"} (5 May 1883 – 24 May 1950) was a senior officer of the British Army. He served in the Second Boer War, the Bazar Valley Campaign and the Great War, during which he was wounded in the Second Battle of Ypres. He served in the Second World War, initially as Commander-in-Chief Middle East, in which role he led British forces to victory over the Italians in western Egypt and eastern Libya during Operation \"Compass\" in December 1940, only to be defeated by the German Army in the Western Desert in April 1941. He served as Commander-in-Chief, India, from July 1941 until June 1943 (apart from a brief tour as Commander of ABDACOM) and then served as Viceroy of India until his retirement in February 1947.»\n",+ "[2] «John French, 1st Earl of Ypres | Field Marshal John Denton Pinkstone French, 1st Earl of Ypres (28 September 1852 – 22 May 1925), known as Sir John French from 1901 to 1916, and as The Viscount French between 1916 and 1922, was a senior British Army officer. Born in Kent to an Anglo-Irish family, he saw brief service as a midshipman in the Royal Navy, before becoming a cavalry officer. He achieved rapid promotion and distinguished himself on the Gordon Relief Expedition. French had a considerable reputation as a womaniser throughout his life and his career nearly ended when he was cited in the divorce of a brother officer whilst in India in the early 1890s.»\n",+ "[3] «Ripple, Kent | John French, 1st Earl of Ypres, the commander of the first British Expeditionary Force was born there in 1852, and is buried at the village church. His sister Charlotte Despard, the suffragist, novelist and Sinn Féin activist was also born in Ripple in 1844.»\n",+ "[4] «John French, 2nd Earl of Ypres | John Richard Lowndes French, 2nd Earl of Ypres (6 July 1881 – 5 April 1958) was the son of the British field marshal and the first commander of the British Expeditionary Force (BEF) in World War I Sir John French.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Answer. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to find the 1st Earl of Ypres who attacked the Boer force from the rear while mounted infantry. We need to find the 1st Earl of Ypres who was born in 1852, and we can find this information in the context.\n",+ "Answer: Archibald Wavell, 1st Earl Wavell\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Given the context and question, generate a specific search query that would lead to relevant passages for answering the question, and provide a detailed reasoning for this search query.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Search Query: ${search_query}\n",+ "\n",+ "---\n",+ "\n",+ "Context: N/A\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Search Query. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to produce the output fields. We need to extract the key information from the question, which is the title of the person, the event, and the action. This will help us formulate a search query that can provide the answer to the question.\n",+ "Search Query: 1st Earl of Ypres birth Boer force mounted infantry\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Generate a subsequent search query based on the provided context and question, aiming to narrow down the scope of relevant information and guide the retrieval of precise passages to support the forthcoming answer.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Search Query: ${search_query}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Will Finn | Will Finn (born November 1, 1958) is an American animator, voice actor, and director. His work in animation includes characters from Disney and Don Bluth films such as \"The Secret of NIMH\", \"Oliver & Company\", \"The Little Mermaid\", \"The Rescuers Down Under\", and \"Pocahontas\". His characters includes Cogsworth in \"Beauty and the Beast\", Iago in \"Aladdin\", and Laverne in \"The Hunchback of Notre Dame\". Finn wrote and directed \"Home on the Range\" and did some voice acting the Hollywood Fish in \"Chicken Little\". In 2006, Finn directed the computer animated short Hammy's Boomerang Adventure, a spin-off of \"Over the Hedge\".»\n",+ "[2] «Over the Hedge (film) | Over the Hedge is a 2006 American computer-animated comedy film, based on the characters from the United Media comic strip of the same name. Directed by Tim Johnson and Karey Kirkpatrick, and produced by Bonnie Arnold, it was released in the United States on May 19, 2006. The film was produced by DreamWorks Animation and distributed through Paramount Pictures. The film features the voices of Bruce Willis, Garry Shandling, Steve Carell, William Shatner, Wanda Sykes, and Nick Nolte. It is the first DreamWorks Animation film to be distributed by Paramount Pictures, which acquired the live-action DreamWorks studio in 2006. The film earned $336 million on an $80 million budget.»\n",+ "[3] «Tim Johnson (film director) | Tim Johnson (born August 27, 1961) is an American animation director. Johnson has directed in many films such as \"Antz\", \"\" and \"Over the Hedge\". Recently, he directed the DreamWorks 3D film \"Home.\"»\n",+ "\n",+ "Question: When was the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed?\n",+ "\n",+ "Reasoning: Let's think step by step in order to find the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed. The context mentions that Will Finn directed the computer animated short Hammy's Boomerang Adventure, a spin-off of \"Over the Hedge\".\n",+ "\n",+ "Search Query: Hammy's Boomerang Adventure release date\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Archibald Wavell, 1st Earl Wavell | Field Marshal Archibald Percival Wavell, 1st Earl Wavell, {'1': \", '2': \", '3': \", '4': \"} (5 May 1883 – 24 May 1950) was a senior officer of the British Army. He served in the Second Boer War, the Bazar Valley Campaign and the Great War, during which he was wounded in the Second Battle of Ypres. He served in the Second World War, initially as Commander-in-Chief Middle East, in which role he led British forces to victory over the Italians in western Egypt and eastern Libya during Operation \"Compass\" in December 1940, only to be defeated by the German Army in the Western Desert in April 1941. He served as Commander-in-Chief, India, from July 1941 until June 1943 (apart from a brief tour as Commander of ABDACOM) and then served as Viceroy of India until his retirement in February 1947.»\n",+ "[2] «John French, 1st Earl of Ypres | Field Marshal John Denton Pinkstone French, 1st Earl of Ypres (28 September 1852 – 22 May 1925), known as Sir John French from 1901 to 1916, and as The Viscount French between 1916 and 1922, was a senior British Army officer. Born in Kent to an Anglo-Irish family, he saw brief service as a midshipman in the Royal Navy, before becoming a cavalry officer. He achieved rapid promotion and distinguished himself on the Gordon Relief Expedition. French had a considerable reputation as a womaniser throughout his life and his career nearly ended when he was cited in the divorce of a brother officer whilst in India in the early 1890s.»\n",+ "[3] «Ripple, Kent | John French, 1st Earl of Ypres, the commander of the first British Expeditionary Force was born there in 1852, and is buried at the village church. His sister Charlotte Despard, the suffragist, novelist and Sinn Féin activist was also born in Ripple in 1844.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Search Query. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to find the 1st Earl of Ypres born who attacked the Boer force from the rear while mounted infantry. The context mentions that Archibald Wavell, 1st Earl Wavell, was a senior officer of the British Army who served in the Second Boer War. We need to find the 1st Earl of Ypres who was born in the same war.\n",+ "Search Query: John French, 1st Earl of Ypres birthdate\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Given a context and a question, generate a logical and coherent answer that accurately addresses the query by meticulously analyzing the provided context, identifying relevant information, and applying critical thinking to provide a precise and informative response.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context: ${context}\n",+ "\n",+ "Question: ${question}\n",+ "\n",+ "Reasoning: Let's think step by step in order to ${produce the output fields}. We ...\n",+ "\n",+ "Answer: ${answer}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Simon Arora | Simon Arora (born November 1969) is a British billionaire businessman, CEO of the retail chain B & M.»\n",+ "[2] «Samir Arora | Samir Arora (born November 5, 1965) is an American businessman and former CEO of Mode Media (formerly Glam Media) from 2003 to April 2016. He was CEO and Chairman of the web design company NetObjects, Inc. from 1995 to 2001.»\n",+ "[3] «Dinesh Arora | Dinesh Arora (born 23 December 1976) is an Indian Administrative Service (IAS) officer, 2002 Kerala cadre, who previously worked as an Executive Director, Rural Electrification Corporation (REC) and CEO RECPDCL.»\n",+ "[4] «Bolliger & Mabillard | Bolliger & Mabillard, officially Bolliger & Mabillard Consulting Engineers Inc., and often abbreviated B&M, is a roller coaster design consultancy based in Monthey, Switzerland. The company was founded in 1988 by Walter Bolliger and Claude Mabillard, with Bolliger as president and Mabillard as vice-president. B&M has built over a hundred roller coasters around the world and pioneered several new ride technologies, most notably the inverted roller coaster. In North America, B&M coaster designs have been manufactured by Ohio company Clermont Steel Fabricators since 1990. B&M has grown significantly since its founding, originally employing four to as many as 37 in 2012, consisting primarily of engineers and draftsmen. In 2016, the company completed its 100th roller coaster.»\n",+ "[5] «Rakefet Russak-Aminoach | Rakefet Russak-Aminoach (Hebrew: רקפת רוסק עמינח ; born February 1, 1966) is the current President and CEO of Bank Leumi le-Israel B.M., a position she has held since May 1, 2012. In September 2015, she was listed in Fortune Magazine’s list of The World's 100 Most Powerful Women.»\n",+ "\n",+ "Question: What company formed in 1978 has Simon Arora as its CEO?\n",+ "\n",+ "Reasoning: Let's think step by step in order to There is no information about Simon Arora being the CEO of a company formed in 1978 in the context. However, we can find the information about Simon Arora being the CEO of B & M in the context.\n",+ "\n",+ "Answer: B & M\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Electoral district of Bragg | Bragg is an electoral district of the House of Assembly in the Australian state of South Australia. The seat of Bragg is named after the eminent physicists Bragg – William Henry and his son, William Lawrence. The electorate is largely urban and encompasses a significant portion of the City of Burnside, stretching from the east parklands of Adelaide into the Adelaide Hills. After a redistribution following the 2006 election, the boundary moved eastwards to include suburbs that had formerly been in the seat of Heysen and now borders Kavel. Bragg currently includes the metropolitan suburbs of Beaumont, Burnside, Dulwich, Erindale, Hazelwood Park, Heathpool, Kensington Park, Leabrook, Linden Park, Rose Park, Marryatville, Skye, St Georges, Stonyfell, Toorak Gardens, Tusmore, Wattle Park and parts of Glen Osmond, and in the hills it includes Crafers, Cleland, Greenhill, Mount Osmond, Piccadilly, Summertown and parts of Ashton, Basket Range, Carey Gully, Horsnell Gully, Leawood Gardens and Uraidla.»\n",+ "[2] «Cleland, South Australia | Cleland is a suburb in South Australia located in the Adelaide metropolitan area about 10 km south-east of the Adelaide city centre. Its boundaries were created in October 2001, with additional land being added in 2010 from the adjoining suburb of Crafers. Its name is derived from the Cleland Conservation Park. The principal land use within the locality is conservation with the majority of its land area being occupied by the Cleland Conservation Park. Places within its extent include the summit of Mount Lofty. Cleland is located within the federal Division of Mayo, the state electoral district of Bragg and the local government area of the Adelaide Hills Council.»\n",+ "[3] «Electoral results for the district of Bragg | This is a list of electoral results for the Electoral district of Bragg in South Australian state elections.»\n",+ "[4] «Electoral district of Burnside | Burnside was an electoral district of the House of Assembly in the Australian state of South Australia from 1938 to 1970. Before 1938 the Burnside area was represented by the three-seat multi-member electorate of Sturt. It was abolished in the 1970 parliamentary reforms, and was replaced with Bragg.»\n",+ "\n",+ "Question: Bragg electoral district, incorporating the Kensington Park suburb, are associated with which South Australian city?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "Context: N/A\n",+ "\n",+ "Question: Bragg electoral district, incorporating the Kensington Park suburb, are associated with which South Australian city?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "\n",+ "Answer: Adelaide\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.»\n",+ "[2] «Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.»\n",+ "[3] «Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world's third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.»\n",+ "\n",+ "Question: The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "Context: [1] «Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.»\n",+ "\n",+ "[2] «Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.»\n",+ "\n",+ "[3] «Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world's third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.»\n",+ "\n",+ "Reasoning: Let's think step by step in order to\n",+ "\n",+ "Answer: India\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Archibald Wavell, 1st Earl Wavell | Field Marshal Archibald Percival Wavell, 1st Earl Wavell, {'1': \", '2': \", '3': \", '4': \"} (5 May 1883 – 24 May 1950) was a senior officer of the British Army. He served in the Second Boer War, the Bazar Valley Campaign and the Great War, during which he was wounded in the Second Battle of Ypres. He served in the Second World War, initially as Commander-in-Chief Middle East, in which role he led British forces to victory over the Italians in western Egypt and eastern Libya during Operation \"Compass\" in December 1940, only to be defeated by the German Army in the Western Desert in April 1941. He served as Commander-in-Chief, India, from July 1941 until June 1943 (apart from a brief tour as Commander of ABDACOM) and then served as Viceroy of India until his retirement in February 1947.»\n",+ "[2] «John French, 1st Earl of Ypres | Field Marshal John Denton Pinkstone French, 1st Earl of Ypres (28 September 1852 – 22 May 1925), known as Sir John French from 1901 to 1916, and as The Viscount French between 1916 and 1922, was a senior British Army officer. Born in Kent to an Anglo-Irish family, he saw brief service as a midshipman in the Royal Navy, before becoming a cavalry officer. He achieved rapid promotion and distinguished himself on the Gordon Relief Expedition. French had a considerable reputation as a womaniser throughout his life and his career nearly ended when he was cited in the divorce of a brother officer whilst in India in the early 1890s.»\n",+ "[3] «Ripple, Kent | John French, 1st Earl of Ypres, the commander of the first British Expeditionary Force was born there in 1852, and is buried at the village church. His sister Charlotte Despard, the suffragist, novelist and Sinn Féin activist was also born in Ripple in 1844.»\n",+ "[4] «John French, 2nd Earl of Ypres | John Richard Lowndes French, 2nd Earl of Ypres (6 July 1881 – 5 April 1958) was the son of the British field marshal and the first commander of the British Expeditionary Force (BEF) in World War I Sir John French.»\n",+ "\n",+ "Question: When was the 1st Earl of Ypres born who attacked attacked the Boer force from the rear while mounted infantry?\n",+ "\n",+ "Please provide the output fields Reasoning then Answer. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n",+ "\n",+ "Reasoning: Let's think step by step in order to find the 1st Earl of Ypres who attacked the Boer force from the rear while mounted infantry. We need to find the 1st Earl of Ypres who was born in 1852, and we can find this information in the context.\n",+ "Answer: Archibald Wavell, 1st Earl Wavell\n",+ "\n",+ "\n",+ "\n",+ "...\n",+ "Score 48.0\n",+ "Best Combination: 9,{0: [[], [Example({'question': 'Oh Yeon-seo is one of the former members of the group LUV, which stands for Love yoUR Voice, is what kind of group?', 'answer': 'K-pop girls group'}) (input_keys={'question'}), Example({'question': 'Melbourne Central railway station feeds into which main metro station that serves the entire metropolitan rail network?', 'answer': 'Flinders Street railway station'}) (input_keys={'question'}), Example({'question': 'When was the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed?', 'answer': '2006'}) (input_keys={'question'})], [Example({'augmented': True, 'context': [], 'question': 'What race is the lieutenant governor of the state that held its primary for her election on May 19th, 2015?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'lieutenant governor race May 19 2015'}) (input_keys=None), Example({'augmented': True, 'context': [], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'clothing company founded 1889 college bass fishing'}) (input_keys=None), Example({'augmented': True, 'context': [], 'question': \"Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\", 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Forrest Tucker height Walt Disney TV miniseries'}) (input_keys=None)], [Example({'augmented': True, 'context': [], 'question': \"Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\", 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Forrest Tucker height Walt Disney TV miniseries'}) (input_keys=None), Example({'augmented': True, 'context': [], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'clothing company founded 1889 college bass fishing'}) (input_keys=None), Example({'question': 'How many years did Lenny Lyles spend with the organization dramatized in the filme \"The insder\"?', 'answer': '27'}) (input_keys={'question'})], [Example({'augmented': True, 'context': [], 'question': 'When was the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Will Finn Over the Hedge short film director'}) (input_keys=None), Example({'question': 'What is the capacity of the La Liga stadium in which The Madrid Derby team plays their home games?', 'answer': '68,000'}) (input_keys={'question'}), Example({'question': 'The Equestrian statue of Marshal Mannerheim was erected as a monument to the leader of the Whites during what event?', 'answer': 'Finnish Civil War'}) (input_keys={'question'})], [Example({'augmented': True, 'context': [], 'question': 'The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Kangchenjunga Singhi Kangri country'}) (input_keys=None), Example({'question': 'Are Beth Gibbons and Ian Curtis the same nationality?', 'answer': 'yes'}) (input_keys={'question'}), Example({'question': \"When did Taylor Swift's satirical song get released to the radio?\", 'answer': 'February 9, 2015'}) (input_keys={'question'})], [Example({'augmented': True, 'context': [], 'question': 'What was the series inspired by the appreciation of manual laborers that Wheat Field with Cypresses is a part of?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Wheat Field with Cypresses series'}) (input_keys=None), Example({'question': 'What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?', 'answer': '2016'}) (input_keys={'question'}), Example({'question': 'How many shops does the larger of the Miami International Mall and Dolphin Mall have?', 'answer': '240'}) (input_keys={'question'})], [Example({'augmented': True, 'context': [], 'question': 'What other sport did Butch Reed, of the wrestling tag team Doom, play professionally?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Butch Reed sports'}) (input_keys=None), Example({'question': 'Which magazine is still in print American Heritage or High Times?', 'answer': 'High Times'}) (input_keys={'question'}), Example({'question': 'What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?', 'answer': '2016'}) (input_keys={'question'})], [Example({'augmented': True, 'context': [], 'question': 'Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Garden City Hotel founder'}) (input_keys=None), Example({'augmented': True, 'context': [], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'clothing company founded 1889 college bass fishing'}) (input_keys=None), Example({'augmented': True, 'context': [], 'question': 'Which theme park has Stitch’s Great Escape which replaced the attraction ExtraTERRORestial Alien Encounter?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': \"Stitch's Great Escape theme park\"}) (input_keys=None)], [Example({'augmented': True, 'context': [], 'question': 'What company formed in 1978 has Simon Arora as its CEO?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Simon Arora CEO company 1978'}) (input_keys=None), Example({'augmented': True, 'context': [], 'question': 'Bragg electoral district, incorporating the Kensington Park suburb, are associated with which South Australian city?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Bragg electoral district Kensington Park suburb South Australian city'}) (input_keys=None), Example({'augmented': True, 'context': [], 'question': 'The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Kangchenjunga Singhi Kangri country'}) (input_keys=None)]], 1: [[], [Example({'question': 'Born Again starred the American film actor who played what role in The Best Years of Our Lives?', 'answer': 'Fred Derry'}) (input_keys={'question'}), Example({'question': 'What type of compilation does Chris Harris and Into the Labyrinth have in common?', 'answer': 'television series'}) (input_keys={'question'}), Example({'question': 'Are both Paul Schattel and Francis Ford Coppola American film directors?', 'answer': 'yes'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Alberta general election, 2015 | The 29th general election of Alberta, Canada, took place on May 5, 2015, following a request of Premier Jim Prentice to the Lieutenant Governor of Alberta Donald Ethell to dissolve the Legislative Assembly on April 7, 2015. This election elected members to the 29th Alberta Legislature. It was one of only four times that Alberta has changed governments.', \"Jenean Hampton | Jenean Michelle Hampton (born May 12, 1958) is the 57th and current Lieutenant Governor of Kentucky since December 8, 2015 after having won office as Republican gubernatorial nominee Matt Bevin's running-mate, in the 2015 gubernatorial election. She is the first African-American to hold any statewide office in Kentucky history, and only the third African-American woman to serve as lieutenant-governor of any U.S. state (after Jennette Bradley and Jennifer Carroll).\", 'Louisiana lieutenant gubernatorial election, 2015 | The Louisiana lieutenant gubernatorial election took place on October 24, 2015, to elect the Lieutenant Governor of Louisiana, with a runoff election held on November 21, 2015. Incumbent Republican Lieutenant Governor Jay Dardenne did not for re-election to a second full term in office. He instead ran for Governor. Billy Nungesser won the election defeating Kip Holden, despite a Democratic victory in the gubernatorial election, in which John Bel Edwards defeated David Vitter by a similar margin.'], 'question': 'What race is the lieutenant governor of the state that held its primary for her election on May 19th, 2015?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'lieutenant governor of the state that held its primary for her election on May 19th 2015'}) (input_keys=None), Example({'augmented': True, 'context': ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.'], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Carhartt bass fishing'}) (input_keys=None), Example({'augmented': True, 'context': ['Forrest Tucker | Forrest Meredith Tucker (February 12, 1919 – October 25, 1986) was an American actor in both movies and television who appeared in nearly a hundred films. Tucker worked as a vaudeville straight man, aged fifteen years old. A mentor provided funds and contacts for a trip to California, where party hostess Cobina Wright persuaded guest Wesley Ruggles to give Tucker a screen test, based on his photogenic good looks, thick wavy hair and height of six feet, five inches - putting him into the same league as John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness\\'s younger brother. Tucker was a sight reader who needed only one take. His film career started well despite a perception in most Hollywood studios that blond men were not photogenic. After twenty years spent mainly in Westerns and action roles, he returned to his roots, showing versatility as a comedic and stage musical actor. In the TV show \"F Troop\", he became identified with the character of Cavalry Sgt. Morgan O\\'Rourke (a manipulative character quite similar to Phil Silvers\\' role as MSgt Ernie Bilko). Tucker struggled with a drinking problem that began to affect his performances in the later years of his career.', 'Emergency Landing (1941 film) | Emergency Landing (a.k.a. Robot Pilot) is a 1941 American aviation spy-fi romantic screwball comedy film directed by William Beaudine. The film stars Forrest Tucker in his second film and in his first leading role with co-stars Carol Hughes and Evelyn Brent. \"Emergency Landing\" features lots of mismatched stock footage of various types of aircraft.', 'Forrest Tucker (criminal) | Forrest Silva \"Woody\" Tucker (June 23, 1920 – May 29, 2004) was an American career criminal first imprisoned at age 15 who spent the rest of his life in and out of jail. He is best known as an escape artist, having escaped from prison 18 times successfully and 12 times unsuccessfully.'], 'question': \"Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\", 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Walt Disney TV miniseries 1955-1956 character'}) (input_keys=None)], [Example({'augmented': True, 'context': ['Forrest Tucker | Forrest Meredith Tucker (February 12, 1919 – October 25, 1986) was an American actor in both movies and television who appeared in nearly a hundred films. Tucker worked as a vaudeville straight man, aged fifteen years old. A mentor provided funds and contacts for a trip to California, where party hostess Cobina Wright persuaded guest Wesley Ruggles to give Tucker a screen test, based on his photogenic good looks, thick wavy hair and height of six feet, five inches - putting him into the same league as John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness\\'s younger brother. Tucker was a sight reader who needed only one take. His film career started well despite a perception in most Hollywood studios that blond men were not photogenic. After twenty years spent mainly in Westerns and action roles, he returned to his roots, showing versatility as a comedic and stage musical actor. In the TV show \"F Troop\", he became identified with the character of Cavalry Sgt. Morgan O\\'Rourke (a manipulative character quite similar to Phil Silvers\\' role as MSgt Ernie Bilko). Tucker struggled with a drinking problem that began to affect his performances in the later years of his career.', 'Emergency Landing (1941 film) | Emergency Landing (a.k.a. Robot Pilot) is a 1941 American aviation spy-fi romantic screwball comedy film directed by William Beaudine. The film stars Forrest Tucker in his second film and in his first leading role with co-stars Carol Hughes and Evelyn Brent. \"Emergency Landing\" features lots of mismatched stock footage of various types of aircraft.', 'Forrest Tucker (criminal) | Forrest Silva \"Woody\" Tucker (June 23, 1920 – May 29, 2004) was an American career criminal first imprisoned at age 15 who spent the rest of his life in and out of jail. He is best known as an escape artist, having escaped from prison 18 times successfully and 12 times unsuccessfully.'], 'question': \"Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\", 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Walt Disney TV miniseries 1955-1956 character'}) (input_keys=None), Example({'augmented': True, 'context': ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.'], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Carhartt bass fishing'}) (input_keys=None), Example({'question': 'How many years did Lenny Lyles spend with the organization dramatized in the filme \"The insder\"?', 'answer': '27'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Will Finn | Will Finn (born November 1, 1958) is an American animator, voice actor, and director. His work in animation includes characters from Disney and Don Bluth films such as \"The Secret of NIMH\", \"Oliver & Company\", \"The Little Mermaid\", \"The Rescuers Down Under\", and \"Pocahontas\". His characters includes Cogsworth in \"Beauty and the Beast\", Iago in \"Aladdin\", and Laverne in \"The Hunchback of Notre Dame\". Finn wrote and directed \"Home on the Range\" and did some voice acting the Hollywood Fish in \"Chicken Little\". In 2006, Finn directed the computer animated short Hammy\\'s Boomerang Adventure, a spin-off of \"Over the Hedge\".', 'Over the Hedge (film) | Over the Hedge is a 2006 American computer-animated comedy film, based on the characters from the United Media comic strip of the same name. Directed by Tim Johnson and Karey Kirkpatrick, and produced by Bonnie Arnold, it was released in the United States on May 19, 2006. The film was produced by DreamWorks Animation and distributed through Paramount Pictures. The film features the voices of Bruce Willis, Garry Shandling, Steve Carell, William Shatner, Wanda Sykes, and Nick Nolte. It is the first DreamWorks Animation film to be distributed by Paramount Pictures, which acquired the live-action DreamWorks studio in 2006. The film earned $336 million on an $80 million budget.', 'Tim Johnson (film director) | Tim Johnson (born August 27, 1961) is an American animation director. Johnson has directed in many films such as \"Antz\", \"\" and \"Over the Hedge\". Recently, he directed the DreamWorks 3D film \"Home.\"'], 'question': 'When was the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed?', 'rationale': 'find the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed. The context mentions that Will Finn directed the computer animated short Hammy\\'s Boomerang Adventure, a spin-off of \"Over the Hedge\".', 'search_query': \"Hammy's Boomerang Adventure release date\"}) (input_keys=None), Example({'question': 'The Equestrian statue of Marshal Mannerheim was erected as a monument to the leader of the Whites during what event?', 'answer': 'Finnish Civil War'}) (input_keys={'question'}), Example({'question': 'What is the capacity of the La Liga stadium in which The Madrid Derby team plays their home games?', 'answer': '68,000'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.', 'Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.', \"Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world's third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.\"], 'question': 'The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Kangchenjunga Singhi Kangri country'}) (input_keys=None), Example({'question': \"When did Taylor Swift's satirical song get released to the radio?\", 'answer': 'February 9, 2015'}) (input_keys={'question'}), Example({'question': 'Are Beth Gibbons and Ian Curtis the same nationality?', 'answer': 'yes'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Wheat Field with Cypresses | A Wheatfield with Cypresses (occasionally called \"A Cornfield with Cypresses\") is any of three similar 1889 oil paintings by Vincent van Gogh, as part of his wheat field series. All were executed at the Saint-Paul-de-Mausole mental asylum at Saint-Rémy near Arles, France, where Van Gogh was voluntarily a patient from May 1889 to May 1890. The works were inspired by the view from the window at the asylum towards the Alpilles mountains.', 'Green Wheat Field with Cypress | Green Wheat Field with Cypress (French: Champ de blé vert avec cyprès) is an oil on canvas painting by a Dutch Post-Impressionist Vincent van Gogh. It was completed in 1889, while van Gogh was voluntarily incarcerated at the asylum of St. Paul near Saint-Rémy in Provence. Several paintings of wheat fields with cypresses were made when van Gogh was able to leave the asylum grounds and explore the local landscape. Besides a fondness for cypresses, van Gogh had a special affinty with wheat fields; he depicted them dozens of times over the years; to Vincent they symbolized the cycle of life and death, and he found in them both solace and inspiration.', 'Olive Trees (Van Gogh series) | Vincent van Gogh painted at least 18 paintings of olive trees, mostly in Saint-Rémy-de-Provence in 1889. At his own request, he lived at an asylum there from May 1889 through May 1890 painting the gardens of the asylum and, when he had permission to venture outside its walls, nearby olive trees, cypresses and wheat fields.'], 'question': 'What was the series inspired by the appreciation of manual laborers that Wheat Field with Cypresses is a part of?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Wheat Field with Cypresses series inspiration'}) (input_keys=None), Example({'question': 'How many shops does the larger of the Miami International Mall and Dolphin Mall have?', 'answer': '240'}) (input_keys={'question'}), Example({'question': 'What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?', 'answer': '2016'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Butch Reed | Bruce Franklin Reed (born July 11, 1954) is an American retired professional wrestler and football player, better known by the ring name Butch Reed. He is best known for his appearances with the World Wrestling Federation and World Championship Wrestling between 1986 and 1992.', 'Doug Reed (futsal player) | Doug Reed is a professional futsal player who is currently playing for Futsal Dinamo and England national futsal team. Previously he has played for Lanzarote Tias Yaiza (Spain), AEL Limassol, MNK Uspinjača, MNK Smederevo and Manchester Futsal Club. In 2011, he won the FA Futsal Cup with Manchester Futsal Club and was awarded Player of the Tournament in the same competition. In the FA National Futsal League, he won several FA National League Championships and FA Futsal Cup medals. Reed also works as a commentator for Sony SIX, Eurosport and BT Sport, and has worked with the emerging US Professional Futsal League as well as Indian Premier Futsal.', 'Doom (professional wrestling) | Doom was a professional wrestling tag team composed of Ron Simmons and Butch Reed. They competed from 1989 to 1991 in the National Wrestling Alliance (NWA) and World Championship Wrestling (WCW).'], 'question': 'What other sport did Butch Reed, of the wrestling tag team Doom, play professionally?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Butch Reed football'}) (input_keys=None), Example({'question': 'What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?', 'answer': '2016'}) (input_keys={'question'}), Example({'question': 'Which magazine is still in print American Heritage or High Times?', 'answer': 'High Times'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.', 'Ebenezer Howard | Sir Ebenezer Howard {\\'1\\': \", \\'2\\': \", \\'3\\': \", \\'4\\': \"} ( ; 29 January 1850 – 1 May 1928), the English founder of the garden city movement, is known for his publication \"\" (1898), the description of a utopian city in which people live harmoniously together with nature. The publication resulted in the founding of the garden city movement, and the building of the First Garden City, Letchworth Garden City, commenced in 1903.', 'Eugene Reynal | Eugene Reynal was an American Publisher and in 1933 Founder of Blue Ribbon Books of Garden City, New York and Reynal and Hitchcock of New York, New York.'], 'question': 'Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Garden City Hotel founder'}) (input_keys=None), Example({'augmented': True, 'context': ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.'], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Carhartt bass fishing'}) (input_keys=None), Example({'augmented': True, 'context': ['Stitch\\'s Great Escape! | Stitch\\'s Great Escape! is a Tomorrowland attraction at the Magic Kingdom theme park within the Walt Disney World Resort. It is a \"theater-in-the-round\" experience starring the title alien from Walt Disney Animation Studios\\' 2002 film \"Lilo & Stitch\". It opened November 16, 2004 and is the fourth attraction to occupy the site in Tomorrowland. Many of the animators who worked on \"Lilo & Stitch\" partnered with Walt Disney Imagineering for Stitch\\'s Great Escape! The attraction is a replacement of The ExtraTERRORestrial Alien Encounter, which formerly occupied the building the attraction is housed in. On September 21, 2016, it was announced that the attraction would be switching from a daily operated attraction to a seasonally operated one, depending on attendance, starting October 2, 2016.', 'Great Escape (amusement park) | Great Escape (sometimes referred to as Six Flags Great Escape) is an amusement and water park owned and operated by Six Flags Entertainment Corp. It is located approximately 60 mi north of Albany, in Queensbury, New York, but is advertised as being in Lake George, New York, a popular tourist and vacation spot nearby. It is one of two Six Flags parks not to be officially branded with the \"Six Flags\" name (La Ronde in Montreal, Quebec, Canada being the other).', 'Houdini\\'s Great Escape | Houdini\\'s Great Escape, also known as \"Boo-dini\" during Fright Fest and \"Houdini - The Great Escape\" at Six Flags New England, are indoor Vekoma Madhouse attractions at Six Flags Great Adventure and Six Flags New England. The ride is located in Great Adventure\\'s Fantasy Forest section and New England\\'s Crack Axle Canyon area. The attraction opened at both parks in 1999, and was Standing but not operating (SBNO) at Great Adventure from 2008 - 2009 but was opened again due to popular demand.'], 'question': 'Which theme park has Stitch’s Great Escape which replaced the attraction ExtraTERRORestial Alien Encounter?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Magic Kingdom theme park'}) (input_keys=None)], [Example({'augmented': True, 'context': ['Simon Arora | Simon Arora (born November 1969) is a British billionaire businessman, CEO of the retail chain B & M.', 'Samir Arora | Samir Arora (born November 5, 1965) is an American businessman and former CEO of Mode Media (formerly Glam Media) from 2003 to April 2016. He was CEO and Chairman of the web design company NetObjects, Inc. from 1995 to 2001.', 'Dinesh Arora | Dinesh Arora (born 23 December 1976) is an Indian Administrative Service (IAS) officer, 2002 Kerala cadre, who previously worked as an Executive Director, Rural Electrification Corporation (REC) and CEO RECPDCL.'], 'question': 'What company formed in 1978 has Simon Arora as its CEO?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'B & M CEO'}) (input_keys=None), Example({'augmented': True, 'context': ['Electoral district of Bragg | Bragg is an electoral district of the House of Assembly in the Australian state of South Australia. The seat of Bragg is named after the eminent physicists Bragg – William Henry and his son, William Lawrence. The electorate is largely urban and encompasses a significant portion of the City of Burnside, stretching from the east parklands of Adelaide into the Adelaide Hills. After a redistribution following the 2006 election, the boundary moved eastwards to include suburbs that had formerly been in the seat of Heysen and now borders Kavel. Bragg currently includes the metropolitan suburbs of Beaumont, Burnside, Dulwich, Erindale, Hazelwood Park, Heathpool, Kensington Park, Leabrook, Linden Park, Rose Park, Marryatville, Skye, St Georges, Stonyfell, Toorak Gardens, Tusmore, Wattle Park and parts of Glen Osmond, and in the hills it includes Crafers, Cleland, Greenhill, Mount Osmond, Piccadilly, Summertown and parts of Ashton, Basket Range, Carey Gully, Horsnell Gully, Leawood Gardens and Uraidla.', 'Cleland, South Australia | Cleland is a suburb in South Australia located in the Adelaide metropolitan area about 10 km south-east of the Adelaide city centre. Its boundaries were created in October 2001, with additional land being added in 2010 from the adjoining suburb of Crafers. Its name is derived from the Cleland Conservation Park. The principal land use within the locality is conservation with the majority of its land area being occupied by the Cleland Conservation Park. Places within its extent include the summit of Mount Lofty. Cleland is located within the federal Division of Mayo, the state electoral district of Bragg and the local government area of the Adelaide Hills Council.', 'Electoral results for the district of Bragg | This is a list of electoral results for the Electoral district of Bragg in South Australian state elections.'], 'question': 'Bragg electoral district, incorporating the Kensington Park suburb, are associated with which South Australian city?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'South Australian city associated with Bragg electoral district'}) (input_keys=None), Example({'augmented': True, 'context': ['Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.', 'Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.', \"Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world's third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.\"], 'question': 'The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Kangchenjunga Singhi Kangri country'}) (input_keys=None)]], 2: [[], [Example({'question': 'What character did this actor, who also played a supporting role in Sleepy Hollow, play in the final six Harry Potter films?', 'answer': 'Professor Albus Dumbledore'}) (input_keys={'question'}), Example({'question': 'The musical sitcom with an album by the same name has what special feature on the album cover?', 'answer': 'actual crossword puzzle'}) (input_keys={'question'}), Example({'question': 'What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?', 'answer': '2016'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Alberta general election, 2015 | The 29th general election of Alberta, Canada, took place on May 5, 2015, following a request of Premier Jim Prentice to the Lieutenant Governor of Alberta Donald Ethell to dissolve the Legislative Assembly on April 7, 2015. This election elected members to the 29th Alberta Legislature. It was one of only four times that Alberta has changed governments.', \"Jenean Hampton | Jenean Michelle Hampton (born May 12, 1958) is the 57th and current Lieutenant Governor of Kentucky since December 8, 2015 after having won office as Republican gubernatorial nominee Matt Bevin's running-mate, in the 2015 gubernatorial election. She is the first African-American to hold any statewide office in Kentucky history, and only the third African-American woman to serve as lieutenant-governor of any U.S. state (after Jennette Bradley and Jennifer Carroll).\", 'Louisiana lieutenant gubernatorial election, 2015 | The Louisiana lieutenant gubernatorial election took place on October 24, 2015, to elect the Lieutenant Governor of Louisiana, with a runoff election held on November 21, 2015. Incumbent Republican Lieutenant Governor Jay Dardenne did not for re-election to a second full term in office. He instead ran for Governor. Billy Nungesser won the election defeating Kip Holden, despite a Democratic victory in the gubernatorial election, in which John Bel Edwards defeated David Vitter by a similar margin.', 'Indiana gubernatorial election, 2016 | The 2016 Indiana gubernatorial election was held November 8, 2016, to elect the Governor and Lieutenant Governor of Indiana, concurrently with the 2016 U.S. presidential election as well as elections to the United States Senate and elections to the United States House of Representatives and various state and local elections. The primaries were held on May 3, 2016. Republican Lieutenant Governor Eric Holcomb won the race with 51.4% of the vote.', 'Virginia lieutenant gubernatorial election, 2017 | The Virginia lieutenant gubernatorial election of 2017 will take place on November 7, 2017. After the party primary elections were held, the major party nominees are Jill Vogel (Republican) and Justin Fairfax (Democrat). The incumbent Lieutenant Governor, Democrat Ralph Northam, is not running for re-election in order to run for Governor.'], 'question': 'What race is the lieutenant governor of the state that held its primary for her election on May 19th, 2015?', 'rationale': 'The lieutenant governor of the state that held its primary for her election on May 19th, 2015 is Jenean Hampton, who is the 57th and current Lieutenant Governor of Kentucky since December 8, 2015. The primary election was held in Kentucky, and Jenean Hampton is the lieutenant governor of Kentucky.', 'answer': 'African-American'}) (input_keys=None), Example({'augmented': True, 'context': ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.', \"College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour\", 'Bass fishing | Bass fishing is the activity of angling for the North American gamefish known colloquially as the \"black bass\". There are numerous black bass species considered as gamefish in North America, including largemouth bass (\"Micropterus salmoides\"), smallmouth bass (\"Micropterus dolomieui\"), spotted bass or Kentucky bass (\"Micropterus punctulatus\"), and Guadalupe bass (order \"Perciformes\").', 'Cliff Crochet | Cliff Crochet (born August 1, 1983), also known as \"The Cajun Baby\", is a professional bass fisherman from Pierre Part, Louisiana. Crochet is a competitor on Bass Angler Sportsman Society\\'s (B.A.S.S.) Bassmaster Elite Series and has $368,837 in B.A.S.S. tournament winnings as of August 22, 2016. Crochet\\'s sponsors include: K2 Coolers, Huk, Skeeter, Yamaha Motor Company, Lowrance Electronics, Falcon Rods, Luck E Strike, Seaguar, Rat-L-Trap, Humdinger Spinnerbaits, Power-Pole. Since 2010, Crochet has competed in three Bassmaster Classic tournaments. Crochet\\'s boat for the 2016 season is a 2016 Skeeter FX 21 with a 250 hp Yamaha SHO outboard engine.'], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': 'The context mentions Carhartt Bassmaster College Bass Fishing Series, which is a college bass fishing tournament series. This series is associated with the clothing company Carhartt, which was founded in 1889.', 'answer': 'Carhartt'}) (input_keys=None), Example({'augmented': True, 'context': ['Forrest Tucker | Forrest Meredith Tucker (February 12, 1919 – October 25, 1986) was an American actor in both movies and television who appeared in nearly a hundred films. Tucker worked as a vaudeville straight man, aged fifteen years old. A mentor provided funds and contacts for a trip to California, where party hostess Cobina Wright persuaded guest Wesley Ruggles to give Tucker a screen test, based on his photogenic good looks, thick wavy hair and height of six feet, five inches - putting him into the same league as John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness\\'s younger brother. Tucker was a sight reader who needed only one take. His film career started well despite a perception in most Hollywood studios that blond men were not photogenic. After twenty years spent mainly in Westerns and action roles, he returned to his roots, showing versatility as a comedic and stage musical actor. In the TV show \"F Troop\", he became identified with the character of Cavalry Sgt. Morgan O\\'Rourke (a manipulative character quite similar to Phil Silvers\\' role as MSgt Ernie Bilko). Tucker struggled with a drinking problem that began to affect his performances in the later years of his career.', 'Emergency Landing (1941 film) | Emergency Landing (a.k.a. Robot Pilot) is a 1941 American aviation spy-fi romantic screwball comedy film directed by William Beaudine. The film stars Forrest Tucker in his second film and in his first leading role with co-stars Carol Hughes and Evelyn Brent. \"Emergency Landing\" features lots of mismatched stock footage of various types of aircraft.', 'Forrest Tucker (criminal) | Forrest Silva \"Woody\" Tucker (June 23, 1920 – May 29, 2004) was an American career criminal first imprisoned at age 15 who spent the rest of his life in and out of jail. He is best known as an escape artist, having escaped from prison 18 times successfully and 12 times unsuccessfully.', 'Fess Parker | Fess Elisha Parker Jr. (August 16, 1924 – March 18, 2010) was an American film and television actor best known for his portrayals of Davy Crockett in the Walt Disney 1955–1956 TV miniseries and as Daniel Boone in a television series from 1964 to 1970. He was also known as a winemaker and resort owner-operator.', 'Davy Crockett, King of the Wild Frontier | Davy Crockett, King of the Wild Frontier is a 1955 live-action Walt Disney adventure film starring Fess Parker as Davy Crockett. This film is an edited compilation of the first three stories from the Disney television miniseries \"Davy Crockett\" :', 'Davy Crockett (miniseries) | Davy Crockett is a five-part serial which aired on ABC from 1954–1955 in one-hour episodes, on the \"Disneyland\" series. The series stars Fess Parker as real-life frontiersman Davy Crockett and Buddy Ebsen as his friend, George Russel.'], 'question': \"Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\", 'rationale': \"From the context, we know that Forrest Tucker was an American actor who appeared in nearly a hundred films. We also know that he was compared to John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness's younger brother. We need to find the actor that played a character in the 1955-1956 Walt Disney TV miniseries, which is Fess Parker.\", 'answer': 'Davy Crockett'}) (input_keys=None)], [Example({'augmented': True, 'context': ['Forrest Tucker | Forrest Meredith Tucker (February 12, 1919 – October 25, 1986) was an American actor in both movies and television who appeared in nearly a hundred films. Tucker worked as a vaudeville straight man, aged fifteen years old. A mentor provided funds and contacts for a trip to California, where party hostess Cobina Wright persuaded guest Wesley Ruggles to give Tucker a screen test, based on his photogenic good looks, thick wavy hair and height of six feet, five inches - putting him into the same league as John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness\\'s younger brother. Tucker was a sight reader who needed only one take. His film career started well despite a perception in most Hollywood studios that blond men were not photogenic. After twenty years spent mainly in Westerns and action roles, he returned to his roots, showing versatility as a comedic and stage musical actor. In the TV show \"F Troop\", he became identified with the character of Cavalry Sgt. Morgan O\\'Rourke (a manipulative character quite similar to Phil Silvers\\' role as MSgt Ernie Bilko). Tucker struggled with a drinking problem that began to affect his performances in the later years of his career.', 'Emergency Landing (1941 film) | Emergency Landing (a.k.a. Robot Pilot) is a 1941 American aviation spy-fi romantic screwball comedy film directed by William Beaudine. The film stars Forrest Tucker in his second film and in his first leading role with co-stars Carol Hughes and Evelyn Brent. \"Emergency Landing\" features lots of mismatched stock footage of various types of aircraft.', 'Forrest Tucker (criminal) | Forrest Silva \"Woody\" Tucker (June 23, 1920 – May 29, 2004) was an American career criminal first imprisoned at age 15 who spent the rest of his life in and out of jail. He is best known as an escape artist, having escaped from prison 18 times successfully and 12 times unsuccessfully.', 'Fess Parker | Fess Elisha Parker Jr. (August 16, 1924 – March 18, 2010) was an American film and television actor best known for his portrayals of Davy Crockett in the Walt Disney 1955–1956 TV miniseries and as Daniel Boone in a television series from 1964 to 1970. He was also known as a winemaker and resort owner-operator.', 'Davy Crockett, King of the Wild Frontier | Davy Crockett, King of the Wild Frontier is a 1955 live-action Walt Disney adventure film starring Fess Parker as Davy Crockett. This film is an edited compilation of the first three stories from the Disney television miniseries \"Davy Crockett\" :', 'Davy Crockett (miniseries) | Davy Crockett is a five-part serial which aired on ABC from 1954–1955 in one-hour episodes, on the \"Disneyland\" series. The series stars Fess Parker as real-life frontiersman Davy Crockett and Buddy Ebsen as his friend, George Russel.'], 'question': \"Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\", 'rationale': 'We need to find the actor who played the character in the 1955-1956 Walt Disney TV miniseries. From the context, we know that the miniseries was about Davy Crockett and starred Fess Parker as Davy Crockett.', 'answer': 'Davy Crockett'}) (input_keys=None), Example({'augmented': True, 'context': ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.', \"College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour\", 'Bass fishing | Bass fishing is the activity of angling for the North American gamefish known colloquially as the \"black bass\". There are numerous black bass species considered as gamefish in North America, including largemouth bass (\"Micropterus salmoides\"), smallmouth bass (\"Micropterus dolomieui\"), spotted bass or Kentucky bass (\"Micropterus punctulatus\"), and Guadalupe bass (order \"Perciformes\").', 'Cliff Crochet | Cliff Crochet (born August 1, 1983), also known as \"The Cajun Baby\", is a professional bass fisherman from Pierre Part, Louisiana. Crochet is a competitor on Bass Angler Sportsman Society\\'s (B.A.S.S.) Bassmaster Elite Series and has $368,837 in B.A.S.S. tournament winnings as of August 22, 2016. Crochet\\'s sponsors include: K2 Coolers, Huk, Skeeter, Yamaha Motor Company, Lowrance Electronics, Falcon Rods, Luck E Strike, Seaguar, Rat-L-Trap, Humdinger Spinnerbaits, Power-Pole. Since 2010, Crochet has competed in three Bassmaster Classic tournaments. Crochet\\'s boat for the 2016 season is a 2016 Skeeter FX 21 with a 250 hp Yamaha SHO outboard engine.'], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Context: [1] «Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.»\\n\\n[2] «College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour»\\n\\n[3] «G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.»\\n\\nReasoning: Let's think step by step in order to\", 'answer': 'Carhartt'}) (input_keys=None), Example({'question': 'How many years did Lenny Lyles spend with the organization dramatized in the filme \"The insder\"?', 'answer': '27'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Will Finn | Will Finn (born November 1, 1958) is an American animator, voice actor, and director. His work in animation includes characters from Disney and Don Bluth films such as \"The Secret of NIMH\", \"Oliver & Company\", \"The Little Mermaid\", \"The Rescuers Down Under\", and \"Pocahontas\". His characters includes Cogsworth in \"Beauty and the Beast\", Iago in \"Aladdin\", and Laverne in \"The Hunchback of Notre Dame\". Finn wrote and directed \"Home on the Range\" and did some voice acting the Hollywood Fish in \"Chicken Little\". In 2006, Finn directed the computer animated short Hammy\\'s Boomerang Adventure, a spin-off of \"Over the Hedge\".', 'Over the Hedge (film) | Over the Hedge is a 2006 American computer-animated comedy film, based on the characters from the United Media comic strip of the same name. Directed by Tim Johnson and Karey Kirkpatrick, and produced by Bonnie Arnold, it was released in the United States on May 19, 2006. The film was produced by DreamWorks Animation and distributed through Paramount Pictures. The film features the voices of Bruce Willis, Garry Shandling, Steve Carell, William Shatner, Wanda Sykes, and Nick Nolte. It is the first DreamWorks Animation film to be distributed by Paramount Pictures, which acquired the live-action DreamWorks studio in 2006. The film earned $336 million on an $80 million budget.', 'Tim Johnson (film director) | Tim Johnson (born August 27, 1961) is an American animation director. Johnson has directed in many films such as \"Antz\", \"\" and \"Over the Hedge\". Recently, he directed the DreamWorks 3D film \"Home.\"', 'Hammy\\'s Boomerang Adventure | Hammy\\'s Boomerang Adventure is a computer-animated short comedy film based on the film \"Over the Hedge\", which was based on a comic strip of the same name. It can be found on the movie\\'s DVD or the \"Madly Madagascar\" DVD as a bonus feature.', 'Boomerang (2015 film) | Boomerang is a 2015 French drama film directed by François Favrat and starring Laurent Lafitte, Mélanie Laurent, Audrey Dana and Wladimir Yordanoff. It is an adaptation of Tatiana de Rosnay\\'s novel \"A Secret Kept\" (alternative title:\"Boomerang\"). The film was released in theatres on 23 September 2015.', 'Over the Hedge: Hammy Goes Nuts! | Over the Hedge: Hammy Goes Nuts! is a video game published by Activision based on characters from the film \"Over the Hedge\". The Game Boy Advance (GBA) version, a miniature golf game, was developed by Vicarious Visions and released on October 10, 2006 in North America. The Nintendo DS and PlayStation Portable (PSP) versions, an action adventure game and a platform game respectively, were both developed by Amaze Entertainment and were released on October 24, 2006 and November 21, 2006. It is the second game to be based on the film.'], 'question': 'When was the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed?', 'rationale': 'Context: N/A\\n\\nQuestion: When was the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed?\\n\\nReasoning: Let\\'s think step by step in order to', 'answer': '2006'}) (input_keys=None), Example({'question': 'What is the capacity of the La Liga stadium in which The Madrid Derby team plays their home games?', 'answer': '68,000'}) (input_keys={'question'}), Example({'question': 'The Equestrian statue of Marshal Mannerheim was erected as a monument to the leader of the Whites during what event?', 'answer': 'Finnish Civil War'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.', 'Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.', \"Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world's third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.\"], 'question': 'The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?', 'rationale': 'Context: [1] «Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.»\\n\\n[2] «Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.»\\n\\n[3] «Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world\\'s third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.»\\n\\nReasoning: Let\\'s think step by step in order to', 'answer': 'India'}) (input_keys=None), Example({'question': 'Are Beth Gibbons and Ian Curtis the same nationality?', 'answer': 'yes'}) (input_keys={'question'}), Example({'question': \"When did Taylor Swift's satirical song get released to the radio?\", 'answer': 'February 9, 2015'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Wheat Field with Cypresses | A Wheatfield with Cypresses (occasionally called \"A Cornfield with Cypresses\") is any of three similar 1889 oil paintings by Vincent van Gogh, as part of his wheat field series. All were executed at the Saint-Paul-de-Mausole mental asylum at Saint-Rémy near Arles, France, where Van Gogh was voluntarily a patient from May 1889 to May 1890. The works were inspired by the view from the window at the asylum towards the Alpilles mountains.', 'Green Wheat Field with Cypress | Green Wheat Field with Cypress (French: Champ de blé vert avec cyprès) is an oil on canvas painting by a Dutch Post-Impressionist Vincent van Gogh. It was completed in 1889, while van Gogh was voluntarily incarcerated at the asylum of St. Paul near Saint-Rémy in Provence. Several paintings of wheat fields with cypresses were made when van Gogh was able to leave the asylum grounds and explore the local landscape. Besides a fondness for cypresses, van Gogh had a special affinty with wheat fields; he depicted them dozens of times over the years; to Vincent they symbolized the cycle of life and death, and he found in them both solace and inspiration.', 'Olive Trees (Van Gogh series) | Vincent van Gogh painted at least 18 paintings of olive trees, mostly in Saint-Rémy-de-Provence in 1889. At his own request, he lived at an asylum there from May 1889 through May 1890 painting the gardens of the asylum and, when he had permission to venture outside its walls, nearby olive trees, cypresses and wheat fields.', 'Wheat Fields (Van Gogh series) | The Wheat Fields is a series of dozens of paintings by Vincent van Gogh, borne out of his religious studies and sermons, connection to nature, appreciation of manual laborers and desire to provide a means of offering comfort to others. The wheat field works demonstrate his progression as an artist from the drab \"Wheat Sheaves\" made in 1885 in the Netherlands to the colorful, dramatic paintings from Arles, Saint-Rémy and Auvers-sur-Oise of rural France.'], 'question': 'What was the series inspired by the appreciation of manual laborers that Wheat Field with Cypresses is a part of?', 'rationale': \"Context: N/A\\n\\nQuestion: What was the series inspired by the appreciation of manual laborers that Wheat Field with Cypresses is a part of?\\n\\nReasoning: Let's think step by step in order to\", 'answer': 'Wheat Fields'}) (input_keys=None), Example({'question': 'What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?', 'answer': '2016'}) (input_keys={'question'}), Example({'question': 'How many shops does the larger of the Miami International Mall and Dolphin Mall have?', 'answer': '240'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Butch Reed | Bruce Franklin Reed (born July 11, 1954) is an American retired professional wrestler and football player, better known by the ring name Butch Reed. He is best known for his appearances with the World Wrestling Federation and World Championship Wrestling between 1986 and 1992.', 'Doug Reed (futsal player) | Doug Reed is a professional futsal player who is currently playing for Futsal Dinamo and England national futsal team. Previously he has played for Lanzarote Tias Yaiza (Spain), AEL Limassol, MNK Uspinjača, MNK Smederevo and Manchester Futsal Club. In 2011, he won the FA Futsal Cup with Manchester Futsal Club and was awarded Player of the Tournament in the same competition. In the FA National Futsal League, he won several FA National League Championships and FA Futsal Cup medals. Reed also works as a commentator for Sony SIX, Eurosport and BT Sport, and has worked with the emerging US Professional Futsal League as well as Indian Premier Futsal.', 'Doom (professional wrestling) | Doom was a professional wrestling tag team composed of Ron Simmons and Butch Reed. They competed from 1989 to 1991 in the National Wrestling Alliance (NWA) and World Championship Wrestling (WCW).', 'Robert Reed (gridiron football) | Robert E. Reed, Jr. (born February 23, 1943) is a former American football offensive lineman in the National Football League for the Washington Redskins. He played college football at Tennessee State University and was drafted in the sixteenth round of the 1965 NFL Draft. Reed was also selected in the 12th round of the 1965 AFL Draft by the Houston Oilers.', 'Reed Doughty | Reed Doughty (born November 4, 1982) is a former American football safety who played in the National Football League. He was drafted by the Washington Redskins in the sixth round, with the 173rd overall pick, of the 2006 NFL Draft. He played college football at the University of Northern Colorado.'], 'question': 'What other sport did Butch Reed, of the wrestling tag team Doom, play professionally?', 'rationale': \"Context: N/A\\n\\nQuestion: What other sport did Butch Reed, of the wrestling tag team Doom, play professionally?\\n\\nReasoning: Let's think step by step in order to\", 'answer': 'football'}) (input_keys=None), Example({'question': 'Which magazine is still in print American Heritage or High Times?', 'answer': 'High Times'}) (input_keys={'question'}), Example({'question': 'What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?', 'answer': '2016'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.', 'Ebenezer Howard | Sir Ebenezer Howard {\\'1\\': \", \\'2\\': \", \\'3\\': \", \\'4\\': \"} ( ; 29 January 1850 – 1 May 1928), the English founder of the garden city movement, is known for his publication \"\" (1898), the description of a utopian city in which people live harmoniously together with nature. The publication resulted in the founding of the garden city movement, and the building of the First Garden City, Letchworth Garden City, commenced in 1903.', 'Eugene Reynal | Eugene Reynal was an American Publisher and in 1933 Founder of Blue Ribbon Books of Garden City, New York and Reynal and Hitchcock of New York, New York.'], 'question': 'Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?', 'rationale': 'Context: [1] «Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.»\\n\\n[2] «A.T. Stewart | Alexander Turney Stewart (October 2, 1792 – April 10, 1876) was an American businessman, politician, and philanthropist. He was a prominent figure in New York City during the mid-19th century, and was known as the \"Merchant Prince\".»\\n\\n[3] «Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.»\\n\\nQuestion: Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?\\n\\nReasoning: Let\\'s think step by step in order to', 'answer': 'A.T. Stewart'}) (input_keys=None), Example({'augmented': True, 'context': ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.', \"College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour\", 'Bass fishing | Bass fishing is the activity of angling for the North American gamefish known colloquially as the \"black bass\". There are numerous black bass species considered as gamefish in North America, including largemouth bass (\"Micropterus salmoides\"), smallmouth bass (\"Micropterus dolomieui\"), spotted bass or Kentucky bass (\"Micropterus punctulatus\"), and Guadalupe bass (order \"Perciformes\").', 'Cliff Crochet | Cliff Crochet (born August 1, 1983), also known as \"The Cajun Baby\", is a professional bass fisherman from Pierre Part, Louisiana. Crochet is a competitor on Bass Angler Sportsman Society\\'s (B.A.S.S.) Bassmaster Elite Series and has $368,837 in B.A.S.S. tournament winnings as of August 22, 2016. Crochet\\'s sponsors include: K2 Coolers, Huk, Skeeter, Yamaha Motor Company, Lowrance Electronics, Falcon Rods, Luck E Strike, Seaguar, Rat-L-Trap, Humdinger Spinnerbaits, Power-Pole. Since 2010, Crochet has competed in three Bassmaster Classic tournaments. Crochet\\'s boat for the 2016 season is a 2016 Skeeter FX 21 with a 250 hp Yamaha SHO outboard engine.'], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Context: [1] «Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.»\\n\\n[2] «College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour»\\n\\n[3] «G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.»\\n\\nReasoning: Let's think step by step in order to\", 'answer': 'Carhartt'}) (input_keys=None), Example({'augmented': True, 'context': ['Stitch\\'s Great Escape! | Stitch\\'s Great Escape! is a Tomorrowland attraction at the Magic Kingdom theme park within the Walt Disney World Resort. It is a \"theater-in-the-round\" experience starring the title alien from Walt Disney Animation Studios\\' 2002 film \"Lilo & Stitch\". It opened November 16, 2004 and is the fourth attraction to occupy the site in Tomorrowland. Many of the animators who worked on \"Lilo & Stitch\" partnered with Walt Disney Imagineering for Stitch\\'s Great Escape! The attraction is a replacement of The ExtraTERRORestrial Alien Encounter, which formerly occupied the building the attraction is housed in. On September 21, 2016, it was announced that the attraction would be switching from a daily operated attraction to a seasonally operated one, depending on attendance, starting October 2, 2016.', 'Great Escape (amusement park) | Great Escape (sometimes referred to as Six Flags Great Escape) is an amusement and water park owned and operated by Six Flags Entertainment Corp. It is located approximately 60 mi north of Albany, in Queensbury, New York, but is advertised as being in Lake George, New York, a popular tourist and vacation spot nearby. It is one of two Six Flags parks not to be officially branded with the \"Six Flags\" name (La Ronde in Montreal, Quebec, Canada being the other).', 'Houdini\\'s Great Escape | Houdini\\'s Great Escape, also known as \"Boo-dini\" during Fright Fest and \"Houdini - The Great Escape\" at Six Flags New England, are indoor Vekoma Madhouse attractions at Six Flags Great Adventure and Six Flags New England. The ride is located in Great Adventure\\'s Fantasy Forest section and New England\\'s Crack Axle Canyon area. The attraction opened at both parks in 1999, and was Standing but not operating (SBNO) at Great Adventure from 2008 - 2009 but was opened again due to popular demand.', 'Magic Kingdom | Magic Kingdom is a theme park at the Walt Disney World Resort in Bay Lake, Florida, near Orlando. Owned and operated by The Walt Disney Company through its Parks and Resorts division, the park opened on October 1, 1971, as the first of four theme parks at the resort. Initialized by Walt Disney and designed by WED Enterprises, its layout and attractions are based on Disneyland Park in Anaheim, California, and is dedicated to fairy tales and Disney characters.', 'Magic Kingdom (disambiguation) | The Magic Kingdom is a theme park at the Walt Disney World Resort in Florida.', 'List of Magic Kingdom attractions | The Magic Kingdom is a theme park located at the Walt Disney World Resort in Florida. Below is a list of the current attractions found therein, arranged by \"land\" and with brief descriptions.'], 'question': 'Which theme park has Stitch’s Great Escape which replaced the attraction ExtraTERRORestial Alien Encounter?', 'rationale': \"We need to find the theme park that has Stitch's Great Escape, which replaced the attraction ExtraTERRORestial Alien Encounter. The context mentions that Stitch's Great Escape is located in the Magic Kingdom theme park.\", 'answer': 'Magic Kingdom'}) (input_keys=None)], [Example({'augmented': True, 'context': ['Simon Arora | Simon Arora (born November 1969) is a British billionaire businessman, CEO of the retail chain B & M.', 'Samir Arora | Samir Arora (born November 5, 1965) is an American businessman and former CEO of Mode Media (formerly Glam Media) from 2003 to April 2016. He was CEO and Chairman of the web design company NetObjects, Inc. from 1995 to 2001.', 'Dinesh Arora | Dinesh Arora (born 23 December 1976) is an Indian Administrative Service (IAS) officer, 2002 Kerala cadre, who previously worked as an Executive Director, Rural Electrification Corporation (REC) and CEO RECPDCL.', 'Bolliger & Mabillard | Bolliger & Mabillard, officially Bolliger & Mabillard Consulting Engineers Inc., and often abbreviated B&M, is a roller coaster design consultancy based in Monthey, Switzerland. The company was founded in 1988 by Walter Bolliger and Claude Mabillard, with Bolliger as president and Mabillard as vice-president. B&M has built over a hundred roller coasters around the world and pioneered several new ride technologies, most notably the inverted roller coaster. In North America, B&M coaster designs have been manufactured by Ohio company Clermont Steel Fabricators since 1990. B&M has grown significantly since its founding, originally employing four to as many as 37 in 2012, consisting primarily of engineers and draftsmen. In 2016, the company completed its 100th roller coaster.', \"Rakefet Russak-Aminoach | Rakefet Russak-Aminoach (Hebrew: רקפת רוסק עמינח\\u200e \\u200e ; born February 1, 1966) is the current President and CEO of Bank Leumi le-Israel B.M., a position she has held since May 1, 2012. In September 2015, she was listed in Fortune Magazine’s list of The World's 100 Most Powerful Women.\"], 'question': 'What company formed in 1978 has Simon Arora as its CEO?', 'rationale': 'There is no information about Simon Arora being the CEO of a company formed in 1978 in the context. However, we can find the information about Simon Arora being the CEO of B & M in the context.', 'answer': 'B & M'}) (input_keys=None), Example({'augmented': True, 'context': ['Electoral district of Bragg | Bragg is an electoral district of the House of Assembly in the Australian state of South Australia. The seat of Bragg is named after the eminent physicists Bragg – William Henry and his son, William Lawrence. The electorate is largely urban and encompasses a significant portion of the City of Burnside, stretching from the east parklands of Adelaide into the Adelaide Hills. After a redistribution following the 2006 election, the boundary moved eastwards to include suburbs that had formerly been in the seat of Heysen and now borders Kavel. Bragg currently includes the metropolitan suburbs of Beaumont, Burnside, Dulwich, Erindale, Hazelwood Park, Heathpool, Kensington Park, Leabrook, Linden Park, Rose Park, Marryatville, Skye, St Georges, Stonyfell, Toorak Gardens, Tusmore, Wattle Park and parts of Glen Osmond, and in the hills it includes Crafers, Cleland, Greenhill, Mount Osmond, Piccadilly, Summertown and parts of Ashton, Basket Range, Carey Gully, Horsnell Gully, Leawood Gardens and Uraidla.', 'Cleland, South Australia | Cleland is a suburb in South Australia located in the Adelaide metropolitan area about 10 km south-east of the Adelaide city centre. Its boundaries were created in October 2001, with additional land being added in 2010 from the adjoining suburb of Crafers. Its name is derived from the Cleland Conservation Park. The principal land use within the locality is conservation with the majority of its land area being occupied by the Cleland Conservation Park. Places within its extent include the summit of Mount Lofty. Cleland is located within the federal Division of Mayo, the state electoral district of Bragg and the local government area of the Adelaide Hills Council.', 'Electoral results for the district of Bragg | This is a list of electoral results for the Electoral district of Bragg in South Australian state elections.', 'Electoral district of Burnside | Burnside was an electoral district of the House of Assembly in the Australian state of South Australia from 1938 to 1970. Before 1938 the Burnside area was represented by the three-seat multi-member electorate of Sturt. It was abolished in the 1970 parliamentary reforms, and was replaced with Bragg.'], 'question': 'Bragg electoral district, incorporating the Kensington Park suburb, are associated with which South Australian city?', 'rationale': \"Context: N/A\\n\\nQuestion: Bragg electoral district, incorporating the Kensington Park suburb, are associated with which South Australian city?\\n\\nReasoning: Let's think step by step in order to\", 'answer': 'Adelaide'}) (input_keys=None), Example({'augmented': True, 'context': ['Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.', 'Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.', \"Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world's third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.\"], 'question': 'The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?', 'rationale': 'Context: [1] «Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.»\\n\\n[2] «Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.»\\n\\n[3] «Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world\\'s third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.»\\n\\nReasoning: Let\\'s think step by step in order to', 'answer': 'India'}) (input_keys=None)]]},3,{0: [[], [Example({'question': 'Oh Yeon-seo is one of the former members of the group LUV, which stands for Love yoUR Voice, is what kind of group?', 'answer': 'K-pop girls group'}) (input_keys={'question'}), Example({'question': 'Melbourne Central railway station feeds into which main metro station that serves the entire metropolitan rail network?', 'answer': 'Flinders Street railway station'}) (input_keys={'question'}), Example({'question': 'When was the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed?', 'answer': '2006'}) (input_keys={'question'})], [Example({'augmented': True, 'context': [], 'question': 'What race is the lieutenant governor of the state that held its primary for her election on May 19th, 2015?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'lieutenant governor race May 19 2015'}) (input_keys=None), Example({'augmented': True, 'context': [], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'clothing company founded 1889 college bass fishing'}) (input_keys=None), Example({'augmented': True, 'context': [], 'question': \"Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\", 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Forrest Tucker height Walt Disney TV miniseries'}) (input_keys=None)], [Example({'augmented': True, 'context': [], 'question': \"Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\", 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Forrest Tucker height Walt Disney TV miniseries'}) (input_keys=None), Example({'augmented': True, 'context': [], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'clothing company founded 1889 college bass fishing'}) (input_keys=None), Example({'question': 'How many years did Lenny Lyles spend with the organization dramatized in the filme \"The insder\"?', 'answer': '27'}) (input_keys={'question'})], [Example({'augmented': True, 'context': [], 'question': 'When was the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Will Finn Over the Hedge short film director'}) (input_keys=None), Example({'question': 'What is the capacity of the La Liga stadium in which The Madrid Derby team plays their home games?', 'answer': '68,000'}) (input_keys={'question'}), Example({'question': 'The Equestrian statue of Marshal Mannerheim was erected as a monument to the leader of the Whites during what event?', 'answer': 'Finnish Civil War'}) (input_keys={'question'})], [Example({'augmented': True, 'context': [], 'question': 'The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Kangchenjunga Singhi Kangri country'}) (input_keys=None), Example({'question': 'Are Beth Gibbons and Ian Curtis the same nationality?', 'answer': 'yes'}) (input_keys={'question'}), Example({'question': \"When did Taylor Swift's satirical song get released to the radio?\", 'answer': 'February 9, 2015'}) (input_keys={'question'})], [Example({'augmented': True, 'context': [], 'question': 'What was the series inspired by the appreciation of manual laborers that Wheat Field with Cypresses is a part of?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Wheat Field with Cypresses series'}) (input_keys=None), Example({'question': 'What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?', 'answer': '2016'}) (input_keys={'question'}), Example({'question': 'How many shops does the larger of the Miami International Mall and Dolphin Mall have?', 'answer': '240'}) (input_keys={'question'})], [Example({'augmented': True, 'context': [], 'question': 'What other sport did Butch Reed, of the wrestling tag team Doom, play professionally?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Butch Reed sports'}) (input_keys=None), Example({'question': 'Which magazine is still in print American Heritage or High Times?', 'answer': 'High Times'}) (input_keys={'question'}), Example({'question': 'What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?', 'answer': '2016'}) (input_keys={'question'})], [Example({'augmented': True, 'context': [], 'question': 'Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Garden City Hotel founder'}) (input_keys=None), Example({'augmented': True, 'context': [], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'clothing company founded 1889 college bass fishing'}) (input_keys=None), Example({'augmented': True, 'context': [], 'question': 'Which theme park has Stitch’s Great Escape which replaced the attraction ExtraTERRORestial Alien Encounter?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': \"Stitch's Great Escape theme park\"}) (input_keys=None)], [Example({'augmented': True, 'context': [], 'question': 'What company formed in 1978 has Simon Arora as its CEO?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Simon Arora CEO company 1978'}) (input_keys=None), Example({'augmented': True, 'context': [], 'question': 'Bragg electoral district, incorporating the Kensington Park suburb, are associated with which South Australian city?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Bragg electoral district Kensington Park suburb South Australian city'}) (input_keys=None), Example({'augmented': True, 'context': [], 'question': 'The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Kangchenjunga Singhi Kangri country'}) (input_keys=None)]], 1: [[], [Example({'question': 'Born Again starred the American film actor who played what role in The Best Years of Our Lives?', 'answer': 'Fred Derry'}) (input_keys={'question'}), Example({'question': 'What type of compilation does Chris Harris and Into the Labyrinth have in common?', 'answer': 'television series'}) (input_keys={'question'}), Example({'question': 'Are both Paul Schattel and Francis Ford Coppola American film directors?', 'answer': 'yes'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Alberta general election, 2015 | The 29th general election of Alberta, Canada, took place on May 5, 2015, following a request of Premier Jim Prentice to the Lieutenant Governor of Alberta Donald Ethell to dissolve the Legislative Assembly on April 7, 2015. This election elected members to the 29th Alberta Legislature. It was one of only four times that Alberta has changed governments.', \"Jenean Hampton | Jenean Michelle Hampton (born May 12, 1958) is the 57th and current Lieutenant Governor of Kentucky since December 8, 2015 after having won office as Republican gubernatorial nominee Matt Bevin's running-mate, in the 2015 gubernatorial election. She is the first African-American to hold any statewide office in Kentucky history, and only the third African-American woman to serve as lieutenant-governor of any U.S. state (after Jennette Bradley and Jennifer Carroll).\", 'Louisiana lieutenant gubernatorial election, 2015 | The Louisiana lieutenant gubernatorial election took place on October 24, 2015, to elect the Lieutenant Governor of Louisiana, with a runoff election held on November 21, 2015. Incumbent Republican Lieutenant Governor Jay Dardenne did not for re-election to a second full term in office. He instead ran for Governor. Billy Nungesser won the election defeating Kip Holden, despite a Democratic victory in the gubernatorial election, in which John Bel Edwards defeated David Vitter by a similar margin.'], 'question': 'What race is the lieutenant governor of the state that held its primary for her election on May 19th, 2015?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'lieutenant governor of the state that held its primary for her election on May 19th 2015'}) (input_keys=None), Example({'augmented': True, 'context': ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.'], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Carhartt bass fishing'}) (input_keys=None), Example({'augmented': True, 'context': ['Forrest Tucker | Forrest Meredith Tucker (February 12, 1919 – October 25, 1986) was an American actor in both movies and television who appeared in nearly a hundred films. Tucker worked as a vaudeville straight man, aged fifteen years old. A mentor provided funds and contacts for a trip to California, where party hostess Cobina Wright persuaded guest Wesley Ruggles to give Tucker a screen test, based on his photogenic good looks, thick wavy hair and height of six feet, five inches - putting him into the same league as John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness\\'s younger brother. Tucker was a sight reader who needed only one take. His film career started well despite a perception in most Hollywood studios that blond men were not photogenic. After twenty years spent mainly in Westerns and action roles, he returned to his roots, showing versatility as a comedic and stage musical actor. In the TV show \"F Troop\", he became identified with the character of Cavalry Sgt. Morgan O\\'Rourke (a manipulative character quite similar to Phil Silvers\\' role as MSgt Ernie Bilko). Tucker struggled with a drinking problem that began to affect his performances in the later years of his career.', 'Emergency Landing (1941 film) | Emergency Landing (a.k.a. Robot Pilot) is a 1941 American aviation spy-fi romantic screwball comedy film directed by William Beaudine. The film stars Forrest Tucker in his second film and in his first leading role with co-stars Carol Hughes and Evelyn Brent. \"Emergency Landing\" features lots of mismatched stock footage of various types of aircraft.', 'Forrest Tucker (criminal) | Forrest Silva \"Woody\" Tucker (June 23, 1920 – May 29, 2004) was an American career criminal first imprisoned at age 15 who spent the rest of his life in and out of jail. He is best known as an escape artist, having escaped from prison 18 times successfully and 12 times unsuccessfully.'], 'question': \"Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\", 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Walt Disney TV miniseries 1955-1956 character'}) (input_keys=None)], [Example({'augmented': True, 'context': ['Forrest Tucker | Forrest Meredith Tucker (February 12, 1919 – October 25, 1986) was an American actor in both movies and television who appeared in nearly a hundred films. Tucker worked as a vaudeville straight man, aged fifteen years old. A mentor provided funds and contacts for a trip to California, where party hostess Cobina Wright persuaded guest Wesley Ruggles to give Tucker a screen test, based on his photogenic good looks, thick wavy hair and height of six feet, five inches - putting him into the same league as John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness\\'s younger brother. Tucker was a sight reader who needed only one take. His film career started well despite a perception in most Hollywood studios that blond men were not photogenic. After twenty years spent mainly in Westerns and action roles, he returned to his roots, showing versatility as a comedic and stage musical actor. In the TV show \"F Troop\", he became identified with the character of Cavalry Sgt. Morgan O\\'Rourke (a manipulative character quite similar to Phil Silvers\\' role as MSgt Ernie Bilko). Tucker struggled with a drinking problem that began to affect his performances in the later years of his career.', 'Emergency Landing (1941 film) | Emergency Landing (a.k.a. Robot Pilot) is a 1941 American aviation spy-fi romantic screwball comedy film directed by William Beaudine. The film stars Forrest Tucker in his second film and in his first leading role with co-stars Carol Hughes and Evelyn Brent. \"Emergency Landing\" features lots of mismatched stock footage of various types of aircraft.', 'Forrest Tucker (criminal) | Forrest Silva \"Woody\" Tucker (June 23, 1920 – May 29, 2004) was an American career criminal first imprisoned at age 15 who spent the rest of his life in and out of jail. He is best known as an escape artist, having escaped from prison 18 times successfully and 12 times unsuccessfully.'], 'question': \"Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\", 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Walt Disney TV miniseries 1955-1956 character'}) (input_keys=None), Example({'augmented': True, 'context': ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.'], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Carhartt bass fishing'}) (input_keys=None), Example({'question': 'How many years did Lenny Lyles spend with the organization dramatized in the filme \"The insder\"?', 'answer': '27'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Will Finn | Will Finn (born November 1, 1958) is an American animator, voice actor, and director. His work in animation includes characters from Disney and Don Bluth films such as \"The Secret of NIMH\", \"Oliver & Company\", \"The Little Mermaid\", \"The Rescuers Down Under\", and \"Pocahontas\". His characters includes Cogsworth in \"Beauty and the Beast\", Iago in \"Aladdin\", and Laverne in \"The Hunchback of Notre Dame\". Finn wrote and directed \"Home on the Range\" and did some voice acting the Hollywood Fish in \"Chicken Little\". In 2006, Finn directed the computer animated short Hammy\\'s Boomerang Adventure, a spin-off of \"Over the Hedge\".', 'Over the Hedge (film) | Over the Hedge is a 2006 American computer-animated comedy film, based on the characters from the United Media comic strip of the same name. Directed by Tim Johnson and Karey Kirkpatrick, and produced by Bonnie Arnold, it was released in the United States on May 19, 2006. The film was produced by DreamWorks Animation and distributed through Paramount Pictures. The film features the voices of Bruce Willis, Garry Shandling, Steve Carell, William Shatner, Wanda Sykes, and Nick Nolte. It is the first DreamWorks Animation film to be distributed by Paramount Pictures, which acquired the live-action DreamWorks studio in 2006. The film earned $336 million on an $80 million budget.', 'Tim Johnson (film director) | Tim Johnson (born August 27, 1961) is an American animation director. Johnson has directed in many films such as \"Antz\", \"\" and \"Over the Hedge\". Recently, he directed the DreamWorks 3D film \"Home.\"'], 'question': 'When was the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed?', 'rationale': 'find the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed. The context mentions that Will Finn directed the computer animated short Hammy\\'s Boomerang Adventure, a spin-off of \"Over the Hedge\".', 'search_query': \"Hammy's Boomerang Adventure release date\"}) (input_keys=None), Example({'question': 'The Equestrian statue of Marshal Mannerheim was erected as a monument to the leader of the Whites during what event?', 'answer': 'Finnish Civil War'}) (input_keys={'question'}), Example({'question': 'What is the capacity of the La Liga stadium in which The Madrid Derby team plays their home games?', 'answer': '68,000'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.', 'Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.', \"Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world's third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.\"], 'question': 'The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Kangchenjunga Singhi Kangri country'}) (input_keys=None), Example({'question': \"When did Taylor Swift's satirical song get released to the radio?\", 'answer': 'February 9, 2015'}) (input_keys={'question'}), Example({'question': 'Are Beth Gibbons and Ian Curtis the same nationality?', 'answer': 'yes'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Wheat Field with Cypresses | A Wheatfield with Cypresses (occasionally called \"A Cornfield with Cypresses\") is any of three similar 1889 oil paintings by Vincent van Gogh, as part of his wheat field series. All were executed at the Saint-Paul-de-Mausole mental asylum at Saint-Rémy near Arles, France, where Van Gogh was voluntarily a patient from May 1889 to May 1890. The works were inspired by the view from the window at the asylum towards the Alpilles mountains.', 'Green Wheat Field with Cypress | Green Wheat Field with Cypress (French: Champ de blé vert avec cyprès) is an oil on canvas painting by a Dutch Post-Impressionist Vincent van Gogh. It was completed in 1889, while van Gogh was voluntarily incarcerated at the asylum of St. Paul near Saint-Rémy in Provence. Several paintings of wheat fields with cypresses were made when van Gogh was able to leave the asylum grounds and explore the local landscape. Besides a fondness for cypresses, van Gogh had a special affinty with wheat fields; he depicted them dozens of times over the years; to Vincent they symbolized the cycle of life and death, and he found in them both solace and inspiration.', 'Olive Trees (Van Gogh series) | Vincent van Gogh painted at least 18 paintings of olive trees, mostly in Saint-Rémy-de-Provence in 1889. At his own request, he lived at an asylum there from May 1889 through May 1890 painting the gardens of the asylum and, when he had permission to venture outside its walls, nearby olive trees, cypresses and wheat fields.'], 'question': 'What was the series inspired by the appreciation of manual laborers that Wheat Field with Cypresses is a part of?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Wheat Field with Cypresses series inspiration'}) (input_keys=None), Example({'question': 'How many shops does the larger of the Miami International Mall and Dolphin Mall have?', 'answer': '240'}) (input_keys={'question'}), Example({'question': 'What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?', 'answer': '2016'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Butch Reed | Bruce Franklin Reed (born July 11, 1954) is an American retired professional wrestler and football player, better known by the ring name Butch Reed. He is best known for his appearances with the World Wrestling Federation and World Championship Wrestling between 1986 and 1992.', 'Doug Reed (futsal player) | Doug Reed is a professional futsal player who is currently playing for Futsal Dinamo and England national futsal team. Previously he has played for Lanzarote Tias Yaiza (Spain), AEL Limassol, MNK Uspinjača, MNK Smederevo and Manchester Futsal Club. In 2011, he won the FA Futsal Cup with Manchester Futsal Club and was awarded Player of the Tournament in the same competition. In the FA National Futsal League, he won several FA National League Championships and FA Futsal Cup medals. Reed also works as a commentator for Sony SIX, Eurosport and BT Sport, and has worked with the emerging US Professional Futsal League as well as Indian Premier Futsal.', 'Doom (professional wrestling) | Doom was a professional wrestling tag team composed of Ron Simmons and Butch Reed. They competed from 1989 to 1991 in the National Wrestling Alliance (NWA) and World Championship Wrestling (WCW).'], 'question': 'What other sport did Butch Reed, of the wrestling tag team Doom, play professionally?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Butch Reed football'}) (input_keys=None), Example({'question': 'What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?', 'answer': '2016'}) (input_keys={'question'}), Example({'question': 'Which magazine is still in print American Heritage or High Times?', 'answer': 'High Times'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.', 'Ebenezer Howard | Sir Ebenezer Howard {\\'1\\': \", \\'2\\': \", \\'3\\': \", \\'4\\': \"} ( ; 29 January 1850 – 1 May 1928), the English founder of the garden city movement, is known for his publication \"\" (1898), the description of a utopian city in which people live harmoniously together with nature. The publication resulted in the founding of the garden city movement, and the building of the First Garden City, Letchworth Garden City, commenced in 1903.', 'Eugene Reynal | Eugene Reynal was an American Publisher and in 1933 Founder of Blue Ribbon Books of Garden City, New York and Reynal and Hitchcock of New York, New York.'], 'question': 'Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Garden City Hotel founder'}) (input_keys=None), Example({'augmented': True, 'context': ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.'], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Carhartt bass fishing'}) (input_keys=None), Example({'augmented': True, 'context': ['Stitch\\'s Great Escape! | Stitch\\'s Great Escape! is a Tomorrowland attraction at the Magic Kingdom theme park within the Walt Disney World Resort. It is a \"theater-in-the-round\" experience starring the title alien from Walt Disney Animation Studios\\' 2002 film \"Lilo & Stitch\". It opened November 16, 2004 and is the fourth attraction to occupy the site in Tomorrowland. Many of the animators who worked on \"Lilo & Stitch\" partnered with Walt Disney Imagineering for Stitch\\'s Great Escape! The attraction is a replacement of The ExtraTERRORestrial Alien Encounter, which formerly occupied the building the attraction is housed in. On September 21, 2016, it was announced that the attraction would be switching from a daily operated attraction to a seasonally operated one, depending on attendance, starting October 2, 2016.', 'Great Escape (amusement park) | Great Escape (sometimes referred to as Six Flags Great Escape) is an amusement and water park owned and operated by Six Flags Entertainment Corp. It is located approximately 60 mi north of Albany, in Queensbury, New York, but is advertised as being in Lake George, New York, a popular tourist and vacation spot nearby. It is one of two Six Flags parks not to be officially branded with the \"Six Flags\" name (La Ronde in Montreal, Quebec, Canada being the other).', 'Houdini\\'s Great Escape | Houdini\\'s Great Escape, also known as \"Boo-dini\" during Fright Fest and \"Houdini - The Great Escape\" at Six Flags New England, are indoor Vekoma Madhouse attractions at Six Flags Great Adventure and Six Flags New England. The ride is located in Great Adventure\\'s Fantasy Forest section and New England\\'s Crack Axle Canyon area. The attraction opened at both parks in 1999, and was Standing but not operating (SBNO) at Great Adventure from 2008 - 2009 but was opened again due to popular demand.'], 'question': 'Which theme park has Stitch’s Great Escape which replaced the attraction ExtraTERRORestial Alien Encounter?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Magic Kingdom theme park'}) (input_keys=None)], [Example({'augmented': True, 'context': ['Simon Arora | Simon Arora (born November 1969) is a British billionaire businessman, CEO of the retail chain B & M.', 'Samir Arora | Samir Arora (born November 5, 1965) is an American businessman and former CEO of Mode Media (formerly Glam Media) from 2003 to April 2016. He was CEO and Chairman of the web design company NetObjects, Inc. from 1995 to 2001.', 'Dinesh Arora | Dinesh Arora (born 23 December 1976) is an Indian Administrative Service (IAS) officer, 2002 Kerala cadre, who previously worked as an Executive Director, Rural Electrification Corporation (REC) and CEO RECPDCL.'], 'question': 'What company formed in 1978 has Simon Arora as its CEO?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'B & M CEO'}) (input_keys=None), Example({'augmented': True, 'context': ['Electoral district of Bragg | Bragg is an electoral district of the House of Assembly in the Australian state of South Australia. The seat of Bragg is named after the eminent physicists Bragg – William Henry and his son, William Lawrence. The electorate is largely urban and encompasses a significant portion of the City of Burnside, stretching from the east parklands of Adelaide into the Adelaide Hills. After a redistribution following the 2006 election, the boundary moved eastwards to include suburbs that had formerly been in the seat of Heysen and now borders Kavel. Bragg currently includes the metropolitan suburbs of Beaumont, Burnside, Dulwich, Erindale, Hazelwood Park, Heathpool, Kensington Park, Leabrook, Linden Park, Rose Park, Marryatville, Skye, St Georges, Stonyfell, Toorak Gardens, Tusmore, Wattle Park and parts of Glen Osmond, and in the hills it includes Crafers, Cleland, Greenhill, Mount Osmond, Piccadilly, Summertown and parts of Ashton, Basket Range, Carey Gully, Horsnell Gully, Leawood Gardens and Uraidla.', 'Cleland, South Australia | Cleland is a suburb in South Australia located in the Adelaide metropolitan area about 10 km south-east of the Adelaide city centre. Its boundaries were created in October 2001, with additional land being added in 2010 from the adjoining suburb of Crafers. Its name is derived from the Cleland Conservation Park. The principal land use within the locality is conservation with the majority of its land area being occupied by the Cleland Conservation Park. Places within its extent include the summit of Mount Lofty. Cleland is located within the federal Division of Mayo, the state electoral district of Bragg and the local government area of the Adelaide Hills Council.', 'Electoral results for the district of Bragg | This is a list of electoral results for the Electoral district of Bragg in South Australian state elections.'], 'question': 'Bragg electoral district, incorporating the Kensington Park suburb, are associated with which South Australian city?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'South Australian city associated with Bragg electoral district'}) (input_keys=None), Example({'augmented': True, 'context': ['Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.', 'Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.', \"Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world's third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.\"], 'question': 'The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Kangchenjunga Singhi Kangri country'}) (input_keys=None)]], 2: [[], [Example({'question': 'What character did this actor, who also played a supporting role in Sleepy Hollow, play in the final six Harry Potter films?', 'answer': 'Professor Albus Dumbledore'}) (input_keys={'question'}), Example({'question': 'The musical sitcom with an album by the same name has what special feature on the album cover?', 'answer': 'actual crossword puzzle'}) (input_keys={'question'}), Example({'question': 'What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?', 'answer': '2016'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Alberta general election, 2015 | The 29th general election of Alberta, Canada, took place on May 5, 2015, following a request of Premier Jim Prentice to the Lieutenant Governor of Alberta Donald Ethell to dissolve the Legislative Assembly on April 7, 2015. This election elected members to the 29th Alberta Legislature. It was one of only four times that Alberta has changed governments.', \"Jenean Hampton | Jenean Michelle Hampton (born May 12, 1958) is the 57th and current Lieutenant Governor of Kentucky since December 8, 2015 after having won office as Republican gubernatorial nominee Matt Bevin's running-mate, in the 2015 gubernatorial election. She is the first African-American to hold any statewide office in Kentucky history, and only the third African-American woman to serve as lieutenant-governor of any U.S. state (after Jennette Bradley and Jennifer Carroll).\", 'Louisiana lieutenant gubernatorial election, 2015 | The Louisiana lieutenant gubernatorial election took place on October 24, 2015, to elect the Lieutenant Governor of Louisiana, with a runoff election held on November 21, 2015. Incumbent Republican Lieutenant Governor Jay Dardenne did not for re-election to a second full term in office. He instead ran for Governor. Billy Nungesser won the election defeating Kip Holden, despite a Democratic victory in the gubernatorial election, in which John Bel Edwards defeated David Vitter by a similar margin.', 'Indiana gubernatorial election, 2016 | The 2016 Indiana gubernatorial election was held November 8, 2016, to elect the Governor and Lieutenant Governor of Indiana, concurrently with the 2016 U.S. presidential election as well as elections to the United States Senate and elections to the United States House of Representatives and various state and local elections. The primaries were held on May 3, 2016. Republican Lieutenant Governor Eric Holcomb won the race with 51.4% of the vote.', 'Virginia lieutenant gubernatorial election, 2017 | The Virginia lieutenant gubernatorial election of 2017 will take place on November 7, 2017. After the party primary elections were held, the major party nominees are Jill Vogel (Republican) and Justin Fairfax (Democrat). The incumbent Lieutenant Governor, Democrat Ralph Northam, is not running for re-election in order to run for Governor.'], 'question': 'What race is the lieutenant governor of the state that held its primary for her election on May 19th, 2015?', 'rationale': 'The lieutenant governor of the state that held its primary for her election on May 19th, 2015 is Jenean Hampton, who is the 57th and current Lieutenant Governor of Kentucky since December 8, 2015. The primary election was held in Kentucky, and Jenean Hampton is the lieutenant governor of Kentucky.', 'answer': 'African-American'}) (input_keys=None), Example({'augmented': True, 'context': ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.', \"College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour\", 'Bass fishing | Bass fishing is the activity of angling for the North American gamefish known colloquially as the \"black bass\". There are numerous black bass species considered as gamefish in North America, including largemouth bass (\"Micropterus salmoides\"), smallmouth bass (\"Micropterus dolomieui\"), spotted bass or Kentucky bass (\"Micropterus punctulatus\"), and Guadalupe bass (order \"Perciformes\").', 'Cliff Crochet | Cliff Crochet (born August 1, 1983), also known as \"The Cajun Baby\", is a professional bass fisherman from Pierre Part, Louisiana. Crochet is a competitor on Bass Angler Sportsman Society\\'s (B.A.S.S.) Bassmaster Elite Series and has $368,837 in B.A.S.S. tournament winnings as of August 22, 2016. Crochet\\'s sponsors include: K2 Coolers, Huk, Skeeter, Yamaha Motor Company, Lowrance Electronics, Falcon Rods, Luck E Strike, Seaguar, Rat-L-Trap, Humdinger Spinnerbaits, Power-Pole. Since 2010, Crochet has competed in three Bassmaster Classic tournaments. Crochet\\'s boat for the 2016 season is a 2016 Skeeter FX 21 with a 250 hp Yamaha SHO outboard engine.'], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': 'The context mentions Carhartt Bassmaster College Bass Fishing Series, which is a college bass fishing tournament series. This series is associated with the clothing company Carhartt, which was founded in 1889.', 'answer': 'Carhartt'}) (input_keys=None), Example({'augmented': True, 'context': ['Forrest Tucker | Forrest Meredith Tucker (February 12, 1919 – October 25, 1986) was an American actor in both movies and television who appeared in nearly a hundred films. Tucker worked as a vaudeville straight man, aged fifteen years old. A mentor provided funds and contacts for a trip to California, where party hostess Cobina Wright persuaded guest Wesley Ruggles to give Tucker a screen test, based on his photogenic good looks, thick wavy hair and height of six feet, five inches - putting him into the same league as John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness\\'s younger brother. Tucker was a sight reader who needed only one take. His film career started well despite a perception in most Hollywood studios that blond men were not photogenic. After twenty years spent mainly in Westerns and action roles, he returned to his roots, showing versatility as a comedic and stage musical actor. In the TV show \"F Troop\", he became identified with the character of Cavalry Sgt. Morgan O\\'Rourke (a manipulative character quite similar to Phil Silvers\\' role as MSgt Ernie Bilko). Tucker struggled with a drinking problem that began to affect his performances in the later years of his career.', 'Emergency Landing (1941 film) | Emergency Landing (a.k.a. Robot Pilot) is a 1941 American aviation spy-fi romantic screwball comedy film directed by William Beaudine. The film stars Forrest Tucker in his second film and in his first leading role with co-stars Carol Hughes and Evelyn Brent. \"Emergency Landing\" features lots of mismatched stock footage of various types of aircraft.', 'Forrest Tucker (criminal) | Forrest Silva \"Woody\" Tucker (June 23, 1920 – May 29, 2004) was an American career criminal first imprisoned at age 15 who spent the rest of his life in and out of jail. He is best known as an escape artist, having escaped from prison 18 times successfully and 12 times unsuccessfully.', 'Fess Parker | Fess Elisha Parker Jr. (August 16, 1924 – March 18, 2010) was an American film and television actor best known for his portrayals of Davy Crockett in the Walt Disney 1955–1956 TV miniseries and as Daniel Boone in a television series from 1964 to 1970. He was also known as a winemaker and resort owner-operator.', 'Davy Crockett, King of the Wild Frontier | Davy Crockett, King of the Wild Frontier is a 1955 live-action Walt Disney adventure film starring Fess Parker as Davy Crockett. This film is an edited compilation of the first three stories from the Disney television miniseries \"Davy Crockett\" :', 'Davy Crockett (miniseries) | Davy Crockett is a five-part serial which aired on ABC from 1954–1955 in one-hour episodes, on the \"Disneyland\" series. The series stars Fess Parker as real-life frontiersman Davy Crockett and Buddy Ebsen as his friend, George Russel.'], 'question': \"Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\", 'rationale': \"From the context, we know that Forrest Tucker was an American actor who appeared in nearly a hundred films. We also know that he was compared to John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness's younger brother. We need to find the actor that played a character in the 1955-1956 Walt Disney TV miniseries, which is Fess Parker.\", 'answer': 'Davy Crockett'}) (input_keys=None)], [Example({'augmented': True, 'context': ['Forrest Tucker | Forrest Meredith Tucker (February 12, 1919 – October 25, 1986) was an American actor in both movies and television who appeared in nearly a hundred films. Tucker worked as a vaudeville straight man, aged fifteen years old. A mentor provided funds and contacts for a trip to California, where party hostess Cobina Wright persuaded guest Wesley Ruggles to give Tucker a screen test, based on his photogenic good looks, thick wavy hair and height of six feet, five inches - putting him into the same league as John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness\\'s younger brother. Tucker was a sight reader who needed only one take. His film career started well despite a perception in most Hollywood studios that blond men were not photogenic. After twenty years spent mainly in Westerns and action roles, he returned to his roots, showing versatility as a comedic and stage musical actor. In the TV show \"F Troop\", he became identified with the character of Cavalry Sgt. Morgan O\\'Rourke (a manipulative character quite similar to Phil Silvers\\' role as MSgt Ernie Bilko). Tucker struggled with a drinking problem that began to affect his performances in the later years of his career.', 'Emergency Landing (1941 film) | Emergency Landing (a.k.a. Robot Pilot) is a 1941 American aviation spy-fi romantic screwball comedy film directed by William Beaudine. The film stars Forrest Tucker in his second film and in his first leading role with co-stars Carol Hughes and Evelyn Brent. \"Emergency Landing\" features lots of mismatched stock footage of various types of aircraft.', 'Forrest Tucker (criminal) | Forrest Silva \"Woody\" Tucker (June 23, 1920 – May 29, 2004) was an American career criminal first imprisoned at age 15 who spent the rest of his life in and out of jail. He is best known as an escape artist, having escaped from prison 18 times successfully and 12 times unsuccessfully.', 'Fess Parker | Fess Elisha Parker Jr. (August 16, 1924 – March 18, 2010) was an American film and television actor best known for his portrayals of Davy Crockett in the Walt Disney 1955–1956 TV miniseries and as Daniel Boone in a television series from 1964 to 1970. He was also known as a winemaker and resort owner-operator.', 'Davy Crockett, King of the Wild Frontier | Davy Crockett, King of the Wild Frontier is a 1955 live-action Walt Disney adventure film starring Fess Parker as Davy Crockett. This film is an edited compilation of the first three stories from the Disney television miniseries \"Davy Crockett\" :', 'Davy Crockett (miniseries) | Davy Crockett is a five-part serial which aired on ABC from 1954–1955 in one-hour episodes, on the \"Disneyland\" series. The series stars Fess Parker as real-life frontiersman Davy Crockett and Buddy Ebsen as his friend, George Russel.'], 'question': \"Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\", 'rationale': 'We need to find the actor who played the character in the 1955-1956 Walt Disney TV miniseries. From the context, we know that the miniseries was about Davy Crockett and starred Fess Parker as Davy Crockett.', 'answer': 'Davy Crockett'}) (input_keys=None), Example({'augmented': True, 'context': ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.', \"College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour\", 'Bass fishing | Bass fishing is the activity of angling for the North American gamefish known colloquially as the \"black bass\". There are numerous black bass species considered as gamefish in North America, including largemouth bass (\"Micropterus salmoides\"), smallmouth bass (\"Micropterus dolomieui\"), spotted bass or Kentucky bass (\"Micropterus punctulatus\"), and Guadalupe bass (order \"Perciformes\").', 'Cliff Crochet | Cliff Crochet (born August 1, 1983), also known as \"The Cajun Baby\", is a professional bass fisherman from Pierre Part, Louisiana. Crochet is a competitor on Bass Angler Sportsman Society\\'s (B.A.S.S.) Bassmaster Elite Series and has $368,837 in B.A.S.S. tournament winnings as of August 22, 2016. Crochet\\'s sponsors include: K2 Coolers, Huk, Skeeter, Yamaha Motor Company, Lowrance Electronics, Falcon Rods, Luck E Strike, Seaguar, Rat-L-Trap, Humdinger Spinnerbaits, Power-Pole. Since 2010, Crochet has competed in three Bassmaster Classic tournaments. Crochet\\'s boat for the 2016 season is a 2016 Skeeter FX 21 with a 250 hp Yamaha SHO outboard engine.'], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Context: [1] «Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.»\\n\\n[2] «College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour»\\n\\n[3] «G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.»\\n\\nReasoning: Let's think step by step in order to\", 'answer': 'Carhartt'}) (input_keys=None), Example({'question': 'How many years did Lenny Lyles spend with the organization dramatized in the filme \"The insder\"?', 'answer': '27'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Will Finn | Will Finn (born November 1, 1958) is an American animator, voice actor, and director. His work in animation includes characters from Disney and Don Bluth films such as \"The Secret of NIMH\", \"Oliver & Company\", \"The Little Mermaid\", \"The Rescuers Down Under\", and \"Pocahontas\". His characters includes Cogsworth in \"Beauty and the Beast\", Iago in \"Aladdin\", and Laverne in \"The Hunchback of Notre Dame\". Finn wrote and directed \"Home on the Range\" and did some voice acting the Hollywood Fish in \"Chicken Little\". In 2006, Finn directed the computer animated short Hammy\\'s Boomerang Adventure, a spin-off of \"Over the Hedge\".', 'Over the Hedge (film) | Over the Hedge is a 2006 American computer-animated comedy film, based on the characters from the United Media comic strip of the same name. Directed by Tim Johnson and Karey Kirkpatrick, and produced by Bonnie Arnold, it was released in the United States on May 19, 2006. The film was produced by DreamWorks Animation and distributed through Paramount Pictures. The film features the voices of Bruce Willis, Garry Shandling, Steve Carell, William Shatner, Wanda Sykes, and Nick Nolte. It is the first DreamWorks Animation film to be distributed by Paramount Pictures, which acquired the live-action DreamWorks studio in 2006. The film earned $336 million on an $80 million budget.', 'Tim Johnson (film director) | Tim Johnson (born August 27, 1961) is an American animation director. Johnson has directed in many films such as \"Antz\", \"\" and \"Over the Hedge\". Recently, he directed the DreamWorks 3D film \"Home.\"', 'Hammy\\'s Boomerang Adventure | Hammy\\'s Boomerang Adventure is a computer-animated short comedy film based on the film \"Over the Hedge\", which was based on a comic strip of the same name. It can be found on the movie\\'s DVD or the \"Madly Madagascar\" DVD as a bonus feature.', 'Boomerang (2015 film) | Boomerang is a 2015 French drama film directed by François Favrat and starring Laurent Lafitte, Mélanie Laurent, Audrey Dana and Wladimir Yordanoff. It is an adaptation of Tatiana de Rosnay\\'s novel \"A Secret Kept\" (alternative title:\"Boomerang\"). The film was released in theatres on 23 September 2015.', 'Over the Hedge: Hammy Goes Nuts! | Over the Hedge: Hammy Goes Nuts! is a video game published by Activision based on characters from the film \"Over the Hedge\". The Game Boy Advance (GBA) version, a miniature golf game, was developed by Vicarious Visions and released on October 10, 2006 in North America. The Nintendo DS and PlayStation Portable (PSP) versions, an action adventure game and a platform game respectively, were both developed by Amaze Entertainment and were released on October 24, 2006 and November 21, 2006. It is the second game to be based on the film.'], 'question': 'When was the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed?', 'rationale': 'Context: N/A\\n\\nQuestion: When was the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed?\\n\\nReasoning: Let\\'s think step by step in order to', 'answer': '2006'}) (input_keys=None), Example({'question': 'What is the capacity of the La Liga stadium in which The Madrid Derby team plays their home games?', 'answer': '68,000'}) (input_keys={'question'}), Example({'question': 'The Equestrian statue of Marshal Mannerheim was erected as a monument to the leader of the Whites during what event?', 'answer': 'Finnish Civil War'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.', 'Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.', \"Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world's third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.\"], 'question': 'The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?', 'rationale': 'Context: [1] «Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.»\\n\\n[2] «Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.»\\n\\n[3] «Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world\\'s third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.»\\n\\nReasoning: Let\\'s think step by step in order to', 'answer': 'India'}) (input_keys=None), Example({'question': 'Are Beth Gibbons and Ian Curtis the same nationality?', 'answer': 'yes'}) (input_keys={'question'}), Example({'question': \"When did Taylor Swift's satirical song get released to the radio?\", 'answer': 'February 9, 2015'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Wheat Field with Cypresses | A Wheatfield with Cypresses (occasionally called \"A Cornfield with Cypresses\") is any of three similar 1889 oil paintings by Vincent van Gogh, as part of his wheat field series. All were executed at the Saint-Paul-de-Mausole mental asylum at Saint-Rémy near Arles, France, where Van Gogh was voluntarily a patient from May 1889 to May 1890. The works were inspired by the view from the window at the asylum towards the Alpilles mountains.', 'Green Wheat Field with Cypress | Green Wheat Field with Cypress (French: Champ de blé vert avec cyprès) is an oil on canvas painting by a Dutch Post-Impressionist Vincent van Gogh. It was completed in 1889, while van Gogh was voluntarily incarcerated at the asylum of St. Paul near Saint-Rémy in Provence. Several paintings of wheat fields with cypresses were made when van Gogh was able to leave the asylum grounds and explore the local landscape. Besides a fondness for cypresses, van Gogh had a special affinty with wheat fields; he depicted them dozens of times over the years; to Vincent they symbolized the cycle of life and death, and he found in them both solace and inspiration.', 'Olive Trees (Van Gogh series) | Vincent van Gogh painted at least 18 paintings of olive trees, mostly in Saint-Rémy-de-Provence in 1889. At his own request, he lived at an asylum there from May 1889 through May 1890 painting the gardens of the asylum and, when he had permission to venture outside its walls, nearby olive trees, cypresses and wheat fields.', 'Wheat Fields (Van Gogh series) | The Wheat Fields is a series of dozens of paintings by Vincent van Gogh, borne out of his religious studies and sermons, connection to nature, appreciation of manual laborers and desire to provide a means of offering comfort to others. The wheat field works demonstrate his progression as an artist from the drab \"Wheat Sheaves\" made in 1885 in the Netherlands to the colorful, dramatic paintings from Arles, Saint-Rémy and Auvers-sur-Oise of rural France.'], 'question': 'What was the series inspired by the appreciation of manual laborers that Wheat Field with Cypresses is a part of?', 'rationale': \"Context: N/A\\n\\nQuestion: What was the series inspired by the appreciation of manual laborers that Wheat Field with Cypresses is a part of?\\n\\nReasoning: Let's think step by step in order to\", 'answer': 'Wheat Fields'}) (input_keys=None), Example({'question': 'What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?', 'answer': '2016'}) (input_keys={'question'}), Example({'question': 'How many shops does the larger of the Miami International Mall and Dolphin Mall have?', 'answer': '240'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Butch Reed | Bruce Franklin Reed (born July 11, 1954) is an American retired professional wrestler and football player, better known by the ring name Butch Reed. He is best known for his appearances with the World Wrestling Federation and World Championship Wrestling between 1986 and 1992.', 'Doug Reed (futsal player) | Doug Reed is a professional futsal player who is currently playing for Futsal Dinamo and England national futsal team. Previously he has played for Lanzarote Tias Yaiza (Spain), AEL Limassol, MNK Uspinjača, MNK Smederevo and Manchester Futsal Club. In 2011, he won the FA Futsal Cup with Manchester Futsal Club and was awarded Player of the Tournament in the same competition. In the FA National Futsal League, he won several FA National League Championships and FA Futsal Cup medals. Reed also works as a commentator for Sony SIX, Eurosport and BT Sport, and has worked with the emerging US Professional Futsal League as well as Indian Premier Futsal.', 'Doom (professional wrestling) | Doom was a professional wrestling tag team composed of Ron Simmons and Butch Reed. They competed from 1989 to 1991 in the National Wrestling Alliance (NWA) and World Championship Wrestling (WCW).', 'Robert Reed (gridiron football) | Robert E. Reed, Jr. (born February 23, 1943) is a former American football offensive lineman in the National Football League for the Washington Redskins. He played college football at Tennessee State University and was drafted in the sixteenth round of the 1965 NFL Draft. Reed was also selected in the 12th round of the 1965 AFL Draft by the Houston Oilers.', 'Reed Doughty | Reed Doughty (born November 4, 1982) is a former American football safety who played in the National Football League. He was drafted by the Washington Redskins in the sixth round, with the 173rd overall pick, of the 2006 NFL Draft. He played college football at the University of Northern Colorado.'], 'question': 'What other sport did Butch Reed, of the wrestling tag team Doom, play professionally?', 'rationale': \"Context: N/A\\n\\nQuestion: What other sport did Butch Reed, of the wrestling tag team Doom, play professionally?\\n\\nReasoning: Let's think step by step in order to\", 'answer': 'football'}) (input_keys=None), Example({'question': 'Which magazine is still in print American Heritage or High Times?', 'answer': 'High Times'}) (input_keys={'question'}), Example({'question': 'What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?', 'answer': '2016'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.', 'Ebenezer Howard | Sir Ebenezer Howard {\\'1\\': \", \\'2\\': \", \\'3\\': \", \\'4\\': \"} ( ; 29 January 1850 – 1 May 1928), the English founder of the garden city movement, is known for his publication \"\" (1898), the description of a utopian city in which people live harmoniously together with nature. The publication resulted in the founding of the garden city movement, and the building of the First Garden City, Letchworth Garden City, commenced in 1903.', 'Eugene Reynal | Eugene Reynal was an American Publisher and in 1933 Founder of Blue Ribbon Books of Garden City, New York and Reynal and Hitchcock of New York, New York.'], 'question': 'Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?', 'rationale': 'Context: [1] «Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.»\\n\\n[2] «A.T. Stewart | Alexander Turney Stewart (October 2, 1792 – April 10, 1876) was an American businessman, politician, and philanthropist. He was a prominent figure in New York City during the mid-19th century, and was known as the \"Merchant Prince\".»\\n\\n[3] «Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.»\\n\\nQuestion: Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?\\n\\nReasoning: Let\\'s think step by step in order to', 'answer': 'A.T. Stewart'}) (input_keys=None), Example({'augmented': True, 'context': ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.', \"College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour\", 'Bass fishing | Bass fishing is the activity of angling for the North American gamefish known colloquially as the \"black bass\". There are numerous black bass species considered as gamefish in North America, including largemouth bass (\"Micropterus salmoides\"), smallmouth bass (\"Micropterus dolomieui\"), spotted bass or Kentucky bass (\"Micropterus punctulatus\"), and Guadalupe bass (order \"Perciformes\").', 'Cliff Crochet | Cliff Crochet (born August 1, 1983), also known as \"The Cajun Baby\", is a professional bass fisherman from Pierre Part, Louisiana. Crochet is a competitor on Bass Angler Sportsman Society\\'s (B.A.S.S.) Bassmaster Elite Series and has $368,837 in B.A.S.S. tournament winnings as of August 22, 2016. Crochet\\'s sponsors include: K2 Coolers, Huk, Skeeter, Yamaha Motor Company, Lowrance Electronics, Falcon Rods, Luck E Strike, Seaguar, Rat-L-Trap, Humdinger Spinnerbaits, Power-Pole. Since 2010, Crochet has competed in three Bassmaster Classic tournaments. Crochet\\'s boat for the 2016 season is a 2016 Skeeter FX 21 with a 250 hp Yamaha SHO outboard engine.'], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Context: [1] «Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.»\\n\\n[2] «College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour»\\n\\n[3] «G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.»\\n\\nReasoning: Let's think step by step in order to\", 'answer': 'Carhartt'}) (input_keys=None), Example({'augmented': True, 'context': ['Stitch\\'s Great Escape! | Stitch\\'s Great Escape! is a Tomorrowland attraction at the Magic Kingdom theme park within the Walt Disney World Resort. It is a \"theater-in-the-round\" experience starring the title alien from Walt Disney Animation Studios\\' 2002 film \"Lilo & Stitch\". It opened November 16, 2004 and is the fourth attraction to occupy the site in Tomorrowland. Many of the animators who worked on \"Lilo & Stitch\" partnered with Walt Disney Imagineering for Stitch\\'s Great Escape! The attraction is a replacement of The ExtraTERRORestrial Alien Encounter, which formerly occupied the building the attraction is housed in. On September 21, 2016, it was announced that the attraction would be switching from a daily operated attraction to a seasonally operated one, depending on attendance, starting October 2, 2016.', 'Great Escape (amusement park) | Great Escape (sometimes referred to as Six Flags Great Escape) is an amusement and water park owned and operated by Six Flags Entertainment Corp. It is located approximately 60 mi north of Albany, in Queensbury, New York, but is advertised as being in Lake George, New York, a popular tourist and vacation spot nearby. It is one of two Six Flags parks not to be officially branded with the \"Six Flags\" name (La Ronde in Montreal, Quebec, Canada being the other).', 'Houdini\\'s Great Escape | Houdini\\'s Great Escape, also known as \"Boo-dini\" during Fright Fest and \"Houdini - The Great Escape\" at Six Flags New England, are indoor Vekoma Madhouse attractions at Six Flags Great Adventure and Six Flags New England. The ride is located in Great Adventure\\'s Fantasy Forest section and New England\\'s Crack Axle Canyon area. The attraction opened at both parks in 1999, and was Standing but not operating (SBNO) at Great Adventure from 2008 - 2009 but was opened again due to popular demand.', 'Magic Kingdom | Magic Kingdom is a theme park at the Walt Disney World Resort in Bay Lake, Florida, near Orlando. Owned and operated by The Walt Disney Company through its Parks and Resorts division, the park opened on October 1, 1971, as the first of four theme parks at the resort. Initialized by Walt Disney and designed by WED Enterprises, its layout and attractions are based on Disneyland Park in Anaheim, California, and is dedicated to fairy tales and Disney characters.', 'Magic Kingdom (disambiguation) | The Magic Kingdom is a theme park at the Walt Disney World Resort in Florida.', 'List of Magic Kingdom attractions | The Magic Kingdom is a theme park located at the Walt Disney World Resort in Florida. Below is a list of the current attractions found therein, arranged by \"land\" and with brief descriptions.'], 'question': 'Which theme park has Stitch’s Great Escape which replaced the attraction ExtraTERRORestial Alien Encounter?', 'rationale': \"We need to find the theme park that has Stitch's Great Escape, which replaced the attraction ExtraTERRORestial Alien Encounter. The context mentions that Stitch's Great Escape is located in the Magic Kingdom theme park.\", 'answer': 'Magic Kingdom'}) (input_keys=None)], [Example({'augmented': True, 'context': ['Simon Arora | Simon Arora (born November 1969) is a British billionaire businessman, CEO of the retail chain B & M.', 'Samir Arora | Samir Arora (born November 5, 1965) is an American businessman and former CEO of Mode Media (formerly Glam Media) from 2003 to April 2016. He was CEO and Chairman of the web design company NetObjects, Inc. from 1995 to 2001.', 'Dinesh Arora | Dinesh Arora (born 23 December 1976) is an Indian Administrative Service (IAS) officer, 2002 Kerala cadre, who previously worked as an Executive Director, Rural Electrification Corporation (REC) and CEO RECPDCL.', 'Bolliger & Mabillard | Bolliger & Mabillard, officially Bolliger & Mabillard Consulting Engineers Inc., and often abbreviated B&M, is a roller coaster design consultancy based in Monthey, Switzerland. The company was founded in 1988 by Walter Bolliger and Claude Mabillard, with Bolliger as president and Mabillard as vice-president. B&M has built over a hundred roller coasters around the world and pioneered several new ride technologies, most notably the inverted roller coaster. In North America, B&M coaster designs have been manufactured by Ohio company Clermont Steel Fabricators since 1990. B&M has grown significantly since its founding, originally employing four to as many as 37 in 2012, consisting primarily of engineers and draftsmen. In 2016, the company completed its 100th roller coaster.', \"Rakefet Russak-Aminoach | Rakefet Russak-Aminoach (Hebrew: רקפת רוסק עמינח\\u200e \\u200e ; born February 1, 1966) is the current President and CEO of Bank Leumi le-Israel B.M., a position she has held since May 1, 2012. In September 2015, she was listed in Fortune Magazine’s list of The World's 100 Most Powerful Women.\"], 'question': 'What company formed in 1978 has Simon Arora as its CEO?', 'rationale': 'There is no information about Simon Arora being the CEO of a company formed in 1978 in the context. However, we can find the information about Simon Arora being the CEO of B & M in the context.', 'answer': 'B & M'}) (input_keys=None), Example({'augmented': True, 'context': ['Electoral district of Bragg | Bragg is an electoral district of the House of Assembly in the Australian state of South Australia. The seat of Bragg is named after the eminent physicists Bragg – William Henry and his son, William Lawrence. The electorate is largely urban and encompasses a significant portion of the City of Burnside, stretching from the east parklands of Adelaide into the Adelaide Hills. After a redistribution following the 2006 election, the boundary moved eastwards to include suburbs that had formerly been in the seat of Heysen and now borders Kavel. Bragg currently includes the metropolitan suburbs of Beaumont, Burnside, Dulwich, Erindale, Hazelwood Park, Heathpool, Kensington Park, Leabrook, Linden Park, Rose Park, Marryatville, Skye, St Georges, Stonyfell, Toorak Gardens, Tusmore, Wattle Park and parts of Glen Osmond, and in the hills it includes Crafers, Cleland, Greenhill, Mount Osmond, Piccadilly, Summertown and parts of Ashton, Basket Range, Carey Gully, Horsnell Gully, Leawood Gardens and Uraidla.', 'Cleland, South Australia | Cleland is a suburb in South Australia located in the Adelaide metropolitan area about 10 km south-east of the Adelaide city centre. Its boundaries were created in October 2001, with additional land being added in 2010 from the adjoining suburb of Crafers. Its name is derived from the Cleland Conservation Park. The principal land use within the locality is conservation with the majority of its land area being occupied by the Cleland Conservation Park. Places within its extent include the summit of Mount Lofty. Cleland is located within the federal Division of Mayo, the state electoral district of Bragg and the local government area of the Adelaide Hills Council.', 'Electoral results for the district of Bragg | This is a list of electoral results for the Electoral district of Bragg in South Australian state elections.', 'Electoral district of Burnside | Burnside was an electoral district of the House of Assembly in the Australian state of South Australia from 1938 to 1970. Before 1938 the Burnside area was represented by the three-seat multi-member electorate of Sturt. It was abolished in the 1970 parliamentary reforms, and was replaced with Bragg.'], 'question': 'Bragg electoral district, incorporating the Kensington Park suburb, are associated with which South Australian city?', 'rationale': \"Context: N/A\\n\\nQuestion: Bragg electoral district, incorporating the Kensington Park suburb, are associated with which South Australian city?\\n\\nReasoning: Let's think step by step in order to\", 'answer': 'Adelaide'}) (input_keys=None), Example({'augmented': True, 'context': ['Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.', 'Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.', \"Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world's third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.\"], 'question': 'The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?', 'rationale': 'Context: [1] «Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.»\\n\\n[2] «Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.»\\n\\n[3] «Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world\\'s third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.»\\n\\nReasoning: Let\\'s think step by step in order to', 'answer': 'India'}) (input_keys=None)]]},2,{0: [[], [Example({'question': 'Oh Yeon-seo is one of the former members of the group LUV, which stands for Love yoUR Voice, is what kind of group?', 'answer': 'K-pop girls group'}) (input_keys={'question'}), Example({'question': 'Melbourne Central railway station feeds into which main metro station that serves the entire metropolitan rail network?', 'answer': 'Flinders Street railway station'}) (input_keys={'question'}), Example({'question': 'When was the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed?', 'answer': '2006'}) (input_keys={'question'})], [Example({'augmented': True, 'context': [], 'question': 'What race is the lieutenant governor of the state that held its primary for her election on May 19th, 2015?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'lieutenant governor race May 19 2015'}) (input_keys=None), Example({'augmented': True, 'context': [], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'clothing company founded 1889 college bass fishing'}) (input_keys=None), Example({'augmented': True, 'context': [], 'question': \"Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\", 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Forrest Tucker height Walt Disney TV miniseries'}) (input_keys=None)], [Example({'augmented': True, 'context': [], 'question': \"Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\", 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Forrest Tucker height Walt Disney TV miniseries'}) (input_keys=None), Example({'augmented': True, 'context': [], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'clothing company founded 1889 college bass fishing'}) (input_keys=None), Example({'question': 'How many years did Lenny Lyles spend with the organization dramatized in the filme \"The insder\"?', 'answer': '27'}) (input_keys={'question'})], [Example({'augmented': True, 'context': [], 'question': 'When was the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Will Finn Over the Hedge short film director'}) (input_keys=None), Example({'question': 'What is the capacity of the La Liga stadium in which The Madrid Derby team plays their home games?', 'answer': '68,000'}) (input_keys={'question'}), Example({'question': 'The Equestrian statue of Marshal Mannerheim was erected as a monument to the leader of the Whites during what event?', 'answer': 'Finnish Civil War'}) (input_keys={'question'})], [Example({'augmented': True, 'context': [], 'question': 'The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Kangchenjunga Singhi Kangri country'}) (input_keys=None), Example({'question': 'Are Beth Gibbons and Ian Curtis the same nationality?', 'answer': 'yes'}) (input_keys={'question'}), Example({'question': \"When did Taylor Swift's satirical song get released to the radio?\", 'answer': 'February 9, 2015'}) (input_keys={'question'})], [Example({'augmented': True, 'context': [], 'question': 'What was the series inspired by the appreciation of manual laborers that Wheat Field with Cypresses is a part of?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Wheat Field with Cypresses series'}) (input_keys=None), Example({'question': 'What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?', 'answer': '2016'}) (input_keys={'question'}), Example({'question': 'How many shops does the larger of the Miami International Mall and Dolphin Mall have?', 'answer': '240'}) (input_keys={'question'})], [Example({'augmented': True, 'context': [], 'question': 'What other sport did Butch Reed, of the wrestling tag team Doom, play professionally?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Butch Reed sports'}) (input_keys=None), Example({'question': 'Which magazine is still in print American Heritage or High Times?', 'answer': 'High Times'}) (input_keys={'question'}), Example({'question': 'What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?', 'answer': '2016'}) (input_keys={'question'})], [Example({'augmented': True, 'context': [], 'question': 'Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Garden City Hotel founder'}) (input_keys=None), Example({'augmented': True, 'context': [], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'clothing company founded 1889 college bass fishing'}) (input_keys=None), Example({'augmented': True, 'context': [], 'question': 'Which theme park has Stitch’s Great Escape which replaced the attraction ExtraTERRORestial Alien Encounter?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': \"Stitch's Great Escape theme park\"}) (input_keys=None)], [Example({'augmented': True, 'context': [], 'question': 'What company formed in 1978 has Simon Arora as its CEO?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Simon Arora CEO company 1978'}) (input_keys=None), Example({'augmented': True, 'context': [], 'question': 'Bragg electoral district, incorporating the Kensington Park suburb, are associated with which South Australian city?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Bragg electoral district Kensington Park suburb South Australian city'}) (input_keys=None), Example({'augmented': True, 'context': [], 'question': 'The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Kangchenjunga Singhi Kangri country'}) (input_keys=None)]], 1: [[], [Example({'question': 'Born Again starred the American film actor who played what role in The Best Years of Our Lives?', 'answer': 'Fred Derry'}) (input_keys={'question'}), Example({'question': 'What type of compilation does Chris Harris and Into the Labyrinth have in common?', 'answer': 'television series'}) (input_keys={'question'}), Example({'question': 'Are both Paul Schattel and Francis Ford Coppola American film directors?', 'answer': 'yes'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Alberta general election, 2015 | The 29th general election of Alberta, Canada, took place on May 5, 2015, following a request of Premier Jim Prentice to the Lieutenant Governor of Alberta Donald Ethell to dissolve the Legislative Assembly on April 7, 2015. This election elected members to the 29th Alberta Legislature. It was one of only four times that Alberta has changed governments.', \"Jenean Hampton | Jenean Michelle Hampton (born May 12, 1958) is the 57th and current Lieutenant Governor of Kentucky since December 8, 2015 after having won office as Republican gubernatorial nominee Matt Bevin's running-mate, in the 2015 gubernatorial election. She is the first African-American to hold any statewide office in Kentucky history, and only the third African-American woman to serve as lieutenant-governor of any U.S. state (after Jennette Bradley and Jennifer Carroll).\", 'Louisiana lieutenant gubernatorial election, 2015 | The Louisiana lieutenant gubernatorial election took place on October 24, 2015, to elect the Lieutenant Governor of Louisiana, with a runoff election held on November 21, 2015. Incumbent Republican Lieutenant Governor Jay Dardenne did not for re-election to a second full term in office. He instead ran for Governor. Billy Nungesser won the election defeating Kip Holden, despite a Democratic victory in the gubernatorial election, in which John Bel Edwards defeated David Vitter by a similar margin.'], 'question': 'What race is the lieutenant governor of the state that held its primary for her election on May 19th, 2015?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'lieutenant governor of the state that held its primary for her election on May 19th 2015'}) (input_keys=None), Example({'augmented': True, 'context': ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.'], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Carhartt bass fishing'}) (input_keys=None), Example({'augmented': True, 'context': ['Forrest Tucker | Forrest Meredith Tucker (February 12, 1919 – October 25, 1986) was an American actor in both movies and television who appeared in nearly a hundred films. Tucker worked as a vaudeville straight man, aged fifteen years old. A mentor provided funds and contacts for a trip to California, where party hostess Cobina Wright persuaded guest Wesley Ruggles to give Tucker a screen test, based on his photogenic good looks, thick wavy hair and height of six feet, five inches - putting him into the same league as John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness\\'s younger brother. Tucker was a sight reader who needed only one take. His film career started well despite a perception in most Hollywood studios that blond men were not photogenic. After twenty years spent mainly in Westerns and action roles, he returned to his roots, showing versatility as a comedic and stage musical actor. In the TV show \"F Troop\", he became identified with the character of Cavalry Sgt. Morgan O\\'Rourke (a manipulative character quite similar to Phil Silvers\\' role as MSgt Ernie Bilko). Tucker struggled with a drinking problem that began to affect his performances in the later years of his career.', 'Emergency Landing (1941 film) | Emergency Landing (a.k.a. Robot Pilot) is a 1941 American aviation spy-fi romantic screwball comedy film directed by William Beaudine. The film stars Forrest Tucker in his second film and in his first leading role with co-stars Carol Hughes and Evelyn Brent. \"Emergency Landing\" features lots of mismatched stock footage of various types of aircraft.', 'Forrest Tucker (criminal) | Forrest Silva \"Woody\" Tucker (June 23, 1920 – May 29, 2004) was an American career criminal first imprisoned at age 15 who spent the rest of his life in and out of jail. He is best known as an escape artist, having escaped from prison 18 times successfully and 12 times unsuccessfully.'], 'question': \"Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\", 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Walt Disney TV miniseries 1955-1956 character'}) (input_keys=None)], [Example({'augmented': True, 'context': ['Forrest Tucker | Forrest Meredith Tucker (February 12, 1919 – October 25, 1986) was an American actor in both movies and television who appeared in nearly a hundred films. Tucker worked as a vaudeville straight man, aged fifteen years old. A mentor provided funds and contacts for a trip to California, where party hostess Cobina Wright persuaded guest Wesley Ruggles to give Tucker a screen test, based on his photogenic good looks, thick wavy hair and height of six feet, five inches - putting him into the same league as John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness\\'s younger brother. Tucker was a sight reader who needed only one take. His film career started well despite a perception in most Hollywood studios that blond men were not photogenic. After twenty years spent mainly in Westerns and action roles, he returned to his roots, showing versatility as a comedic and stage musical actor. In the TV show \"F Troop\", he became identified with the character of Cavalry Sgt. Morgan O\\'Rourke (a manipulative character quite similar to Phil Silvers\\' role as MSgt Ernie Bilko). Tucker struggled with a drinking problem that began to affect his performances in the later years of his career.', 'Emergency Landing (1941 film) | Emergency Landing (a.k.a. Robot Pilot) is a 1941 American aviation spy-fi romantic screwball comedy film directed by William Beaudine. The film stars Forrest Tucker in his second film and in his first leading role with co-stars Carol Hughes and Evelyn Brent. \"Emergency Landing\" features lots of mismatched stock footage of various types of aircraft.', 'Forrest Tucker (criminal) | Forrest Silva \"Woody\" Tucker (June 23, 1920 – May 29, 2004) was an American career criminal first imprisoned at age 15 who spent the rest of his life in and out of jail. He is best known as an escape artist, having escaped from prison 18 times successfully and 12 times unsuccessfully.'], 'question': \"Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\", 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Walt Disney TV miniseries 1955-1956 character'}) (input_keys=None), Example({'augmented': True, 'context': ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.'], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Carhartt bass fishing'}) (input_keys=None), Example({'question': 'How many years did Lenny Lyles spend with the organization dramatized in the filme \"The insder\"?', 'answer': '27'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Will Finn | Will Finn (born November 1, 1958) is an American animator, voice actor, and director. His work in animation includes characters from Disney and Don Bluth films such as \"The Secret of NIMH\", \"Oliver & Company\", \"The Little Mermaid\", \"The Rescuers Down Under\", and \"Pocahontas\". His characters includes Cogsworth in \"Beauty and the Beast\", Iago in \"Aladdin\", and Laverne in \"The Hunchback of Notre Dame\". Finn wrote and directed \"Home on the Range\" and did some voice acting the Hollywood Fish in \"Chicken Little\". In 2006, Finn directed the computer animated short Hammy\\'s Boomerang Adventure, a spin-off of \"Over the Hedge\".', 'Over the Hedge (film) | Over the Hedge is a 2006 American computer-animated comedy film, based on the characters from the United Media comic strip of the same name. Directed by Tim Johnson and Karey Kirkpatrick, and produced by Bonnie Arnold, it was released in the United States on May 19, 2006. The film was produced by DreamWorks Animation and distributed through Paramount Pictures. The film features the voices of Bruce Willis, Garry Shandling, Steve Carell, William Shatner, Wanda Sykes, and Nick Nolte. It is the first DreamWorks Animation film to be distributed by Paramount Pictures, which acquired the live-action DreamWorks studio in 2006. The film earned $336 million on an $80 million budget.', 'Tim Johnson (film director) | Tim Johnson (born August 27, 1961) is an American animation director. Johnson has directed in many films such as \"Antz\", \"\" and \"Over the Hedge\". Recently, he directed the DreamWorks 3D film \"Home.\"'], 'question': 'When was the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed?', 'rationale': 'find the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed. The context mentions that Will Finn directed the computer animated short Hammy\\'s Boomerang Adventure, a spin-off of \"Over the Hedge\".', 'search_query': \"Hammy's Boomerang Adventure release date\"}) (input_keys=None), Example({'question': 'The Equestrian statue of Marshal Mannerheim was erected as a monument to the leader of the Whites during what event?', 'answer': 'Finnish Civil War'}) (input_keys={'question'}), Example({'question': 'What is the capacity of the La Liga stadium in which The Madrid Derby team plays their home games?', 'answer': '68,000'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.', 'Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.', \"Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world's third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.\"], 'question': 'The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Kangchenjunga Singhi Kangri country'}) (input_keys=None), Example({'question': \"When did Taylor Swift's satirical song get released to the radio?\", 'answer': 'February 9, 2015'}) (input_keys={'question'}), Example({'question': 'Are Beth Gibbons and Ian Curtis the same nationality?', 'answer': 'yes'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Wheat Field with Cypresses | A Wheatfield with Cypresses (occasionally called \"A Cornfield with Cypresses\") is any of three similar 1889 oil paintings by Vincent van Gogh, as part of his wheat field series. All were executed at the Saint-Paul-de-Mausole mental asylum at Saint-Rémy near Arles, France, where Van Gogh was voluntarily a patient from May 1889 to May 1890. The works were inspired by the view from the window at the asylum towards the Alpilles mountains.', 'Green Wheat Field with Cypress | Green Wheat Field with Cypress (French: Champ de blé vert avec cyprès) is an oil on canvas painting by a Dutch Post-Impressionist Vincent van Gogh. It was completed in 1889, while van Gogh was voluntarily incarcerated at the asylum of St. Paul near Saint-Rémy in Provence. Several paintings of wheat fields with cypresses were made when van Gogh was able to leave the asylum grounds and explore the local landscape. Besides a fondness for cypresses, van Gogh had a special affinty with wheat fields; he depicted them dozens of times over the years; to Vincent they symbolized the cycle of life and death, and he found in them both solace and inspiration.', 'Olive Trees (Van Gogh series) | Vincent van Gogh painted at least 18 paintings of olive trees, mostly in Saint-Rémy-de-Provence in 1889. At his own request, he lived at an asylum there from May 1889 through May 1890 painting the gardens of the asylum and, when he had permission to venture outside its walls, nearby olive trees, cypresses and wheat fields.'], 'question': 'What was the series inspired by the appreciation of manual laborers that Wheat Field with Cypresses is a part of?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Wheat Field with Cypresses series inspiration'}) (input_keys=None), Example({'question': 'How many shops does the larger of the Miami International Mall and Dolphin Mall have?', 'answer': '240'}) (input_keys={'question'}), Example({'question': 'What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?', 'answer': '2016'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Butch Reed | Bruce Franklin Reed (born July 11, 1954) is an American retired professional wrestler and football player, better known by the ring name Butch Reed. He is best known for his appearances with the World Wrestling Federation and World Championship Wrestling between 1986 and 1992.', 'Doug Reed (futsal player) | Doug Reed is a professional futsal player who is currently playing for Futsal Dinamo and England national futsal team. Previously he has played for Lanzarote Tias Yaiza (Spain), AEL Limassol, MNK Uspinjača, MNK Smederevo and Manchester Futsal Club. In 2011, he won the FA Futsal Cup with Manchester Futsal Club and was awarded Player of the Tournament in the same competition. In the FA National Futsal League, he won several FA National League Championships and FA Futsal Cup medals. Reed also works as a commentator for Sony SIX, Eurosport and BT Sport, and has worked with the emerging US Professional Futsal League as well as Indian Premier Futsal.', 'Doom (professional wrestling) | Doom was a professional wrestling tag team composed of Ron Simmons and Butch Reed. They competed from 1989 to 1991 in the National Wrestling Alliance (NWA) and World Championship Wrestling (WCW).'], 'question': 'What other sport did Butch Reed, of the wrestling tag team Doom, play professionally?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Butch Reed football'}) (input_keys=None), Example({'question': 'What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?', 'answer': '2016'}) (input_keys={'question'}), Example({'question': 'Which magazine is still in print American Heritage or High Times?', 'answer': 'High Times'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.', 'Ebenezer Howard | Sir Ebenezer Howard {\\'1\\': \", \\'2\\': \", \\'3\\': \", \\'4\\': \"} ( ; 29 January 1850 – 1 May 1928), the English founder of the garden city movement, is known for his publication \"\" (1898), the description of a utopian city in which people live harmoniously together with nature. The publication resulted in the founding of the garden city movement, and the building of the First Garden City, Letchworth Garden City, commenced in 1903.', 'Eugene Reynal | Eugene Reynal was an American Publisher and in 1933 Founder of Blue Ribbon Books of Garden City, New York and Reynal and Hitchcock of New York, New York.'], 'question': 'Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Garden City Hotel founder'}) (input_keys=None), Example({'augmented': True, 'context': ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.'], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Carhartt bass fishing'}) (input_keys=None), Example({'augmented': True, 'context': ['Stitch\\'s Great Escape! | Stitch\\'s Great Escape! is a Tomorrowland attraction at the Magic Kingdom theme park within the Walt Disney World Resort. It is a \"theater-in-the-round\" experience starring the title alien from Walt Disney Animation Studios\\' 2002 film \"Lilo & Stitch\". It opened November 16, 2004 and is the fourth attraction to occupy the site in Tomorrowland. Many of the animators who worked on \"Lilo & Stitch\" partnered with Walt Disney Imagineering for Stitch\\'s Great Escape! The attraction is a replacement of The ExtraTERRORestrial Alien Encounter, which formerly occupied the building the attraction is housed in. On September 21, 2016, it was announced that the attraction would be switching from a daily operated attraction to a seasonally operated one, depending on attendance, starting October 2, 2016.', 'Great Escape (amusement park) | Great Escape (sometimes referred to as Six Flags Great Escape) is an amusement and water park owned and operated by Six Flags Entertainment Corp. It is located approximately 60 mi north of Albany, in Queensbury, New York, but is advertised as being in Lake George, New York, a popular tourist and vacation spot nearby. It is one of two Six Flags parks not to be officially branded with the \"Six Flags\" name (La Ronde in Montreal, Quebec, Canada being the other).', 'Houdini\\'s Great Escape | Houdini\\'s Great Escape, also known as \"Boo-dini\" during Fright Fest and \"Houdini - The Great Escape\" at Six Flags New England, are indoor Vekoma Madhouse attractions at Six Flags Great Adventure and Six Flags New England. The ride is located in Great Adventure\\'s Fantasy Forest section and New England\\'s Crack Axle Canyon area. The attraction opened at both parks in 1999, and was Standing but not operating (SBNO) at Great Adventure from 2008 - 2009 but was opened again due to popular demand.'], 'question': 'Which theme park has Stitch’s Great Escape which replaced the attraction ExtraTERRORestial Alien Encounter?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Magic Kingdom theme park'}) (input_keys=None)], [Example({'augmented': True, 'context': ['Simon Arora | Simon Arora (born November 1969) is a British billionaire businessman, CEO of the retail chain B & M.', 'Samir Arora | Samir Arora (born November 5, 1965) is an American businessman and former CEO of Mode Media (formerly Glam Media) from 2003 to April 2016. He was CEO and Chairman of the web design company NetObjects, Inc. from 1995 to 2001.', 'Dinesh Arora | Dinesh Arora (born 23 December 1976) is an Indian Administrative Service (IAS) officer, 2002 Kerala cadre, who previously worked as an Executive Director, Rural Electrification Corporation (REC) and CEO RECPDCL.'], 'question': 'What company formed in 1978 has Simon Arora as its CEO?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'B & M CEO'}) (input_keys=None), Example({'augmented': True, 'context': ['Electoral district of Bragg | Bragg is an electoral district of the House of Assembly in the Australian state of South Australia. The seat of Bragg is named after the eminent physicists Bragg – William Henry and his son, William Lawrence. The electorate is largely urban and encompasses a significant portion of the City of Burnside, stretching from the east parklands of Adelaide into the Adelaide Hills. After a redistribution following the 2006 election, the boundary moved eastwards to include suburbs that had formerly been in the seat of Heysen and now borders Kavel. Bragg currently includes the metropolitan suburbs of Beaumont, Burnside, Dulwich, Erindale, Hazelwood Park, Heathpool, Kensington Park, Leabrook, Linden Park, Rose Park, Marryatville, Skye, St Georges, Stonyfell, Toorak Gardens, Tusmore, Wattle Park and parts of Glen Osmond, and in the hills it includes Crafers, Cleland, Greenhill, Mount Osmond, Piccadilly, Summertown and parts of Ashton, Basket Range, Carey Gully, Horsnell Gully, Leawood Gardens and Uraidla.', 'Cleland, South Australia | Cleland is a suburb in South Australia located in the Adelaide metropolitan area about 10 km south-east of the Adelaide city centre. Its boundaries were created in October 2001, with additional land being added in 2010 from the adjoining suburb of Crafers. Its name is derived from the Cleland Conservation Park. The principal land use within the locality is conservation with the majority of its land area being occupied by the Cleland Conservation Park. Places within its extent include the summit of Mount Lofty. Cleland is located within the federal Division of Mayo, the state electoral district of Bragg and the local government area of the Adelaide Hills Council.', 'Electoral results for the district of Bragg | This is a list of electoral results for the Electoral district of Bragg in South Australian state elections.'], 'question': 'Bragg electoral district, incorporating the Kensington Park suburb, are associated with which South Australian city?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'South Australian city associated with Bragg electoral district'}) (input_keys=None), Example({'augmented': True, 'context': ['Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.', 'Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.', \"Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world's third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.\"], 'question': 'The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?', 'rationale': \"Reasoning: Let's think step by step in order to\", 'search_query': 'Kangchenjunga Singhi Kangri country'}) (input_keys=None)]], 2: [[], [Example({'question': 'What character did this actor, who also played a supporting role in Sleepy Hollow, play in the final six Harry Potter films?', 'answer': 'Professor Albus Dumbledore'}) (input_keys={'question'}), Example({'question': 'The musical sitcom with an album by the same name has what special feature on the album cover?', 'answer': 'actual crossword puzzle'}) (input_keys={'question'}), Example({'question': 'What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?', 'answer': '2016'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Alberta general election, 2015 | The 29th general election of Alberta, Canada, took place on May 5, 2015, following a request of Premier Jim Prentice to the Lieutenant Governor of Alberta Donald Ethell to dissolve the Legislative Assembly on April 7, 2015. This election elected members to the 29th Alberta Legislature. It was one of only four times that Alberta has changed governments.', \"Jenean Hampton | Jenean Michelle Hampton (born May 12, 1958) is the 57th and current Lieutenant Governor of Kentucky since December 8, 2015 after having won office as Republican gubernatorial nominee Matt Bevin's running-mate, in the 2015 gubernatorial election. She is the first African-American to hold any statewide office in Kentucky history, and only the third African-American woman to serve as lieutenant-governor of any U.S. state (after Jennette Bradley and Jennifer Carroll).\", 'Louisiana lieutenant gubernatorial election, 2015 | The Louisiana lieutenant gubernatorial election took place on October 24, 2015, to elect the Lieutenant Governor of Louisiana, with a runoff election held on November 21, 2015. Incumbent Republican Lieutenant Governor Jay Dardenne did not for re-election to a second full term in office. He instead ran for Governor. Billy Nungesser won the election defeating Kip Holden, despite a Democratic victory in the gubernatorial election, in which John Bel Edwards defeated David Vitter by a similar margin.', 'Indiana gubernatorial election, 2016 | The 2016 Indiana gubernatorial election was held November 8, 2016, to elect the Governor and Lieutenant Governor of Indiana, concurrently with the 2016 U.S. presidential election as well as elections to the United States Senate and elections to the United States House of Representatives and various state and local elections. The primaries were held on May 3, 2016. Republican Lieutenant Governor Eric Holcomb won the race with 51.4% of the vote.', 'Virginia lieutenant gubernatorial election, 2017 | The Virginia lieutenant gubernatorial election of 2017 will take place on November 7, 2017. After the party primary elections were held, the major party nominees are Jill Vogel (Republican) and Justin Fairfax (Democrat). The incumbent Lieutenant Governor, Democrat Ralph Northam, is not running for re-election in order to run for Governor.'], 'question': 'What race is the lieutenant governor of the state that held its primary for her election on May 19th, 2015?', 'rationale': 'The lieutenant governor of the state that held its primary for her election on May 19th, 2015 is Jenean Hampton, who is the 57th and current Lieutenant Governor of Kentucky since December 8, 2015. The primary election was held in Kentucky, and Jenean Hampton is the lieutenant governor of Kentucky.', 'answer': 'African-American'}) (input_keys=None), Example({'augmented': True, 'context': ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.', \"College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour\", 'Bass fishing | Bass fishing is the activity of angling for the North American gamefish known colloquially as the \"black bass\". There are numerous black bass species considered as gamefish in North America, including largemouth bass (\"Micropterus salmoides\"), smallmouth bass (\"Micropterus dolomieui\"), spotted bass or Kentucky bass (\"Micropterus punctulatus\"), and Guadalupe bass (order \"Perciformes\").', 'Cliff Crochet | Cliff Crochet (born August 1, 1983), also known as \"The Cajun Baby\", is a professional bass fisherman from Pierre Part, Louisiana. Crochet is a competitor on Bass Angler Sportsman Society\\'s (B.A.S.S.) Bassmaster Elite Series and has $368,837 in B.A.S.S. tournament winnings as of August 22, 2016. Crochet\\'s sponsors include: K2 Coolers, Huk, Skeeter, Yamaha Motor Company, Lowrance Electronics, Falcon Rods, Luck E Strike, Seaguar, Rat-L-Trap, Humdinger Spinnerbaits, Power-Pole. Since 2010, Crochet has competed in three Bassmaster Classic tournaments. Crochet\\'s boat for the 2016 season is a 2016 Skeeter FX 21 with a 250 hp Yamaha SHO outboard engine.'], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': 'The context mentions Carhartt Bassmaster College Bass Fishing Series, which is a college bass fishing tournament series. This series is associated with the clothing company Carhartt, which was founded in 1889.', 'answer': 'Carhartt'}) (input_keys=None), Example({'augmented': True, 'context': ['Forrest Tucker | Forrest Meredith Tucker (February 12, 1919 – October 25, 1986) was an American actor in both movies and television who appeared in nearly a hundred films. Tucker worked as a vaudeville straight man, aged fifteen years old. A mentor provided funds and contacts for a trip to California, where party hostess Cobina Wright persuaded guest Wesley Ruggles to give Tucker a screen test, based on his photogenic good looks, thick wavy hair and height of six feet, five inches - putting him into the same league as John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness\\'s younger brother. Tucker was a sight reader who needed only one take. His film career started well despite a perception in most Hollywood studios that blond men were not photogenic. After twenty years spent mainly in Westerns and action roles, he returned to his roots, showing versatility as a comedic and stage musical actor. In the TV show \"F Troop\", he became identified with the character of Cavalry Sgt. Morgan O\\'Rourke (a manipulative character quite similar to Phil Silvers\\' role as MSgt Ernie Bilko). Tucker struggled with a drinking problem that began to affect his performances in the later years of his career.', 'Emergency Landing (1941 film) | Emergency Landing (a.k.a. Robot Pilot) is a 1941 American aviation spy-fi romantic screwball comedy film directed by William Beaudine. The film stars Forrest Tucker in his second film and in his first leading role with co-stars Carol Hughes and Evelyn Brent. \"Emergency Landing\" features lots of mismatched stock footage of various types of aircraft.', 'Forrest Tucker (criminal) | Forrest Silva \"Woody\" Tucker (June 23, 1920 – May 29, 2004) was an American career criminal first imprisoned at age 15 who spent the rest of his life in and out of jail. He is best known as an escape artist, having escaped from prison 18 times successfully and 12 times unsuccessfully.', 'Fess Parker | Fess Elisha Parker Jr. (August 16, 1924 – March 18, 2010) was an American film and television actor best known for his portrayals of Davy Crockett in the Walt Disney 1955–1956 TV miniseries and as Daniel Boone in a television series from 1964 to 1970. He was also known as a winemaker and resort owner-operator.', 'Davy Crockett, King of the Wild Frontier | Davy Crockett, King of the Wild Frontier is a 1955 live-action Walt Disney adventure film starring Fess Parker as Davy Crockett. This film is an edited compilation of the first three stories from the Disney television miniseries \"Davy Crockett\" :', 'Davy Crockett (miniseries) | Davy Crockett is a five-part serial which aired on ABC from 1954–1955 in one-hour episodes, on the \"Disneyland\" series. The series stars Fess Parker as real-life frontiersman Davy Crockett and Buddy Ebsen as his friend, George Russel.'], 'question': \"Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\", 'rationale': \"From the context, we know that Forrest Tucker was an American actor who appeared in nearly a hundred films. We also know that he was compared to John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness's younger brother. We need to find the actor that played a character in the 1955-1956 Walt Disney TV miniseries, which is Fess Parker.\", 'answer': 'Davy Crockett'}) (input_keys=None)], [Example({'augmented': True, 'context': ['Forrest Tucker | Forrest Meredith Tucker (February 12, 1919 – October 25, 1986) was an American actor in both movies and television who appeared in nearly a hundred films. Tucker worked as a vaudeville straight man, aged fifteen years old. A mentor provided funds and contacts for a trip to California, where party hostess Cobina Wright persuaded guest Wesley Ruggles to give Tucker a screen test, based on his photogenic good looks, thick wavy hair and height of six feet, five inches - putting him into the same league as John Wayne, Chuck Connors, Fess Parker, James Arness, and Peter Graves, Arness\\'s younger brother. Tucker was a sight reader who needed only one take. His film career started well despite a perception in most Hollywood studios that blond men were not photogenic. After twenty years spent mainly in Westerns and action roles, he returned to his roots, showing versatility as a comedic and stage musical actor. In the TV show \"F Troop\", he became identified with the character of Cavalry Sgt. Morgan O\\'Rourke (a manipulative character quite similar to Phil Silvers\\' role as MSgt Ernie Bilko). Tucker struggled with a drinking problem that began to affect his performances in the later years of his career.', 'Emergency Landing (1941 film) | Emergency Landing (a.k.a. Robot Pilot) is a 1941 American aviation spy-fi romantic screwball comedy film directed by William Beaudine. The film stars Forrest Tucker in his second film and in his first leading role with co-stars Carol Hughes and Evelyn Brent. \"Emergency Landing\" features lots of mismatched stock footage of various types of aircraft.', 'Forrest Tucker (criminal) | Forrest Silva \"Woody\" Tucker (June 23, 1920 – May 29, 2004) was an American career criminal first imprisoned at age 15 who spent the rest of his life in and out of jail. He is best known as an escape artist, having escaped from prison 18 times successfully and 12 times unsuccessfully.', 'Fess Parker | Fess Elisha Parker Jr. (August 16, 1924 – March 18, 2010) was an American film and television actor best known for his portrayals of Davy Crockett in the Walt Disney 1955–1956 TV miniseries and as Daniel Boone in a television series from 1964 to 1970. He was also known as a winemaker and resort owner-operator.', 'Davy Crockett, King of the Wild Frontier | Davy Crockett, King of the Wild Frontier is a 1955 live-action Walt Disney adventure film starring Fess Parker as Davy Crockett. This film is an edited compilation of the first three stories from the Disney television miniseries \"Davy Crockett\" :', 'Davy Crockett (miniseries) | Davy Crockett is a five-part serial which aired on ABC from 1954–1955 in one-hour episodes, on the \"Disneyland\" series. The series stars Fess Parker as real-life frontiersman Davy Crockett and Buddy Ebsen as his friend, George Russel.'], 'question': \"Forrest Tucker's height put him in the same league as at actor that played what character in the 1955-1956 Walt Disney TV miniseries?\", 'rationale': 'We need to find the actor who played the character in the 1955-1956 Walt Disney TV miniseries. From the context, we know that the miniseries was about Davy Crockett and starred Fess Parker as Davy Crockett.', 'answer': 'Davy Crockett'}) (input_keys=None), Example({'augmented': True, 'context': ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.', \"College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour\", 'Bass fishing | Bass fishing is the activity of angling for the North American gamefish known colloquially as the \"black bass\". There are numerous black bass species considered as gamefish in North America, including largemouth bass (\"Micropterus salmoides\"), smallmouth bass (\"Micropterus dolomieui\"), spotted bass or Kentucky bass (\"Micropterus punctulatus\"), and Guadalupe bass (order \"Perciformes\").', 'Cliff Crochet | Cliff Crochet (born August 1, 1983), also known as \"The Cajun Baby\", is a professional bass fisherman from Pierre Part, Louisiana. Crochet is a competitor on Bass Angler Sportsman Society\\'s (B.A.S.S.) Bassmaster Elite Series and has $368,837 in B.A.S.S. tournament winnings as of August 22, 2016. Crochet\\'s sponsors include: K2 Coolers, Huk, Skeeter, Yamaha Motor Company, Lowrance Electronics, Falcon Rods, Luck E Strike, Seaguar, Rat-L-Trap, Humdinger Spinnerbaits, Power-Pole. Since 2010, Crochet has competed in three Bassmaster Classic tournaments. Crochet\\'s boat for the 2016 season is a 2016 Skeeter FX 21 with a 250 hp Yamaha SHO outboard engine.'], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Context: [1] «Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.»\\n\\n[2] «College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour»\\n\\n[3] «G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.»\\n\\nReasoning: Let's think step by step in order to\", 'answer': 'Carhartt'}) (input_keys=None), Example({'question': 'How many years did Lenny Lyles spend with the organization dramatized in the filme \"The insder\"?', 'answer': '27'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Will Finn | Will Finn (born November 1, 1958) is an American animator, voice actor, and director. His work in animation includes characters from Disney and Don Bluth films such as \"The Secret of NIMH\", \"Oliver & Company\", \"The Little Mermaid\", \"The Rescuers Down Under\", and \"Pocahontas\". His characters includes Cogsworth in \"Beauty and the Beast\", Iago in \"Aladdin\", and Laverne in \"The Hunchback of Notre Dame\". Finn wrote and directed \"Home on the Range\" and did some voice acting the Hollywood Fish in \"Chicken Little\". In 2006, Finn directed the computer animated short Hammy\\'s Boomerang Adventure, a spin-off of \"Over the Hedge\".', 'Over the Hedge (film) | Over the Hedge is a 2006 American computer-animated comedy film, based on the characters from the United Media comic strip of the same name. Directed by Tim Johnson and Karey Kirkpatrick, and produced by Bonnie Arnold, it was released in the United States on May 19, 2006. The film was produced by DreamWorks Animation and distributed through Paramount Pictures. The film features the voices of Bruce Willis, Garry Shandling, Steve Carell, William Shatner, Wanda Sykes, and Nick Nolte. It is the first DreamWorks Animation film to be distributed by Paramount Pictures, which acquired the live-action DreamWorks studio in 2006. The film earned $336 million on an $80 million budget.', 'Tim Johnson (film director) | Tim Johnson (born August 27, 1961) is an American animation director. Johnson has directed in many films such as \"Antz\", \"\" and \"Over the Hedge\". Recently, he directed the DreamWorks 3D film \"Home.\"', 'Hammy\\'s Boomerang Adventure | Hammy\\'s Boomerang Adventure is a computer-animated short comedy film based on the film \"Over the Hedge\", which was based on a comic strip of the same name. It can be found on the movie\\'s DVD or the \"Madly Madagascar\" DVD as a bonus feature.', 'Boomerang (2015 film) | Boomerang is a 2015 French drama film directed by François Favrat and starring Laurent Lafitte, Mélanie Laurent, Audrey Dana and Wladimir Yordanoff. It is an adaptation of Tatiana de Rosnay\\'s novel \"A Secret Kept\" (alternative title:\"Boomerang\"). The film was released in theatres on 23 September 2015.', 'Over the Hedge: Hammy Goes Nuts! | Over the Hedge: Hammy Goes Nuts! is a video game published by Activision based on characters from the film \"Over the Hedge\". The Game Boy Advance (GBA) version, a miniature golf game, was developed by Vicarious Visions and released on October 10, 2006 in North America. The Nintendo DS and PlayStation Portable (PSP) versions, an action adventure game and a platform game respectively, were both developed by Amaze Entertainment and were released on October 24, 2006 and November 21, 2006. It is the second game to be based on the film.'], 'question': 'When was the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed?', 'rationale': 'Context: N/A\\n\\nQuestion: When was the computer-animated short comedy film based on the film \"Over the Hedge\" by Will Finn directed?\\n\\nReasoning: Let\\'s think step by step in order to', 'answer': '2006'}) (input_keys=None), Example({'question': 'What is the capacity of the La Liga stadium in which The Madrid Derby team plays their home games?', 'answer': '68,000'}) (input_keys={'question'}), Example({'question': 'The Equestrian statue of Marshal Mannerheim was erected as a monument to the leader of the Whites during what event?', 'answer': 'Finnish Civil War'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.', 'Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.', \"Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world's third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.\"], 'question': 'The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?', 'rationale': 'Context: [1] «Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.»\\n\\n[2] «Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.»\\n\\n[3] «Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world\\'s third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.»\\n\\nReasoning: Let\\'s think step by step in order to', 'answer': 'India'}) (input_keys=None), Example({'question': 'Are Beth Gibbons and Ian Curtis the same nationality?', 'answer': 'yes'}) (input_keys={'question'}), Example({'question': \"When did Taylor Swift's satirical song get released to the radio?\", 'answer': 'February 9, 2015'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Wheat Field with Cypresses | A Wheatfield with Cypresses (occasionally called \"A Cornfield with Cypresses\") is any of three similar 1889 oil paintings by Vincent van Gogh, as part of his wheat field series. All were executed at the Saint-Paul-de-Mausole mental asylum at Saint-Rémy near Arles, France, where Van Gogh was voluntarily a patient from May 1889 to May 1890. The works were inspired by the view from the window at the asylum towards the Alpilles mountains.', 'Green Wheat Field with Cypress | Green Wheat Field with Cypress (French: Champ de blé vert avec cyprès) is an oil on canvas painting by a Dutch Post-Impressionist Vincent van Gogh. It was completed in 1889, while van Gogh was voluntarily incarcerated at the asylum of St. Paul near Saint-Rémy in Provence. Several paintings of wheat fields with cypresses were made when van Gogh was able to leave the asylum grounds and explore the local landscape. Besides a fondness for cypresses, van Gogh had a special affinty with wheat fields; he depicted them dozens of times over the years; to Vincent they symbolized the cycle of life and death, and he found in them both solace and inspiration.', 'Olive Trees (Van Gogh series) | Vincent van Gogh painted at least 18 paintings of olive trees, mostly in Saint-Rémy-de-Provence in 1889. At his own request, he lived at an asylum there from May 1889 through May 1890 painting the gardens of the asylum and, when he had permission to venture outside its walls, nearby olive trees, cypresses and wheat fields.', 'Wheat Fields (Van Gogh series) | The Wheat Fields is a series of dozens of paintings by Vincent van Gogh, borne out of his religious studies and sermons, connection to nature, appreciation of manual laborers and desire to provide a means of offering comfort to others. The wheat field works demonstrate his progression as an artist from the drab \"Wheat Sheaves\" made in 1885 in the Netherlands to the colorful, dramatic paintings from Arles, Saint-Rémy and Auvers-sur-Oise of rural France.'], 'question': 'What was the series inspired by the appreciation of manual laborers that Wheat Field with Cypresses is a part of?', 'rationale': \"Context: N/A\\n\\nQuestion: What was the series inspired by the appreciation of manual laborers that Wheat Field with Cypresses is a part of?\\n\\nReasoning: Let's think step by step in order to\", 'answer': 'Wheat Fields'}) (input_keys=None), Example({'question': 'What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?', 'answer': '2016'}) (input_keys={'question'}), Example({'question': 'How many shops does the larger of the Miami International Mall and Dolphin Mall have?', 'answer': '240'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Butch Reed | Bruce Franklin Reed (born July 11, 1954) is an American retired professional wrestler and football player, better known by the ring name Butch Reed. He is best known for his appearances with the World Wrestling Federation and World Championship Wrestling between 1986 and 1992.', 'Doug Reed (futsal player) | Doug Reed is a professional futsal player who is currently playing for Futsal Dinamo and England national futsal team. Previously he has played for Lanzarote Tias Yaiza (Spain), AEL Limassol, MNK Uspinjača, MNK Smederevo and Manchester Futsal Club. In 2011, he won the FA Futsal Cup with Manchester Futsal Club and was awarded Player of the Tournament in the same competition. In the FA National Futsal League, he won several FA National League Championships and FA Futsal Cup medals. Reed also works as a commentator for Sony SIX, Eurosport and BT Sport, and has worked with the emerging US Professional Futsal League as well as Indian Premier Futsal.', 'Doom (professional wrestling) | Doom was a professional wrestling tag team composed of Ron Simmons and Butch Reed. They competed from 1989 to 1991 in the National Wrestling Alliance (NWA) and World Championship Wrestling (WCW).', 'Robert Reed (gridiron football) | Robert E. Reed, Jr. (born February 23, 1943) is a former American football offensive lineman in the National Football League for the Washington Redskins. He played college football at Tennessee State University and was drafted in the sixteenth round of the 1965 NFL Draft. Reed was also selected in the 12th round of the 1965 AFL Draft by the Houston Oilers.', 'Reed Doughty | Reed Doughty (born November 4, 1982) is a former American football safety who played in the National Football League. He was drafted by the Washington Redskins in the sixth round, with the 173rd overall pick, of the 2006 NFL Draft. He played college football at the University of Northern Colorado.'], 'question': 'What other sport did Butch Reed, of the wrestling tag team Doom, play professionally?', 'rationale': \"Context: N/A\\n\\nQuestion: What other sport did Butch Reed, of the wrestling tag team Doom, play professionally?\\n\\nReasoning: Let's think step by step in order to\", 'answer': 'football'}) (input_keys=None), Example({'question': 'Which magazine is still in print American Heritage or High Times?', 'answer': 'High Times'}) (input_keys={'question'}), Example({'question': 'What year saw the release of the first album by the boy band featured in a remix of Todo Cambio?', 'answer': '2016'}) (input_keys={'question'})], [Example({'augmented': True, 'context': ['Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.', 'Ebenezer Howard | Sir Ebenezer Howard {\\'1\\': \", \\'2\\': \", \\'3\\': \", \\'4\\': \"} ( ; 29 January 1850 – 1 May 1928), the English founder of the garden city movement, is known for his publication \"\" (1898), the description of a utopian city in which people live harmoniously together with nature. The publication resulted in the founding of the garden city movement, and the building of the First Garden City, Letchworth Garden City, commenced in 1903.', 'Eugene Reynal | Eugene Reynal was an American Publisher and in 1933 Founder of Blue Ribbon Books of Garden City, New York and Reynal and Hitchcock of New York, New York.'], 'question': 'Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?', 'rationale': 'Context: [1] «Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.»\\n\\n[2] «A.T. Stewart | Alexander Turney Stewart (October 2, 1792 – April 10, 1876) was an American businessman, politician, and philanthropist. He was a prominent figure in New York City during the mid-19th century, and was known as the \"Merchant Prince\".»\\n\\n[3] «Garden City Hotel | The Garden City Hotel is a hotel in Garden City, New York. The first incarnation was built in 1874 by A.T. Stewart and the current fourth incarnation was built in 1983 by the late Myron Nelkin. It is famous for having hosted many world leaders and celebrities, including John F. Kennedy, Margaret Thatcher, Hillary Clinton, George H. Bush, Prince Khalid of Saudi Arabia, and Irish Taoiseach Garrett FitzGerald. Charles Lindbergh, rented a room at the hotel the night before his famous transatlantic flight to Paris, although he did not actually use it, instead taking a three-hour nap at the home of his friend on 105 Third Street.»\\n\\nQuestion: Who built the first Garden City Hotel that is situated across the street from Garden City Railroad?\\n\\nReasoning: Let\\'s think step by step in order to', 'answer': 'A.T. Stewart'}) (input_keys=None), Example({'augmented': True, 'context': ['Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.', 'Dry-Fly Fishing in Theory and Practice | Dry-Fly Fishing in Theory and Practice (1889) is British author and angler Frederic M. Halford\\'s second and most influential book on dry-fly fishing. It followed \"Floating Flies and How to Dress Them\" (1886) and this pair of books initiated some 40 years of a rigid, and sometimes dogmatic school, the Halfordian school, of dry-fly fishing, especially on English chalk streams. The work also played a significant role in the development of dry-fly fishing in America.', 'G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.', \"College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour\", 'Bass fishing | Bass fishing is the activity of angling for the North American gamefish known colloquially as the \"black bass\". There are numerous black bass species considered as gamefish in North America, including largemouth bass (\"Micropterus salmoides\"), smallmouth bass (\"Micropterus dolomieui\"), spotted bass or Kentucky bass (\"Micropterus punctulatus\"), and Guadalupe bass (order \"Perciformes\").', 'Cliff Crochet | Cliff Crochet (born August 1, 1983), also known as \"The Cajun Baby\", is a professional bass fisherman from Pierre Part, Louisiana. Crochet is a competitor on Bass Angler Sportsman Society\\'s (B.A.S.S.) Bassmaster Elite Series and has $368,837 in B.A.S.S. tournament winnings as of August 22, 2016. Crochet\\'s sponsors include: K2 Coolers, Huk, Skeeter, Yamaha Motor Company, Lowrance Electronics, Falcon Rods, Luck E Strike, Seaguar, Rat-L-Trap, Humdinger Spinnerbaits, Power-Pole. Since 2010, Crochet has competed in three Bassmaster Classic tournaments. Crochet\\'s boat for the 2016 season is a 2016 Skeeter FX 21 with a 250 hp Yamaha SHO outboard engine.'], 'question': 'Which clothing company that focuses on work clothes founded in 1889 has association with college bass fishing?', 'rationale': \"Context: [1] «Carhartt | Carhartt, Inc., is a U.S.-based clothing company founded in 1889 that focuses on work clothes. It is still a family-owned company, owned by the descendants of company founder Hamilton Carhartt, with its headquarters in Dearborn, Michigan.»\\n\\n[2] «College bass fishing | College bass fishing is competitive bass fishing that is played by teams from universities and colleges. The NCAA is not involved in college bass fishing, allowing teams to use sponsorships to their advantage and keep their winnings. College bass fishing consists of mostly club teams, but there are a select few varsity bass fishing programs. There are an estimated 610 college bass fishing teams in the United States. The major college bass fishing tournament series are the FLW Outdoors College Series, Carhartt Bassmaster College Bass Fishing Series,[Cabela's]] Collegiate Bass Fishing Series, and The Fishlife Collegiate Tour»\\n\\n[3] «G.H. Bass & Co. | G.H. Bass & Co. is an American footwear brand founded by George Henry Bass in 1876.»\\n\\nReasoning: Let's think step by step in order to\", 'answer': 'Carhartt'}) (input_keys=None), Example({'augmented': True, 'context': ['Stitch\\'s Great Escape! | Stitch\\'s Great Escape! is a Tomorrowland attraction at the Magic Kingdom theme park within the Walt Disney World Resort. It is a \"theater-in-the-round\" experience starring the title alien from Walt Disney Animation Studios\\' 2002 film \"Lilo & Stitch\". It opened November 16, 2004 and is the fourth attraction to occupy the site in Tomorrowland. Many of the animators who worked on \"Lilo & Stitch\" partnered with Walt Disney Imagineering for Stitch\\'s Great Escape! The attraction is a replacement of The ExtraTERRORestrial Alien Encounter, which formerly occupied the building the attraction is housed in. On September 21, 2016, it was announced that the attraction would be switching from a daily operated attraction to a seasonally operated one, depending on attendance, starting October 2, 2016.', 'Great Escape (amusement park) | Great Escape (sometimes referred to as Six Flags Great Escape) is an amusement and water park owned and operated by Six Flags Entertainment Corp. It is located approximately 60 mi north of Albany, in Queensbury, New York, but is advertised as being in Lake George, New York, a popular tourist and vacation spot nearby. It is one of two Six Flags parks not to be officially branded with the \"Six Flags\" name (La Ronde in Montreal, Quebec, Canada being the other).', 'Houdini\\'s Great Escape | Houdini\\'s Great Escape, also known as \"Boo-dini\" during Fright Fest and \"Houdini - The Great Escape\" at Six Flags New England, are indoor Vekoma Madhouse attractions at Six Flags Great Adventure and Six Flags New England. The ride is located in Great Adventure\\'s Fantasy Forest section and New England\\'s Crack Axle Canyon area. The attraction opened at both parks in 1999, and was Standing but not operating (SBNO) at Great Adventure from 2008 - 2009 but was opened again due to popular demand.', 'Magic Kingdom | Magic Kingdom is a theme park at the Walt Disney World Resort in Bay Lake, Florida, near Orlando. Owned and operated by The Walt Disney Company through its Parks and Resorts division, the park opened on October 1, 1971, as the first of four theme parks at the resort. Initialized by Walt Disney and designed by WED Enterprises, its layout and attractions are based on Disneyland Park in Anaheim, California, and is dedicated to fairy tales and Disney characters.', 'Magic Kingdom (disambiguation) | The Magic Kingdom is a theme park at the Walt Disney World Resort in Florida.', 'List of Magic Kingdom attractions | The Magic Kingdom is a theme park located at the Walt Disney World Resort in Florida. Below is a list of the current attractions found therein, arranged by \"land\" and with brief descriptions.'], 'question': 'Which theme park has Stitch’s Great Escape which replaced the attraction ExtraTERRORestial Alien Encounter?', 'rationale': \"We need to find the theme park that has Stitch's Great Escape, which replaced the attraction ExtraTERRORestial Alien Encounter. The context mentions that Stitch's Great Escape is located in the Magic Kingdom theme park.\", 'answer': 'Magic Kingdom'}) (input_keys=None)], [Example({'augmented': True, 'context': ['Simon Arora | Simon Arora (born November 1969) is a British billionaire businessman, CEO of the retail chain B & M.', 'Samir Arora | Samir Arora (born November 5, 1965) is an American businessman and former CEO of Mode Media (formerly Glam Media) from 2003 to April 2016. He was CEO and Chairman of the web design company NetObjects, Inc. from 1995 to 2001.', 'Dinesh Arora | Dinesh Arora (born 23 December 1976) is an Indian Administrative Service (IAS) officer, 2002 Kerala cadre, who previously worked as an Executive Director, Rural Electrification Corporation (REC) and CEO RECPDCL.', 'Bolliger & Mabillard | Bolliger & Mabillard, officially Bolliger & Mabillard Consulting Engineers Inc., and often abbreviated B&M, is a roller coaster design consultancy based in Monthey, Switzerland. The company was founded in 1988 by Walter Bolliger and Claude Mabillard, with Bolliger as president and Mabillard as vice-president. B&M has built over a hundred roller coasters around the world and pioneered several new ride technologies, most notably the inverted roller coaster. In North America, B&M coaster designs have been manufactured by Ohio company Clermont Steel Fabricators since 1990. B&M has grown significantly since its founding, originally employing four to as many as 37 in 2012, consisting primarily of engineers and draftsmen. In 2016, the company completed its 100th roller coaster.', \"Rakefet Russak-Aminoach | Rakefet Russak-Aminoach (Hebrew: רקפת רוסק עמינח\\u200e \\u200e ; born February 1, 1966) is the current President and CEO of Bank Leumi le-Israel B.M., a position she has held since May 1, 2012. In September 2015, she was listed in Fortune Magazine’s list of The World's 100 Most Powerful Women.\"], 'question': 'What company formed in 1978 has Simon Arora as its CEO?', 'rationale': 'There is no information about Simon Arora being the CEO of a company formed in 1978 in the context. However, we can find the information about Simon Arora being the CEO of B & M in the context.', 'answer': 'B & M'}) (input_keys=None), Example({'augmented': True, 'context': ['Electoral district of Bragg | Bragg is an electoral district of the House of Assembly in the Australian state of South Australia. The seat of Bragg is named after the eminent physicists Bragg – William Henry and his son, William Lawrence. The electorate is largely urban and encompasses a significant portion of the City of Burnside, stretching from the east parklands of Adelaide into the Adelaide Hills. After a redistribution following the 2006 election, the boundary moved eastwards to include suburbs that had formerly been in the seat of Heysen and now borders Kavel. Bragg currently includes the metropolitan suburbs of Beaumont, Burnside, Dulwich, Erindale, Hazelwood Park, Heathpool, Kensington Park, Leabrook, Linden Park, Rose Park, Marryatville, Skye, St Georges, Stonyfell, Toorak Gardens, Tusmore, Wattle Park and parts of Glen Osmond, and in the hills it includes Crafers, Cleland, Greenhill, Mount Osmond, Piccadilly, Summertown and parts of Ashton, Basket Range, Carey Gully, Horsnell Gully, Leawood Gardens and Uraidla.', 'Cleland, South Australia | Cleland is a suburb in South Australia located in the Adelaide metropolitan area about 10 km south-east of the Adelaide city centre. Its boundaries were created in October 2001, with additional land being added in 2010 from the adjoining suburb of Crafers. Its name is derived from the Cleland Conservation Park. The principal land use within the locality is conservation with the majority of its land area being occupied by the Cleland Conservation Park. Places within its extent include the summit of Mount Lofty. Cleland is located within the federal Division of Mayo, the state electoral district of Bragg and the local government area of the Adelaide Hills Council.', 'Electoral results for the district of Bragg | This is a list of electoral results for the Electoral district of Bragg in South Australian state elections.', 'Electoral district of Burnside | Burnside was an electoral district of the House of Assembly in the Australian state of South Australia from 1938 to 1970. Before 1938 the Burnside area was represented by the three-seat multi-member electorate of Sturt. It was abolished in the 1970 parliamentary reforms, and was replaced with Bragg.'], 'question': 'Bragg electoral district, incorporating the Kensington Park suburb, are associated with which South Australian city?', 'rationale': \"Context: N/A\\n\\nQuestion: Bragg electoral district, incorporating the Kensington Park suburb, are associated with which South Australian city?\\n\\nReasoning: Let's think step by step in order to\", 'answer': 'Adelaide'}) (input_keys=None), Example({'augmented': True, 'context': ['Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.', 'Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.', \"Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world's third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.\"], 'question': 'The high-peaked mountains Kangchenjunga and Singhi Kangri are both partially in which country?', 'rationale': 'Context: [1] «Kangchenjunga | Kangchenjunga (Nepali: कञ्चनजङ्घा , \"Kañcanajaṅghā\"), also spelled Kanchenjunga, is the third highest mountain in the world, and lies partly in Nepal and partly in Sikkim, India. It rises with an elevation of 8586 m in a section of the Himalayas called \"Kangchenjunga Himal\" that is limited in the west by the Tamur River, in the north by the Lhonak Chu and Jongsang La, and in the east by the Teesta River.»\\n\\n[2] «Singhi Kangri | Singhi Kangri is a 7202 m peak in the Karakoram range. It is located on the border between China and the Siachen Glacier region (controlled by India, claimed by Pakistan.) The mountain was first climbed in 1976 by a Japanese expedition.»\\n\\n[3] «Kaluk | Kaluk is the small village located in the Himalayan foothills of West Sikkim, near the West Sikkim capital of Gyalshing in North India. Kangchenjunga, the world\\'s third highest peak, is visible from Kaluk. It is one of the windiest places in Sikkim and noted for destinations like the Durga Mandir, Rinchenpong Gumba and Megi Dara. The people of this area are dependent on agriculture and horticulture for their livelihood. Tourism also plays an economic role in the region.»\\n\\nReasoning: Let\\'s think step by step in order to', 'answer': 'India'}) (input_keys=None)]]} with Mean = 60.0\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "Average Metric: 49 / 100 (49.0): 100%|██████████| 100/100 [00:41<00:00, 2.40it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "UPDATING BEST SCORE WITH 49.0\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ " 0%| | 0/150 [00:00<?, ?it/s]"+ ]+ }+ ],+ "source": [+ "COMPILE_PROGRAM = True\n",+ "with dspy.context(lm=mhqa_llama_8b, rm=retriever):\n",+ " vanilla_program = BasicMH()\n",+ " if COMPILE_PROGRAM:\n",+ " teleprompter = MIPROv2(prompt_model=llama_8b, task_model=mhqa_llama_8b, metric=metric, num_candidates=10, init_temperature=0.9, verbose=True)\n",+ " compiled_program = teleprompter.compile(vanilla_program, trainset=optimizer_trainset, valset=optimizer_valset, num_batches=30, max_bootstrapped_demos=MAX_BOOTSTRAPPED_DEMOS,max_labeled_demos=MAX_LABELED_DEMOS, eval_kwargs=eval_kwargs, requires_permission_to_run=False)\n",+ " compiled_program.save(f\"basicmh_8b_ft_MIPROv2_{MAX_BOOTSTRAPPED_DEMOS}_{MAX_LABELED_DEMOS}.json\")\n",+ " else:\n",+ " compiled_program = BasicMH()\n",+ " compiled_program.load(f\"basicmh_8b_ft_MIPROv2_{MAX_BOOTSTRAPPED_DEMOS}_{MAX_LABELED_DEMOS}.json\")\n",+ " llama_8b_ft_mipro_eval = evaluate_devset(compiled_program)"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "Lastly, lets give the base 8B model a fair chance by prompt optimizing it."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "COMPILE_PROGRAM = False\n",+ "\n",+ "with dspy.context(lm=llama_8b, rm=retriever):\n",+ " vanilla_program = BasicMH()\n",+ " if COMPILE_PROGRAM:\n",+ " bfrs_program = bfrs_optimizer.compile(vanilla_program, trainset=optimizer_trainset, valset=optimizer_valset)\n",+ " bfrs_program.save(f\"basicmh_8b_bfrs_{MAX_BOOTSTRAPPED_DEMOS}_{MAX_LABELED_DEMOS}_{NUM_CANDIDATE_PROGRAMS}.json\")\n",+ " else:\n",+ " bfrs_program = BasicMH()\n",+ " bfrs_program.load(f\"basicmh_8b_bfrs_{MAX_BOOTSTRAPPED_DEMOS}_{MAX_LABELED_DEMOS}_{NUM_CANDIDATE_PROGRAMS}.json\")\n",+ " llama_8b_bfrs_eval = evaluate_devset(bfrs_program)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "# Now we can compare all iterations of this pipeline\n",+ "print(f\"Results for HotPotQA fine-tuning LLaMa 8B with a starting trainset\")\n",+ "print(f\" 70B model (vanilla program): {llama_70b_base_eval}\")\n",+ "print(f\" 70B model (bfrs program): {llama_70b_bfrs_eval}\")\n",+ "print(f\" 8B model (vanilla program): {vanilla_8b_base_eval}\")\n",+ "print(f\" 8B model (bfrs program): {llama_8b_bfrs_eval}\")\n",+ "print(f\" 8B model (finetuned program): {llama_8b_finetuned_eval}\")\n",+ "print(f\" 8B model (finetuned bfrs program): {llama_8b_bfrs_finetuned_eval}\")\n",+ "print(f\" 8B model (finetuned mipro program): {llama_8b_ft_mipro_eval}\")"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "TODO: Let's now use the new offline batch inference to evaluate the finetuned model with optimized program on the entire devset"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "# TODO: implement once done"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "raise NotImplementedError(\"Stop here\")"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "# Serving\n",+ "\n",+ "This is the second biggest unknown\n",+ "I imagine it to be easy, but crazier things have happened\n",+ "\n",+ "I need to keep a reference or link to the LLM forge job inside the LM.finetune method"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "how do I get the ray llm image!\n",+ "\n",+ "We'll start by running the rayllm CLI command below to start the workflow to generate the service yaml configuration:\n",+ "```bash\n",+ "mkdir /home/ray/default/deploy/services\n",+ "cd /home/ray/default/deploy/services\n",+ "rayllm gen-config \n",+ "```\n",+ "\n",+ "<img src=\"assets/cli.png\" width=500 alt=\"todo! get this inage of what I need to serve\">\n"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "## Batch offline inference\n",+ "- Compare running inference using \n",+ " - Ray Data \n",+ " - multithreading on local VLLM thru HTTP\n",+ " - Multithreading to Ray Serve instance thru HTTP\n",+ "- Dev time estimate: 7 days"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": []+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "<b style=\"background-color: yellow;\"> 🛑 IMPORTANT </b>: Please `Terminate` your service from the Service page to avoid depleting your free trial credits."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "# Clean up\n",+ "!python src/clear_cell_nums.py\n",+ "!find . | grep -E \".ipynb_checkpoints\" | xargs rm -rf\n",+ "!find . | grep -E \"(__pycache__|\\.pyc|\\.pyo)\" | xargs rm -rf\n",+ "!rm -rf __pycache__ data .HF_TOKEN deploy/services"+ ]+ }+ ],+ "metadata": {+ "kernelspec": {+ "display_name": "base",+ "language": "python",+ "name": "python3"+ },+ "language_info": {+ "codemirror_mode": {+ "name": "ipython",+ "version": 3+ },+ "file_extension": ".py",+ "mimetype": "text/x-python",+ "name": "python",+ "nbconvert_exporter": "python",+ "pygments_lexer": "ipython3",+ "version": "3.11.9"+ }+ },+ "nbformat": 4,+ "nbformat_minor": 2+}diff --git a/examples/anyscale/e2e/sampling_curve.png b/examples/anyscale/e2e/sampling_curve.pngnew file mode 100644index 000000000..60049205fBinary files /dev/null and b/examples/anyscale/e2e/sampling_curve.png differdiff --git a/examples/anyscale/utils/ft.py b/examples/anyscale/e2e/utils/ft.pysimilarity index 100%rename from examples/anyscale/utils/ft.pyrename to examples/anyscale/e2e/utils/ft.pydiff --git a/examples/anyscale/utils/ft.yaml b/examples/anyscale/e2e/utils/ft.yamlsimilarity index 100%rename from examples/anyscale/utils/ft.yamlrename to examples/anyscale/e2e/utils/ft.yamldiff --git a/examples/anyscale/job_runner_config.yaml b/examples/anyscale/job_runner_config.yamlindex 7e7611368..44e789873 100644--- a/examples/anyscale/job_runner_config.yaml+++ b/examples/anyscale/job_runner_config.yaml@@ -1,4 +1,4 @@-entrypoint: llmforge anyscale finetune model_config_dspy_2600252322592036946.yaml+entrypoint: llmforge anyscale finetune model_config_dspy_1711773604051130730.yamlimage_uri: localhost:5555/anyscale/llm-forge:0.5.5name: dspy-llmforge-fine-tuning-jobworking_dir: .diff --git a/examples/anyscale/just_get_working/e2e_dspy.ipynb b/examples/anyscale/just_get_working/e2e_dspy.ipynbdeleted file mode 100644index 27025179b..000000000--- a/examples/anyscale/just_get_working/e2e_dspy.ipynb+++ /dev/null@@ -1,854 +0,0 @@-{- "cells": [- {- "cell_type": "markdown",- "metadata": {},- "source": [- "# End-to-end DSPy Workflows Guide "- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": [- "- **Setup and Data Gathering**: gather and preprocess a dataset for a finetuning a DSPy pipeline using LLaMa 3 70B\n",- "- **Fine-tuning**: tune a smaller (LLama 3 8B) LLM (LoRA / full param) on the pipeline\n",- "- **Serving**: serve the pipeline as a production application that can autoscale, etc.\n",- "- **Evaluation**: apply batch offline inference with Ray data and VLLM to quickly evaluate the pipeline."- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": [- "## Set up"- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": [- "Node Set up:\n",- "\n",- "We will be running everything on a head node that uses 8xL4 GPUs. I find that they are usually available and suitable for this usecase. You can also use any more powerful node.\n",- "\n",- "To change to use L4 GPUs, click the \"1 active node\" in the top right corner, then for workspace node, click the pencil icon and navigate to the L4 tab and select the 8xL4 option. If you do not see L4 in the list of GPUs, they may not be available on your cloud. Choose another kind of GPU (This notebook has been tested on X, and Y as alternatives) (TODO)"- ]- },- {- "cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],- "source": [- "# TODO(work): DSPy installation cell\n",- "# TODO(decision): are these changes going to be merged into DSPy main\n",- "\n",- "# Either pull from pip or install locally if using a non-public version\n",- "\n",- "# !pip install -e dspy-d\n",- "# !pip install -r dspy-d/requirements.txt\n",- "# !pip install vllm"- ]- },- {- "cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],- "source": [- "import dspy\n",- "import dsp\n",- "import os\n",- "\n",- "# TODO: include cache in notebook\n",- "cache_dir = \"/home/ray/default/dspy_cache\"\n",- "if not os.path.exists(cache_dir):\n",- " os.makedirs(cache_dir)\n",- " \n",- "os.environ[\"DSP_CACHEDIR\"] = cache_dir"- ]- },- {- "cell_type": "code",- "execution_count": 2,- "metadata": {},- "outputs": [- {- "ename": "NameError",- "evalue": "name 'os' is not defined",- "output_type": "error",- "traceback": [- "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",- "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)",- "Cell \u001b[0;32mIn[2], line 17\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;66;03m# Here are the different options for setting the OPENAI_API_KEY.\u001b[39;00m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;66;03m# (1) If you are running this notebook locally, you can set it in your shell:\u001b[39;00m\n\u001b[1;32m 3\u001b[0m \u001b[38;5;66;03m# export OPENAI_API_KEY=<YOUR_OPENAI_API_KEY>\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 15\u001b[0m \n\u001b[1;32m 16\u001b[0m \u001b[38;5;66;03m# Ensure that the OPENAI_API_KEY is set before continuing\u001b[39;00m\n\u001b[0;32m---> 17\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[43mos\u001b[49m\u001b[38;5;241m.\u001b[39menviron[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mOPENAI_API_KEY\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[1;32m 18\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m os\u001b[38;5;241m.\u001b[39menviron[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mAPI_BASE\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[1;32m 19\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m os\u001b[38;5;241m.\u001b[39menviron[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mDSP_CACHEDIR\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n",- "\u001b[0;31mNameError\u001b[0m: name 'os' is not defined"- ]- }- ],- "source": [- "# Here are the different options for setting the OPENAI_API_KEY.\n",- "# (1) If you are running this notebook locally, you can set it in your shell:\n",- "# export OPENAI_API_KEY=<YOUR_OPENAI_API_KEY>\n",- "\n",- "# (2) If you are running this notebook in a Google Colab, you can set it in\n",- "# the secrets panel and import by uncommenting the following lines:\n",- "# from google.colab import userdata\n",- "# os.environ[\"OPENAI_API_KEY\"] = userdata.get(\"OPENAI_API_KEY\")\n",- "\n",- "# (3) You can set it directly in the notebook by uncommenting the following\n",- "# line with the appropriate key. If you pick this option, make sure to\n",- "# remove your key before sharing your notebook or committing to a\n",- "# repository.\n",- "# os.environ[\"OPENAI_API_KEY\"] = <OPENAI_API_KEY>\n",- "\n",- "\n",- "necessary_env_vars = [\n",- " # \"OPENAI_API_KEY\",\n",- " # \"API_BASE\",\n",- " \"DSP_CACHEDIR\",\n",- " \"HF_TOKEN\"\n",- "]\n",- "\n",- "for var in necessary_env_vars:\n",- " assert os.environ[var], f\"{var} is not set\""- ]- },- {- "cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],- "source": [- "import ray\n",- "\n",- "if not ray.is_initialized():\n",- " ray.init(runtime_env={\"env_vars\": os.environ, \"py_modules\": [dspy, dsp]})"- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": [- "Now we need to download the models that we will use for this notebook.\n",- "\n",- "In this case, it is the Instruct version of the Meta Llama 3 70B and 8B models.\n",- "\n",- "We can use the public Anyscale S3 bucket to download the models. We will use the cluster storage as the download location. (TODO: Is cluster storage okay for this?)"- ]- },- {- "cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],- "source": [- "!aws s3 cp s3://large-dl-models-mirror/models--meta-llama--Meta-Llama-3-70B-Instruct/main-safetensors/ /mnt/cluster_storage/meta-llama--Meta-Llama-3-70B-Instruct --recursive\n",- "!aws s3 cp s3://large-dl-models-mirror/models--meta-llama--Meta-Llama-3-8B-Instruct/main-safetensors/ /mnt/cluster_storage/meta-llama--Meta-Llama-3-8B-Instruct --recursive"- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": [- "We will make use of a random number generator in this notebook. We are creating a Random object here to ensure that our notebook is reproducible."- ]- },- {- "cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],- "source": [- "import random\n",- "\n",- "rng = random.Random()"- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": [- "# Data preparation"- ]- },- {- "cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],- "source": [- "from dspy.datasets import HotPotQA\n",- "from dspy.evaluate import Evaluate\n",- "from dsp.utils.utils import deduplicate\n",- "\n",- "\n",- "# We are setting the experimental flag to True to make use of the fine-tuning\n",- "# features that are still in development.\n",- "dspy.settings.configure(experimental=True)\n",- "\n",- "# Define the program\n",- "class BasicMH(dspy.Module):\n",- " def __init__(self, passages_per_hop=3, num_hops=2):\n",- " super().__init__()\n",- " self.num_hops = num_hops\n",- " self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",- " self.generate_query = [dspy.ChainOfThought(\"context, question -> search_query\") for _ in range(self.num_hops)]\n",- " self.generate_answer = dspy.ChainOfThought(\"context, question -> answer\")\n",- " \n",- " def forward(self, question):\n",- " context = []\n",- " \n",- " for hop in range(self.num_hops):\n",- " search_query = self.generate_query[hop](context=context, question=question).search_query\n",- " passages = self.retrieve(search_query).passages\n",- " context = deduplicate(context + passages)\n",- "\n",- " answer = self.generate_answer(context=context, question=question).copy(context=context)\n",- " return answer"- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": [- "Lets breakdown what BasicMH does\n",- "\n",- "TODO: add explanation of what each part does"- ]- },- {- "cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],- "source": [- "# Prepare the dataset\n",- "TRAIN_SIZE = 1000\n",- "DEV_SIZE = 500\n",- "dataset = HotPotQA(train_seed=1, eval_seed=2023, test_size=0, only_hard_examples=True)\n",- "trainset = [x.with_inputs('question') for x in dataset.train][:TRAIN_SIZE]\n",- "devset = [x.with_inputs('question') for x in dataset.dev][:DEV_SIZE]"- ]- },- {- "cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],- "source": [- "# Prepare the metric and evaluator\n",- "NUM_THREADS = 12\n",- "metric = dspy.evaluate.answer_exact_match\n",- "evaluate = Evaluate(devset=devset, metric=metric, num_threads=NUM_THREADS, display_progress=True)"- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": [- "TODO(optional): Discuss LLM as judge vs exact match"- ]- },- {- "cell_type": "code",- "execution_count": 3,- "metadata": {},- "outputs": [- {- "ename": "NameError",- "evalue": "name 'dspy' is not defined",- "output_type": "error",- "traceback": [- "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",- "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)",- "Cell \u001b[0;32mIn[3], line 4\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;66;03m# Prepare the retriever model\u001b[39;00m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;66;03m# Note that this is not hosted on Anyscale on purpose to represent a real-world scenario where you arent hosting your DB on Anyscale\u001b[39;00m\n\u001b[1;32m 3\u001b[0m COLBERT_V2_ENDPOINT \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhttp://20.102.90.50:2017/wiki17_abstracts\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m----> 4\u001b[0m retriever \u001b[38;5;241m=\u001b[39m \u001b[43mdspy\u001b[49m\u001b[38;5;241m.\u001b[39mColBERTv2(url\u001b[38;5;241m=\u001b[39mCOLBERT_V2_ENDPOINT)\n",- "\u001b[0;31mNameError\u001b[0m: name 'dspy' is not defined"- ]- }- ],- "source": [- "# Prepare the retriever model\n",- "# Note that this is not hosted on Anyscale on purpose to represent a real-world scenario where you arent hosting your DB on Anyscale\n",- "COLBERT_V2_ENDPOINT = \"http://20.102.90.50:2017/wiki17_abstracts\"\n",- "retriever = dspy.ColBERTv2(url=COLBERT_V2_ENDPOINT)"- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": [- "## Gathering baseline performance\n",- "\n",- "run evaluate on a base pipeline"- ]- },- {- "cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],- "source": [- "MAX_TOKENS = 1024\n",- "MODEL_PARAMETERS = {\n",- " \"max_tokens\": MAX_TOKENS,\n",- " \"temperature\": 0,\n",- "}\n",- "\n",- "vanilla_program = BasicMH()"- ]- },- {- "cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],- "source": []- },- {- "cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],- "source": [- "from math import ceil\n",- "from deploy_dspy.async_llm import \n",- "def evaluate_ray(program, devset, model, tokenizer, dspy_context_kwargs, tensor_parallel_size, num_gpus):\n",- " # need to get number of GPUs somehow\n",- " concurrency = num_gpus\n",- " count = devset.count()\n",- " batch_size = ceil(count / concurrency)\n",- " print(devset.map_batches(DSPyActor, \n",- " batch_size=batch_size,\n",- " num_gpus=1,\n",- " concurrency=concurrency,\n",- " fn_constructor_kwargs={\"batch_size\": batch_size}\n",- " ).take_all()"- ]- },- {- "cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],- "source": [- "# Let's first get a baseline with the 8B model\n",- "from deploy_dspy.async_llm import AsyncLLMWrapper\n",- "from transformers import AutoTokenizer\n",- "# We first need to create an instance of the LLM in order for DSPy to interact with it\n",- "# We will discuss it in detail later, but this is an opportunity to take advantage of Anyscale specific integrations with DSPy and use the new VLLMOfflineEngine class. Because we have 8 GPUs and each one can fit a model instance, we can use 8 instances of the 8B model to run inference in parallel."- ]- },- {- "cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],- "source": [- "llama_8b = dspy.VLLMOfflineEngine(model=\"meta-llama/Meta-Llama-3-8B\", async_mode=False, **MODEL_PARAMETERS)\n",- "\n",- "with dspy.context(lm=llama_8b, rm=retriever):\n",- " print(\"Evaluating the vanilla program on the devset using the model to be trained (llama 8B)...\")\n",- " vanilla_8b_base_eval = evaluate(vanilla_program)"- ]- },- {- "cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],- "source": [- "# Now we will \n",- "\n",- "llm = AsyncLLMWrapper(\"/mnt/cluster_storage/hf/cache/Meta-Llama-3-70B-Instruct\",\n",- " max_pending_requests=512,\n",- " tokenizer=AutoTokenizer.from_pretrained(\"meta-llama/Meta-Llama-3-70B-Instruct\"),\n",- " enforce_eager=True,\n",- " engine_use_ray=False,\n",- " worker_use_ray=False,\n",- " enable_prefix_caching=True,\n",- " tensor_parallel_size=8\n",- " )"- ]- },- {- "cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],- "source": [- "# Prepare the language model\n",- "llama_70b = dspy.VLLMOfflineEngine(model=\"meta-llama/Meta-Llama-3-70B\", async_mode=False, **MODEL_PARAMETERS)\n",- "\n",- "with dspy.context(lm=llama_70b, rm=retriever):\n",- " print(\"Evaluating the vanilla program on the devset using llama 70B...\")\n",- " vanilla_70b_base_eval = evaluate(vanilla_program)"- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": [- "We hope to bring the 8B performance up to at least 70B level"- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": [- "## Optimizing the LLaMa 70B pipeline\n",- "\n",- "run BSFS and MIPROv2 on the pipeline with the playground model, choose whichever one gets the best performance\n",- "\n",- "TODO: add in MIPROv2 ( If desired)"- ]- },- {- "cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],- "source": [- "# Optimization hyperparameters\n",- "from dspy.teleprompt.random_search import BootstrapFewShotWithRandomSearch\n",- "\n",- "# Define the hyperparameters for prompt optimization\n",- "MAX_BOOTSTRAPPED_DEMOS = 3\n",- "MAX_LABELED_DEMOS = 3\n",- "NUM_CANDIDATE_PROGRAMS = 6\n",- "OPTIMIZER_NUM_TRAIN = 100\n",- "OPTIMIZER_NUM_VAL = 150"- ]- },- {- "cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],- "source": [- "# Prepare the training and validation sets for the optimizer using the original\n",- "# trainset. This ensures that our \"devset\" is left untouched.\n",- "shuffled_trainset = [d for d in trainset]\n",- "rng.shuffle(shuffled_trainset)\n",- "optimizer_trainset = shuffled_trainset[:OPTIMIZER_NUM_TRAIN]\n",- "optimizer_valset = shuffled_trainset[OPTIMIZER_NUM_TRAIN:OPTIMIZER_NUM_TRAIN+OPTIMIZER_NUM_VAL]"- ]- },- {- "cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],- "source": [- "# Initialize the optimizer\n",- "bfrs_optimizer = BootstrapFewShotWithRandomSearch(\n",- " metric=metric,\n",- " max_bootstrapped_demos=MAX_BOOTSTRAPPED_DEMOS,\n",- " max_labeled_demos=MAX_LABELED_DEMOS,\n",- " num_candidate_programs=NUM_CANDIDATE_PROGRAMS,\n",- " num_threads=NUM_THREADS\n",- ")\n",- "\n",- "# Compile the optimizer and evaluate\n",- "with dspy.context(lm=llama_70b, rm=retriever):\n",- " vanilla_program = BasicMH()\n",- " bfrs_base_program = bfrs_optimizer.compile(vanilla_program, trainset=optimizer_trainset, valset=optimizer_valset)\n",- " bfrs_base_eval = evaluate(bfrs_base_program, devset=devset)"- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": [- "To get a baseline for the finetuned model, we can run BSFS on the vanilla program with the smaller 8B model"- ]- },- {- "cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],- "source": [- "with dspy.context(lm=llama_8b, rm=retriever):\n",- " vanilla_program = BasicMH()\n",- " bfrs_8b_program = bfrs_optimizer.compile(vanilla_program, trainset=optimizer_trainset, valset=optimizer_valset)\n",- " bfrs_8b_eval = evaluate(bfrs_8b_program, devset=devset)"- ]- },- {- "cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],- "source": [- "raise SystemExit(\"Stop right there!\")"- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": [- "### Bootstrap Data\n",- "\n",- "\n",- "In this section, we bootstrap data for fine-tuning. In the code block below, we are deciding which program should be used to collect the bootstraps. We are setting this to the prompt optimized program, but one could also set this to the vanilla program, though doing so would lead to lower quality bootstraps."- ]- },- {- "cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],- "source": [- "bootstrap_program = bfrs_base_program"- ]- },- {- "cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],- "source": [- "from dspy.teleprompt.finetune_teleprompter import bootstrap_data\n",- "\n",- "\n",- "# Bootstrap traces using the prompt optimized program\n",- "# TODO: Change to using the version with multiple retries\n",- "with dspy.context(lm=llama_70b, rm=retriever):\n",- " bootstrapped_data = bootstrap_data(program=bootstrap_program, dataset=trainset, metric=metric, num_threads=NUM_THREADS)\n",- "\n",- "# The bootstrap_data function returns a list of dictionaries, each dictionary\n",- "# corresponding to an exmaple in the data, with the the following keys and\n",- "# values:\n",- "# - The `example` field corresponding to the example itself\n",- "# - The `prediction` field corresponding to the prediction made by the program\n",- "# on the example.\n",- "# - The `trace` field corresponding to the trace generated by the program on the\n",- "# example.\n",- "# - The `score` field corresponding to the metric score of the example, if the\n",- "# metric is provided. Otherwise, it is not included in the data.\n",- "print(f\"\\nTrainset has {len(trainset)} examples.\")\n",- "print(f\"Bootstrapped data has {len(bootstrapped_data)} examples.\")\n",- "print(f\"Bootstrapped data dictionaries have the following keys: {list(bootstrapped_data[0].keys())}\")"- ]- },- {- "cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],- "source": [- "rng.shuffle(bootstrapped_data)"- ]- },- {- "cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],- "source": [- "# Filter the bootstrapped data to only include examples where the metric score\n",- "# is 1\n",- "filtered_bootstrapped_data = [d for d in bootstrapped_data if d['score']]\n",- "print(f\"Filtered bootstrapped data has {len(filtered_bootstrapped_data)} examples.\")"- ]- },- {- "cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],- "source": [- "from dspy.teleprompt.finetune_teleprompter import convert_to_module_level_prompt_completion_data\n",- "\n",- "# To train a model on our program traces, we need a dataset that consists of\n",- "# the prompts and the completions for each module in our program. We can use the\n",- "# helper method `convert_to_prompt_completion_data` to generate these traces.\n",- "# This function takes in a list of dictionaries, each of which must contain the\n",- "# \"trace\" field, which is used for generating the prompts and completions for\n",- "# each module in the program. The function returns a list of dictionaries, each\n",- "# of which contains the keys \"prompt\" and \"completion\", among others. Refer to\n",- "# the documentation for more information.\n",- "#\n",- "# The `exclude_demos` flag controls whether the demonstrations should be\n",- "# included in the prompts. We are setting it to True here to have the \"vanilla\"\n",- "# prompts as part of our finetuning data.\n",- "finetune_data = convert_to_module_level_prompt_completion_data(filtered_bootstrapped_data, exclude_demos=True)\n",- "print(f\"For each datapoint, we get one prompt completion pair for each module in our program of {len(vanilla_program.predictors())} modules.\")\n",- "print(f\"As a result, finetune data has {len(filtered_bootstrapped_data)} * {len(bootstrap_program.predictors())} = {len(finetune_data)} examples.\")\n",- "print(f\"The finetune data dictionaries have the following keys: {list(finetune_data[0].keys())}\")"- ]- },- {- "cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],- "source": [- "# Let's look at an example prompt completion pair!\n",- "print(\"Example prompt:\")\n",- "print(finetune_data[0]['prompt'])\n",- "print(\"-\"*50)\n",- "print(\"Example completion:\")\n",- "print(finetune_data[0]['completion'])\n",- "print(\"-\"*50)"- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": [- "# Fine-tuning\n",- "\n",- "Take the best pipeline from the previous step and run BSFT on it with the LLama 3 8B model thru LLM forge.\n",- "\n",- "NOTE: I think this is the part that needs the most work\n",- "\n",- "TODO: Outline what steps are still needed\n",- "\n",- "- Dev time estimate TBD based on outlining steps"- ]- },- {- "cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],- "source": [- "student_llama_8b = dspy.TrainableAnyscaleLM(model=\"meta-llama/Meta-Llama-3-8B\", **MODEL_PARAMETERS)\n",- "\n",- "# Showing that the model is an instance of the TrainableLM class\n",- "isinstance(student_llama_8b, dspy.TrainableLM)"- ]- },- {- "cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],- "source": [- "import ujson\n",- "\n",- "train_path = \"trainset_data.jsonl\"\n",- "\n",- "print(f\"Writing dataset with length {len(finetune_data)} to {train_path}\")\n",- "with open(train_path, \"w\") as f:\n",- " ujson.dump(finetune_data, f)"- ]- },- {- "cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],- "source": [- "from dsp.modules.lm import TrainingMethod\n",- "\n",- "USE_CACHED_MODEL = False\n",- "\n",- "if USE_CACHED_MODEL:\n",- " CACHED_FINETUNED_MODEL_ID = \"\"\n",- " # Support this case\n",- " finetuned_lm = dspy.TrainableAnyscaleLM(model=CACHED_FINETUNED_MODEL_ID, **MODEL_PARAMETERS)\n",- "else:\n",- " # Define the hyperparameters for finetuning. To look at the specifics of\n",- " # what kinds of arguments should be passed to `get_finetune`, refer to the\n",- " # documentation for the particular subclass you are using, in this case,\n",- " # `dspy.TrainableAnyscaleLM`.\n",- " hyperparameters = {\n",- " \"n_epochs\": 3\n",- " }\n",- "\n",- " # Select the training method. Must be one of the options in the\n",- " # `TrainingMethod` enum.\n",- " method = TrainingMethod.SFT\n",- "\n",- " # Get a future object for the finetuned language model. This will get populated\n",- " # with the finetuned model once the training is complete.\n",- " # TODO: This is a big point of work\n",- " # Need to grab this from the other workspace\n",- " future_finetuned_lm = student_llama_8b.get_finetune(\n",- " method=method,\n",- " train_path=train_path,\n",- " hyperparameters=hyperparameters\n",- " )\n",- "\n",- " # Call the result method on the future object to get the finetuned language.\n",- " # This will block the execution until the training is complete.\n",- " finetuned_lm = future_finetuned_lm.result()\n",- "\n",- "# Record the model ID to access the model again in the future. You can directly\n",- "# substitute this model ID for the `CACHED_FINETUNED_MODEL_ID` variable above,\n",- "# and the next line will load the model directly.\n",- "finetuned_lm.kwargs['model']"- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": [- "# Evaluation\n",- "\n",- "Throughout this section, anything using 8B model (or technically 70B too) should use the new evaluate with ray data batch offline(or technically online) inference.\n",- "\n",- "Probably worth testing offline with 8x8 threads vs just 64 threads to see if it makes a meaningful difference.\n",- "\n",- "## Performance comparisons\n",- "\n",- "- 70B\n",- "- 70B BSFS\n",- "- 8B\n",- "- 8B BSFT\n",- "- 8B BSFT + BSFS\n",- "\n",- "---\n",- "- Note: Allegedly really easy\n",- "- Dev time estimate: 1 day"- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": [- "Let's see how well the vanilla program performs with the finetuned model"- ]- },- {- "cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],- "source": [- "with dspy.context(lm=finetuned_lm, rm=retriever):\n",- " vanilla_program = BasicMH()\n",- " vanilla_finetuned_eval = evaluate(vanilla_program, devset=devset)"- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": [- "Now let's try optimizing the program with the finetuned model"- ]- },- {- "cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],- "source": [- "with dspy.context(lm=finetuned_lm, rm=retriever):\n",- " vanilla_program = BasicMH()\n",- " bfrs_finetuned_program = bfrs_optimizer.compile(vanilla_program, trainset=optimizer_trainset, valset=optimizer_valset)\n",- " bfrs_finetuned_eval = evaluate(bfrs_finetuned_program, devset=devset)"- ]- },- {- "cell_type": "code",- "execution_count": 4,- "metadata": {},- "outputs": [- {- "ename": "NameError",- "evalue": "name 'trainset' is not defined",- "output_type": "error",- "traceback": [- "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",- "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)",- "Cell \u001b[0;32mIn[4], line 2\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;66;03m# Now we can compare all iterations of this pipeline\u001b[39;00m\n\u001b[0;32m----> 2\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mResults for HotPotQA fine-tuning with a starting trainset with \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mlen\u001b[39m(\u001b[43mtrainset\u001b[49m)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m. Out of the \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mlen\u001b[39m(trainset)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m examples, only \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mlen\u001b[39m(filtered_bootstrapped_data)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m examples were judged as successful using a prompt optimized with BootstrapFewShotRandomSearch prompt optimized (bfrs) program on the base model. Traces from these examples are used for SFT. The reported results are computed on a held-out devset of \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mlen\u001b[39m(devset)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m examples.\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 3\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m Base model (vanilla program): \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mvanilla_base_eval\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 4\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m Base model (bfrs program): \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mbfrs_base_eval\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n",- "\u001b[0;31mNameError\u001b[0m: name 'trainset' is not defined"- ]- }- ],- "source": [- "# Now we can compare all iterations of this pipeline\n",- "print(f\"Results for HotPotQA fine-tuning LLaMa 8Bwith a starting trainset with {len(trainset)}. Out of the {len(trainset)} examples, only {len(filtered_bootstrapped_data)} examples were judged as successful using a prompt optimized with BootstrapFewShotRandomSearch prompt optimized (bfrs) program on the base model. Traces from these examples are used for SFT. The reported results are computed on a held-out devset of {len(devset)} examples.\\n\")\n",- "print(f\" 70B model (vanilla program): {vanilla_70b_base_eval}\")\n",- "print(f\" 70B model (bfrs program): {bfrs_base_eval}\")\n",- "print(f\" 8B model (vanilla program): {vanilla_8b_base_eval}\")\n",- "print(f\" 8B model (bfrs program): {bfrs_8b_eval}\")\n",- "print(f\" 8B model (finetuned program): {vanilla_finetuned_eval}\")\n",- "print(f\" 8B model (finetuned bfrs program): {bfrs_finetuned_eval}\")"- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": [- "Let's now use the new offline batch inference to evaluate the finetuned model with optimized program on the entire devset"- ]- },- {- "cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],- "source": [- "# TODO: implement once done"- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": [- "# Serving\n",- "\n",- "This is the second biggest unknown\n",- "I imagine it to be easy, but crazier things have happened\n",- "\n",- "I need to keep a reference or link to the LLM forge job inside the LM.finetune method"- ]- },- {- "cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],- "source": [- "vanilla_program = BasicMH()\n",- "\n",- "with dspy.context(lm=finetuned_lm, rm=retriever):\n",- " print(\"Evaluating the vanilla program on the devset using the finetuned model...\")\n",- " vanilla_finetune_eval = evaluate(vanilla_program, devset=devset)"- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": [- "## Batch offline inference\n",- "- Compare running inference using \n",- " - Ray Data \n",- " - multithreading on local VLLM thru HTTP\n",- " - Multithreading to Ray Serve instance thru HTTP\n",- "- Dev time estimate: 7 days"- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": []- }- ],- "metadata": {- "kernelspec": {- "display_name": "base",- "language": "python",- "name": "python3"- },- "language_info": {- "codemirror_mode": {- "name": "ipython",- "version": 3- },- "file_extension": ".py",- "mimetype": "text/x-python",- "name": "python",- "nbconvert_exporter": "python",- "pygments_lexer": "ipython3",- "version": "3.11.8"- }- },- "nbformat": 4,- "nbformat_minor": 2-}diff --git a/examples/anyscale/just_get_working/scratch_offline_engine_2.py b/examples/anyscale/just_get_working/scratch_offline_engine_2.pynew file mode 100644index 000000000..0d8ba48ba--- /dev/null+++ b/examples/anyscale/just_get_working/scratch_offline_engine_2.py@@ -0,0 +1,54 @@+import dspy+from dspy.datasets import HotPotQA+from dsp.utils import deduplicate+from dspy.teleprompt import BootstrapFewShotWithRandomSearch+from dspy.evaluate import answer_exact_match, Evaluate+from dspy.teleprompt.finetune_teleprompter import convert_to_module_level_prompt_completion_data, bootstrap_data+import json++engine_args = {+ "enforce_eager": True,+ # "engine_use_ray": False,+ # "worker_use_ray": False,+ "enable_prefix_caching": True,+ "tensor_parallel_size": 1,+ "enable_lora": True+ }++NUM_THREADS = 70+lm = dspy.VLLMOfflineEngine3.instantiate_with_llm(model="meta-llama/Meta-Llama-3-8B-Instruct", engine_args=engine_args, batch_size=NUM_THREADS)+COLBERT_V2_ENDPOINT = "http://20.102.90.50:2017/wiki17_abstracts"+rm = dspy.ColBERTv2(url=COLBERT_V2_ENDPOINT)++dspy.settings.configure(lm=lm, rm=rm)++class BasicMH(dspy.Module):+ def __init__(self, passages_per_hop=3, num_hops=2):+ super().__init__()+ self.num_hops = num_hops+ self.retrieve = dspy.Retrieve(k=passages_per_hop)+ self.generate_query = [dspy.ChainOfThought("context, question -> search_query") for _ in range(self.num_hops)]+ self.generate_answer = dspy.ChainOfThought("context, question -> answer")++ def forward(self, question):+ context = []++ for hop in range(self.num_hops):+ search_query = self.generate_query[hop](context=context, question=question).search_query+ passages = self.retrieve(search_query).passages+ context = deduplicate(context + passages)++ answer = self.generate_answer(context=context, question=question).copy(context=context)+ return answer++predict = BasicMH()++# print(predict(question="what is the capital of France?"))++dataset = HotPotQA(train_seed=1, eval_seed=2023, test_size=0, only_hard_examples=True)++trainset = [x.with_inputs('question') for x in dataset.train]+devset = [x.with_inputs('question') for x in dataset.dev]++evaluate = Evaluate(devset=devset[:100], metric=answer_exact_match, num_threads=NUM_THREADS, display_progress=True)+evaluate(predict)\ No newline at end of file