Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Run full eval on valset not trainset

Michael Ryan3d024bae875d
diff --git a/dspy/teleprompt/mipro_optimizer_v2.py b/dspy/teleprompt/mipro_optimizer_v2.py
index f5c7043d5..fa4293d00 100644
--- a/dspy/teleprompt/mipro_optimizer_v2.py
+++ b/dspy/teleprompt/mipro_optimizer_v2.py
@@ -386,8 +386,10 @@ class MIPROv2(Teleprompter):
# Evaluate the candidate program with relevant batch size
batch_size = minibatch_size if minibatch else len(trainset)
+ evalset = trainset if minibatch else valset
+
score = eval_candidate_program(
- batch_size, trainset, candidate_program,
+ batch_size, evalset, candidate_program,
evaluate,
)
diff --git a/testing/log_dir/HotPotQA_default_20240916_121736/evaluated_programs/program_6 b/testing/log_dir/HotPotQA_default_20240916_121736/evaluated_programs/program_6
new file mode 100644
index 000000000..4998543b7
--- /dev/null
+++ b/testing/log_dir/HotPotQA_default_20240916_121736/evaluated_programs/program_6
@@ -0,0 +1,130 @@
+{
+ "retrieve": {
+ "k": 3
+ },
+ "generate_query": {
+ "lm": null,
+ "traces": [],
+ "train": [],
+ "demos": [],
+ "signature_instructions": "Given the fields `context`, `question`, produce the fields `search_query`.",
+ "signature_prefix": "Search Query:",
+ "extended_signature_instructions": "Please generate a search query that accurately captures the essence of the question, considering the context provided, and is likely to yield relevant results. The search query should be concise, clear, and free of ambiguity. Please keep in mind that the goal is to find the answer to the question, and the search query should be designed to facilitate this process.\".",
+ "extended_signature_prefix": "Search Query:"
+ },
+ "generate_answer": {
+ "lm": null,
+ "traces": [],
+ "train": [],
+ "demos": [
+ {
+ "augmented": true,
+ "context": [
+ "Fig Trees | Fig Trees is a 2009 Canadian operatic documentary film written and directed by John Greyson. It follows South African AIDS activist Zackie Achmat and Canadian AIDS activist Tim McCaskell as they fight for access to treatment for HIV\/AIDS. It was also inspired by Gertrude Stein and Virgil Thomson's opera \"Four Saints in Three Acts\". The film premiered at the 59th Berlin International Film Festival where it won the Teddy Award for Best Documentary.",
+ "Pond Hockey (film) | Pond Hockey is a 2008 American documentary film, directed by Tommy Haines, and produced by Northland Films. The film is an examination of the changing culture of pond hockey.",
+ "Fig Leaves | Fig Leaves is a 1926 American silent comedy film directed by Howard Hawks, released by Fox Film Corporation, and starring George O'Brien and Olive Borden. The film had a sequence, a fashion show, filmed in Technicolor. A print of the film survives in the film archive of the Museum of Modern Art.",
+ "Pond Hockey (film) | Pond Hockey is a 2008 American documentary film, directed by Tommy Haines, and produced by Northland Films. The film is an examination of the changing culture of pond hockey.",
+ "When Ponds Freeze Over | When Ponds Freeze Over is a Canadian short film, directed by Mary Lewis and released in 1998.",
+ "In the Crease | In the Crease (the youth hockey experience) is a feature-length documentary film that follows the real-life story of a teenage hockey team's quest to win a national championship and also stars over a dozen NHL players sharing their own hockey triumphs. The film was produced and directed by Matthew Gannon and Michael Sarner, distributed by Stickmen Pictures LLC, aired on Versus on January 20, 2008 and was released on DVD on October 26, 2006."
+ ],
+ "question": "Which film was created first, Fig Trees or Pond Hockey?",
+ "rationale": "determine which film was created first. We can look at the release dates of the films. According to the context, Fig Trees was released in 2009, while Pond Hockey was released in 2008. Therefore, Pond Hockey was created first.",
+ "answer": "Pond Hockey"
+ },
+ {
+ "question": "What head of state position was held by Harry S Truman when he gave Harold E Wilson the Medal of Honor?",
+ "answer": "President of the United States",
+ "dspy_uuid": "6fffe5a1-916d-4ac1-b39e-2d9993f65055",
+ "dspy_split": "train"
+ },
+ {
+ "question": "Are the Carlisle Cathedral and Bangor Cathedral both located in the United Kingdom?",
+ "answer": "yes",
+ "dspy_uuid": "ed690e53-ca05-4df0-8acb-e9ac900e3e1d",
+ "dspy_split": "train"
+ },
+ {
+ "question": "Tombstone stared an actor born May 17, 1955 known as who?",
+ "answer": "Bill Paxton",
+ "dspy_uuid": "750d8ea1-9ba7-4f2b-b8c7-4705d55e23e2",
+ "dspy_split": "train"
+ },
+ {
+ "question": "Wing warping, a system for lateral aircraft control, was pioneered by the Wright brothers during their first controlled, sustained flight near which now-famous location?",
+ "answer": "Kitty Hawk, North Carolina",
+ "dspy_uuid": "015071db-85e3-4936-8071-2070b0f5b9dd",
+ "dspy_split": "train"
+ },
+ {
+ "question": "Who has been on a British television music competition show and was was most popular in the 80's with the pop band 'Culture Club'?",
+ "answer": "George Alan O'Dowd",
+ "dspy_uuid": "82cc66f2-ac13-48f4-9bb5-e6763f685acd",
+ "dspy_split": "train"
+ },
+ {
+ "question": "Johnny Narron is a friend of this center fielder whom he helped in his recovery from drug addiction who won the MVP award in what year?",
+ "answer": "2010",
+ "dspy_uuid": "d2ad5b70-73ad-46f7-9e0b-a713992dabff",
+ "dspy_split": "train"
+ },
+ {
+ "question": "Who composed the song from the film \"Zindagi Na Milegi Dobara\" that was shot in the Spanish town of Alajar?",
+ "answer": "Javed Akhtar",
+ "dspy_uuid": "419d7dc2-6a55-4830-b3d8-6653eb8f1345",
+ "dspy_split": "train"
+ },
+ {
+ "question": "Which movie was released first, Son of Flubber or Davy Crockett, King of the Wild Frontier?",
+ "answer": "Davy Crockett, King of the Wild Frontier",
+ "dspy_uuid": "8eec55a6-7cdf-4f09-b9e9-c173ff065f72",
+ "dspy_split": "train"
+ },
+ {
+ "question": "Who released the album that \"Stylo\" was a single on in the United States?",
+ "answer": "Virgin Records",
+ "dspy_uuid": "ba19746b-f1f2-4c5c-b6e6-9f7a87f1a420",
+ "dspy_split": "train"
+ },
+ {
+ "question": "The second place finisher of the 2011 Gran Premio Santander d'Italia drove for who when he won the 2009 FIA Formula One World Championship?",
+ "answer": "Brawn GP",
+ "dspy_uuid": "3eff0e9a-a9c7-4edc-8b6e-c3e0a17a49ee",
+ "dspy_split": "train"
+ },
+ {
+ "question": "Juan Pablo Di Pace had a role on the sequel series to what Jeff Franklin-created American sitcom?",
+ "answer": "Full House",
+ "dspy_uuid": "4e2ab377-42d7-425b-965d-772ac1f0c029",
+ "dspy_split": "train"
+ },
+ {
+ "question": "Carlo Ignazio Monza's music formed a ballet based on what 18th century play? ",
+ "answer": "Commedia dell'arte",
+ "dspy_uuid": "0ba486b3-6709-4234-969a-e3dcfa16044b",
+ "dspy_split": "train"
+ },
+ {
+ "question": "What part of the world do the Viezenbeek and Lindemans Brewery hale from?",
+ "answer": "Brussels",
+ "dspy_uuid": "1506cb1a-988a-4b62-a286-674f646efa84",
+ "dspy_split": "train"
+ },
+ {
+ "question": "For which college team did this American professional basketball player for the Miami Heat of the National Basketball Association (NBA) who earned praise for being the NBA's leading rebounder during the 2016\u201317 Miami Heat season play?",
+ "answer": "the Marshall Thundering Herd",
+ "dspy_uuid": "6fbe8aa1-e4bb-4abb-b460-cb8f113b6398",
+ "dspy_split": "train"
+ },
+ {
+ "question": "The Sting was an episode of The Office written by the American actress who created what other sitcom?",
+ "answer": "The Mindy Project",
+ "dspy_uuid": "3e33d9ad-acc2-477e-b0ad-88dca44cb807",
+ "dspy_split": "train"
+ }
+ ],
+ "signature_instructions": "Given the fields `context`, `question`, produce the fields `answer`.",
+ "signature_prefix": "Answer:",
+ "extended_signature_instructions": "You are a trivia expert. Given the context and question, generate the answer and provide a brief reasoning to support your answer. Use the context to help you find the answer. For example, if the question is about a character in a book or movie, look for relevant information in the context to help you identify the correct answer.",
+ "extended_signature_prefix": "Answer:"
+ }
+}
\ No newline at end of file