DSPy: The framework for programming—not prompting—language models
Python
Initial commit
Omar Khattabf9d6db96fa65
diff --git a/.gitignore b/.gitignorenew file mode 100644index 000000000..d19467c29--- /dev/null+++ b/.gitignore@@ -0,0 +1,30 @@+/experiments/+/checkpoints/+/data/+/logs/+/mlruns/+/profiler/+/logs/+/docs/downloads/+/docs/experiments/++# Byte-compiled / optimized / DLL files+__pycache__/+*.py[cod]+*$py.class++# Jupyter Notebook+.ipynb_checkpoints+# notebooks/++# mac+.DS_Store++# Other+.vscode+*.tsv+*.pt+*.json+gpt*.txt+*.env+local/\ No newline at end of filediff --git a/README.md b/README.mdnew file mode 100644index 000000000..da84787cd--- /dev/null+++ b/README.md@@ -0,0 +1,26 @@+# 🤝🎓 𝗗𝗦𝗣: 𝗗EMONSTRATE–𝗦EARCH–𝗣REDICT++A framework for composing retrieval models and language models into powerful pipelines that tackle knowledge-intensive tasks.++## 💾 Installation++```pip install .```++## 🏃 Getting Started++Our [intro notebook](intro.ipynb) provides examples of five "multi-hop" question answering programs of increasing complexity written in DSP.++Once you go through the notebook, you'll be ready to create your own DSP pipelines!++## ✍️ Reference++If you use DSP in a research paper, please cite our work as follows:++```+@article{khattab2022demonstrate,+ title={Demonstrate-Search-Predict: Composing Retrieval and Language Models for Knowledge-Intensive {NLP}},+ author={Khattab, Omar and Santhanam, Keshav and Li, Xiang Lisa and Hall, David and Liang, Percy and Potts, Christopher and Zaharia, Matei},+ journal={arXiv preprint arXiv:2212.14024},+ year={2022}+}+```diff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/02d447c34c62da81a2429548a8be5bac/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/02d447c34c62da81a2429548a8be5bac/output.pklnew file mode 100644index 000000000..92e9a9a06Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/02d447c34c62da81a2429548a8be5bac/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/04a418289cf53e61ab1cddf225610389/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/04a418289cf53e61ab1cddf225610389/output.pklnew file mode 100644index 000000000..391220236Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/04a418289cf53e61ab1cddf225610389/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/06d1703048f3c2f80f6d63d10c7367e9/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/06d1703048f3c2f80f6d63d10c7367e9/output.pklnew file mode 100644index 000000000..07ad238bdBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/06d1703048f3c2f80f6d63d10c7367e9/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/07340ee80f4ade05413c56786aa733ee/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/07340ee80f4ade05413c56786aa733ee/output.pklnew file mode 100644index 000000000..0dc6d8a8bBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/07340ee80f4ade05413c56786aa733ee/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/075814d14f7d5be0da690cc77b743b2d/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/075814d14f7d5be0da690cc77b743b2d/output.pklnew file mode 100644index 000000000..2d92e319aBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/075814d14f7d5be0da690cc77b743b2d/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/07787b955212dc715e3a375f07338c05/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/07787b955212dc715e3a375f07338c05/output.pklnew file mode 100644index 000000000..e6a85b712Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/07787b955212dc715e3a375f07338c05/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/086e0acfada14e13859a7cd6597a1405/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/086e0acfada14e13859a7cd6597a1405/output.pklnew file mode 100644index 000000000..1f519eb1fBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/086e0acfada14e13859a7cd6597a1405/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/0ebaa9028705c79e32458a8dde436121/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/0ebaa9028705c79e32458a8dde436121/output.pklnew file mode 100644index 000000000..d46e22b83Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/0ebaa9028705c79e32458a8dde436121/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/0fe464b5d6792de7e0e1b61421184c48/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/0fe464b5d6792de7e0e1b61421184c48/output.pklnew file mode 100644index 000000000..353a99207Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/0fe464b5d6792de7e0e1b61421184c48/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/12e6c79f12634ee15c884c96f0ccc601/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/12e6c79f12634ee15c884c96f0ccc601/output.pklnew file mode 100644index 000000000..487bf4569Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/12e6c79f12634ee15c884c96f0ccc601/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/1bac797a504591624236a3303d182ed6/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/1bac797a504591624236a3303d182ed6/output.pklnew file mode 100644index 000000000..aef2e906fBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/1bac797a504591624236a3303d182ed6/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/1be442ffecb211d74e1ac8073c5fca5f/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/1be442ffecb211d74e1ac8073c5fca5f/output.pklnew file mode 100644index 000000000..31e6ad984Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/1be442ffecb211d74e1ac8073c5fca5f/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/1c80fbce72ee61d55dbf281fcbe46989/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/1c80fbce72ee61d55dbf281fcbe46989/output.pklnew file mode 100644index 000000000..2c1a6dd18Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/1c80fbce72ee61d55dbf281fcbe46989/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/1d07052202b35bdb92e535b2ba20912d/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/1d07052202b35bdb92e535b2ba20912d/output.pklnew file mode 100644index 000000000..ec02fd625Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/1d07052202b35bdb92e535b2ba20912d/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/1de769f204bd83bcdb4ddfc0df891e65/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/1de769f204bd83bcdb4ddfc0df891e65/output.pklnew file mode 100644index 000000000..0033a26a1Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/1de769f204bd83bcdb4ddfc0df891e65/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/1ede3e71a81df5e8ab9f3b1683aeade3/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/1ede3e71a81df5e8ab9f3b1683aeade3/output.pklnew file mode 100644index 000000000..8ec9a62efBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/1ede3e71a81df5e8ab9f3b1683aeade3/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/1f6e6213508fe9d598f3c6915cac9262/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/1f6e6213508fe9d598f3c6915cac9262/output.pklnew file mode 100644index 000000000..f32c4206eBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/1f6e6213508fe9d598f3c6915cac9262/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/205190e3ed9c67eee67be8013c0bb6d9/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/205190e3ed9c67eee67be8013c0bb6d9/output.pklnew file mode 100644index 000000000..f6c7f8606Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/205190e3ed9c67eee67be8013c0bb6d9/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/2080746da03dfab5ebe5e3e4f23dff5d/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/2080746da03dfab5ebe5e3e4f23dff5d/output.pklnew file mode 100644index 000000000..5c16e854fBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/2080746da03dfab5ebe5e3e4f23dff5d/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/2306c19f83985f8330c5a3c7c7537f5e/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/2306c19f83985f8330c5a3c7c7537f5e/output.pklnew file mode 100644index 000000000..cf3cf40d4Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/2306c19f83985f8330c5a3c7c7537f5e/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/23a8ef200742c19b342ee00b97505ee8/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/23a8ef200742c19b342ee00b97505ee8/output.pklnew file mode 100644index 000000000..cd2f172d8Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/23a8ef200742c19b342ee00b97505ee8/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/26d2272f596859489ddd8207bd912b7b/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/26d2272f596859489ddd8207bd912b7b/output.pklnew file mode 100644index 000000000..81dcb94ceBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/26d2272f596859489ddd8207bd912b7b/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/2ca503f36721c83d1b96e706b0bfffb5/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/2ca503f36721c83d1b96e706b0bfffb5/output.pklnew file mode 100644index 000000000..3fb57c73dBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/2ca503f36721c83d1b96e706b0bfffb5/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/2d149eec6b27568b86a8bd7162bb9a4b/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/2d149eec6b27568b86a8bd7162bb9a4b/output.pklnew file mode 100644index 000000000..fe7a49464Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/2d149eec6b27568b86a8bd7162bb9a4b/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/2d2287d68ad94f2041a5c4ff4fc1a397/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/2d2287d68ad94f2041a5c4ff4fc1a397/output.pklnew file mode 100644index 000000000..a8100710dBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/2d2287d68ad94f2041a5c4ff4fc1a397/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/2d25a1d4cb7fab5e4724fd1156fb01ec/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/2d25a1d4cb7fab5e4724fd1156fb01ec/output.pklnew file mode 100644index 000000000..d85084b0fBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/2d25a1d4cb7fab5e4724fd1156fb01ec/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/2d9ec6c75cb00760f06f8d2b86127673/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/2d9ec6c75cb00760f06f8d2b86127673/output.pklnew file mode 100644index 000000000..ce1c32ee4Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/2d9ec6c75cb00760f06f8d2b86127673/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/2f7269f24a1eb0fae3e606be17879fa4/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/2f7269f24a1eb0fae3e606be17879fa4/output.pklnew file mode 100644index 000000000..ed3ff7b39Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/2f7269f24a1eb0fae3e606be17879fa4/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/2fff163014e9aab20020dac4eadc762e/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/2fff163014e9aab20020dac4eadc762e/output.pklnew file mode 100644index 000000000..f18422233Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/2fff163014e9aab20020dac4eadc762e/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/33ccd298b84a2a558bbc10d2b99355d5/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/33ccd298b84a2a558bbc10d2b99355d5/output.pklnew file mode 100644index 000000000..ed3ff7b39Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/33ccd298b84a2a558bbc10d2b99355d5/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/3760a471c440877ba8d7f5502192c5f9/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/3760a471c440877ba8d7f5502192c5f9/output.pklnew file mode 100644index 000000000..a6eccd630Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/3760a471c440877ba8d7f5502192c5f9/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/3902c3a0d5869c78427180ccd94443ba/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/3902c3a0d5869c78427180ccd94443ba/output.pklnew file mode 100644index 000000000..ffb9281d5Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/3902c3a0d5869c78427180ccd94443ba/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/392129b4c4cb9cfeb7dbb75c2f6a9de1/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/392129b4c4cb9cfeb7dbb75c2f6a9de1/output.pklnew file mode 100644index 000000000..64ec7629aBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/392129b4c4cb9cfeb7dbb75c2f6a9de1/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/3a5b1b47a1c082ec998a2d3931c21259/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/3a5b1b47a1c082ec998a2d3931c21259/output.pklnew file mode 100644index 000000000..a3b42bd9aBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/3a5b1b47a1c082ec998a2d3931c21259/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/3b7a00f5dc49133a1bd83bb7b323f034/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/3b7a00f5dc49133a1bd83bb7b323f034/output.pklnew file mode 100644index 000000000..4d7878f1dBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/3b7a00f5dc49133a1bd83bb7b323f034/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/3bc6aa5a21c57829861d45421b284d1e/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/3bc6aa5a21c57829861d45421b284d1e/output.pklnew file mode 100644index 000000000..c60a203e9Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/3bc6aa5a21c57829861d45421b284d1e/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/4351507a3774fb0b329597dbeb6adc6e/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/4351507a3774fb0b329597dbeb6adc6e/output.pklnew file mode 100644index 000000000..dbe97c9ddBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/4351507a3774fb0b329597dbeb6adc6e/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/44ae442020661bb94788e752653b1436/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/44ae442020661bb94788e752653b1436/output.pklnew file mode 100644index 000000000..03f8bb101Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/44ae442020661bb94788e752653b1436/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/483fd7cb0f67da5cc9a3cdb36313a08c/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/483fd7cb0f67da5cc9a3cdb36313a08c/output.pklnew file mode 100644index 000000000..e7e55aff7Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/483fd7cb0f67da5cc9a3cdb36313a08c/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/4e4eb62bf02ee33b854d4a70a9cccd51/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/4e4eb62bf02ee33b854d4a70a9cccd51/output.pklnew file mode 100644index 000000000..0c05098a2Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/4e4eb62bf02ee33b854d4a70a9cccd51/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/50bedef0339230a65bdd4cd0e3171fd0/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/50bedef0339230a65bdd4cd0e3171fd0/output.pklnew file mode 100644index 000000000..641699a30Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/50bedef0339230a65bdd4cd0e3171fd0/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/54179545dc387cb7eaa51b7fa9b81e0a/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/54179545dc387cb7eaa51b7fa9b81e0a/output.pklnew file mode 100644index 000000000..881a2cbcbBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/54179545dc387cb7eaa51b7fa9b81e0a/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/566d0e3f3de4bb941c5255bee945ca29/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/566d0e3f3de4bb941c5255bee945ca29/output.pklnew file mode 100644index 000000000..d8c2b2cb0Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/566d0e3f3de4bb941c5255bee945ca29/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/57e432e9b3c46680dc4e53210bac0f3c/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/57e432e9b3c46680dc4e53210bac0f3c/output.pklnew file mode 100644index 000000000..269b00c0fBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/57e432e9b3c46680dc4e53210bac0f3c/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/588e602eea3ca24ad410ea39d3682e4d/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/588e602eea3ca24ad410ea39d3682e4d/output.pklnew file mode 100644index 000000000..80a1a8190Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/588e602eea3ca24ad410ea39d3682e4d/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/5fb4559e3b8aad45a22959eb719228e4/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/5fb4559e3b8aad45a22959eb719228e4/output.pklnew file mode 100644index 000000000..39894304cBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/5fb4559e3b8aad45a22959eb719228e4/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/5fed267af48965ad7786bba4b6d7278d/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/5fed267af48965ad7786bba4b6d7278d/output.pklnew file mode 100644index 000000000..90b79c4b1Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/5fed267af48965ad7786bba4b6d7278d/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/64687e114095eaf4b4424b86d0c04dbc/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/64687e114095eaf4b4424b86d0c04dbc/output.pklnew file mode 100644index 000000000..ecdf5d82aBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/64687e114095eaf4b4424b86d0c04dbc/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/67e54b45d4ea360c4feaeb52df270825/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/67e54b45d4ea360c4feaeb52df270825/output.pklnew file mode 100644index 000000000..a7635f090Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/67e54b45d4ea360c4feaeb52df270825/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/6881da725177a3216cca089348365f95/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/6881da725177a3216cca089348365f95/output.pklnew file mode 100644index 000000000..a2bd34897Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/6881da725177a3216cca089348365f95/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/6c541bc2696d02c5fd2f15388dee08b6/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/6c541bc2696d02c5fd2f15388dee08b6/output.pklnew file mode 100644index 000000000..97bb2e57aBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/6c541bc2696d02c5fd2f15388dee08b6/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/6febb3b21a6adb0ca4e1c4c4f94842fd/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/6febb3b21a6adb0ca4e1c4c4f94842fd/output.pklnew file mode 100644index 000000000..7bb8bf4dbBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/6febb3b21a6adb0ca4e1c4c4f94842fd/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/70001aeae94a32c67c969b2df7ac92b8/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/70001aeae94a32c67c969b2df7ac92b8/output.pklnew file mode 100644index 000000000..524948c78Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/70001aeae94a32c67c969b2df7ac92b8/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/766dadff57a55aa0fc01682a87bf83a5/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/766dadff57a55aa0fc01682a87bf83a5/output.pklnew file mode 100644index 000000000..af3660f65Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/766dadff57a55aa0fc01682a87bf83a5/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/783ed511b7293118d6449e20c136c75e/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/783ed511b7293118d6449e20c136c75e/output.pklnew file mode 100644index 000000000..f950a27a6Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/783ed511b7293118d6449e20c136c75e/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/7924e011ee488c5e3a289d33ce0acbff/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/7924e011ee488c5e3a289d33ce0acbff/output.pklnew file mode 100644index 000000000..eac591813Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/7924e011ee488c5e3a289d33ce0acbff/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/79f24f088e0336cc276308f44bef8295/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/79f24f088e0336cc276308f44bef8295/output.pklnew file mode 100644index 000000000..9c74cc315Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/79f24f088e0336cc276308f44bef8295/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/7acdd13c88b13edf6e0e1d7bf865773b/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/7acdd13c88b13edf6e0e1d7bf865773b/output.pklnew file mode 100644index 000000000..70ab11603Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/7acdd13c88b13edf6e0e1d7bf865773b/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/7c201f0ea2feb03848f712453654c4bd/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/7c201f0ea2feb03848f712453654c4bd/output.pklnew file mode 100644index 000000000..7bede2688Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/7c201f0ea2feb03848f712453654c4bd/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/7ddff18cb7e46a6595493a5452e5fcb4/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/7ddff18cb7e46a6595493a5452e5fcb4/output.pklnew file mode 100644index 000000000..9dfe45d8fBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/7ddff18cb7e46a6595493a5452e5fcb4/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/824af5ca15c263ab537f5d2b35e681ce/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/824af5ca15c263ab537f5d2b35e681ce/output.pklnew file mode 100644index 000000000..b31032ce5Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/824af5ca15c263ab537f5d2b35e681ce/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/84be06e0b748e01909b18081161b98b6/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/84be06e0b748e01909b18081161b98b6/output.pklnew file mode 100644index 000000000..5f4aafb2dBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/84be06e0b748e01909b18081161b98b6/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/854d697ecfe3e1d54af4a3d5a413f137/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/854d697ecfe3e1d54af4a3d5a413f137/output.pklnew file mode 100644index 000000000..4d9f1fea3Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/854d697ecfe3e1d54af4a3d5a413f137/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/85ade75ba961a337b856c74be7abdc05/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/85ade75ba961a337b856c74be7abdc05/output.pklnew file mode 100644index 000000000..724de499cBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/85ade75ba961a337b856c74be7abdc05/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/8b5a777d45d9e2999240ac0697d6e3aa/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/8b5a777d45d9e2999240ac0697d6e3aa/output.pklnew file mode 100644index 000000000..9dfe45d8fBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/8b5a777d45d9e2999240ac0697d6e3aa/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/8bad96807b5daf25f9ca338e61b69384/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/8bad96807b5daf25f9ca338e61b69384/output.pklnew file mode 100644index 000000000..b2f3c9260Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/8bad96807b5daf25f9ca338e61b69384/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/8d225cb7b60fd792ca50258a6924635f/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/8d225cb7b60fd792ca50258a6924635f/output.pklnew file mode 100644index 000000000..9d3160b6eBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/8d225cb7b60fd792ca50258a6924635f/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/8ed9f15f822c33db8f7686b0955ff48a/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/8ed9f15f822c33db8f7686b0955ff48a/output.pklnew file mode 100644index 000000000..407d1235aBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/8ed9f15f822c33db8f7686b0955ff48a/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/919570f506618dea12aa5fa8b8fbcfa4/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/919570f506618dea12aa5fa8b8fbcfa4/output.pklnew file mode 100644index 000000000..2030cc4d7Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/919570f506618dea12aa5fa8b8fbcfa4/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/91b150bb0a3cc43fc1fbe0cded7687f5/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/91b150bb0a3cc43fc1fbe0cded7687f5/output.pklnew file mode 100644index 000000000..c1b6ceba2Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/91b150bb0a3cc43fc1fbe0cded7687f5/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/921baaf1ff58ed8a625528cbb3858baf/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/921baaf1ff58ed8a625528cbb3858baf/output.pklnew file mode 100644index 000000000..4427c042fBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/921baaf1ff58ed8a625528cbb3858baf/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/9448e5042af0f32c15cd6ae56493db15/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/9448e5042af0f32c15cd6ae56493db15/output.pklnew file mode 100644index 000000000..fcbb4558bBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/9448e5042af0f32c15cd6ae56493db15/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/95da4988837f8c6c01fb28503153d711/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/95da4988837f8c6c01fb28503153d711/output.pklnew file mode 100644index 000000000..b17c2a977Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/95da4988837f8c6c01fb28503153d711/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/9bf7d66fb82c5c93f1568d7f801d27fd/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/9bf7d66fb82c5c93f1568d7f801d27fd/output.pklnew file mode 100644index 000000000..f637cacc7Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/9bf7d66fb82c5c93f1568d7f801d27fd/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/9c1370f85fe247f2fe64905799337aea/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/9c1370f85fe247f2fe64905799337aea/output.pklnew file mode 100644index 000000000..c4df67578Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/9c1370f85fe247f2fe64905799337aea/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/9ca46d034558ddb9e0910f8ad19cd68d/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/9ca46d034558ddb9e0910f8ad19cd68d/output.pklnew file mode 100644index 000000000..3239920afBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/9ca46d034558ddb9e0910f8ad19cd68d/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/9e023405755ab4129b94367a6b954572/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/9e023405755ab4129b94367a6b954572/output.pklnew file mode 100644index 000000000..69e521172Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/9e023405755ab4129b94367a6b954572/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/9e29ea2e07a148fe474cab0f549eb3a1/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/9e29ea2e07a148fe474cab0f549eb3a1/output.pklnew file mode 100644index 000000000..43e7d3247Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/9e29ea2e07a148fe474cab0f549eb3a1/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/a017b8e5c539e0c3ff79536ac42712e4/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/a017b8e5c539e0c3ff79536ac42712e4/output.pklnew file mode 100644index 000000000..c46d816bfBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/a017b8e5c539e0c3ff79536ac42712e4/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/a086118422ba5b82b8ab5e21fbf27671/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/a086118422ba5b82b8ab5e21fbf27671/output.pklnew file mode 100644index 000000000..22bf13618Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/a086118422ba5b82b8ab5e21fbf27671/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/a4805777d4621bd094ab3905861e848d/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/a4805777d4621bd094ab3905861e848d/output.pklnew file mode 100644index 000000000..46ca2fbf1Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/a4805777d4621bd094ab3905861e848d/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/a4d6df88b5fe07bcfee40166aeecf6f1/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/a4d6df88b5fe07bcfee40166aeecf6f1/output.pklnew file mode 100644index 000000000..dde5da5f0Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/a4d6df88b5fe07bcfee40166aeecf6f1/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/a8ea27e6153dec26ad5061cc57f20e0e/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/a8ea27e6153dec26ad5061cc57f20e0e/output.pklnew file mode 100644index 000000000..76b5bd728Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/a8ea27e6153dec26ad5061cc57f20e0e/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/aa954ff25b4ea0126ad61d883a0b25e9/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/aa954ff25b4ea0126ad61d883a0b25e9/output.pklnew file mode 100644index 000000000..cbadc6df9Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/aa954ff25b4ea0126ad61d883a0b25e9/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/aada5a35826f9c51606b52cecc20b632/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/aada5a35826f9c51606b52cecc20b632/output.pklnew file mode 100644index 000000000..af24b1b40Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/aada5a35826f9c51606b52cecc20b632/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/acd87217c1c6ff60d99ab632a4c07dbf/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/acd87217c1c6ff60d99ab632a4c07dbf/output.pklnew file mode 100644index 000000000..82b8c578cBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/acd87217c1c6ff60d99ab632a4c07dbf/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/b08e67d75934a82dbad2daf53f97e17f/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/b08e67d75934a82dbad2daf53f97e17f/output.pklnew file mode 100644index 000000000..885613628Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/b08e67d75934a82dbad2daf53f97e17f/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/b14d2249afe4ff025ffb93a11b6f475e/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/b14d2249afe4ff025ffb93a11b6f475e/output.pklnew file mode 100644index 000000000..15269b4aeBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/b14d2249afe4ff025ffb93a11b6f475e/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/b1aef1c57bc9719deb6ed12ef589a18c/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/b1aef1c57bc9719deb6ed12ef589a18c/output.pklnew file mode 100644index 000000000..6410b15e2Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/b1aef1c57bc9719deb6ed12ef589a18c/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/b2088cbe7f71ca3ca6610a9cce414780/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/b2088cbe7f71ca3ca6610a9cce414780/output.pklnew file mode 100644index 000000000..042b684d6Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/b2088cbe7f71ca3ca6610a9cce414780/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/b24ebd21a4bf73d0b30f74d6e81a0045/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/b24ebd21a4bf73d0b30f74d6e81a0045/output.pklnew file mode 100644index 000000000..a1869d74bBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/b24ebd21a4bf73d0b30f74d6e81a0045/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/b334417965763018740591a7e647d178/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/b334417965763018740591a7e647d178/output.pklnew file mode 100644index 000000000..5ebb90172Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/b334417965763018740591a7e647d178/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/b3f0aec26aaff1785b271ec737330336/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/b3f0aec26aaff1785b271ec737330336/output.pklnew file mode 100644index 000000000..57d63d4fdBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/b3f0aec26aaff1785b271ec737330336/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/b4067a514b48bb9b6447a9dff06ec010/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/b4067a514b48bb9b6447a9dff06ec010/output.pklnew file mode 100644index 000000000..50248219bBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/b4067a514b48bb9b6447a9dff06ec010/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/b59758fbadd354df7a802247dd7c3dc2/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/b59758fbadd354df7a802247dd7c3dc2/output.pklnew file mode 100644index 000000000..61a36ad10Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/b59758fbadd354df7a802247dd7c3dc2/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/b5bba828afadb6f71ec6b2639e695413/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/b5bba828afadb6f71ec6b2639e695413/output.pklnew file mode 100644index 000000000..85a056ee8Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/b5bba828afadb6f71ec6b2639e695413/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/bcd9111d5c46214d2b26abd8c58e10c8/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/bcd9111d5c46214d2b26abd8c58e10c8/output.pklnew file mode 100644index 000000000..8422d9dceBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/bcd9111d5c46214d2b26abd8c58e10c8/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/bea5145cc6935509c64d5fd647c1994d/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/bea5145cc6935509c64d5fd647c1994d/output.pklnew file mode 100644index 000000000..3f3ea7d16Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/bea5145cc6935509c64d5fd647c1994d/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/c085c6145149f13727eb1f776e83a9fa/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/c085c6145149f13727eb1f776e83a9fa/output.pklnew file mode 100644index 000000000..6034ac4efBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/c085c6145149f13727eb1f776e83a9fa/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/c1b9c4e77bab56b49c9a242f142bcf6f/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/c1b9c4e77bab56b49c9a242f142bcf6f/output.pklnew file mode 100644index 000000000..0dc6d8a8bBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/c1b9c4e77bab56b49c9a242f142bcf6f/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/c276308483776378809107872268489b/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/c276308483776378809107872268489b/output.pklnew file mode 100644index 000000000..71b1a8618Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/c276308483776378809107872268489b/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/c926111fbf37964f1e55994f997fa509/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/c926111fbf37964f1e55994f997fa509/output.pklnew file mode 100644index 000000000..536d28881Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/c926111fbf37964f1e55994f997fa509/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/cb5279a4783759ffe7bbd1a3aebe3522/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/cb5279a4783759ffe7bbd1a3aebe3522/output.pklnew file mode 100644index 000000000..59716423bBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/cb5279a4783759ffe7bbd1a3aebe3522/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/ce2c3125438d932f8c2fb6be3442cc43/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/ce2c3125438d932f8c2fb6be3442cc43/output.pklnew file mode 100644index 000000000..74b09c019Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/ce2c3125438d932f8c2fb6be3442cc43/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/ce7a36198caa33b66fa55b692abb082f/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/ce7a36198caa33b66fa55b692abb082f/output.pklnew file mode 100644index 000000000..e496078baBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/ce7a36198caa33b66fa55b692abb082f/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/cff16546307c69e74fe74a9c0d2b8392/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/cff16546307c69e74fe74a9c0d2b8392/output.pklnew file mode 100644index 000000000..5dd87691cBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/cff16546307c69e74fe74a9c0d2b8392/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/d06bfcd864311ff8c92dcd5f3456a44f/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/d06bfcd864311ff8c92dcd5f3456a44f/output.pklnew file mode 100644index 000000000..bc4667f65Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/d06bfcd864311ff8c92dcd5f3456a44f/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/d12d68205d02bc216624e9543c7f4124/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/d12d68205d02bc216624e9543c7f4124/output.pklnew file mode 100644index 000000000..2c8e04b21Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/d12d68205d02bc216624e9543c7f4124/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/d2ae46244168c1d1dd99616e1e98a5c7/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/d2ae46244168c1d1dd99616e1e98a5c7/output.pklnew file mode 100644index 000000000..9e289448bBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/d2ae46244168c1d1dd99616e1e98a5c7/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/d386b235bbfcf6f2d932dd66f2d621b6/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/d386b235bbfcf6f2d932dd66f2d621b6/output.pklnew file mode 100644index 000000000..d2dbc4fa9Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/d386b235bbfcf6f2d932dd66f2d621b6/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/d5faa190b14440293e1135df73b2ab04/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/d5faa190b14440293e1135df73b2ab04/output.pklnew file mode 100644index 000000000..89ff41118Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/d5faa190b14440293e1135df73b2ab04/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/d6af6b810fb2aeadc3f5c7e05863ed36/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/d6af6b810fb2aeadc3f5c7e05863ed36/output.pklnew file mode 100644index 000000000..57d63d4fdBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/d6af6b810fb2aeadc3f5c7e05863ed36/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/d713de91ac4df3d6d794716673df8612/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/d713de91ac4df3d6d794716673df8612/output.pklnew file mode 100644index 000000000..a2dc9a5bcBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/d713de91ac4df3d6d794716673df8612/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/d74e50e9fa15a39d056c31c7876635fc/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/d74e50e9fa15a39d056c31c7876635fc/output.pklnew file mode 100644index 000000000..951dd2bf9Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/d74e50e9fa15a39d056c31c7876635fc/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/da4747c1558b46c697fdae8b142949b3/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/da4747c1558b46c697fdae8b142949b3/output.pklnew file mode 100644index 000000000..63ee21385Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/da4747c1558b46c697fdae8b142949b3/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/db7c667bae0058f4965c449f34e50c44/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/db7c667bae0058f4965c449f34e50c44/output.pklnew file mode 100644index 000000000..a54b61b50Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/db7c667bae0058f4965c449f34e50c44/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/dccaae07b29b8abdb006b0ed7e177a5d/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/dccaae07b29b8abdb006b0ed7e177a5d/output.pklnew file mode 100644index 000000000..269b00c0fBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/dccaae07b29b8abdb006b0ed7e177a5d/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/dd90c68eb54b6bc12609579e12e1f9f0/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/dd90c68eb54b6bc12609579e12e1f9f0/output.pklnew file mode 100644index 000000000..aeefb864bBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/dd90c68eb54b6bc12609579e12e1f9f0/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/e5eb079eee15cb44084bf4f04a633afc/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/e5eb079eee15cb44084bf4f04a633afc/output.pklnew file mode 100644index 000000000..8104704a6Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/e5eb079eee15cb44084bf4f04a633afc/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/e923ba4a005ddf84974b00fb5004797a/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/e923ba4a005ddf84974b00fb5004797a/output.pklnew file mode 100644index 000000000..c288dd7feBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/e923ba4a005ddf84974b00fb5004797a/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/ea9d3d8c86c5d8881e2dcfd40248dacb/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/ea9d3d8c86c5d8881e2dcfd40248dacb/output.pklnew file mode 100644index 000000000..afa9aa7a0Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/ea9d3d8c86c5d8881e2dcfd40248dacb/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/eb6c391a2e2156587dfbbaa00ad938b6/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/eb6c391a2e2156587dfbbaa00ad938b6/output.pklnew file mode 100644index 000000000..b3caf0b03Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/eb6c391a2e2156587dfbbaa00ad938b6/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/eba341b04943c33d6750f660d6f012e8/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/eba341b04943c33d6750f660d6f012e8/output.pklnew file mode 100644index 000000000..346f4459fBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/eba341b04943c33d6750f660d6f012e8/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/ebd82006380260a1a0851e61e70e807f/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/ebd82006380260a1a0851e61e70e807f/output.pklnew file mode 100644index 000000000..22a931a67Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/ebd82006380260a1a0851e61e70e807f/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/ecde603dac0265ba424dd42d0ee939c3/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/ecde603dac0265ba424dd42d0ee939c3/output.pklnew file mode 100644index 000000000..bfe5e5175Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/ecde603dac0265ba424dd42d0ee939c3/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/ef43abef160e4cbfbc1577bf0fe3c106/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/ef43abef160e4cbfbc1577bf0fe3c106/output.pklnew file mode 100644index 000000000..7548c7091Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/ef43abef160e4cbfbc1577bf0fe3c106/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/f148049d4bbb3d951cce39a88ee00e57/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/f148049d4bbb3d951cce39a88ee00e57/output.pklnew file mode 100644index 000000000..54ecec009Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/f148049d4bbb3d951cce39a88ee00e57/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/f1632b421a1e4a80db8781ca3029872c/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/f1632b421a1e4a80db8781ca3029872c/output.pklnew file mode 100644index 000000000..e957e68baBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/f1632b421a1e4a80db8781ca3029872c/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/f323e207a4657ac7194d8f315d638902/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/f323e207a4657ac7194d8f315d638902/output.pklnew file mode 100644index 000000000..d8837df21Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/f323e207a4657ac7194d8f315d638902/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/f6077de587846981a8041e425ac00085/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/f6077de587846981a8041e425ac00085/output.pklnew file mode 100644index 000000000..b42afc4a0Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/f6077de587846981a8041e425ac00085/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/f6ce721bd511707ad32c21b379a7b4fa/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/f6ce721bd511707ad32c21b379a7b4fa/output.pklnew file mode 100644index 000000000..e2f7e7f26Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/f6ce721bd511707ad32c21b379a7b4fa/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/f7db402b0d1b9266cc846220a37d3786/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/f7db402b0d1b9266cc846220a37d3786/output.pklnew file mode 100644index 000000000..f59b2906eBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/f7db402b0d1b9266cc846220a37d3786/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/f87b1cc73c45e04a550a402a999702b2/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/f87b1cc73c45e04a550a402a999702b2/output.pklnew file mode 100644index 000000000..3e5d82bf6Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/f87b1cc73c45e04a550a402a999702b2/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/f9d754c6d5cb805813f8155c123c10e5/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/f9d754c6d5cb805813f8155c123c10e5/output.pklnew file mode 100644index 000000000..408e78500Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/f9d754c6d5cb805813f8155c123c10e5/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/fab366fa097683e9b6b519aa090ca821/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/fab366fa097683e9b6b519aa090ca821/output.pklnew file mode 100644index 000000000..bb326c4f5Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/fab366fa097683e9b6b519aa090ca821/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/fc9af8091140a7af2e86f19b0a04cc3e/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/fc9af8091140a7af2e86f19b0a04cc3e/output.pklnew file mode 100644index 000000000..f78adc865Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/fc9af8091140a7af2e86f19b0a04cc3e/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/fd42564f1a6aa1f6ad8e0976fca10789/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/fd42564f1a6aa1f6ad8e0976fca10789/output.pklnew file mode 100644index 000000000..a393ec33fBinary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/fd42564f1a6aa1f6ad8e0976fca10789/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/fee98c433da15d0931619725ef781d24/output.pkl b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/fee98c433da15d0931619725ef781d24/output.pklnew file mode 100644index 000000000..5c97c4242Binary files /dev/null and b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/fee98c433da15d0931619725ef781d24/output.pkl differdiff --git a/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/func_code.py b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/func_code.pynew file mode 100644index 000000000..3373e98a3--- /dev/null+++ b/cache/joblib/dsp/modules/colbertv2/colbertv2_get_request_v2_wrapped/func_code.py@@ -0,0 +1,5 @@+# first line: 45[email protected]_cache(maxsize=None)+def colbertv2_get_request_v2_wrapped(*args, **kwargs):+ return colbertv2_get_request_v2(*args, **kwargs)diff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/00982d6a230eb089a5ff5a982cf2d9c3/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/00982d6a230eb089a5ff5a982cf2d9c3/output.pklnew file mode 100644index 000000000..33fb04fb1Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/00982d6a230eb089a5ff5a982cf2d9c3/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/02580511d9bac3c1b328269104bcd671/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/02580511d9bac3c1b328269104bcd671/output.pklnew file mode 100644index 000000000..381d59691Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/02580511d9bac3c1b328269104bcd671/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/07b7e72a76d66152915a494722b7e9e7/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/07b7e72a76d66152915a494722b7e9e7/output.pklnew file mode 100644index 000000000..8d6a35533Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/07b7e72a76d66152915a494722b7e9e7/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/088d39890501b62e23c72ba5027e3753/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/088d39890501b62e23c72ba5027e3753/output.pklnew file mode 100644index 000000000..9450c2b7aBinary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/088d39890501b62e23c72ba5027e3753/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/08fb38aa51dd0cc1bfbf8f862c0fa691/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/08fb38aa51dd0cc1bfbf8f862c0fa691/output.pklnew file mode 100644index 000000000..210b93cb3Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/08fb38aa51dd0cc1bfbf8f862c0fa691/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/0aee18a9a269f32c423ff72584d6955c/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/0aee18a9a269f32c423ff72584d6955c/output.pklnew file mode 100644index 000000000..55b418c51Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/0aee18a9a269f32c423ff72584d6955c/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/0d174251a849e2fe9d4627a21d4bf236/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/0d174251a849e2fe9d4627a21d4bf236/output.pklnew file mode 100644index 000000000..2f5f641d4Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/0d174251a849e2fe9d4627a21d4bf236/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/0e6afffad380f6b7cccb0ee5176b9ce9/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/0e6afffad380f6b7cccb0ee5176b9ce9/output.pklnew file mode 100644index 000000000..25d9f25a1Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/0e6afffad380f6b7cccb0ee5176b9ce9/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/10f71844d31cc1d16ef44fc15e124469/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/10f71844d31cc1d16ef44fc15e124469/output.pklnew file mode 100644index 000000000..8966d5eafBinary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/10f71844d31cc1d16ef44fc15e124469/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/11a364fe1fa548a5d3acc5b0bc44f414/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/11a364fe1fa548a5d3acc5b0bc44f414/output.pklnew file mode 100644index 000000000..f91f5c71cBinary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/11a364fe1fa548a5d3acc5b0bc44f414/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/185eefe7f341c01fa711431bc9eed940/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/185eefe7f341c01fa711431bc9eed940/output.pklnew file mode 100644index 000000000..7c4a48761Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/185eefe7f341c01fa711431bc9eed940/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/1ba43da77ffbdf2bd91bd44dd82e2515/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/1ba43da77ffbdf2bd91bd44dd82e2515/output.pklnew file mode 100644index 000000000..702ae6e70Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/1ba43da77ffbdf2bd91bd44dd82e2515/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/1bfa725fd43f71d9a713096d06ab8107/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/1bfa725fd43f71d9a713096d06ab8107/output.pklnew file mode 100644index 000000000..a45fe4bceBinary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/1bfa725fd43f71d9a713096d06ab8107/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/23a8351cbdee22ec65b8726cbce62c0a/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/23a8351cbdee22ec65b8726cbce62c0a/output.pklnew file mode 100644index 000000000..9b8fc275dBinary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/23a8351cbdee22ec65b8726cbce62c0a/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/24ddc7fb8beb6585d194abdbd0f3968c/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/24ddc7fb8beb6585d194abdbd0f3968c/output.pklnew file mode 100644index 000000000..56b82f971Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/24ddc7fb8beb6585d194abdbd0f3968c/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/26dc0eec7ac4fcab2c28901f952cd9f4/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/26dc0eec7ac4fcab2c28901f952cd9f4/output.pklnew file mode 100644index 000000000..123222bc3Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/26dc0eec7ac4fcab2c28901f952cd9f4/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/2b109d7991d9d9a94d1997e7d67bdf99/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/2b109d7991d9d9a94d1997e7d67bdf99/output.pklnew file mode 100644index 000000000..2bb0bb140Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/2b109d7991d9d9a94d1997e7d67bdf99/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/2b641043e82b2810fb344080e0d3f4c1/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/2b641043e82b2810fb344080e0d3f4c1/output.pklnew file mode 100644index 000000000..5e7f60dfbBinary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/2b641043e82b2810fb344080e0d3f4c1/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/2b714f957570c7babd08b8ae7e5e5f67/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/2b714f957570c7babd08b8ae7e5e5f67/output.pklnew file mode 100644index 000000000..e8e98395aBinary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/2b714f957570c7babd08b8ae7e5e5f67/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/2c6555207e6e7fa27247dff0238f2202/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/2c6555207e6e7fa27247dff0238f2202/output.pklnew file mode 100644index 000000000..d2e418558Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/2c6555207e6e7fa27247dff0238f2202/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/2ca83d0596078f472da30d974bac98e4/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/2ca83d0596078f472da30d974bac98e4/output.pklnew file mode 100644index 000000000..1a6735f02Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/2ca83d0596078f472da30d974bac98e4/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/2de2b29c7940f02050f723da03849f20/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/2de2b29c7940f02050f723da03849f20/output.pklnew file mode 100644index 000000000..8ba6c57a4Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/2de2b29c7940f02050f723da03849f20/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/2e180799e579d2f84b2608d7c55e66ac/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/2e180799e579d2f84b2608d7c55e66ac/output.pklnew file mode 100644index 000000000..2d8d3f9c5Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/2e180799e579d2f84b2608d7c55e66ac/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/2f29728cd4566a52861e424e0973117b/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/2f29728cd4566a52861e424e0973117b/output.pklnew file mode 100644index 000000000..ff3c5d26fBinary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/2f29728cd4566a52861e424e0973117b/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/2f405605e0231de14373d94323d10d77/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/2f405605e0231de14373d94323d10d77/output.pklnew file mode 100644index 000000000..8be68250eBinary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/2f405605e0231de14373d94323d10d77/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/2f777ebf96f2476d4a3cb1c990ebcd52/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/2f777ebf96f2476d4a3cb1c990ebcd52/output.pklnew file mode 100644index 000000000..e84696d5eBinary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/2f777ebf96f2476d4a3cb1c990ebcd52/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/31684bf4effea4443ba4779f26b2d09f/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/31684bf4effea4443ba4779f26b2d09f/output.pklnew file mode 100644index 000000000..07139f5e3Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/31684bf4effea4443ba4779f26b2d09f/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/320e28d1fa467b361922497cecaf6e4d/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/320e28d1fa467b361922497cecaf6e4d/output.pklnew file mode 100644index 000000000..ecef9f02dBinary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/320e28d1fa467b361922497cecaf6e4d/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/3549e5addd140ffb0aa60b5902376382/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/3549e5addd140ffb0aa60b5902376382/output.pklnew file mode 100644index 000000000..1bfaa38f5Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/3549e5addd140ffb0aa60b5902376382/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/3878b416c741f72fec14d7788c2d2ebe/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/3878b416c741f72fec14d7788c2d2ebe/output.pklnew file mode 100644index 000000000..567365bdaBinary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/3878b416c741f72fec14d7788c2d2ebe/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/392a142e44ffb5155ce9799b065904c2/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/392a142e44ffb5155ce9799b065904c2/output.pklnew file mode 100644index 000000000..c55f92207Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/392a142e44ffb5155ce9799b065904c2/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/3b162193b1569b0773e19cb55935d687/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/3b162193b1569b0773e19cb55935d687/output.pklnew file mode 100644index 000000000..7e12f85e9Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/3b162193b1569b0773e19cb55935d687/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/3b23200f9e897f2c770e53526925d0f8/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/3b23200f9e897f2c770e53526925d0f8/output.pklnew file mode 100644index 000000000..01dc40612Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/3b23200f9e897f2c770e53526925d0f8/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/3c44b5648d8cae3f55d672a0102177c7/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/3c44b5648d8cae3f55d672a0102177c7/output.pklnew file mode 100644index 000000000..f2b531552Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/3c44b5648d8cae3f55d672a0102177c7/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/435725d6d77c3db16e5cc72e4fec2312/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/435725d6d77c3db16e5cc72e4fec2312/output.pklnew file mode 100644index 000000000..2d4382401Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/435725d6d77c3db16e5cc72e4fec2312/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/47ac353b1709a1597b6bfe8c7bb08509/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/47ac353b1709a1597b6bfe8c7bb08509/output.pklnew file mode 100644index 000000000..0fc23bcafBinary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/47ac353b1709a1597b6bfe8c7bb08509/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/48e0d344c2c50eb434a1a422af9ae484/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/48e0d344c2c50eb434a1a422af9ae484/output.pklnew file mode 100644index 000000000..88b3b2114Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/48e0d344c2c50eb434a1a422af9ae484/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/4991226e832089f7dd7e85253f5d79a2/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/4991226e832089f7dd7e85253f5d79a2/output.pklnew file mode 100644index 000000000..a79eaf455Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/4991226e832089f7dd7e85253f5d79a2/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/4b5f3605dc48b1f651164916c634e23f/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/4b5f3605dc48b1f651164916c634e23f/output.pklnew file mode 100644index 000000000..08a0e20f2Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/4b5f3605dc48b1f651164916c634e23f/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/4b637825ab44a58dab29881c783e1dae/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/4b637825ab44a58dab29881c783e1dae/output.pklnew file mode 100644index 000000000..e2b78414cBinary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/4b637825ab44a58dab29881c783e1dae/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/57bef1f6487f47959d5b563ce0fd287a/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/57bef1f6487f47959d5b563ce0fd287a/output.pklnew file mode 100644index 000000000..c4c539500Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/57bef1f6487f47959d5b563ce0fd287a/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/5802fef57d3203cfe2a438d86ee3c432/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/5802fef57d3203cfe2a438d86ee3c432/output.pklnew file mode 100644index 000000000..f9f64a6ccBinary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/5802fef57d3203cfe2a438d86ee3c432/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/5aa4983159d610920e5c5fe429f7a58e/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/5aa4983159d610920e5c5fe429f7a58e/output.pklnew file mode 100644index 000000000..f6f3c09c2Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/5aa4983159d610920e5c5fe429f7a58e/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/5b04de99c846fac2b9a0970ded9efe3c/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/5b04de99c846fac2b9a0970ded9efe3c/output.pklnew file mode 100644index 000000000..acb86c2b9Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/5b04de99c846fac2b9a0970ded9efe3c/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/5b75aef552ed734fd97a45674a4edfbe/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/5b75aef552ed734fd97a45674a4edfbe/output.pklnew file mode 100644index 000000000..39e6c7e07Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/5b75aef552ed734fd97a45674a4edfbe/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/5ca14940b7964f629c9769d37c816740/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/5ca14940b7964f629c9769d37c816740/output.pklnew file mode 100644index 000000000..e0ac28c3fBinary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/5ca14940b7964f629c9769d37c816740/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/5dca5c09668f4020ab28fae04b123c35/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/5dca5c09668f4020ab28fae04b123c35/output.pklnew file mode 100644index 000000000..fa1d6be26Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/5dca5c09668f4020ab28fae04b123c35/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/5de52ed7db1a7cffcf613bdafe8c9d1a/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/5de52ed7db1a7cffcf613bdafe8c9d1a/output.pklnew file mode 100644index 000000000..02ac87d2aBinary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/5de52ed7db1a7cffcf613bdafe8c9d1a/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/5e0b6f706d5cef690aa5f219e372f708/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/5e0b6f706d5cef690aa5f219e372f708/output.pklnew file mode 100644index 000000000..3667f14a6Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/5e0b6f706d5cef690aa5f219e372f708/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/60080b7f08567f4ee49a00efaaef6e31/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/60080b7f08567f4ee49a00efaaef6e31/output.pklnew file mode 100644index 000000000..e3471d8eeBinary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/60080b7f08567f4ee49a00efaaef6e31/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/603d3784ee39e659598e14f9a7a96842/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/603d3784ee39e659598e14f9a7a96842/output.pklnew file mode 100644index 000000000..d727248bcBinary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/603d3784ee39e659598e14f9a7a96842/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/61a63221fb8324af44d897fc54fb7f11/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/61a63221fb8324af44d897fc54fb7f11/output.pklnew file mode 100644index 000000000..26ab6a382Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/61a63221fb8324af44d897fc54fb7f11/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/63e6a87d65a1df801a212dc0f0903b0b/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/63e6a87d65a1df801a212dc0f0903b0b/output.pklnew file mode 100644index 000000000..81cf9bc85Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/63e6a87d65a1df801a212dc0f0903b0b/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/642bd8a13bb484c6d33a407bf34aeb2b/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/642bd8a13bb484c6d33a407bf34aeb2b/output.pklnew file mode 100644index 000000000..82cadb4bbBinary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/642bd8a13bb484c6d33a407bf34aeb2b/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/6889b6b9ed8c0d7fd64159041d1f1068/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/6889b6b9ed8c0d7fd64159041d1f1068/output.pklnew file mode 100644index 000000000..ae2a415bfBinary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/6889b6b9ed8c0d7fd64159041d1f1068/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/696bbfc05bd6b13cf4d7fbdf7f3a2d22/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/696bbfc05bd6b13cf4d7fbdf7f3a2d22/output.pklnew file mode 100644index 000000000..b973596b7Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/696bbfc05bd6b13cf4d7fbdf7f3a2d22/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/6999c3fbab592e630c0614e03f143808/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/6999c3fbab592e630c0614e03f143808/output.pklnew file mode 100644index 000000000..5efb75e2eBinary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/6999c3fbab592e630c0614e03f143808/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/6b0135d3b799f50d278d1d06fb883d69/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/6b0135d3b799f50d278d1d06fb883d69/output.pklnew file mode 100644index 000000000..bea3488b5Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/6b0135d3b799f50d278d1d06fb883d69/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/6d10fab6e3b74042bddf2cd08f68f891/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/6d10fab6e3b74042bddf2cd08f68f891/output.pklnew file mode 100644index 000000000..e95be011cBinary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/6d10fab6e3b74042bddf2cd08f68f891/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/70b4d6b8fa24bb282fee39384dfe0bf1/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/70b4d6b8fa24bb282fee39384dfe0bf1/output.pklnew file mode 100644index 000000000..5e39c9290Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/70b4d6b8fa24bb282fee39384dfe0bf1/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/7305e12a58523a7825cd833b2c1521ed/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/7305e12a58523a7825cd833b2c1521ed/output.pklnew file mode 100644index 000000000..9510222b9Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/7305e12a58523a7825cd833b2c1521ed/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/74c387c31a1ae052c9e6ccdc24628212/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/74c387c31a1ae052c9e6ccdc24628212/output.pklnew file mode 100644index 000000000..40fbe53c1Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/74c387c31a1ae052c9e6ccdc24628212/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/74f6e6b0b2b9de1514ffa7216026c57e/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/74f6e6b0b2b9de1514ffa7216026c57e/output.pklnew file mode 100644index 000000000..2b923a405Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/74f6e6b0b2b9de1514ffa7216026c57e/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/78c36ede7a8d1d123a51d381a0ff8618/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/78c36ede7a8d1d123a51d381a0ff8618/output.pklnew file mode 100644index 000000000..5871eb586Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/78c36ede7a8d1d123a51d381a0ff8618/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/7c3b23be3f74578078483bc9af63abfd/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/7c3b23be3f74578078483bc9af63abfd/output.pklnew file mode 100644index 000000000..a5f0ceaa2Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/7c3b23be3f74578078483bc9af63abfd/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/7cb5ab258cf4a389d7def995de306150/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/7cb5ab258cf4a389d7def995de306150/output.pklnew file mode 100644index 000000000..dbf42cbf8Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/7cb5ab258cf4a389d7def995de306150/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/7e58df075ef5d9cb157e984c8e7cc80e/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/7e58df075ef5d9cb157e984c8e7cc80e/output.pklnew file mode 100644index 000000000..04bdd44d2Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/7e58df075ef5d9cb157e984c8e7cc80e/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/7eebd40c87bb50ce9194ebad0b94cbee/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/7eebd40c87bb50ce9194ebad0b94cbee/output.pklnew file mode 100644index 000000000..5c5d588caBinary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/7eebd40c87bb50ce9194ebad0b94cbee/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/82f0136832c0d5c8eff1b1d95c4c8aaf/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/82f0136832c0d5c8eff1b1d95c4c8aaf/output.pklnew file mode 100644index 000000000..df213d503Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/82f0136832c0d5c8eff1b1d95c4c8aaf/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/848faa4359b0a07634fe513865cddde5/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/848faa4359b0a07634fe513865cddde5/output.pklnew file mode 100644index 000000000..602931c5aBinary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/848faa4359b0a07634fe513865cddde5/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/8aed85b2894b05d8fc5d35274841293b/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/8aed85b2894b05d8fc5d35274841293b/output.pklnew file mode 100644index 000000000..209122de3Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/8aed85b2894b05d8fc5d35274841293b/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/8da52adec7b8469bd583b8260b69842c/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/8da52adec7b8469bd583b8260b69842c/output.pklnew file mode 100644index 000000000..ad743d316Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/8da52adec7b8469bd583b8260b69842c/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/8e466ccaed785616b22a5beb42f4c7b1/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/8e466ccaed785616b22a5beb42f4c7b1/output.pklnew file mode 100644index 000000000..4a52f05c9Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/8e466ccaed785616b22a5beb42f4c7b1/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/90ddf4d3ad7190aa756fe0ac57ce2416/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/90ddf4d3ad7190aa756fe0ac57ce2416/output.pklnew file mode 100644index 000000000..5ca9fdc5cBinary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/90ddf4d3ad7190aa756fe0ac57ce2416/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/91084736981a29d4ce377846a3d59b01/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/91084736981a29d4ce377846a3d59b01/output.pklnew file mode 100644index 000000000..5af7f4eccBinary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/91084736981a29d4ce377846a3d59b01/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/92c61ccf63a0f373aa9eb3b15b995f22/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/92c61ccf63a0f373aa9eb3b15b995f22/output.pklnew file mode 100644index 000000000..18ffb7e9aBinary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/92c61ccf63a0f373aa9eb3b15b995f22/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/9627555958a5f2e88d558d471ad1b00e/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/9627555958a5f2e88d558d471ad1b00e/output.pklnew file mode 100644index 000000000..9bdc09904Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/9627555958a5f2e88d558d471ad1b00e/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/96736f793658916f312feb5fbc371f6f/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/96736f793658916f312feb5fbc371f6f/output.pklnew file mode 100644index 000000000..af990a752Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/96736f793658916f312feb5fbc371f6f/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/99a8f7a01484533ce41b88612f28d051/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/99a8f7a01484533ce41b88612f28d051/output.pklnew file mode 100644index 000000000..d2b679230Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/99a8f7a01484533ce41b88612f28d051/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/99d6f8c280a97fa1fd8a5766aebd6b21/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/99d6f8c280a97fa1fd8a5766aebd6b21/output.pklnew file mode 100644index 000000000..26e6b01f4Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/99d6f8c280a97fa1fd8a5766aebd6b21/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/a9a891291b033e68979613a220f3f289/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/a9a891291b033e68979613a220f3f289/output.pklnew file mode 100644index 000000000..6fa2b2867Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/a9a891291b033e68979613a220f3f289/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/aafbd3a25728c0d5f8760da25d926453/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/aafbd3a25728c0d5f8760da25d926453/output.pklnew file mode 100644index 000000000..dee5f066dBinary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/aafbd3a25728c0d5f8760da25d926453/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/ab833fe4f1ea3b3371c3dfb5af293d05/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/ab833fe4f1ea3b3371c3dfb5af293d05/output.pklnew file mode 100644index 000000000..1cb6dca71Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/ab833fe4f1ea3b3371c3dfb5af293d05/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/ab8d806b62e6d6c27e8e77585168985c/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/ab8d806b62e6d6c27e8e77585168985c/output.pklnew file mode 100644index 000000000..0ce5596e4Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/ab8d806b62e6d6c27e8e77585168985c/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/ab908cbdbf4684c76c44fecdb69407c3/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/ab908cbdbf4684c76c44fecdb69407c3/output.pklnew file mode 100644index 000000000..b5f57c6d8Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/ab908cbdbf4684c76c44fecdb69407c3/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/abda9f7c254f46317fcbe283aa372864/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/abda9f7c254f46317fcbe283aa372864/output.pklnew file mode 100644index 000000000..95d75917fBinary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/abda9f7c254f46317fcbe283aa372864/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/aef964d1771b5b015baf0912c95a2cc0/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/aef964d1771b5b015baf0912c95a2cc0/output.pklnew file mode 100644index 000000000..0fa271c74Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/aef964d1771b5b015baf0912c95a2cc0/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/b0e22b41aee081dcf66a71a81fbd53e1/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/b0e22b41aee081dcf66a71a81fbd53e1/output.pklnew file mode 100644index 000000000..95a4f745fBinary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/b0e22b41aee081dcf66a71a81fbd53e1/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/b74070d27c600075c2a54705bca68a06/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/b74070d27c600075c2a54705bca68a06/output.pklnew file mode 100644index 000000000..ac50fc256Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/b74070d27c600075c2a54705bca68a06/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/b78e6fdb4bf0f215c12c396bb27bd2da/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/b78e6fdb4bf0f215c12c396bb27bd2da/output.pklnew file mode 100644index 000000000..74c09d0d4Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/b78e6fdb4bf0f215c12c396bb27bd2da/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/b8b4c90d2daaa004c5e455608220dffb/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/b8b4c90d2daaa004c5e455608220dffb/output.pklnew file mode 100644index 000000000..f84af81a3Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/b8b4c90d2daaa004c5e455608220dffb/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/b98373298739b8c414be9dd175973960/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/b98373298739b8c414be9dd175973960/output.pklnew file mode 100644index 000000000..6ffac4f4fBinary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/b98373298739b8c414be9dd175973960/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/c35074aba97c27c97fea4e255cfbbdc1/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/c35074aba97c27c97fea4e255cfbbdc1/output.pklnew file mode 100644index 000000000..108978c30Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/c35074aba97c27c97fea4e255cfbbdc1/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/c3cd5b2d203837a101c5f913d210104a/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/c3cd5b2d203837a101c5f913d210104a/output.pklnew file mode 100644index 000000000..3b67ea2fbBinary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/c3cd5b2d203837a101c5f913d210104a/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/c445855f2e2a81d3ae52cb83605795da/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/c445855f2e2a81d3ae52cb83605795da/output.pklnew file mode 100644index 000000000..ce9142072Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/c445855f2e2a81d3ae52cb83605795da/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/c4588a3ac634959c5bbe5781e8f3df27/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/c4588a3ac634959c5bbe5781e8f3df27/output.pklnew file mode 100644index 000000000..03aba4f11Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/c4588a3ac634959c5bbe5781e8f3df27/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/c4674b8f419ccbcf78a90af1775b39be/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/c4674b8f419ccbcf78a90af1775b39be/output.pklnew file mode 100644index 000000000..0f4247a5dBinary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/c4674b8f419ccbcf78a90af1775b39be/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/c505232fbef27944f651b7f19d00961f/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/c505232fbef27944f651b7f19d00961f/output.pklnew file mode 100644index 000000000..9d84b2040Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/c505232fbef27944f651b7f19d00961f/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/c54f5dd4420a2668a952ec41eea754fe/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/c54f5dd4420a2668a952ec41eea754fe/output.pklnew file mode 100644index 000000000..a6fda7c47Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/c54f5dd4420a2668a952ec41eea754fe/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/c6e00959938239de7fc15d270ae4d092/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/c6e00959938239de7fc15d270ae4d092/output.pklnew file mode 100644index 000000000..7641b325bBinary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/c6e00959938239de7fc15d270ae4d092/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/c779e845b33337c58d28b1b754e373ba/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/c779e845b33337c58d28b1b754e373ba/output.pklnew file mode 100644index 000000000..e18138f2dBinary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/c779e845b33337c58d28b1b754e373ba/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/c8480a05537f415818c65e0b518aca8a/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/c8480a05537f415818c65e0b518aca8a/output.pklnew file mode 100644index 000000000..f47977099Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/c8480a05537f415818c65e0b518aca8a/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/cac624f8e6949341413109413638dc9f/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/cac624f8e6949341413109413638dc9f/output.pklnew file mode 100644index 000000000..66a8dbc30Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/cac624f8e6949341413109413638dc9f/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/cb1729766ecfa414697e78727ff6e67f/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/cb1729766ecfa414697e78727ff6e67f/output.pklnew file mode 100644index 000000000..9e8bcf6a5Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/cb1729766ecfa414697e78727ff6e67f/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/cb7fa9645097133ec3727587e916e9c4/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/cb7fa9645097133ec3727587e916e9c4/output.pklnew file mode 100644index 000000000..e60780da5Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/cb7fa9645097133ec3727587e916e9c4/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/ceff9076e088b8af5a19e651172752a6/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/ceff9076e088b8af5a19e651172752a6/output.pklnew file mode 100644index 000000000..477f69fe4Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/ceff9076e088b8af5a19e651172752a6/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/d082d006e8c448ef18cd1cd0b9eb0fc3/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/d082d006e8c448ef18cd1cd0b9eb0fc3/output.pklnew file mode 100644index 000000000..940d3b479Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/d082d006e8c448ef18cd1cd0b9eb0fc3/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/d08af08e158c24aba789b35bb7e5ccbe/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/d08af08e158c24aba789b35bb7e5ccbe/output.pklnew file mode 100644index 000000000..14c8f5372Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/d08af08e158c24aba789b35bb7e5ccbe/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/d5d65aa0b39bce23517f84839bbec4a6/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/d5d65aa0b39bce23517f84839bbec4a6/output.pklnew file mode 100644index 000000000..24682a007Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/d5d65aa0b39bce23517f84839bbec4a6/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/d7bf986aa2971456abef64b9bcb632d3/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/d7bf986aa2971456abef64b9bcb632d3/output.pklnew file mode 100644index 000000000..fc3919d9cBinary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/d7bf986aa2971456abef64b9bcb632d3/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/d82a00e33be556923fac55c1f0b4fef6/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/d82a00e33be556923fac55c1f0b4fef6/output.pklnew file mode 100644index 000000000..1e8d14386Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/d82a00e33be556923fac55c1f0b4fef6/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/da785596a1e9a0da8b16b0b9de9d74e1/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/da785596a1e9a0da8b16b0b9de9d74e1/output.pklnew file mode 100644index 000000000..a82164f72Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/da785596a1e9a0da8b16b0b9de9d74e1/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/dcde536323be3d04f6a6889709f83136/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/dcde536323be3d04f6a6889709f83136/output.pklnew file mode 100644index 000000000..6f80d0278Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/dcde536323be3d04f6a6889709f83136/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/de8c0fbe452420182e944522c6dd5033/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/de8c0fbe452420182e944522c6dd5033/output.pklnew file mode 100644index 000000000..2c0b6f4ffBinary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/de8c0fbe452420182e944522c6dd5033/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/df0ee824fdf28beffa4e611a94b00292/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/df0ee824fdf28beffa4e611a94b00292/output.pklnew file mode 100644index 000000000..bf7b849a6Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/df0ee824fdf28beffa4e611a94b00292/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/e120b12483176160d10847b4b5605b0a/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/e120b12483176160d10847b4b5605b0a/output.pklnew file mode 100644index 000000000..534906606Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/e120b12483176160d10847b4b5605b0a/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/e32fa0cb57528cd646518c0624f9dace/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/e32fa0cb57528cd646518c0624f9dace/output.pklnew file mode 100644index 000000000..28cbdbe2dBinary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/e32fa0cb57528cd646518c0624f9dace/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/e39948d66d86b1c2936f733de83cd478/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/e39948d66d86b1c2936f733de83cd478/output.pklnew file mode 100644index 000000000..444076e1aBinary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/e39948d66d86b1c2936f733de83cd478/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/e433e0df0cf214cac539999748d2c711/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/e433e0df0cf214cac539999748d2c711/output.pklnew file mode 100644index 000000000..dd24626bbBinary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/e433e0df0cf214cac539999748d2c711/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/e6776ef318460d2246850d65d8692a72/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/e6776ef318460d2246850d65d8692a72/output.pklnew file mode 100644index 000000000..1b2bd5302Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/e6776ef318460d2246850d65d8692a72/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/e99a1766a42911153df6e2c032c4ceed/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/e99a1766a42911153df6e2c032c4ceed/output.pklnew file mode 100644index 000000000..778473f39Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/e99a1766a42911153df6e2c032c4ceed/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/edb612a0b216a1a8d01b8f1255eab6ba/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/edb612a0b216a1a8d01b8f1255eab6ba/output.pklnew file mode 100644index 000000000..2d72d0687Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/edb612a0b216a1a8d01b8f1255eab6ba/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/ee9f4d6b74e6d2ae412cab05e86d81d5/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/ee9f4d6b74e6d2ae412cab05e86d81d5/output.pklnew file mode 100644index 000000000..68d0b02c7Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/ee9f4d6b74e6d2ae412cab05e86d81d5/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/f16fa122cfa22cfa349bf3190ecfe64a/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/f16fa122cfa22cfa349bf3190ecfe64a/output.pklnew file mode 100644index 000000000..181a6f3dfBinary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/f16fa122cfa22cfa349bf3190ecfe64a/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/f30dc9c9b82e5ad3107be10d608b85c5/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/f30dc9c9b82e5ad3107be10d608b85c5/output.pklnew file mode 100644index 000000000..8e94dea1dBinary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/f30dc9c9b82e5ad3107be10d608b85c5/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/f3a5e47fe4a3675b6950160e804d6910/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/f3a5e47fe4a3675b6950160e804d6910/output.pklnew file mode 100644index 000000000..ee52c143fBinary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/f3a5e47fe4a3675b6950160e804d6910/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/f9f844beba88b14ae5d8fd6a8a55cc4c/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/f9f844beba88b14ae5d8fd6a8a55cc4c/output.pklnew file mode 100644index 000000000..232de3b49Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/f9f844beba88b14ae5d8fd6a8a55cc4c/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/fbdfd41e5ba84cba1775280ae7b7073a/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/fbdfd41e5ba84cba1775280ae7b7073a/output.pklnew file mode 100644index 000000000..0b1205728Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/fbdfd41e5ba84cba1775280ae7b7073a/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/fcaad534bfcf109b14615002be89813d/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/fcaad534bfcf109b14615002be89813d/output.pklnew file mode 100644index 000000000..67150ff0cBinary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/fcaad534bfcf109b14615002be89813d/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/fdc429e313892b92beaedf4ca754b2ac/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/fdc429e313892b92beaedf4ca754b2ac/output.pklnew file mode 100644index 000000000..645265e42Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/fdc429e313892b92beaedf4ca754b2ac/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/fe2132f9db7daac915bfc7c6540bd691/output.pkl b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/fe2132f9db7daac915bfc7c6540bd691/output.pklnew file mode 100644index 000000000..e10a48cc9Binary files /dev/null and b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/fe2132f9db7daac915bfc7c6540bd691/output.pkl differdiff --git a/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/func_code.py b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/func_code.pynew file mode 100644index 000000000..f83427c73--- /dev/null+++ b/cache/joblib/dsp/modules/gpt3/cached_gpt3_request_v2_wrapped/func_code.py@@ -0,0 +1,5 @@+# first line: 117[email protected]_cache(maxsize=None if cache_turn_on else 0)+def cached_gpt3_request_v2_wrapped(**kwargs):+ return cached_gpt3_request_v2(**kwargs)diff --git a/dsp/__init__.py b/dsp/__init__.pynew file mode 100644index 000000000..cf5378343--- /dev/null+++ b/dsp/__init__.py@@ -0,0 +1,4 @@+from .modules import *+from .primitives import *+from .templates import *+from .utils import settings\ No newline at end of filediff --git a/dsp/evaluation/utils.py b/dsp/evaluation/utils.pynew file mode 100644index 000000000..9d059437f--- /dev/null+++ b/dsp/evaluation/utils.py@@ -0,0 +1,26 @@+import tqdm+import pandas as pd++from IPython.display import display+from dsp.utils import EM++def evaluate(fn, dev, metric=EM):+ data = []++ for example in tqdm.tqdm(dev):+ question = example.question+ prediction = fn(question)++ d = dict(example)++ d['prediction'] = prediction+ d['correct'] = metric(prediction, example.answer)+ data.append(d)++ df = pd.DataFrame(data)++ print(f"Answered {df['correct'].sum()} / {len(dev)} correctly.")+ df['correct'] = df['correct'].apply(lambda x: '✔️' if x else '❌')++ pd.options.display.max_colwidth = None+ display(df.style.set_table_styles([{'selector': 'th', 'props': [('text-align', 'left')]}, {'selector': 'td', 'props': [('text-align', 'left')]}]))diff --git a/dsp/modules/__init__.py b/dsp/modules/__init__.pynew file mode 100644index 000000000..b2b5f3049--- /dev/null+++ b/dsp/modules/__init__.py@@ -0,0 +1,3 @@+from .cache_utils import *+from .gpt3 import *+from .colbertv2 import *\ No newline at end of filediff --git a/dsp/modules/cache_utils.py b/dsp/modules/cache_utils.pynew file mode 100644index 000000000..a5c36c9f9--- /dev/null+++ b/dsp/modules/cache_utils.py@@ -0,0 +1,36 @@+import os++from pathlib import Path+from joblib import Memory+from functools import wraps++from dsp.utils import dotdict+++cache_turn_on = True+++def noop_decorator(func):+ @wraps(func)+ def wrapper(*args, **kwargs):+ return func(*args, **kwargs)+ return wrapper+++cachedir = os.environ.get('DSP_CACHEDIR') or os.path.join(Path.home(), 'cachedir_joblib')+CacheMemory = Memory(location=cachedir, verbose=0)++cachedir2 = os.environ.get('DSP_NOTEBOOK_CACHEDIR')+NotebookCacheMemory = dotdict()+NotebookCacheMemory.cache = noop_decorator++if cachedir2:+ NotebookCacheMemory = Memory(location=cachedir2, verbose=0)+++if not cache_turn_on:+ CacheMemory = dotdict()+ CacheMemory.cache = noop_decorator++ NotebookCacheMemory = dotdict()+ NotebookCacheMemory.cache = noop_decoratordiff --git a/dsp/modules/colbertv2.py b/dsp/modules/colbertv2.pynew file mode 100644index 000000000..4881b8cc4--- /dev/null+++ b/dsp/modules/colbertv2.py@@ -0,0 +1,51 @@+import functools+import requests++from dsp.modules.cache_utils import CacheMemory, NotebookCacheMemory+from dsp.utils import dotdict+++# TODO: Ideally, this takes the name of the index and looks up its port.+++class ColBERTv2:+ def __init__(self, url='http://0.0.0.0', port=None):+ self.url = f'{url}:{port}' if port else url+ self.headers = {"Content-Type": "application/json; charset=utf-8"}++ def post(self, query: str, k: int):+ payload = {"query": query, "k": k}+ res = requests.post(self.url, json=payload, headers=self.headers)++ return res.json()['topk'][:k]++ def __call__(self, query: str, k=10, simplify=False):+ topk = colbertv2_get_request(self.url, query, k)+ topk = [dotdict(psg) for psg in topk]++ if simplify:+ topk = [psg.long_text for psg in topk]++ return topk+++def colbertv2_get_request_v2(url: str, query: str, k: int):+ assert k <= 100, f'Only k <= 100 is supported for the hosted ColBERTv2 server at the moment.'++ payload = {"query": query, "k": k}+ res = requests.get(url, params=payload)++ topk = res.json()['topk'][:k]+ topk = [{**d, 'long_text': d['text']} for d in topk]++ return topk[:k]++[email protected]_cache(maxsize=None)+def colbertv2_get_request_v2_wrapped(*args, **kwargs):+ return colbertv2_get_request_v2(*args, **kwargs)+++colbertv2_get_request = colbertv2_get_request_v2_wrappeddiff --git a/dsp/modules/gpt3.py b/dsp/modules/gpt3.pynew file mode 100644index 000000000..e9a08a598--- /dev/null+++ b/dsp/modules/gpt3.py@@ -0,0 +1,123 @@+import openai+import openai.error+import backoff+import functools++from dsp.modules.cache_utils import CacheMemory, NotebookCacheMemory, cache_turn_on+++def backoff_hdlr(details):+ # Handler from https://pypi.org/project/backoff/+ print("Backing off {wait:0.1f} seconds after {tries} tries "+ "calling function {target} with args {args} and kwargs "+ "{kwargs}".format(**details))+++class GPT3:+ def __init__(self, model='text-davinci-002', api_key=None):+ if api_key:+ openai.api_key = api_key++ self.kwargs = {'model': model, 'temperature': 0.0, 'max_tokens': 150, 'top_p': 1,+ 'frequency_penalty': 0, 'presence_penalty': 0, 'n': 1}++ self.history = []++ def basic_request(self, prompt, **kwargs):+ raw_kwargs = kwargs+ kwargs = {**self.kwargs, 'prompt': prompt, **kwargs}+ response = cached_gpt3_request(**kwargs)++ history = {'prompt': prompt, 'response': response,+ 'kwargs': kwargs, 'raw_kwargs': raw_kwargs}+ self.history.append(history)++ return response++ @backoff.on_exception(backoff.expo,+ (openai.error.RateLimitError,+ openai.error.ServiceUnavailableError),+ max_time=1000,+ on_backoff=backoff_hdlr)+ def request(self, prompt, **kwargs):+ return self.basic_request(prompt, **kwargs)++ def print_green(self, text, end='\n'):+ print("\x1b[32m" + str(text) + "\x1b[0m", end=end)++ def print_red(self, text, end='\n'):+ print("\x1b[31m" + str(text) + "\x1b[0m", end=end)++ def inspect_history(self, n=1):+ last_prompt = None+ printed = []++ for x in reversed(self.history[-100:]):+ prompt = x['prompt']++ if prompt != last_prompt:+ printed.append((prompt, x['response']['choices']))++ last_prompt = prompt++ if len(printed) >= n:+ break++ for prompt, choices in reversed(printed):+ print('\n\n\n')+ print(prompt, end='')+ self.print_green(choices[0]['text'], end='')++ if len(choices) > 1:+ self.print_red(f" \t (and {len(choices)-1} other completions)", end='')+ print('\n\n\n')+++ def __call__(self, prompt, only_completed=True, return_sorted=False, **kwargs):+ assert only_completed, "for now"+ assert return_sorted is False, "for now"++ if kwargs.get('n', 1) > 1:+ kwargs = {**kwargs, 'logprobs': 5}++ response = self.request(prompt, **kwargs)+ choices = response['choices']++ completed_choices = [c for c in choices if c['finish_reason'] != 'length']++ if only_completed and len(completed_choices):+ choices = completed_choices++ completions = [c['text'] for c in choices]++ if return_sorted and kwargs.get('n', 1) > 1:+ scored_completions = []++ for c in choices:+ tokens, logprobs = c['logprobs']['tokens'], c['logprobs']['token_logprobs']++ if '<|endoftext|>' in tokens:+ index = tokens.index('<|endoftext|>') + 1+ tokens, logprobs = tokens[:index], logprobs[:index]++ avglog = sum(logprobs) / len(logprobs)+ scored_completions.append((avglog, c['text']))++ scored_completions = sorted(scored_completions, reverse=True)+ completions = [c for _, c in scored_completions]++ return completions+++def cached_gpt3_request_v2(**kwargs):+ return openai.Completion.create(**kwargs)++[email protected]_cache(maxsize=None if cache_turn_on else 0)+def cached_gpt3_request_v2_wrapped(**kwargs):+ return cached_gpt3_request_v2(**kwargs)+++cached_gpt3_request = cached_gpt3_request_v2_wrappeddiff --git a/dsp/primitives/__init__.py b/dsp/primitives/__init__.pynew file mode 100644index 000000000..224061fa2--- /dev/null+++ b/dsp/primitives/__init__.py@@ -0,0 +1,4 @@+from .primitives import *+from .demonstrate import *+from .search import *+from .predict import *\ No newline at end of filediff --git a/dsp/primitives/demonstrate.py b/dsp/primitives/demonstrate.pynew file mode 100644index 000000000..485ccfdac--- /dev/null+++ b/dsp/primitives/demonstrate.py@@ -0,0 +1,121 @@+from dsp.utils import dotdict, has_answer, DPR_normalize, normalize_text, EM+++class Example(dotdict):+ def __init__(self, *args, **kwargs):+ assert len(args) <= 1+ super().__init__()++ if len(args):+ assert len(args) == 1+ self.update(args[0])++ self.update(**kwargs)++ def copy(self, **kwargs):+ the_copy = Example(**{**dict(self), **kwargs})++ return the_copy++ def without(self, *keys):+ keys = set(keys)+ return Example({k: v for k, v in self.items() if k not in keys})++ def demos_at(self, fn):+ def at(d):+ try:+ # print("HERE")+ return fn(d).without('augmented')+ except:+ # print("NOT HERE")+ return {}++ demos = [d.copy(**at(d)) for d in self.demos]+ return self.copy(demos=demos)++ # def __str__(self):+ # return self.get('question', self.get('query'))++++import copy+import tqdm+import random++# from colbertfm.metrics.qa import EM, normalize_text+# from colbert.utils.utils import print_message+# from utility.utils.dpr import DPR_normalize, has_answer++import dsp+++def augment(*transformations):+ def do_augment(train, k=None, return_all=False):+ rdemos = []+ ademos = []++ for example in (train): # tqdm.tqdm+ raw_example = dsp.Example(example)++ if k and len(ademos) >= k:+ example = None++ for f in transformations:+ if example is None:+ break++ example = f(example)++ if example is not None:+ example.augmented = True+ ademos.append(example)+ else:+ raw_example.augmented = False+ rdemos.append(raw_example)++ if return_all:+ return ademos + rdemos++ return ademos++ return do_augment++def annotate(*transformations):+ return augment(*transformations)++def sample(train, k: int):+ branch_idx = hasattr(dsp.settings, 'branch_idx') and isinstance(dsp.settings.branch_idx, int) and dsp.settings.branch_idx+ branch_idx = branch_idx or 0+ # print(f"branch_idx = {branch_idx}")++ rng = random.Random(branch_idx)+ shuffled_train = list(train)+ rng.shuffle(shuffled_train)++ subset = shuffled_train[:k]+ subset = [dsp.Example(x) for x in subset] # augmented=False++ return subset++def all_but(train, x):+ # output = [y for y in train if y.question != x.question]++ output = [y for y in train+ if not set.intersection(set(x.get('history', []) + [x.question]), set(y.get('history', []) + [y.question]))]+ # print(len(output))++ return output+++def passage_match(passages, answers):+ # passages = example.passages+ # answers = example.answers+ return any(passage_has_answers(psg, answers) for psg in passages)++def answer_match(prediction, answers):+ # pred = example.prediction+ # answers = example.answers+ return EM(prediction, answers)++def passage_has_answers(passage, answers):+ return has_answer([DPR_normalize(normalize_text(ans)) for ans in answers], normalize_text(passage))diff --git a/dsp/primitives/predict.py b/dsp/primitives/predict.pynew file mode 100644index 000000000..28b87e519--- /dev/null+++ b/dsp/primitives/predict.py@@ -0,0 +1,155 @@+# import random+from collections import Counter+import dsp+from dsp.utils import zipstar++# from dsp.primitives.primitive import Transformation+from dsp.utils import normalize_text++from collections import namedtuple+Completion = namedtuple('Completion', ['rationale', 'prediction'])+++class Completions:+ def __init__(self, completions: list, template):+ self.data = completions+ self.template = template++ def __iter__(self):+ return self.data.__iter__()++ def __getitem__(self, item):+ return self.data[item]++ def __len__(self):+ return len(self.data)++ def unpack(self, key=None):+ if key:+ return [getattr(c, key) for c in self.data]++ return zipstar(self.data)++ def __getattr__(self, name):+ assert len(self.data) == 1++ completion = self.data[0]++ if name in completion.keys():+ return getattr(completion, name)++ if name.endswith('s') and name[:-1] in completion.keys():+ pass++ assert False, name++++def generate(template, **kwargs):+ generator = dsp.settings.lm++ def do_generate(example, stage, max_depth=2):+ assert stage is not None++ # Look up the appropriate fields in each demonstration.+ example = example.demos_at(lambda d: d[stage])++ # Generate and extract the fields.+ prompt = template(example)+ completions = generator(prompt, **kwargs)+ completions = [template.extract(example, p) for p in completions]++ # Find the completions that are most complete.+ field_names = [field.input_variable for field in template.fields]++ last_field_idx = 0+ for field_idx, key in enumerate(field_names):+ completions_ = [c for c in completions if key in c.keys() and c[key] is not None]++ # Filter out completions that are missing fields that are present in at least one completion.+ if len(completions_):+ completions = completions_+ last_field_idx = field_idx+1++ # If none of the completions is completed (i.e., none has the final field set).+ if last_field_idx < len(field_names):+ # Pick the first completion that has gone farthest.+ completion = completions[0]+ completion[field_names[last_field_idx]] = ''++ # Recurse with greedy decoding and a shorter length.+ max_tokens = kwargs.get('max_tokens', dsp.settings.lm.kwargs['max_tokens'])+ max_tokens = min(max(75, max_tokens // 2), max_tokens)+ new_kwargs = {**kwargs, 'max_tokens': max_tokens, 'n': 1, 'temperature': 0.0}++ assert max_depth > 0+ return generate(template, **new_kwargs)(completion, stage=stage, max_depth=max_depth-1)++ completions = Completions(completions, template=template)+ example = example.copy(completions=completions)++ if len(completions) == 1:+ completion = completions[0]+ example[stage] = example.copy(**completion)++ return example, completions++ return do_generate+++def generate_sc(example, prompt, normalize=True, extract=None, prediction_field=None, **kwargs):+ kwargs = {'temperature': 0.7, 'max_tokens': 150, 'n': 20, **kwargs}+ completions = dsp.settings.lm(prompt, **kwargs)+ completions = extract_final_answer(example, completions, extract=extract)+ return majority_vote_(completions, normalize=normalize, prediction_field=prediction_field)++def extract_final_answer(example, completions, extract=None):+ if extract:+ completions = [extract(example, p) for p in completions]+ else:+ completions = [p.strip().split('\n')[-1].split(':', 1)[-1].strip() for p in completions]++ dsp.settings.lm.history.append({**dsp.settings.lm.history[-1], 'completions': completions})++ return completions++def majority(completions, normalize=True, field=None):+ field = completions.template.fields[-1].output_variable if field is None else field++ return Completions(majority_vote_(completions, normalize=normalize, prediction_field=field),+ template=completions.template)+++def majority_vote_(completions, normalize: bool, prediction_field: str):+ if normalize:+ original_completions = completions+ completions_ = []+ for pred in completions:+ if prediction_field in pred:+ completions_.append(normalize_text(pred[prediction_field]))+ else:+ completions_.append('')+ completions = completions_+ normalized_to_original = {}++ for completion, normalized_completion in zip(original_completions, completions):+ if normalized_completion not in normalized_to_original:+ normalized_to_original[normalized_completion] = completion++ completions_ = [x for x in completions if x]+ # completions_ = [x for x in completions_ if len(x.split()) < 7]++ if completions_:+ completions = completions_++ topk = Counter(completions).most_common()+ pred, _ = topk[0]++ if normalize:+ pred = normalized_to_original[pred]++ dsp.settings.lm.history.append({**dsp.settings.lm.history[-1], 'topk': topk, 'completions': [pred]})++ return [pred]++diff --git a/dsp/primitives/primitives.py b/dsp/primitives/primitives.pynew file mode 100644index 000000000..0dab744fe--- /dev/null+++ b/dsp/primitives/primitives.py@@ -0,0 +1,14 @@+import dsp+import copy++from functools import wraps++def shallow_copy_example_args(func):+ @wraps(func)+ def wrapper(*args, **kwargs):+ args = [dsp.Example(arg) if isinstance(arg, dsp.Example) else arg for arg in args]+ kwargs = {key: dsp.Example(value) if isinstance(value, dsp.Example) else value for key, value in kwargs.items()}+ return func(*args, **kwargs)+ return wrapper++transformation = shallow_copy_example_argsdiff --git a/dsp/primitives/search.py b/dsp/primitives/search.pynew file mode 100644index 000000000..adf53d33b--- /dev/null+++ b/dsp/primitives/search.py@@ -0,0 +1,29 @@+import dsp+++def retrieve(query: str, k):+ passages = dsp.settings.rm(query, k=k)+ passages = [psg.long_text for psg in passages]++ return passages++def retrieveEnsemble(queries, k, by_prob=True):+ queries_ = queries+ queries = [q for q in queries if q]++ if queries != queries_:+ print('!?!?!?!??!?! \t\t\t ', queries_)++ passages = {}+ for q in queries:+ for psg in dsp.settings.rm(q, k=k*3):+ if by_prob:+ passages[psg.long_text] = passages.get(psg.long_text, 0.0) + psg.prob+ else:+ passages[psg.long_text] = passages.get(psg.long_text, 0.0) + psg.score++ passages = [(score, text) for text, score in passages.items()]+ passages = sorted(passages, reverse=True)[:k]+ passages = [text for _, text in passages]++ return passagesdiff --git a/dsp/templates/__init__.py b/dsp/templates/__init__.pynew file mode 100644index 000000000..240ce9424--- /dev/null+++ b/dsp/templates/__init__.py@@ -0,0 +1,7 @@+from .utils import *+from .template_v2 import *+from .template_v3 import *+# from .template import *+# from .qa import *+# from .first_hop import *+# from .nth_hop import *diff --git a/dsp/templates/template_v2.py b/dsp/templates/template_v2.pynew file mode 100644index 000000000..427d070df--- /dev/null+++ b/dsp/templates/template_v2.py@@ -0,0 +1,210 @@+from collections import namedtuple+import re++import dsp++Field = namedtuple(+ "Field", "name seperator input_variable output_variable description")++# TODO: de-duplicate with dsp/templates/template.py+++def passages2text(passages):+ if type(passages) is str:+ return passages++ assert type(passages) in [list, tuple]++ if len(passages) == 1:+ return f"«{passages[0]}»"++ return "\n".join([f"[{idx+1}] «{txt}»" for idx, txt in enumerate(passages)])+++def format_answers(answers):+ if isinstance(answers, list):+ if len(answers) >= 1:+ return str(answers[0]).strip()+ elif len(answers) == 0:+ raise ValueError(f"No answers found")+ elif isinstance(answers, str):+ return answers+ else:+ raise ValueError(f"Unable to parse answers of type {type(answers)}")+++class TemplateV2:+ def __init__(self, template, format_handlers={"passages": passages2text, "context": passages2text, "answer": format_answers, "answers": format_answers}):+ self.format_handlers = format_handlers++ template = template.strip()++ self.instructions = re.search("(.*)\n", template).group(1)+ template = template[len(self.instructions):].strip()++ self.fields = []+ while len(template) > 0:+ match = re.search("(.*)(\s){(.*)}\s(.*\${.*})", template)+ if match is not None:+ name = match.group(1)+ seperator = match.group(2)+ variable = match.group(3)+ description = match.group(4)+ else:+ match = re.search("(.*)(\s){(.*)}", template)+ if match is not None:+ name = match.group(1)+ seperator = match.group(2)+ variable = match.group(3)+ description = None+ else:+ raise ValueError(f"Could not parse template")++ var_match = re.match("(.*) -> (.*)", variable)+ if var_match is not None:+ input_variable = var_match.group(1)+ output_variable = var_match.group(2)+ else:+ input_variable = variable+ output_variable = variable++ self.fields.append(+ Field(+ name=name,+ seperator=seperator,+ input_variable=input_variable,+ output_variable=output_variable,+ description=description,+ )+ )++ template = template[len(match.group(0)):].strip()++ def query(self, example, is_demo=False):+ result = []++ if not is_demo:+ has_value = [+ field.input_variable in example+ and example[field.input_variable] is not None+ and example[field.input_variable] != ""+ for field in self.fields+ ]++ for i in range(1, len(has_value)):+ if has_value[i - 1] and not any(has_value[i:]):+ example[self.fields[i].input_variable] = ""+ break++ for field in self.fields:+ if (field.input_variable in example and example[field.input_variable] is not None):+ if field.input_variable in self.format_handlers:+ format_handler = self.format_handlers[field.input_variable]+ else:+ def format_handler(x): return ' '.join(x.split())++ result.append(f"{field.name}{field.seperator}{format_handler(example[field.input_variable])}")++ if self._has_augmented_guidelines() and ("augmented" in example and example.augmented):+ return "\n\n".join(result)+ else:+ return "\n".join(result)++ def guidelines(self):+ result = "Follow the following format.\n\n"++ example = dsp.Example()+ for field in self.fields:+ example[field.input_variable] = field.description+ example.augmented = self._has_augmented_guidelines()++ result += self.query(example)+ return result++ def _has_augmented_guidelines(self):+ return len(self.fields) > 3 or any(field.seperator == "\n" for field in self.fields)++ def extract(self, example, raw_pred):+ example = dsp.Example(example)++ raw_pred = raw_pred.strip()++ idx = 0+ while idx < len(self.fields):+ if (self.fields[idx].input_variable not in example or example[self.fields[idx].input_variable] is None):+ break+ idx += 1++ idx = min(idx, len(self.fields) - 1)++ while raw_pred != "" and idx < len(self.fields):+ # print("HERE,", idx)+ if idx < len(self.fields) - 1:+ next_field_name = '\n' + self.fields[idx + 1].name+ offset = raw_pred.find(next_field_name)+ # print(f"Looking for ```{next_field_name}```. Found it at {offset}")++ if offset >= 0:+ example[self.fields[idx].output_variable] = raw_pred[:offset].strip()+ raw_pred = raw_pred[offset + len(next_field_name):].strip()+ idx += 1+ else:+ example[self.fields[idx].output_variable] = raw_pred.strip()+ raw_pred = ""+ idx += 1+ # print("Exiting")+ break++ else: #if idx == len(self.fields) - 1:+ assert idx == len(self.fields) - 1, (idx, len(self.fields))+ example[self.fields[idx].output_variable] = raw_pred.strip()+ break+ # offset = raw_pred.find(self.fields[idx].name)+ # if offset < 0:+ # else:+ # raw_pred = raw_pred[offset + len(self.fields[idx].name):].strip()+ # example[self.fields[idx].output_variable] = raw_pred++ return example++ def __call__(self, example):+ example = dsp.Example(example)++ if self.fields[-1].input_variable in example:+ del example[self.fields[-1].input_variable]++ rdemos = [+ self.query(demo, is_demo=True)+ for demo in example.demos+ if (+ ("augmented" not in demo or not demo.augmented)+ and (+ self.fields[-1].input_variable in demo+ and demo[self.fields[-1].input_variable] is not None+ )+ )+ ]+ ademos = [+ self.query(demo, is_demo=True)+ for demo in example.demos+ if "augmented" in demo and demo.augmented+ ]++ long_query = self._has_augmented_guidelines()+ if long_query:+ example["augmented"] = True+ query = self.query(example)++ rdemos = "\n\n".join(rdemos)+ if len(rdemos) >= 1 and len(ademos) == 0 and not long_query:+ rdemos_and_query = "\n\n".join([rdemos, query])+ parts = [self.instructions, self.guidelines(), rdemos_and_query]+ elif len(rdemos) == 0:+ parts = [self.instructions, self.guidelines(), *ademos, query]+ else:+ parts = [self.instructions, rdemos,+ self.guidelines(), *ademos, query]++ prompt = "\n\n---\n\n".join([p.strip() for p in parts if p])++ return prompt.strip()diff --git a/dsp/templates/template_v3.py b/dsp/templates/template_v3.pynew file mode 100644index 000000000..bf9fe44a9--- /dev/null+++ b/dsp/templates/template_v3.py@@ -0,0 +1,32 @@+from dsp.templates import TemplateV2, passages2text, format_answers, Field++class Type:+ def __init__(self, prefix, desc, format=None, aliases=[]) -> None:+ self.prefix = prefix+ self.desc = desc+ self.format = format+ self.aliases = aliases++ assert aliases == [], "TODO: If aliases is used, handling of input_variable in V2 needs to be richer."++ def __call__(self, **kwargs):+ kwargs = {**self.__dict__, **kwargs}+ return Type(**kwargs)++class Template(TemplateV2):+ def __init__(self, instructions, **kwargs):+ self.instructions = instructions++ self.fields = []+ self.format_handlers = {"context": passages2text, "answers": format_answers}++ for key, value in kwargs.items():+ prefix = value.prefix+ separator = ' ' if prefix.rstrip() == prefix else prefix[len(prefix.rstrip()):]+ field = Field(name=prefix.strip(), description=value.desc,+ input_variable=key, output_variable=key, seperator=separator)+ self.fields.append(field)++ if value.format:+ for name in [key, *value.aliases]:+ self.format_handlers[name] = value.formatdiff --git a/dsp/templates/utils.py b/dsp/templates/utils.pynew file mode 100644index 000000000..8c217d535--- /dev/null+++ b/dsp/templates/utils.py@@ -0,0 +1,10 @@+def passages2text(passages):+ if type(passages) is str:+ return passages++ assert type(passages) in [list, tuple]++ if len(passages) == 1:+ return f"«{passages[0]}»"++ return '\n'.join([f"[{idx+1}] «{txt}»" for idx, txt in enumerate(passages)])diff --git a/dsp/utils/__init__.py b/dsp/utils/__init__.pynew file mode 100644index 000000000..dc24ec025--- /dev/null+++ b/dsp/utils/__init__.py@@ -0,0 +1,4 @@+from dsp.utils.settings import *+from dsp.utils.utils import *+from dsp.utils.metrics import *+from dsp.utils.dpr import *\ No newline at end of filediff --git a/dsp/utils/dpr.py b/dsp/utils/dpr.pynew file mode 100644index 000000000..f268d85be--- /dev/null+++ b/dsp/utils/dpr.py@@ -0,0 +1,237 @@+"""+ Source: DPR Implementation from Facebook Research+ https://github.com/facebookresearch/DPR/tree/master/dpr+"""++import string+import spacy+import regex+import unicodedata+++class Tokens(object):+ """A class to represent a list of tokenized text."""+ TEXT = 0+ TEXT_WS = 1+ SPAN = 2+ POS = 3+ LEMMA = 4+ NER = 5++ def __init__(self, data, annotators, opts=None):+ self.data = data+ self.annotators = annotators+ self.opts = opts or {}++ def __len__(self):+ """The number of tokens."""+ return len(self.data)++ def slice(self, i=None, j=None):+ """Return a view of the list of tokens from [i, j)."""+ new_tokens = copy.copy(self)+ new_tokens.data = self.data[i: j]+ return new_tokens++ def untokenize(self):+ """Returns the original text (with whitespace reinserted)."""+ return ''.join([t[self.TEXT_WS] for t in self.data]).strip()++ def words(self, uncased=False):+ """Returns a list of the text of each token++ Args:+ uncased: lower cases text+ """+ if uncased:+ return [t[self.TEXT].lower() for t in self.data]+ else:+ return [t[self.TEXT] for t in self.data]++ def offsets(self):+ """Returns a list of [start, end) character offsets of each token."""+ return [t[self.SPAN] for t in self.data]++ def pos(self):+ """Returns a list of part-of-speech tags of each token.+ Returns None if this annotation was not included.+ """+ if 'pos' not in self.annotators:+ return None+ return [t[self.POS] for t in self.data]++ def lemmas(self):+ """Returns a list of the lemmatized text of each token.+ Returns None if this annotation was not included.+ """+ if 'lemma' not in self.annotators:+ return None+ return [t[self.LEMMA] for t in self.data]++ def entities(self):+ """Returns a list of named-entity-recognition tags of each token.+ Returns None if this annotation was not included.+ """+ if 'ner' not in self.annotators:+ return None+ return [t[self.NER] for t in self.data]++ def ngrams(self, n=1, uncased=False, filter_fn=None, as_strings=True):+ """Returns a list of all ngrams from length 1 to n.++ Args:+ n: upper limit of ngram length+ uncased: lower cases text+ filter_fn: user function that takes in an ngram list and returns+ True or False to keep or not keep the ngram+ as_string: return the ngram as a string vs list+ """++ def _skip(gram):+ if not filter_fn:+ return False+ return filter_fn(gram)++ words = self.words(uncased)+ ngrams = [(s, e + 1)+ for s in range(len(words))+ for e in range(s, min(s + n, len(words)))+ if not _skip(words[s:e + 1])]++ # Concatenate into strings+ if as_strings:+ ngrams = ['{}'.format(' '.join(words[s:e])) for (s, e) in ngrams]++ return ngrams++ def entity_groups(self):+ """Group consecutive entity tokens with the same NER tag."""+ entities = self.entities()+ if not entities:+ return None+ non_ent = self.opts.get('non_ent', 'O')+ groups = []+ idx = 0+ while idx < len(entities):+ ner_tag = entities[idx]+ # Check for entity tag+ if ner_tag != non_ent:+ # Chomp the sequence+ start = idx+ while (idx < len(entities) and entities[idx] == ner_tag):+ idx += 1+ groups.append((self.slice(start, idx).untokenize(), ner_tag))+ else:+ idx += 1+ return groups+++class Tokenizer(object):+ """Base tokenizer class.+ Tokenizers implement tokenize, which should return a Tokens class.+ """++ def tokenize(self, text):+ raise NotImplementedError++ def shutdown(self):+ pass++ def __del__(self):+ self.shutdown()+++class SimpleTokenizer(Tokenizer):+ ALPHA_NUM = r'[\p{L}\p{N}\p{M}]+'+ NON_WS = r'[^\p{Z}\p{C}]'++ def __init__(self, **kwargs):+ """+ Args:+ annotators: None or empty set (only tokenizes).+ """+ self._regexp = regex.compile(+ '(%s)|(%s)' % (self.ALPHA_NUM, self.NON_WS),+ flags=regex.IGNORECASE + regex.UNICODE + regex.MULTILINE+ )+ if len(kwargs.get('annotators', {})) > 0:+ logger.warning('%s only tokenizes! Skipping annotators: %s' %+ (type(self).__name__, kwargs.get('annotators')))+ self.annotators = set()++ def tokenize(self, text):+ data = []+ matches = [m for m in self._regexp.finditer(text)]+ for i in range(len(matches)):+ # Get text+ token = matches[i].group()++ # Get whitespace+ span = matches[i].span()+ start_ws = span[0]+ if i + 1 < len(matches):+ end_ws = matches[i + 1].span()[0]+ else:+ end_ws = span[1]++ # Format data+ data.append((+ token,+ text[start_ws: end_ws],+ span,+ ))+ return Tokens(data, self.annotators)+++def has_answer(tokenized_answers, text):+ text = DPR_normalize(text)++ for single_answer in tokenized_answers:+ for i in range(0, len(text) - len(single_answer) + 1):+ if single_answer == text[i: i + len(single_answer)]:+ return True++ return False+++def locate_answers(tokenized_answers, text):+ """+ Returns each occurrence of an answer as (offset, endpos) in terms of *characters*.+ """+ tokenized_text = DPR_tokenize(text)+ occurrences = []++ text_words, text_word_positions = tokenized_text.words(uncased=True), tokenized_text.offsets()+ answers_words = [ans.words(uncased=True) for ans in tokenized_answers]++ for single_answer in answers_words:+ for i in range(0, len(text_words) - len(single_answer) + 1):+ if single_answer == text_words[i: i + len(single_answer)]:+ (offset, _), (_, endpos) = text_word_positions[i], text_word_positions[i+len(single_answer)-1]+ occurrences.append((offset, endpos))++ return occurrences+++STokenizer = SimpleTokenizer()+++def DPR_tokenize(text):+ return STokenizer.tokenize(unicodedata.normalize('NFD', text))+++def DPR_normalize(text):+ return DPR_tokenize(text).words(uncased=True)+++# Source: https://github.com/shmsw25/qa-hard-em/blob/master/prepro_util.py+def strip_accents(text):+ """Strips accents from a piece of text."""+ text = unicodedata.normalize("NFD", text)+ output = []+ for char in text:+ cat = unicodedata.category(char)+ if cat == "Mn":+ continue+ output.append(char)+ return "".join(output)diff --git a/dsp/utils/metrics.py b/dsp/utils/metrics.pynew file mode 100644index 000000000..f0d69f00f--- /dev/null+++ b/dsp/utils/metrics.py@@ -0,0 +1,160 @@+import re+import string+import unicodedata++from collections import Counter+from dsp.utils.utils import print_message+++def EM(prediction, answers_list):+ assert type(answers_list) == list++ return max(em_score(prediction, ans) for ans in answers_list)+++def F1(prediction, answers_list):+ assert type(answers_list) == list++ return max(f1_score(prediction, ans) for ans in answers_list)+++def HotPotF1(prediction, answers_list):+ assert type(answers_list) == list++ return max(hotpot_f1_score(prediction, ans) for ans in answers_list)+++def nF1(history, prediction, answers_list, return_recall=False):+ assert type(answers_list) == list++ return max(novel_f1_score(history, prediction, ans, return_recall=return_recall) for ans in answers_list)+++def normalize_text(s):+ s = unicodedata.normalize('NFD', s)++ def remove_articles(text):+ return re.sub(r'\b(a|an|the)\b', ' ', text)++ def white_space_fix(text):+ return ' '.join(text.split())++ def remove_punc(text):+ exclude = set(string.punctuation)+ return ''.join(ch for ch in text if ch not in exclude)++ def lower(text):+ return text.lower()++ return white_space_fix(remove_articles(remove_punc(lower(s))))+++def em_score(prediction, ground_truth):+ return normalize_text(prediction) == normalize_text(ground_truth)+++# See: https://github.com/hotpotqa/hotpot/blob/master/hotpot_evaluate_v1.py+# See: https://rajpurkar.github.io/SQuAD-explorer/ under Evaluation Script+# See: QReCC's++def f1_score(prediction, ground_truth):+ prediction_tokens = normalize_text(prediction).split()+ ground_truth_tokens = normalize_text(ground_truth).split()++ common = Counter(prediction_tokens) & Counter(ground_truth_tokens)+ num_same = sum(common.values())++ if len(prediction_tokens) == len(ground_truth_tokens) == 0:+ # Unlike most tasks, QReCC and SQuAD-2.0 assign 1.0 in this edge case. We don't for uniformity.+ print_message(+ "\n#> F1 Metric: Rare edge case of len(prediction_tokens) == len(ground_truth_tokens) == 0.\n")++ if num_same == 0:+ return 0++ precision = 1.0 * num_same / len(prediction_tokens)+ recall = 1.0 * num_same / len(ground_truth_tokens)+ f1 = (2 * precision * recall) / (precision + recall)++ return f1+++def hotpot_f1_score(prediction, ground_truth):+ normalized_prediction = normalize_text(prediction)+ normalized_ground_truth = normalize_text(ground_truth)++ if normalized_prediction in ['yes', 'no', 'noanswer'] and normalized_prediction != normalized_ground_truth:+ return 0+ if normalized_ground_truth in ['yes', 'no', 'noanswer'] and normalized_prediction != normalized_ground_truth:+ return 0++ prediction_tokens = normalized_prediction.split()+ ground_truth_tokens = normalized_ground_truth.split()+ common = Counter(prediction_tokens) & Counter(ground_truth_tokens)+ num_same = sum(common.values())+ if num_same == 0:+ return 0+ precision = 1.0 * num_same / len(prediction_tokens)+ recall = 1.0 * num_same / len(ground_truth_tokens)+ f1 = (2 * precision * recall) / (precision + recall)+ return f1+++def precision_score(prediction, ground_truth):+ prediction_tokens = normalize_text(prediction).split()+ ground_truth_tokens = normalize_text(ground_truth).split()++ common = Counter(prediction_tokens) & Counter(ground_truth_tokens)+ num_same = sum(common.values())++ if len(prediction_tokens) == len(ground_truth_tokens) == 0:+ # Unlike most tasks, QReCC and SQuAD-2.0 assign 1.0 in this edge case. We don't for uniformity.+ print_message(+ "\n#> F1 Metric: Rare edge case of len(prediction_tokens) == len(ground_truth_tokens) == 0.\n")++ if num_same == 0:+ return 0++ precision = 1.0 * num_same / len(prediction_tokens)++ return precision+++# Source: https://gist.github.com/sebleier/554280+stopwords = ["i", "me", "my", "myself", "we", "our", "ours", "ourselves", "you", "your", "yours", "yourself",+ "yourselves", "he", "him", "his", "himself", "she", "her", "hers", "herself", "it", "its", "itself",+ "they", "them", "their", "theirs", "themselves", "what", "which", "who", "whom", "this", "that", "these",+ "those", "am", "is", "are", "was", "were", "be", "been", "being", "have", "has", "had", "having", "do",+ "does", "did", "doing", "a", "an", "the", "and", "but", "if", "or", "because", "as", "until", "while",+ "of", "at", "by", "for", "with", "about", "against", "between", "into", "through", "during", "before",+ "after", "above", "below", "to", "from", "up", "down", "in", "out", "on", "off", "over", "under", "again",+ "further", "then", "once", "here", "there", "when", "where", "why", "how", "all", "any", "both", "each",+ "few", "more", "most", "other", "some", "such", "no", "nor", "not", "only", "own", "same", "so", "than",+ "too", "very", "s", "t", "can", "will", "just", "don", "should", "now"]+++def novel_f1_score(history, prediction, ground_truth, return_recall=False):+ history_tokens = normalize_text(history).split()+ prediction_tokens = normalize_text(prediction).split()+ ground_truth_tokens = normalize_text(ground_truth).split()++ history_tokens = set(history_tokens + stopwords)++ prediction_tokens = [+ t for t in prediction_tokens if t not in history_tokens]+ ground_truth_tokens = [+ t for t in ground_truth_tokens if t not in history_tokens]++ common = Counter(prediction_tokens) & Counter(ground_truth_tokens)+ num_same = sum(common.values())+ if num_same == 0:+ return 0++ precision = 1.0 * num_same / len(prediction_tokens)+ recall = 1.0 * num_same / len(ground_truth_tokens)+ f1 = (2 * precision * recall) / (precision + recall)++ if return_recall:+ return recall++ return f1diff --git a/dsp/utils/settings.py b/dsp/utils/settings.pynew file mode 100644index 000000000..654402606--- /dev/null+++ b/dsp/utils/settings.py@@ -0,0 +1,59 @@+from contextlib import contextmanager+from dsp.utils.utils import dotdict+++class Settings(object):+ _instance = None++ def __new__(cls):+ """+ Singleton Pattern. See https://python-patterns.guide/gang-of-four/singleton/+ """++ if cls._instance is None:+ cls._instance = super().__new__(cls)+ cls._instance.stack = []++ config = dotdict()+ cls._instance.__append(config)++ return cls._instance++ @property+ def config(self):+ return self.stack[-1]++ def __getattr__(self, name):+ if hasattr(self.config, name):+ return getattr(self.config, name)++ if name in self.config:+ return self.config[name]++ super().__getattr__(name)++ def __append(self, config):+ self.stack.append(config)++ def __pop(self):+ self.stack.pop()++ def configure(self, inherit_config=True, **kwargs):+ if inherit_config:+ config = {**self.config, **kwargs}++ self.__append(config)++ @contextmanager+ def context(self, inherit_config=True, **kwargs):+ self.configure(inherit_config=inherit_config, **kwargs)++ try:+ yield+ finally:+ self.__pop()++ def __repr__(self) -> str:+ return repr(self.config)++settings = Settings()diff --git a/dsp/utils/utils.py b/dsp/utils/utils.pynew file mode 100644index 000000000..ec5978ffe--- /dev/null+++ b/dsp/utils/utils.py@@ -0,0 +1,211 @@+import os+import tqdm+import datetime+import itertools++from collections import defaultdict+++def print_message(*s, condition=True, pad=False, sep=None):+ s = ' '.join([str(x) for x in s])+ msg = "[{}] {}".format(datetime.datetime.now().strftime("%b %d, %H:%M:%S"), s)++ if condition:+ msg = msg if not pad else f'\n{msg}\n'+ print(msg, flush=True, sep=sep)+++ return msg+++def timestamp(daydir=False):+ format_str = f"%Y-%m{'/' if daydir else '-'}%d{'/' if daydir else '_'}%H.%M.%S"+ result = datetime.datetime.now().strftime(format_str)+ return result+++def file_tqdm(file):+ print(f"#> Reading {file.name}")++ with tqdm.tqdm(total=os.path.getsize(file.name) / 1024.0 / 1024.0, unit="MiB") as pbar:+ for line in file:+ yield line+ pbar.update(len(line) / 1024.0 / 1024.0)++ pbar.close()+++def create_directory(path):+ if os.path.exists(path):+ print('\n')+ print_message("#> Note: Output directory", path, 'already exists\n\n')+ else:+ print('\n')+ print_message("#> Creating directory", path, '\n\n')+ os.makedirs(path)+++def deduplicate(seq):+ """+ Source: https://stackoverflow.com/a/480227/1493011+ """++ seen = set()+ return [x for x in seq if not (x in seen or seen.add(x))]+++def batch(group, bsize, provide_offset=False):+ offset = 0+ while offset < len(group):+ L = group[offset: offset + bsize]+ yield ((offset, L) if provide_offset else L)+ offset += len(L)+ return+++class dotdict(dict):+ """+ dot.notation access to dictionary attributes+ Credit: derek73 @ https://stackoverflow.com/questions/2352181+ """+ __getattr__ = dict.__getitem__+ __setattr__ = dict.__setitem__+ __delattr__ = dict.__delitem__+++class dotdict_lax(dict):+ __getattr__ = dict.get+ __setattr__ = dict.__setitem__+ __delattr__ = dict.__delitem__+++def flatten(L):+ # return [x for y in L for x in y]++ result = []+ for _list in L:+ result += _list++ return result+++def zipstar(L, lazy=False):+ """+ A much faster A, B, C = zip(*[(a, b, c), (a, b, c), ...])+ May return lists or tuples.+ """++ if len(L) == 0:+ return L++ width = len(L[0])++ if width < 100:+ return [[elem[idx] for elem in L] for idx in range(width)]++ L = zip(*L)++ return L if lazy else list(L)+++def zip_first(L1, L2):+ length = len(L1) if type(L1) in [tuple, list] else None++ L3 = list(zip(L1, L2))++ assert length in [None, len(L3)], "zip_first() failure: length differs!"++ return L3+++def int_or_float(val):+ if '.' in val:+ return float(val)++ return int(val)+++def groupby_first_item(lst):+ groups = defaultdict(list)++ for first, *rest in lst:+ rest = rest[0] if len(rest) == 1 else rest+ groups[first].append(rest)++ return groups+++def process_grouped_by_first_item(lst):+ """+ Requires items in list to already be grouped by first item.+ """++ groups = defaultdict(list)++ started = False+ last_group = None++ for first, *rest in lst:+ rest = rest[0] if len(rest) == 1 else rest++ if started and first != last_group:+ yield (last_group, groups[last_group])+ assert first not in groups, f"{first} seen earlier --- violates precondition."++ groups[first].append(rest)++ last_group = first+ started = True++ return groups+++def grouper(iterable, n, fillvalue=None):+ """+ Collect data into fixed-length chunks or blocks+ Example: grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx"+ Source: https://docs.python.org/3/library/itertools.html#itertools-recipes+ """++ args = [iter(iterable)] * n+ return itertools.zip_longest(*args, fillvalue=fillvalue)+++def lengths2offsets(lengths):+ offset = 0++ for length in lengths:+ yield (offset, offset + length)+ offset += length++ return+++# see https://stackoverflow.com/a/45187287+class NullContextManager(object):+ def __init__(self, dummy_resource=None):+ self.dummy_resource = dummy_resource+ def __enter__(self):+ return self.dummy_resource+ def __exit__(self, *args):+ pass+++def load_batch_backgrounds(args, qids):+ if args.qid2backgrounds is None:+ return None++ qbackgrounds = []++ for qid in qids:+ back = args.qid2backgrounds[qid]++ if len(back) and type(back[0]) == int:+ x = [args.collection[pid] for pid in back]+ else:+ x = [args.collectionX.get(pid, '') for pid in back]++ x = ' [SEP] '.join(x)+ qbackgrounds.append(x)++ return qbackgroundsdiff --git a/intro.ipynb b/intro.ipynbnew file mode 100644index 000000000..3fe1ded7c--- /dev/null+++ b/intro.ipynb@@ -0,0 +1,2016 @@+{+ "cells": [+ {+ "attachments": {},+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "## 🤝🎓 𝗗𝗦𝗣: 𝗗EMONSTRATE–𝗦EARCH–𝗣REDICT\n",+ "\n",+ "This notebook illustrates the Demonstrate–Search–Predict (𝗗𝗦𝗣) framework on the multi-hop question answering (QA) task.\n",+ "\n",+ "Below, we'll build **five** small 𝗗𝗦𝗣 programs that implement strategies for this task. The programs range from extremely simple ones (programs 1 and 2) to moderate ones (programs 3 and 4). Program 5 considers a fairly advanced pipeline."+ ]+ },+ {+ "attachments": {},+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "### Set up\n",+ "\n",+ "We'll start by setting up the language model (LM) and retrieval model (RM).\n",+ "\n",+ "We will work with the **GPT-3.5** LM (`text-davinci-002`) and the **ColBERTv2** RM.\n",+ "\n",+ "To use GPT-3, you'll need an OpenAI key. For ColBERTv2, we've set up a server hosting a Wikipedia (Dec 2018) search index, so you don't need to worry about setting one up!\n",+ "\n",+ "To make things easy, we've set up a cache in this repository. _If you want to run this notebook without changing the code or examples, you don't need an API key. All examples are cached._"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 1,+ "metadata": {},+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "env: DSP_NOTEBOOK_CACHEDIR=cache\n"+ ]+ }+ ],+ "source": [+ "%set_env DSP_NOTEBOOK_CACHEDIR cache\n",+ "%load_ext autoreload\n",+ "%autoreload 2\n",+ "\n",+ "import os\n",+ "import dsp\n",+ "\n",+ "openai_key = os.getenv('OPENAI_API_KEY') # or replace with your API key (optional)\n",+ "colbert_server = 'http://ec2-44-228-128-229.us-west-2.compute.amazonaws.com:8893/api/search'\n",+ "\n",+ "lm = dsp.GPT3(model='text-davinci-002', api_key=openai_key)\n",+ "rm = dsp.ColBERTv2(url=colbert_server)\n",+ "\n",+ "dsp.settings.configure(lm=lm, rm=rm)"+ ]+ },+ {+ "attachments": {},+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "### Task Examples\n",+ "\n",+ "Next, let's look at a few examples of the task. Each example consists of a question and one or more gold answers.\n",+ "\n",+ "We have six training examples (`train`), which we'll feed into the programs. These will help define the task.\n",+ "\n",+ "Notice that our examples only have input (`question`) and output (`answer`) fields. When our advanced programs build sophisticated pipelines, training \"demonstrations\" for other fields will be constructed automatically."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 2,+ "metadata": {},+ "outputs": [],+ "source": [+ "train = [('Who produced the album that included a re-recording of \"Lithium\"?', ['Butch Vig']),\n",+ " ('Who was the director of the 2009 movie featuring Peter Outerbridge as William Easton?', ['Kevin Greutert']),\n",+ " ('The heir to the Du Pont family fortune sponsored what wrestling team?', ['Foxcatcher', 'Team Foxcatcher', 'Foxcatcher Team']),\n",+ " ('In what year was the star of To Hell and Back born?', ['1925']),\n",+ " ('Which award did the first book of Gary Zukav receive?', ['U.S. National Book Award', 'National Book Award']),\n",+ " ('What city was the victim of Joseph Druces working in?', ['Boston, Massachusetts', 'Boston']),]\n",+ "\n",+ "train = [dsp.Example(question=question, answer=answer) for question, answer in train]"+ ]+ },+ {+ "attachments": {},+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "The development examples (`dev`) will be used to assess the behavior of each program we build. Of course, this tiny set is not meant to be a reliable benchmark, but it'll be instructive to use it for illustration."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 3,+ "metadata": {},+ "outputs": [],+ "source": [+ "dev = [('Who has a broader scope of profession: E. L. Doctorow or Julia Peterkin?', ['E. L. Doctorow', 'E.L. Doctorow', 'Doctorow']),\n",+ " ('What documentary about the Gilgo Beach Killer debuted on A&E?', ['The Killing Season']),\n",+ " ('Right Back At It Again contains lyrics co-written by the singer born in what city?', ['Gainesville, Florida', 'Gainesville']),\n",+ " ('What year was the party of the winner of the 1971 San Francisco mayoral election founded?', ['1828']),\n",+ " ('Which author is English: John Braine or Studs Terkel?', ['John Braine']),\n",+ " ('Anthony Dirrell is the brother of which super middleweight title holder?', ['Andre Dirrell']),\n",+ " ('In which city is the sports nutrition business established by Oliver Cookson based ?', ['Cheshire', 'Cheshire, UK']),\n",+ " ('Find the birth date of the actor who played roles in First Wives Club and Searching for the Elephant.', ['February 13, 1980']),\n",+ " ('Kyle Moran was born in the town on what river?', ['Castletown', 'Castletown River']),\n",+ " (\"What is the name of one branch of Robert D. Braun's speciality?\", ['aeronautical engineering', 'astronautical engineering', 'aeronautics', 'astronautics']),\n",+ " (\"Where was the actress who played the niece in the Priest film born?\", ['Surrey', 'Guildford, Surrey']),\n",+ " ('Name the movie in which the daughter of Noel Harrison plays Violet Trefusis.', ['Portrait of a Marriage']),\n",+ " ('What year was the father of the Princes in the Tower born?', ['1442'])]\n",+ "\n",+ "dev = [dsp.Example(question=question, answer=answer) for question, answer in dev]"+ ]+ },+ {+ "attachments": {},+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "### Program 1: Vanilla GPT-3.5\n",+ "\n",+ "Let's begin by implementing the simplest program for this task. We'll prompt **GPT-3.5** to answer each question based on its internal parameteric knowledge.\n",+ "\n",+ "We'll start by defining the `Template` that defines how we will communicate with the LM.\n",+ "\n",+ "Specifically, the question–answer template (`qa_template`) will include a question and a short answer for each example."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 4,+ "metadata": {},+ "outputs": [],+ "source": [+ "Question = dsp.Type(prefix=\"Question:\", desc=\"${the question to be answered}\")\n",+ "Answer = dsp.dsp.Type(prefix=\"Final Answer:\", desc=\"${a short factoid answer, often between 1 and 5 words}\", format=dsp.format_answers)\n",+ "\n",+ "qa_template = dsp.Template(instructions=\"Answer questions with short factoid answers.\", question=Question(), answer=Answer())"+ ]+ },+ {+ "attachments": {},+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "Then, let's define the actual program, `vanilla_LM_QA`. It'll accept a string (`question`) and returns another string (its short `answer`)."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 5,+ "metadata": {},+ "outputs": [],+ "source": [+ "def vanilla_LM_QA(question: str) -> str:\n",+ " demos = dsp.sample(train, k=7)\n",+ " example = dsp.Example(question=question, demos=demos)\n",+ "\n",+ " example, completions = dsp.generate(qa_template)(example, stage='qa')\n",+ " return completions.answer"+ ]+ },+ {+ "attachments": {},+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "Let's invoke the program on a sample question."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 6,+ "metadata": {},+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Who has a broader scope of profession: E. L. Doctorow or Julia Peterkin?\n",+ "E. L. Doctorow\n"+ ]+ }+ ],+ "source": [+ "print(dev[0].question)\n",+ "print(vanilla_LM_QA(dev[0].question))"+ ]+ },+ {+ "attachments": {},+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "Let's inspect the last call to the LM to learn more about the behavior of the program."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 7,+ "metadata": {},+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "\n",+ "\n",+ "\n",+ "\n",+ "Answer questions with short factoid answers.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Question: ${the question to be answered}\n",+ "Final Answer: ${a short factoid answer, often between 1 and 5 words}\n",+ "\n",+ "---\n",+ "\n",+ "Question: Which award did the first book of Gary Zukav receive?\n",+ "Final Answer: U.S. National Book Award\n",+ "\n",+ "Question: The heir to the Du Pont family fortune sponsored what wrestling team?\n",+ "Final Answer: Foxcatcher\n",+ "\n",+ "Question: Who was the director of the 2009 movie featuring Peter Outerbridge as William Easton?\n",+ "Final Answer: Kevin Greutert\n",+ "\n",+ "Question: Who produced the album that included a re-recording of \"Lithium\"?\n",+ "Final Answer: Butch Vig\n",+ "\n",+ "Question: What city was the victim of Joseph Druces working in?\n",+ "Final Answer: Boston, Massachusetts\n",+ "\n",+ "Question: In what year was the star of To Hell and Back born?\n",+ "Final Answer: 1925\n",+ "\n",+ "Question: Who has a broader scope of profession: E. L. Doctorow or Julia Peterkin?\n",+ "Final Answer:\u001b[32m E. L. Doctorow\u001b[0m\n",+ "\n",+ "\n",+ "\n"+ ]+ }+ ],+ "source": [+ "lm.inspect_history(n=1)"+ ]+ },+ {+ "attachments": {},+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "We can see that the program has organized the examples in a simple prompt and generated an answer for the `dev[0]` question we asked.\n",+ "\n",+ "Let's try on the remaining development examples."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 8,+ "metadata": {},+ "outputs": [+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "100%|██████████| 13/13 [00:00<00:00, 115.31it/s]\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Answered 4 / 13 correctly.\n"+ ]+ },+ {+ "data": {+ "text/html": [+ "<style type=\"text/css\">\n",+ "#T_e645f_ th {\n",+ " text-align: left;\n",+ "}\n",+ "#T_e645f_ td {\n",+ " text-align: left;\n",+ "}\n",+ "</style>\n",+ "<table id=\"T_e645f_\">\n",+ " <thead>\n",+ " <tr>\n",+ " <th class=\"blank level0\" > </th>\n",+ " <th class=\"col_heading level0 col0\" >question</th>\n",+ " <th class=\"col_heading level0 col1\" >answer</th>\n",+ " <th class=\"col_heading level0 col2\" >prediction</th>\n",+ " <th class=\"col_heading level0 col3\" >correct</th>\n",+ " </tr>\n",+ " </thead>\n",+ " <tbody>\n",+ " <tr>\n",+ " <th id=\"T_e645f_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",+ " <td id=\"T_e645f_row0_col0\" class=\"data row0 col0\" >Who has a broader scope of profession: E. L. Doctorow or Julia Peterkin?</td>\n",+ " <td id=\"T_e645f_row0_col1\" class=\"data row0 col1\" >['E. L. Doctorow', 'E.L. Doctorow', 'Doctorow']</td>\n",+ " <td id=\"T_e645f_row0_col2\" class=\"data row0 col2\" >E. L. Doctorow</td>\n",+ " <td id=\"T_e645f_row0_col3\" class=\"data row0 col3\" >✔️</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_e645f_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",+ " <td id=\"T_e645f_row1_col0\" class=\"data row1 col0\" >What documentary about the Gilgo Beach Killer debuted on A&E?</td>\n",+ " <td id=\"T_e645f_row1_col1\" class=\"data row1 col1\" >['The Killing Season']</td>\n",+ " <td id=\"T_e645f_row1_col2\" class=\"data row1 col2\" >The Killing Season</td>\n",+ " <td id=\"T_e645f_row1_col3\" class=\"data row1 col3\" >✔️</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_e645f_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",+ " <td id=\"T_e645f_row2_col0\" class=\"data row2 col0\" >Right Back At It Again contains lyrics co-written by the singer born in what city?</td>\n",+ " <td id=\"T_e645f_row2_col1\" class=\"data row2 col1\" >['Gainesville, Florida', 'Gainesville']</td>\n",+ " <td id=\"T_e645f_row2_col2\" class=\"data row2 col2\" >Melbourne, Australia</td>\n",+ " <td id=\"T_e645f_row2_col3\" class=\"data row2 col3\" >❌</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_e645f_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",+ " <td id=\"T_e645f_row3_col0\" class=\"data row3 col0\" >What year was the party of the winner of the 1971 San Francisco mayoral election founded?</td>\n",+ " <td id=\"T_e645f_row3_col1\" class=\"data row3 col1\" >['1828']</td>\n",+ " <td id=\"T_e645f_row3_col2\" class=\"data row3 col2\" >1966</td>\n",+ " <td id=\"T_e645f_row3_col3\" class=\"data row3 col3\" >❌</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_e645f_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",+ " <td id=\"T_e645f_row4_col0\" class=\"data row4 col0\" >Which author is English: John Braine or Studs Terkel?</td>\n",+ " <td id=\"T_e645f_row4_col1\" class=\"data row4 col1\" >['John Braine']</td>\n",+ " <td id=\"T_e645f_row4_col2\" class=\"data row4 col2\" >John Braine</td>\n",+ " <td id=\"T_e645f_row4_col3\" class=\"data row4 col3\" >✔️</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_e645f_level0_row5\" class=\"row_heading level0 row5\" >5</th>\n",+ " <td id=\"T_e645f_row5_col0\" class=\"data row5 col0\" >Anthony Dirrell is the brother of which super middleweight title holder?</td>\n",+ " <td id=\"T_e645f_row5_col1\" class=\"data row5 col1\" >['Andre Dirrell']</td>\n",+ " <td id=\"T_e645f_row5_col2\" class=\"data row5 col2\" >Andre Dirrell</td>\n",+ " <td id=\"T_e645f_row5_col3\" class=\"data row5 col3\" >✔️</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_e645f_level0_row6\" class=\"row_heading level0 row6\" >6</th>\n",+ " <td id=\"T_e645f_row6_col0\" class=\"data row6 col0\" >In which city is the sports nutrition business established by Oliver Cookson based ?</td>\n",+ " <td id=\"T_e645f_row6_col1\" class=\"data row6 col1\" >['Cheshire', 'Cheshire, UK']</td>\n",+ " <td id=\"T_e645f_row6_col2\" class=\"data row6 col2\" >Manchester, England</td>\n",+ " <td id=\"T_e645f_row6_col3\" class=\"data row6 col3\" >❌</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_e645f_level0_row7\" class=\"row_heading level0 row7\" >7</th>\n",+ " <td id=\"T_e645f_row7_col0\" class=\"data row7 col0\" >Find the birth date of the actor who played roles in First Wives Club and Searching for the Elephant.</td>\n",+ " <td id=\"T_e645f_row7_col1\" class=\"data row7 col1\" >['February 13, 1980']</td>\n",+ " <td id=\"T_e645f_row7_col2\" class=\"data row7 col2\" >July 3, 1961</td>\n",+ " <td id=\"T_e645f_row7_col3\" class=\"data row7 col3\" >❌</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_e645f_level0_row8\" class=\"row_heading level0 row8\" >8</th>\n",+ " <td id=\"T_e645f_row8_col0\" class=\"data row8 col0\" >Kyle Moran was born in the town on what river?</td>\n",+ " <td id=\"T_e645f_row8_col1\" class=\"data row8 col1\" >['Castletown', 'Castletown River']</td>\n",+ " <td id=\"T_e645f_row8_col2\" class=\"data row8 col2\" >Susquehanna River</td>\n",+ " <td id=\"T_e645f_row8_col3\" class=\"data row8 col3\" >❌</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_e645f_level0_row9\" class=\"row_heading level0 row9\" >9</th>\n",+ " <td id=\"T_e645f_row9_col0\" class=\"data row9 col0\" >What is the name of one branch of Robert D. Braun's speciality?</td>\n",+ " <td id=\"T_e645f_row9_col1\" class=\"data row9 col1\" >['aeronautical engineering', 'astronautical engineering', 'aeronautics', 'astronautics']</td>\n",+ " <td id=\"T_e645f_row9_col2\" class=\"data row9 col2\" >Mars exploration</td>\n",+ " <td id=\"T_e645f_row9_col3\" class=\"data row9 col3\" >❌</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_e645f_level0_row10\" class=\"row_heading level0 row10\" >10</th>\n",+ " <td id=\"T_e645f_row10_col0\" class=\"data row10 col0\" >Where was the actress who played the niece in the Priest film born?</td>\n",+ " <td id=\"T_e645f_row10_col1\" class=\"data row10 col1\" >['Surrey', 'Guildford, Surrey']</td>\n",+ " <td id=\"T_e645f_row10_col2\" class=\"data row10 col2\" >Hong Kong</td>\n",+ " <td id=\"T_e645f_row10_col3\" class=\"data row10 col3\" >❌</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_e645f_level0_row11\" class=\"row_heading level0 row11\" >11</th>\n",+ " <td id=\"T_e645f_row11_col0\" class=\"data row11 col0\" >Name the movie in which the daughter of Noel Harrison plays Violet Trefusis.</td>\n",+ " <td id=\"T_e645f_row11_col1\" class=\"data row11 col1\" >['Portrait of a Marriage']</td>\n",+ " <td id=\"T_e645f_row11_col2\" class=\"data row11 col2\" >Venus</td>\n",+ " <td id=\"T_e645f_row11_col3\" class=\"data row11 col3\" >❌</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_e645f_level0_row12\" class=\"row_heading level0 row12\" >12</th>\n",+ " <td id=\"T_e645f_row12_col0\" class=\"data row12 col0\" >What year was the father of the Princes in the Tower born?</td>\n",+ " <td id=\"T_e645f_row12_col1\" class=\"data row12 col1\" >['1442']</td>\n",+ " <td id=\"T_e645f_row12_col2\" class=\"data row12 col2\" >1457</td>\n",+ " <td id=\"T_e645f_row12_col3\" class=\"data row12 col3\" >❌</td>\n",+ " </tr>\n",+ " </tbody>\n",+ "</table>\n"+ ],+ "text/plain": [+ "<pandas.io.formats.style.Styler at 0x7f0914ef0250>"+ ]+ },+ "metadata": {},+ "output_type": "display_data"+ }+ ],+ "source": [+ "from dsp.evaluation.utils import evaluate\n",+ "\n",+ "evaluate(vanilla_LM_QA, dev)"+ ]+ },+ {+ "attachments": {},+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "### Program 2: Retrieve-then-Read w/ GPT-3.5\n",+ "\n",+ "The program above relies entirely on the factual knowledge memorized by GPT-3. As we have seen, it's sometimes correct but it's far from reliable.\n",+ "\n",+ "Let's try to address this. Our first improvement to the vanilla **GPT-3.5** strategy is to include search results in the LM prompt, often called a *retrieve-then-read* approach.\n",+ "\n",+ "In particular, we'll retrieve the most relevant passage to the question (according to the retriever) from the corpus and concatenate it into the prompt. This may help GPT-3.5 answer more factually.\n",+ "\n",+ "To do this, we'll need to modify the `Template`. In particular, we'll define a `qa_template_with_passages` template, which will include a `context` field in addition to a question and an answer."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 9,+ "metadata": {},+ "outputs": [],+ "source": [+ "Context = dsp.Type(\n",+ " prefix=\"Context:\\n\",\n",+ " desc=\"${one or more sources, some of which may contain relevant content or clues}\",\n",+ " format=dsp.passages2text\n",+ ")\n",+ "\n",+ "qa_template_with_passages = dsp.Template(\n",+ " instructions=qa_template.instructions,\n",+ " context=Context(), question=Question(), answer=Answer()\n",+ ")"+ ]+ },+ {+ "attachments": {},+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "Now let's modify the program. We'll call `dsp.retrieve`, which we hooked earlier in this notebook to a **ColBERTv2** retriever serving a Wikipedia (Dec 2018) index."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 10,+ "metadata": {},+ "outputs": [],+ "source": [+ "def retrieve_then_read_QA(question: str) -> str:\n",+ " demos = dsp.sample(train, k=7)\n",+ " passages = dsp.retrieve(question, k=1)\n",+ " \n",+ " example = dsp.Example(question=question, context=passages, demos=demos)\n",+ " example, completions = dsp.generate(qa_template_with_passages)(example, stage='qa')\n",+ "\n",+ " return completions.answer"+ ]+ },+ {+ "attachments": {},+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "Let's invoke this on a sample question and let's inspect the last call to the LM to learn more about the behavior of the program."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 11,+ "metadata": {},+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "\n",+ "\n",+ "\n",+ "\n",+ "Answer questions with short factoid answers.\n",+ "\n",+ "---\n",+ "\n",+ "Question: Which award did the first book of Gary Zukav receive?\n",+ "Final Answer: U.S. National Book Award\n",+ "\n",+ "Question: The heir to the Du Pont family fortune sponsored what wrestling team?\n",+ "Final Answer: Foxcatcher\n",+ "\n",+ "Question: Who was the director of the 2009 movie featuring Peter Outerbridge as William Easton?\n",+ "Final Answer: Kevin Greutert\n",+ "\n",+ "Question: Who produced the album that included a re-recording of \"Lithium\"?\n",+ "Final Answer: Butch Vig\n",+ "\n",+ "Question: What city was the victim of Joseph Druces working in?\n",+ "Final Answer: Boston, Massachusetts\n",+ "\n",+ "Question: In what year was the star of To Hell and Back born?\n",+ "Final Answer: 1925\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context:\n",+ "${one or more sources, some of which may contain relevant content or clues}\n",+ "\n",+ "Question: ${the question to be answered}\n",+ "\n",+ "Final Answer: ${a short factoid answer, often between 1 and 5 words}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "«Julia Peterkin | Julia Peterkin Julia Peterkin (October 31, 1880 – August 10, 1961) was an American author from South Carolina. In 1929 she won the Pulitzer Prize for Novel/Literature, for her novel \"Scarlet Sister Mary.\" She wrote several novels about the plantation South, especially the Gullah people of the Low Country. She was one of the few white authors who wrote about the African-American experience. Julia Mood was born in Laurens County, South Carolina. Her father was a physician, and she was the third of his four children. Her mother died soon after her birth, and her father later married Janie Brogdon.»\n",+ "\n",+ "Question: Who has a broader scope of profession: E. L. Doctorow or Julia Peterkin?\n",+ "\n",+ "Final Answer:\u001b[32m E. L. Doctorow\u001b[0m\n",+ "\n",+ "\n",+ "\n"+ ]+ },+ {+ "data": {+ "text/plain": [+ "('E. L. Doctorow', None)"+ ]+ },+ "execution_count": 11,+ "metadata": {},+ "output_type": "execute_result"+ }+ ],+ "source": [+ "retrieve_then_read_QA(dev[0].question), lm.inspect_history(n=1)"+ ]+ },+ {+ "attachments": {},+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "Now, let's evaluate this on the dev examples."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 12,+ "metadata": {},+ "outputs": [+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "100%|██████████| 13/13 [00:00<00:00, 63.50it/s]"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Answered 5 / 13 correctly.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "\n"+ ]+ },+ {+ "data": {+ "text/html": [+ "<style type=\"text/css\">\n",+ "#T_f97da_ th {\n",+ " text-align: left;\n",+ "}\n",+ "#T_f97da_ td {\n",+ " text-align: left;\n",+ "}\n",+ "</style>\n",+ "<table id=\"T_f97da_\">\n",+ " <thead>\n",+ " <tr>\n",+ " <th class=\"blank level0\" > </th>\n",+ " <th class=\"col_heading level0 col0\" >question</th>\n",+ " <th class=\"col_heading level0 col1\" >answer</th>\n",+ " <th class=\"col_heading level0 col2\" >prediction</th>\n",+ " <th class=\"col_heading level0 col3\" >correct</th>\n",+ " </tr>\n",+ " </thead>\n",+ " <tbody>\n",+ " <tr>\n",+ " <th id=\"T_f97da_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",+ " <td id=\"T_f97da_row0_col0\" class=\"data row0 col0\" >Who has a broader scope of profession: E. L. Doctorow or Julia Peterkin?</td>\n",+ " <td id=\"T_f97da_row0_col1\" class=\"data row0 col1\" >['E. L. Doctorow', 'E.L. Doctorow', 'Doctorow']</td>\n",+ " <td id=\"T_f97da_row0_col2\" class=\"data row0 col2\" >E. L. Doctorow</td>\n",+ " <td id=\"T_f97da_row0_col3\" class=\"data row0 col3\" >✔️</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_f97da_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",+ " <td id=\"T_f97da_row1_col0\" class=\"data row1 col0\" >What documentary about the Gilgo Beach Killer debuted on A&E?</td>\n",+ " <td id=\"T_f97da_row1_col1\" class=\"data row1 col1\" >['The Killing Season']</td>\n",+ " <td id=\"T_f97da_row1_col2\" class=\"data row1 col2\" >The Killing Season</td>\n",+ " <td id=\"T_f97da_row1_col3\" class=\"data row1 col3\" >✔️</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_f97da_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",+ " <td id=\"T_f97da_row2_col0\" class=\"data row2 col0\" >Right Back At It Again contains lyrics co-written by the singer born in what city?</td>\n",+ " <td id=\"T_f97da_row2_col1\" class=\"data row2 col1\" >['Gainesville, Florida', 'Gainesville']</td>\n",+ " <td id=\"T_f97da_row2_col2\" class=\"data row2 col2\" >Daytona Beach, Florida</td>\n",+ " <td id=\"T_f97da_row2_col3\" class=\"data row2 col3\" >❌</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_f97da_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",+ " <td id=\"T_f97da_row3_col0\" class=\"data row3 col0\" >What year was the party of the winner of the 1971 San Francisco mayoral election founded?</td>\n",+ " <td id=\"T_f97da_row3_col1\" class=\"data row3 col1\" >['1828']</td>\n",+ " <td id=\"T_f97da_row3_col2\" class=\"data row3 col2\" >1856</td>\n",+ " <td id=\"T_f97da_row3_col3\" class=\"data row3 col3\" >❌</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_f97da_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",+ " <td id=\"T_f97da_row4_col0\" class=\"data row4 col0\" >Which author is English: John Braine or Studs Terkel?</td>\n",+ " <td id=\"T_f97da_row4_col1\" class=\"data row4 col1\" >['John Braine']</td>\n",+ " <td id=\"T_f97da_row4_col2\" class=\"data row4 col2\" >Studs Terkel</td>\n",+ " <td id=\"T_f97da_row4_col3\" class=\"data row4 col3\" >❌</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_f97da_level0_row5\" class=\"row_heading level0 row5\" >5</th>\n",+ " <td id=\"T_f97da_row5_col0\" class=\"data row5 col0\" >Anthony Dirrell is the brother of which super middleweight title holder?</td>\n",+ " <td id=\"T_f97da_row5_col1\" class=\"data row5 col1\" >['Andre Dirrell']</td>\n",+ " <td id=\"T_f97da_row5_col2\" class=\"data row5 col2\" >Andre Dirrell</td>\n",+ " <td id=\"T_f97da_row5_col3\" class=\"data row5 col3\" >✔️</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_f97da_level0_row6\" class=\"row_heading level0 row6\" >6</th>\n",+ " <td id=\"T_f97da_row6_col0\" class=\"data row6 col0\" >In which city is the sports nutrition business established by Oliver Cookson based ?</td>\n",+ " <td id=\"T_f97da_row6_col1\" class=\"data row6 col1\" >['Cheshire', 'Cheshire, UK']</td>\n",+ " <td id=\"T_f97da_row6_col2\" class=\"data row6 col2\" >Manchester, England</td>\n",+ " <td id=\"T_f97da_row6_col3\" class=\"data row6 col3\" >❌</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_f97da_level0_row7\" class=\"row_heading level0 row7\" >7</th>\n",+ " <td id=\"T_f97da_row7_col0\" class=\"data row7 col0\" >Find the birth date of the actor who played roles in First Wives Club and Searching for the Elephant.</td>\n",+ " <td id=\"T_f97da_row7_col1\" class=\"data row7 col1\" >['February 13, 1980']</td>\n",+ " <td id=\"T_f97da_row7_col2\" class=\"data row7 col2\" >5 August 1862</td>\n",+ " <td id=\"T_f97da_row7_col3\" class=\"data row7 col3\" >❌</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_f97da_level0_row8\" class=\"row_heading level0 row8\" >8</th>\n",+ " <td id=\"T_f97da_row8_col0\" class=\"data row8 col0\" >Kyle Moran was born in the town on what river?</td>\n",+ " <td id=\"T_f97da_row8_col1\" class=\"data row8 col1\" >['Castletown', 'Castletown River']</td>\n",+ " <td id=\"T_f97da_row8_col2\" class=\"data row8 col2\" >Dundalk</td>\n",+ " <td id=\"T_f97da_row8_col3\" class=\"data row8 col3\" >❌</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_f97da_level0_row9\" class=\"row_heading level0 row9\" >9</th>\n",+ " <td id=\"T_f97da_row9_col0\" class=\"data row9 col0\" >What is the name of one branch of Robert D. Braun's speciality?</td>\n",+ " <td id=\"T_f97da_row9_col1\" class=\"data row9 col1\" >['aeronautical engineering', 'astronautical engineering', 'aeronautics', 'astronautics']</td>\n",+ " <td id=\"T_f97da_row9_col2\" class=\"data row9 col2\" >Jet Propulsion</td>\n",+ " <td id=\"T_f97da_row9_col3\" class=\"data row9 col3\" >❌</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_f97da_level0_row10\" class=\"row_heading level0 row10\" >10</th>\n",+ " <td id=\"T_f97da_row10_col0\" class=\"data row10 col0\" >Where was the actress who played the niece in the Priest film born?</td>\n",+ " <td id=\"T_f97da_row10_col1\" class=\"data row10 col1\" >['Surrey', 'Guildford, Surrey']</td>\n",+ " <td id=\"T_f97da_row10_col2\" class=\"data row10 col2\" >Los Angeles, California</td>\n",+ " <td id=\"T_f97da_row10_col3\" class=\"data row10 col3\" >❌</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_f97da_level0_row11\" class=\"row_heading level0 row11\" >11</th>\n",+ " <td id=\"T_f97da_row11_col0\" class=\"data row11 col0\" >Name the movie in which the daughter of Noel Harrison plays Violet Trefusis.</td>\n",+ " <td id=\"T_f97da_row11_col1\" class=\"data row11 col1\" >['Portrait of a Marriage']</td>\n",+ " <td id=\"T_f97da_row11_col2\" class=\"data row11 col2\" >Portrait of a Marriage</td>\n",+ " <td id=\"T_f97da_row11_col3\" class=\"data row11 col3\" >✔️</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_f97da_level0_row12\" class=\"row_heading level0 row12\" >12</th>\n",+ " <td id=\"T_f97da_row12_col0\" class=\"data row12 col0\" >What year was the father of the Princes in the Tower born?</td>\n",+ " <td id=\"T_f97da_row12_col1\" class=\"data row12 col1\" >['1442']</td>\n",+ " <td id=\"T_f97da_row12_col2\" class=\"data row12 col2\" >1442</td>\n",+ " <td id=\"T_f97da_row12_col3\" class=\"data row12 col3\" >✔️</td>\n",+ " </tr>\n",+ " </tbody>\n",+ "</table>\n"+ ],+ "text/plain": [+ "<pandas.io.formats.style.Styler at 0x7f090f1b0450>"+ ]+ },+ "metadata": {},+ "output_type": "display_data"+ }+ ],+ "source": [+ "evaluate(retrieve_then_read_QA, dev)"+ ]+ },+ {+ "attachments": {},+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "### Program 3: Retrieve-then-Read w/ Self-Consistency\n",+ "\n",+ "It's clear that retrieval has the capacity to help **GPT-3.5** answer a larger number of questions more factually. However, these questions are too complicated for a single retrieved passage to suffice.\n",+ "\n",+ "In this program, we try to make some improvements to the pipeline from **Program 2**.\n",+ "\n",+ "In particular:\n",+ "\n",+ "- We will include 5 passages (instead of a single passage) into the prompt.\n",+ "- We will ask GPT-3.5 to generate a Chain-of-Thought (CoT) rationale to more effectively extract the answer from the passages.\n",+ "- We will use Self-Consistency (SC) to marginalize this prediction across many chains of thought."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 13,+ "metadata": {},+ "outputs": [],+ "source": [+ "Rationale = dsp.Type(\n",+ " prefix=\"Rationale: Let's think step by step.\",\n",+ " desc=\"${a step-by-step deduction that identifies the correct response, which will be provided below}\"\n",+ ")\n",+ "\n",+ "qa_template_with_CoT = dsp.Template(\n",+ " instructions=qa_template.instructions,\n",+ " context=Context(), question=Question(), rationale=Rationale(), answer=Answer()\n",+ ")"+ ]+ },+ {+ "attachments": {},+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "The snippet above updates our template to include a `rationale` field.\n",+ "\n",+ "Below, we define the new program `retrieve_then_read_QA_v2`. In it, we define a new DSP _transformation_ `QA_predict`, which we will re-use in multiple programs.\n",+ "\n",+ "The transformation `QA_predict` takes an `Example` (which has a question, context, and demonstrations) and generates an answer. It can be configured to use self-consistency (`sc=True`) or not."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 14,+ "metadata": {},+ "outputs": [],+ "source": [+ "@dsp.transformation\n",+ "def QA_predict(example: dsp.Example, sc=True):\n",+ " if sc:\n",+ " example, completions = dsp.generate(qa_template_with_CoT, n=20, temperature=0.7)(example, stage='qa')\n",+ " completions = dsp.majority(completions)\n",+ " else:\n",+ " example, completions = dsp.generate(qa_template_with_CoT)(example, stage='qa')\n",+ " \n",+ " return example.copy(answer=completions.answer)\n",+ "\n",+ "def retrieve_then_read_QA_v2(question: str) -> str:\n",+ " demos = dsp.sample(train, k=7)\n",+ " passages = dsp.retrieve(question, k=5)\n",+ " example = dsp.Example(question=question, context=passages, demos=demos)\n",+ " \n",+ " return QA_predict(example).answer"+ ]+ },+ {+ "attachments": {},+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "Let's inspect an example."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 15,+ "metadata": {},+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "\n",+ "\n",+ "\n",+ "\n",+ "Answer questions with short factoid answers.\n",+ "\n",+ "---\n",+ "\n",+ "Question: Which award did the first book of Gary Zukav receive?\n",+ "Final Answer: U.S. National Book Award\n",+ "\n",+ "Question: The heir to the Du Pont family fortune sponsored what wrestling team?\n",+ "Final Answer: Foxcatcher\n",+ "\n",+ "Question: Who was the director of the 2009 movie featuring Peter Outerbridge as William Easton?\n",+ "Final Answer: Kevin Greutert\n",+ "\n",+ "Question: Who produced the album that included a re-recording of \"Lithium\"?\n",+ "Final Answer: Butch Vig\n",+ "\n",+ "Question: What city was the victim of Joseph Druces working in?\n",+ "Final Answer: Boston, Massachusetts\n",+ "\n",+ "Question: In what year was the star of To Hell and Back born?\n",+ "Final Answer: 1925\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context:\n",+ "${one or more sources, some of which may contain relevant content or clues}\n",+ "\n",+ "Question: ${the question to be answered}\n",+ "\n",+ "Rationale: Let's think step by step. ${a step-by-step deduction that identifies the correct response, which will be provided below}\n",+ "\n",+ "Final Answer: ${a short factoid answer, often between 1 and 5 words}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Right Back at It Again | Right Back at It Again \"Right Back at It Again\" is the second track and the first single from A Day to Remember's fifth album, \"Common Courtesy\" (2013). In October 20, 2015, the song was featured in Activision rhythm-music game, \"\". Vocalist, Jeremy McKinnon wrote the lyrics, while the music was written by McKinnon, former guitarist Tom Denney, guitarist Neil Westfall and producer Andrew Wade. \"Right Back at It Again\" almost wasn't included on the album as it was one of the excess songs the band had recorded, \"we realised that it sounded great, so on it went.\" \"Right Back»\n",+ "[2] «Right Back at It Again | at the Kerrang! Awards. Personnel per digital booklet. Right Back at It Again \"Right Back at It Again\" is the second track and the first single from A Day to Remember's fifth album, \"Common Courtesy\" (2013). In October 20, 2015, the song was featured in Activision rhythm-music game, \"\". Vocalist, Jeremy McKinnon wrote the lyrics, while the music was written by McKinnon, former guitarist Tom Denney, guitarist Neil Westfall and producer Andrew Wade. \"Right Back at It Again\" almost wasn't included on the album as it was one of the excess songs the band had recorded, \"we realised that it»\n",+ "[3] «Right Back at It Again | band played \"Right Back at It Again\" on both occasions. AbsolutePunk writer Thomas Nassiff said the \"mosh call\" lyric \"\"Brace for impact\"\" had \"pretty much ruin[ed] that track\". Tamsyn Wilce for Alter the Press! noted that the way \"Right Back at it Again\" follows on from opening track \"City of Ocala\" \"continues the catchy hardcore beats\" that is present in all of the band's albums. \"Rock Sound\"'s Andy Ritchie called the song \"unmistakably the 'All I Want' of 'Common Courtesy'\". A music video for the song was released on 19 December 2013. The music video was nominated for Best Video»\n",+ "[4] «Born Again in Dixieland | Born Again in Dixieland \"Born Again in Dixieland\" is a song co-written and recorded by Canadian country music artist Jason McCoy. Released in June 1997, it was the lead single from his album \"Playin' for Keeps\". The song reached #3 on the \"RPM\" Country Tracks chart in September 1997. The song won a SOCAN Song of the Year award. McCoy said that he started writing the song with Sheridan and he was playing songs for MCA Canada label and they asked if he had any other songs. He played a verse and the chorus from the song thinking it wasn't»\n",+ "[5] «Right Back Where We Started From | 1974. In a 3 May 2008 interview with Michael Shelley of WFMU, Edwards recalled that after hearing Maxine Nightingale sing on the session for Al Matthews' \"Fool\" that track's producer Pierre Tubbs had come up with \"Right Back Where We Started From\" as a good title for a song for Nightingale herself to record and had invited Edwards to co-write the song. Utilizing a tune which Edwards had written \"a couple of years before\", Tubbs and Edwards wrote \"Right Back Where We Started From\" in about seven minutes while driving to Charing Cross Hospital where Tubbs' wife Gabrielle (née Zimmerman)»\n",+ "\n",+ "Question: Right Back At It Again contains lyrics co-written by the singer born in what city?\n",+ "\n",+ "Rationale: Let's think step by step.\u001b[32m The song was co-written by the singer, Jeremy McKinnon. A quick search of his name reveals that he was born in Ocala, Florida.\n",+ "\n",+ "Final Answer: Ocala, Florida\u001b[0m\u001b[31m \t (and 19 other completions)\u001b[0m\n",+ "\n",+ "\n",+ "\n"+ ]+ },+ {+ "data": {+ "text/plain": [+ "('Ocala, Florida', None)"+ ]+ },+ "execution_count": 15,+ "metadata": {},+ "output_type": "execute_result"+ }+ ],+ "source": [+ "retrieve_then_read_QA_v2(dev[2].question), lm.inspect_history(n=1)"+ ]+ },+ {+ "attachments": {},+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "And let's run this on the other dev examples."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 16,+ "metadata": {},+ "outputs": [+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "100%|██████████| 13/13 [00:01<00:00, 8.49it/s]"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Answered 6 / 13 correctly.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "\n"+ ]+ },+ {+ "data": {+ "text/html": [+ "<style type=\"text/css\">\n",+ "#T_52677_ th {\n",+ " text-align: left;\n",+ "}\n",+ "#T_52677_ td {\n",+ " text-align: left;\n",+ "}\n",+ "</style>\n",+ "<table id=\"T_52677_\">\n",+ " <thead>\n",+ " <tr>\n",+ " <th class=\"blank level0\" > </th>\n",+ " <th class=\"col_heading level0 col0\" >question</th>\n",+ " <th class=\"col_heading level0 col1\" >answer</th>\n",+ " <th class=\"col_heading level0 col2\" >prediction</th>\n",+ " <th class=\"col_heading level0 col3\" >correct</th>\n",+ " </tr>\n",+ " </thead>\n",+ " <tbody>\n",+ " <tr>\n",+ " <th id=\"T_52677_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",+ " <td id=\"T_52677_row0_col0\" class=\"data row0 col0\" >Who has a broader scope of profession: E. L. Doctorow or Julia Peterkin?</td>\n",+ " <td id=\"T_52677_row0_col1\" class=\"data row0 col1\" >['E. L. Doctorow', 'E.L. Doctorow', 'Doctorow']</td>\n",+ " <td id=\"T_52677_row0_col2\" class=\"data row0 col2\" >Julia Peterkin</td>\n",+ " <td id=\"T_52677_row0_col3\" class=\"data row0 col3\" >❌</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_52677_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",+ " <td id=\"T_52677_row1_col0\" class=\"data row1 col0\" >What documentary about the Gilgo Beach Killer debuted on A&E?</td>\n",+ " <td id=\"T_52677_row1_col1\" class=\"data row1 col1\" >['The Killing Season']</td>\n",+ " <td id=\"T_52677_row1_col2\" class=\"data row1 col2\" >The Killing Season</td>\n",+ " <td id=\"T_52677_row1_col3\" class=\"data row1 col3\" >✔️</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_52677_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",+ " <td id=\"T_52677_row2_col0\" class=\"data row2 col0\" >Right Back At It Again contains lyrics co-written by the singer born in what city?</td>\n",+ " <td id=\"T_52677_row2_col1\" class=\"data row2 col1\" >['Gainesville, Florida', 'Gainesville']</td>\n",+ " <td id=\"T_52677_row2_col2\" class=\"data row2 col2\" >Ocala, Florida</td>\n",+ " <td id=\"T_52677_row2_col3\" class=\"data row2 col3\" >❌</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_52677_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",+ " <td id=\"T_52677_row3_col0\" class=\"data row3 col0\" >What year was the party of the winner of the 1971 San Francisco mayoral election founded?</td>\n",+ " <td id=\"T_52677_row3_col1\" class=\"data row3 col1\" >['1828']</td>\n",+ " <td id=\"T_52677_row3_col2\" class=\"data row3 col2\" >1824</td>\n",+ " <td id=\"T_52677_row3_col3\" class=\"data row3 col3\" >❌</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_52677_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",+ " <td id=\"T_52677_row4_col0\" class=\"data row4 col0\" >Which author is English: John Braine or Studs Terkel?</td>\n",+ " <td id=\"T_52677_row4_col1\" class=\"data row4 col1\" >['John Braine']</td>\n",+ " <td id=\"T_52677_row4_col2\" class=\"data row4 col2\" >John Braine</td>\n",+ " <td id=\"T_52677_row4_col3\" class=\"data row4 col3\" >✔️</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_52677_level0_row5\" class=\"row_heading level0 row5\" >5</th>\n",+ " <td id=\"T_52677_row5_col0\" class=\"data row5 col0\" >Anthony Dirrell is the brother of which super middleweight title holder?</td>\n",+ " <td id=\"T_52677_row5_col1\" class=\"data row5 col1\" >['Andre Dirrell']</td>\n",+ " <td id=\"T_52677_row5_col2\" class=\"data row5 col2\" >Andre Dirrell</td>\n",+ " <td id=\"T_52677_row5_col3\" class=\"data row5 col3\" >✔️</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_52677_level0_row6\" class=\"row_heading level0 row6\" >6</th>\n",+ " <td id=\"T_52677_row6_col0\" class=\"data row6 col0\" >In which city is the sports nutrition business established by Oliver Cookson based ?</td>\n",+ " <td id=\"T_52677_row6_col1\" class=\"data row6 col1\" >['Cheshire', 'Cheshire, UK']</td>\n",+ " <td id=\"T_52677_row6_col2\" class=\"data row6 col2\" >Cheshire, UK</td>\n",+ " <td id=\"T_52677_row6_col3\" class=\"data row6 col3\" >✔️</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_52677_level0_row7\" class=\"row_heading level0 row7\" >7</th>\n",+ " <td id=\"T_52677_row7_col0\" class=\"data row7 col0\" >Find the birth date of the actor who played roles in First Wives Club and Searching for the Elephant.</td>\n",+ " <td id=\"T_52677_row7_col1\" class=\"data row7 col1\" >['February 13, 1980']</td>\n",+ " <td id=\"T_52677_row7_col2\" class=\"data row7 col2\" >September 27, 1974</td>\n",+ " <td id=\"T_52677_row7_col3\" class=\"data row7 col3\" >❌</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_52677_level0_row8\" class=\"row_heading level0 row8\" >8</th>\n",+ " <td id=\"T_52677_row8_col0\" class=\"data row8 col0\" >Kyle Moran was born in the town on what river?</td>\n",+ " <td id=\"T_52677_row8_col1\" class=\"data row8 col1\" >['Castletown', 'Castletown River']</td>\n",+ " <td id=\"T_52677_row8_col2\" class=\"data row8 col2\" >Dundalk</td>\n",+ " <td id=\"T_52677_row8_col3\" class=\"data row8 col3\" >❌</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_52677_level0_row9\" class=\"row_heading level0 row9\" >9</th>\n",+ " <td id=\"T_52677_row9_col0\" class=\"data row9 col0\" >What is the name of one branch of Robert D. Braun's speciality?</td>\n",+ " <td id=\"T_52677_row9_col1\" class=\"data row9 col1\" >['aeronautical engineering', 'astronautical engineering', 'aeronautics', 'astronautics']</td>\n",+ " <td id=\"T_52677_row9_col2\" class=\"data row9 col2\" >Aerospace engineering</td>\n",+ " <td id=\"T_52677_row9_col3\" class=\"data row9 col3\" >❌</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_52677_level0_row10\" class=\"row_heading level0 row10\" >10</th>\n",+ " <td id=\"T_52677_row10_col0\" class=\"data row10 col0\" >Where was the actress who played the niece in the Priest film born?</td>\n",+ " <td id=\"T_52677_row10_col1\" class=\"data row10 col1\" >['Surrey', 'Guildford, Surrey']</td>\n",+ " <td id=\"T_52677_row10_col2\" class=\"data row10 col2\" >Italy</td>\n",+ " <td id=\"T_52677_row10_col3\" class=\"data row10 col3\" >❌</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_52677_level0_row11\" class=\"row_heading level0 row11\" >11</th>\n",+ " <td id=\"T_52677_row11_col0\" class=\"data row11 col0\" >Name the movie in which the daughter of Noel Harrison plays Violet Trefusis.</td>\n",+ " <td id=\"T_52677_row11_col1\" class=\"data row11 col1\" >['Portrait of a Marriage']</td>\n",+ " <td id=\"T_52677_row11_col2\" class=\"data row11 col2\" >Portrait of a Marriage</td>\n",+ " <td id=\"T_52677_row11_col3\" class=\"data row11 col3\" >✔️</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_52677_level0_row12\" class=\"row_heading level0 row12\" >12</th>\n",+ " <td id=\"T_52677_row12_col0\" class=\"data row12 col0\" >What year was the father of the Princes in the Tower born?</td>\n",+ " <td id=\"T_52677_row12_col1\" class=\"data row12 col1\" >['1442']</td>\n",+ " <td id=\"T_52677_row12_col2\" class=\"data row12 col2\" >1442</td>\n",+ " <td id=\"T_52677_row12_col3\" class=\"data row12 col3\" >✔️</td>\n",+ " </tr>\n",+ " </tbody>\n",+ "</table>\n"+ ],+ "text/plain": [+ "<pandas.io.formats.style.Styler at 0x7f0915703d50>"+ ]+ },+ "metadata": {},+ "output_type": "display_data"+ }+ ],+ "source": [+ "evaluate(retrieve_then_read_QA_v2, dev)"+ ]+ },+ {+ "attachments": {},+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "### Program 4: Multi-Hop Retrieval\n",+ "\n",+ "From these simple programs, it becomes clear that a single search query is not enough for this task. This can be seen when trying to find the birth city of the writer of \"Right Back At It Again\". A search query identifies the author correctly as \"Jeremy McKinnon\", but it doesn't mention when he was born.\n",+ "\n",+ "The standard approach for this challenge in the retrieval-augmented NLP literature is to build multi-hop search systems, like GoldEn (Qi et al., 2019), IRRR (Qi et al., 2021) and Baleen (Khattab et al., 2021). These systems read the retrieved results and then generate additional queries to gather additional information if necessary."+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "\n",+ "With **DSP**, we can easily simulate such systems in a few lines of code. Let's begin by defining two new templates.\n",+ "\n",+ "First, in `rewrite_template`, we will re-write the input question into a simple search query. This will serve as our \"first hop\" query."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 17,+ "metadata": {},+ "outputs": [],+ "source": [+ "SearchRationale = dsp.Type(\n",+ " prefix=\"Rationale: Let's think step by step. To answer this question, we first need to find out\",\n",+ " desc=\"${missing information}\"\n",+ ")\n",+ "\n",+ "SearchQuery = dsp.Type(\n",+ " prefix=\"Search Query:\",\n",+ " desc=\"${a simple question for seeking the missing or required information}\"\n",+ ")\n",+ "\n",+ "rewrite_template = dsp.Template(\n",+ " instructions=\"Write a search query that will help answer a complex question.\",\n",+ " question=Question(), rationale=SearchRationale(), query=SearchQuery()\n",+ ")"+ ]+ },+ {+ "attachments": {},+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "Next, in `hop_template`, we will use the retrieved information from earlier hops to generate additional search queries for missing information."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 18,+ "metadata": {},+ "outputs": [],+ "source": [+ "CondenseRationale = dsp.Type(\n",+ " prefix=\"Rationale: Let's think step by step. We now know that\",\n",+ " desc=\"${information from the context that may provide useful clues for the question}\"\n",+ ")\n",+ "\n",+ "hop_template = dsp.Template(\n",+ " instructions=rewrite_template.instructions,\n",+ " context=Context(), question=Question(), rationale=CondenseRationale(), query=SearchQuery()\n",+ ")"+ ]+ },+ {+ "attachments": {},+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "Let's build our `multihop_QA_v1` program. The hallmark of this program is the search transformation called `multihop_search_v1`.\n",+ "\n",+ "This will largely simulate the IRRR system. In each hop, we will retrieve `k` (e.g., `k=2`) passages and concatenate them to the context retrieved from earlier hops to form a `context` chain.\n",+ "\n",+ "For simplicity, we will repeat this for a total of `max_hops` hops (e.g., `max_hops=2`). It's easy to add a stopping criteria in which the LM is asked whether more hops are required."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 19,+ "metadata": {},+ "outputs": [],+ "source": [+ "from dsp.utils import deduplicate\n",+ "\n",+ "@dsp.transformation\n",+ "def multihop_search_v1(example: dsp.Example, max_hops=2, k=2) -> dsp.Example:\n",+ " example.context = []\n",+ " \n",+ " for hop in range(max_hops):\n",+ " # Generate a query based\n",+ " template = rewrite_template if hop == 0 else hop_template\n",+ " example, completions = dsp.generate(template)(example, stage=f'h{hop}')\n",+ "\n",+ " # Retrieve k results based on the query generated\n",+ " passages = dsp.retrieve(completions.query, k=k)\n",+ "\n",+ " # Update the context by concatenating old and new passages\n",+ " example.context = deduplicate(example.context + passages)\n",+ "\n",+ " return example\n",+ "\n",+ "\n",+ "def multihop_QA_v1(question: str) -> str:\n",+ " demos = dsp.sample(train, k=7)\n",+ " x = dsp.Example(question=question, demos=demos)\n",+ " \n",+ " x = multihop_search_v1(x)\n",+ " x = QA_predict(x, sc=False)\n",+ "\n",+ " return x.answer"+ ]+ },+ {+ "attachments": {},+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "Let's run this on one example and inspect the last 3 calls to the LM (i.e., search query 1, search query 2, and QA)."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 20,+ "metadata": {},+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "\n",+ "\n",+ "\n",+ "\n",+ "Write a search query that will help answer a complex question.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Question: ${the question to be answered}\n",+ "Rationale: Let's think step by step. To answer this question, we first need to find out ${missing information}\n",+ "Search Query: ${a simple question for seeking the missing or required information}\n",+ "\n",+ "---\n",+ "\n",+ "Question: Right Back At It Again contains lyrics co-written by the singer born in what city?\n",+ "Rationale: Let's think step by step. To answer this question, we first need to find out\u001b[32m the name of the singer\n",+ "Search Query: Who sings Right Back At It Again\u001b[0m\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Write a search query that will help answer a complex question.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context:\n",+ "${one or more sources, some of which may contain relevant content or clues}\n",+ "\n",+ "Question: ${the question to be answered}\n",+ "\n",+ "Rationale: Let's think step by step. We now know that ${information from the context that may provide useful clues for the question}\n",+ "\n",+ "Search Query: ${a simple question for seeking the missing or required information}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Right Back at It Again | at the Kerrang! Awards. Personnel per digital booklet. Right Back at It Again \"Right Back at It Again\" is the second track and the first single from A Day to Remember's fifth album, \"Common Courtesy\" (2013). In October 20, 2015, the song was featured in Activision rhythm-music game, \"\". Vocalist, Jeremy McKinnon wrote the lyrics, while the music was written by McKinnon, former guitarist Tom Denney, guitarist Neil Westfall and producer Andrew Wade. \"Right Back at It Again\" almost wasn't included on the album as it was one of the excess songs the band had recorded, \"we realised that it»\n",+ "[2] «Right Back at It Again | Right Back at It Again \"Right Back at It Again\" is the second track and the first single from A Day to Remember's fifth album, \"Common Courtesy\" (2013). In October 20, 2015, the song was featured in Activision rhythm-music game, \"\". Vocalist, Jeremy McKinnon wrote the lyrics, while the music was written by McKinnon, former guitarist Tom Denney, guitarist Neil Westfall and producer Andrew Wade. \"Right Back at It Again\" almost wasn't included on the album as it was one of the excess songs the band had recorded, \"we realised that it sounded great, so on it went.\" \"Right Back»\n",+ "\n",+ "Question: Right Back At It Again contains lyrics co-written by the singer born in what city?\n",+ "\n",+ "Rationale: Let's think step by step. We now know that\u001b[32m the song was co-written by the singer, Jeremy McKinnon.\n",+ "\n",+ "Search Query: Where was Jeremy McKinnon born?\u001b[0m\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Answer questions with short factoid answers.\n",+ "\n",+ "---\n",+ "\n",+ "Question: Which award did the first book of Gary Zukav receive?\n",+ "Final Answer: U.S. National Book Award\n",+ "\n",+ "Question: The heir to the Du Pont family fortune sponsored what wrestling team?\n",+ "Final Answer: Foxcatcher\n",+ "\n",+ "Question: Who was the director of the 2009 movie featuring Peter Outerbridge as William Easton?\n",+ "Final Answer: Kevin Greutert\n",+ "\n",+ "Question: Who produced the album that included a re-recording of \"Lithium\"?\n",+ "Final Answer: Butch Vig\n",+ "\n",+ "Question: What city was the victim of Joseph Druces working in?\n",+ "Final Answer: Boston, Massachusetts\n",+ "\n",+ "Question: In what year was the star of To Hell and Back born?\n",+ "Final Answer: 1925\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context:\n",+ "${one or more sources, some of which may contain relevant content or clues}\n",+ "\n",+ "Question: ${the question to be answered}\n",+ "\n",+ "Rationale: Let's think step by step. ${a step-by-step deduction that identifies the correct response, which will be provided below}\n",+ "\n",+ "Final Answer: ${a short factoid answer, often between 1 and 5 words}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Right Back at It Again | at the Kerrang! Awards. Personnel per digital booklet. Right Back at It Again \"Right Back at It Again\" is the second track and the first single from A Day to Remember's fifth album, \"Common Courtesy\" (2013). In October 20, 2015, the song was featured in Activision rhythm-music game, \"\". Vocalist, Jeremy McKinnon wrote the lyrics, while the music was written by McKinnon, former guitarist Tom Denney, guitarist Neil Westfall and producer Andrew Wade. \"Right Back at It Again\" almost wasn't included on the album as it was one of the excess songs the band had recorded, \"we realised that it»\n",+ "[2] «Right Back at It Again | Right Back at It Again \"Right Back at It Again\" is the second track and the first single from A Day to Remember's fifth album, \"Common Courtesy\" (2013). In October 20, 2015, the song was featured in Activision rhythm-music game, \"\". Vocalist, Jeremy McKinnon wrote the lyrics, while the music was written by McKinnon, former guitarist Tom Denney, guitarist Neil Westfall and producer Andrew Wade. \"Right Back at It Again\" almost wasn't included on the album as it was one of the excess songs the band had recorded, \"we realised that it sounded great, so on it went.\" \"Right Back»\n",+ "[3] «Jeremy McKinnon | Jeremy McKinnon Jeremy Wade McKinnon (born December 17, 1985) is an American musician and record producer, best known as a founding member and the lead vocalist of A Day to Remember. He has produced full-length albums for The Devil Wears Prada, The Ghost Inside, Neck Deep, and Wage War. Jeremy McKinnon was born in Gainesville, Florida and grew up with two sisters. His parents are originally from Brooklyn, New York and is of Irish and Italian descent. McKinnon first got a job at Boston Market and later went on to do construction work. McKinnon's interest in music came as a»\n",+ "[4] «Martin McKinnon | Martin McKinnon Martin \"Marty\" McKinnon (born 5 July 1975 in Adelaide) is a former Australian rules footballer who played with Adelaide, Geelong and the Brisbane Lions in the Australian Football League (AFL). McKinnon was recruited by Adelaide in the 1992 AFL Draft, with their first ever national draft pick. He was the youngest player on Adelaide's list at the time and played for Central District in the SANFL when not appearing with Adelaide. He made his AFL debut late in the 1994 AFL season and was a regular in the team the following year, playing mostly as a wingman. He»\n",+ "\n",+ "Question: Right Back At It Again contains lyrics co-written by the singer born in what city?\n",+ "\n",+ "Rationale: Let's think step by step.\u001b[32m The song Right Back At It Again was co-written by the singer Jeremy McKinnon. A quick search shows that he was born in Gainesville, Florida.\n",+ "\n",+ "Final Answer: Gainesville, Florida\u001b[0m\n",+ "\n",+ "\n",+ "\n"+ ]+ },+ {+ "data": {+ "text/plain": [+ "('Gainesville, Florida', None)"+ ]+ },+ "execution_count": 20,+ "metadata": {},+ "output_type": "execute_result"+ }+ ],+ "source": [+ "multihop_QA_v1(dev[2].question), lm.inspect_history(n=3)"+ ]+ },+ {+ "attachments": {},+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "Now, let's run this on the remaining dev examples."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 21,+ "metadata": {},+ "outputs": [+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "100%|██████████| 13/13 [00:00<00:00, 23.89it/s]"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Answered 7 / 13 correctly.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "\n"+ ]+ },+ {+ "data": {+ "text/html": [+ "<style type=\"text/css\">\n",+ "#T_2baa7_ th {\n",+ " text-align: left;\n",+ "}\n",+ "#T_2baa7_ td {\n",+ " text-align: left;\n",+ "}\n",+ "</style>\n",+ "<table id=\"T_2baa7_\">\n",+ " <thead>\n",+ " <tr>\n",+ " <th class=\"blank level0\" > </th>\n",+ " <th class=\"col_heading level0 col0\" >question</th>\n",+ " <th class=\"col_heading level0 col1\" >answer</th>\n",+ " <th class=\"col_heading level0 col2\" >prediction</th>\n",+ " <th class=\"col_heading level0 col3\" >correct</th>\n",+ " </tr>\n",+ " </thead>\n",+ " <tbody>\n",+ " <tr>\n",+ " <th id=\"T_2baa7_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",+ " <td id=\"T_2baa7_row0_col0\" class=\"data row0 col0\" >Who has a broader scope of profession: E. L. Doctorow or Julia Peterkin?</td>\n",+ " <td id=\"T_2baa7_row0_col1\" class=\"data row0 col1\" >['E. L. Doctorow', 'E.L. Doctorow', 'Doctorow']</td>\n",+ " <td id=\"T_2baa7_row0_col2\" class=\"data row0 col2\" >E. L. Doctorow</td>\n",+ " <td id=\"T_2baa7_row0_col3\" class=\"data row0 col3\" >✔️</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_2baa7_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",+ " <td id=\"T_2baa7_row1_col0\" class=\"data row1 col0\" >What documentary about the Gilgo Beach Killer debuted on A&E?</td>\n",+ " <td id=\"T_2baa7_row1_col1\" class=\"data row1 col1\" >['The Killing Season']</td>\n",+ " <td id=\"T_2baa7_row1_col2\" class=\"data row1 col2\" >The Killing Season</td>\n",+ " <td id=\"T_2baa7_row1_col3\" class=\"data row1 col3\" >✔️</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_2baa7_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",+ " <td id=\"T_2baa7_row2_col0\" class=\"data row2 col0\" >Right Back At It Again contains lyrics co-written by the singer born in what city?</td>\n",+ " <td id=\"T_2baa7_row2_col1\" class=\"data row2 col1\" >['Gainesville, Florida', 'Gainesville']</td>\n",+ " <td id=\"T_2baa7_row2_col2\" class=\"data row2 col2\" >Gainesville, Florida</td>\n",+ " <td id=\"T_2baa7_row2_col3\" class=\"data row2 col3\" >✔️</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_2baa7_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",+ " <td id=\"T_2baa7_row3_col0\" class=\"data row3 col0\" >What year was the party of the winner of the 1971 San Francisco mayoral election founded?</td>\n",+ " <td id=\"T_2baa7_row3_col1\" class=\"data row3 col1\" >['1828']</td>\n",+ " <td id=\"T_2baa7_row3_col2\" class=\"data row3 col2\" >1947</td>\n",+ " <td id=\"T_2baa7_row3_col3\" class=\"data row3 col3\" >❌</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_2baa7_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",+ " <td id=\"T_2baa7_row4_col0\" class=\"data row4 col0\" >Which author is English: John Braine or Studs Terkel?</td>\n",+ " <td id=\"T_2baa7_row4_col1\" class=\"data row4 col1\" >['John Braine']</td>\n",+ " <td id=\"T_2baa7_row4_col2\" class=\"data row4 col2\" >John Braine</td>\n",+ " <td id=\"T_2baa7_row4_col3\" class=\"data row4 col3\" >✔️</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_2baa7_level0_row5\" class=\"row_heading level0 row5\" >5</th>\n",+ " <td id=\"T_2baa7_row5_col0\" class=\"data row5 col0\" >Anthony Dirrell is the brother of which super middleweight title holder?</td>\n",+ " <td id=\"T_2baa7_row5_col1\" class=\"data row5 col1\" >['Andre Dirrell']</td>\n",+ " <td id=\"T_2baa7_row5_col2\" class=\"data row5 col2\" >Andre Dirrell</td>\n",+ " <td id=\"T_2baa7_row5_col3\" class=\"data row5 col3\" >✔️</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_2baa7_level0_row6\" class=\"row_heading level0 row6\" >6</th>\n",+ " <td id=\"T_2baa7_row6_col0\" class=\"data row6 col0\" >In which city is the sports nutrition business established by Oliver Cookson based ?</td>\n",+ " <td id=\"T_2baa7_row6_col1\" class=\"data row6 col1\" >['Cheshire', 'Cheshire, UK']</td>\n",+ " <td id=\"T_2baa7_row6_col2\" class=\"data row6 col2\" >Manchester, England</td>\n",+ " <td id=\"T_2baa7_row6_col3\" class=\"data row6 col3\" >❌</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_2baa7_level0_row7\" class=\"row_heading level0 row7\" >7</th>\n",+ " <td id=\"T_2baa7_row7_col0\" class=\"data row7 col0\" >Find the birth date of the actor who played roles in First Wives Club and Searching for the Elephant.</td>\n",+ " <td id=\"T_2baa7_row7_col1\" class=\"data row7 col1\" >['February 13, 1980']</td>\n",+ " <td id=\"T_2baa7_row7_col2\" class=\"data row7 col2\" >1965</td>\n",+ " <td id=\"T_2baa7_row7_col3\" class=\"data row7 col3\" >❌</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_2baa7_level0_row8\" class=\"row_heading level0 row8\" >8</th>\n",+ " <td id=\"T_2baa7_row8_col0\" class=\"data row8 col0\" >Kyle Moran was born in the town on what river?</td>\n",+ " <td id=\"T_2baa7_row8_col1\" class=\"data row8 col1\" >['Castletown', 'Castletown River']</td>\n",+ " <td id=\"T_2baa7_row8_col2\" class=\"data row8 col2\" >River Foyle</td>\n",+ " <td id=\"T_2baa7_row8_col3\" class=\"data row8 col3\" >❌</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_2baa7_level0_row9\" class=\"row_heading level0 row9\" >9</th>\n",+ " <td id=\"T_2baa7_row9_col0\" class=\"data row9 col0\" >What is the name of one branch of Robert D. Braun's speciality?</td>\n",+ " <td id=\"T_2baa7_row9_col1\" class=\"data row9 col1\" >['aeronautical engineering', 'astronautical engineering', 'aeronautics', 'astronautics']</td>\n",+ " <td id=\"T_2baa7_row9_col2\" class=\"data row9 col2\" >Aerospace engineering</td>\n",+ " <td id=\"T_2baa7_row9_col3\" class=\"data row9 col3\" >❌</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_2baa7_level0_row10\" class=\"row_heading level0 row10\" >10</th>\n",+ " <td id=\"T_2baa7_row10_col0\" class=\"data row10 col0\" >Where was the actress who played the niece in the Priest film born?</td>\n",+ " <td id=\"T_2baa7_row10_col1\" class=\"data row10 col1\" >['Surrey', 'Guildford, Surrey']</td>\n",+ " <td id=\"T_2baa7_row10_col2\" class=\"data row10 col2\" >Guildford, Surrey</td>\n",+ " <td id=\"T_2baa7_row10_col3\" class=\"data row10 col3\" >✔️</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_2baa7_level0_row11\" class=\"row_heading level0 row11\" >11</th>\n",+ " <td id=\"T_2baa7_row11_col0\" class=\"data row11 col0\" >Name the movie in which the daughter of Noel Harrison plays Violet Trefusis.</td>\n",+ " <td id=\"T_2baa7_row11_col1\" class=\"data row11 col1\" >['Portrait of a Marriage']</td>\n",+ " <td id=\"T_2baa7_row11_col2\" class=\"data row11 col2\" >Carrington</td>\n",+ " <td id=\"T_2baa7_row11_col3\" class=\"data row11 col3\" >❌</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_2baa7_level0_row12\" class=\"row_heading level0 row12\" >12</th>\n",+ " <td id=\"T_2baa7_row12_col0\" class=\"data row12 col0\" >What year was the father of the Princes in the Tower born?</td>\n",+ " <td id=\"T_2baa7_row12_col1\" class=\"data row12 col1\" >['1442']</td>\n",+ " <td id=\"T_2baa7_row12_col2\" class=\"data row12 col2\" >1442</td>\n",+ " <td id=\"T_2baa7_row12_col3\" class=\"data row12 col3\" >✔️</td>\n",+ " </tr>\n",+ " </tbody>\n",+ "</table>\n"+ ],+ "text/plain": [+ "<pandas.io.formats.style.Styler at 0x7f090de2b5d0>"+ ]+ },+ "metadata": {},+ "output_type": "display_data"+ }+ ],+ "source": [+ "evaluate(multihop_QA_v1, dev)"+ ]+ },+ {+ "attachments": {},+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "### Program 5: Multi-Hop Condensed Retrieval w/ Automatic Demos and Query Fusion\n",+ "\n",+ "Through Program 4, we've begun to explore some of the power of the DSP abstraction. However, if you look closely, you will see a few downsides of the previous approach:\n",+ "\n",+ "1. The search transformations invoke the LM without any demonstrations in the prompt. That is because we only have training data for the question–answer pairs and not for the intermediate labels (e.g., search queries).\n",+ "2. The QA prompt uses passages (`context`) and a Chain-of-Thought (`rationale`) for the question to be answered. However, the training demonstrations include neither context nor CoT because they aren't available in our labels.\n",+ "3. The search transformations commit to a single query per hop, which may single out an unproductive chain of passages and hence fail to uncover relevant information."+ ]+ },+ {+ "attachments": {},+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "We address these problems automatically in the program below. In it, we begin by defining `multihop_demonstrate` (which uses `multihop_attempt`) to automatically **annotate** demonstrations for the complex multi-hop pipeline. These demonstrations will be provided to the LM when it's invoked for each transformation."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 22,+ "metadata": {},+ "outputs": [],+ "source": [+ "@dsp.transformation\n",+ "def multihop_attempt(d: dsp.Example) -> dsp.Example:\n",+ " # Prepare unaugmented demonstrations for the example.\n",+ " x = dsp.Example(question=d.question, demos=dsp.all_but(train, d))\n",+ " \n",+ " # Search. And skip examples where search fails.\n",+ " # Annotate demonstrations for multihop_search_v2 with the simpler multihop_search_v1 pipeline.\n",+ " x = multihop_search_v1(x)\n",+ " if not dsp.passage_match(x.context, d.answer): return None\n",+ " \n",+ " # Predict. And skip examples where predict fails.\n",+ " x = QA_predict(x, sc=False)\n",+ " if not dsp.answer_match(x.answer, d.answer): return None\n",+ " \n",+ " return d.copy(**x)\n",+ "\n",+ "@dsp.transformation\n",+ "def multihop_demonstrate(x: dsp.Example) -> dsp.Example:\n",+ " demos = dsp.sample(train, k=7)\n",+ " x.demos = dsp.annotate(multihop_attempt)(demos, k=3, return_all=True)\n",+ " return x"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 23,+ "metadata": {},+ "outputs": [],+ "source": [+ "@dsp.transformation\n",+ "def multihop_search_v2(example: dsp.Example, max_hops=2, k=5) -> dsp.Example:\n",+ " example.context = []\n",+ "\n",+ " for hop in range(max_hops):\n",+ " # Generate queries\n",+ " template = rewrite_template if hop == 0 else hop_template\n",+ " example, completions = dsp.generate(template, n=10, temperature=0.7)(example, stage=f'h{hop}')\n",+ " \n",+ " # Collect the queries and search with result fusion\n",+ " queries = [c.query for c in completions] + [example.question]\n",+ " example.context = dsp.retrieveEnsemble(queries, k=k)\n",+ "\n",+ " # Arrange the passages for the next hop\n",+ " if hop > 0:\n",+ " example.context = [completions[0].rationale] + example.context\n",+ " \n",+ " return example\n",+ "\n",+ "def multihop_QA_v2(question: str) -> str:\n",+ " x = dsp.Example(question=question)\n",+ " x = multihop_demonstrate(x)\n",+ " x = multihop_search_v2(x)\n",+ " x = QA_predict(x)\n",+ " return x.answer"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 24,+ "metadata": {},+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "\n",+ "\n",+ "\n",+ "\n",+ "Write a search query that will help answer a complex question.\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context:\n",+ "${one or more sources, some of which may contain relevant content or clues}\n",+ "\n",+ "Question: ${the question to be answered}\n",+ "\n",+ "Rationale: Let's think step by step. We now know that ${information from the context that may provide useful clues for the question}\n",+ "\n",+ "Search Query: ${a simple question for seeking the missing or required information}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Gary Zukav | Gary Zukav Gary Zukav (born October 17, 1942) is an American spiritual teacher and the author of four consecutive New York Times Best Sellers. Beginning in 1998, he appeared more than 30 times on \"The Oprah Winfrey Show\" to discuss transformation in human consciousness concepts presented in his book \"The Seat of the Soul\". His first book, \"The Dancing Wu Li Masters\" (1979), won a U.S. National Book Award. Gary Zukav was born in Port Arthur, Texas, and spent his early childhood in San Antonio and Houston. His family moved to Pittsburg, Kansas, while he was in fourth grade. In»\n",+ "[2] «Gary Zukav | experience that led to his first book, \"Dancing Wu Li Masters\", written with extensive help from Jack Sarfatti and other physicists he met through Sarfatti as described in David Kaiser's book \"How the Hippies Saved Physics\". He later described this book as his \"first gift to Life\". In 1987 he moved to Mount Shasta, California, where he lived in a cabin as a self-described \"secular monk\" and spent extensive time in the surrounding wilderness. In 1993 he met Linda Francis. They co-founded the Seat of the Soul Institute in 1998 and moved to Ashland, Oregon, in 2000. Christopher Lehmann-Haupt reviewed»\n",+ "\n",+ "Question: Which award did the first book of Gary Zukav receive?\n",+ "\n",+ "Rationale: Let's think step by step. We now know that Gary Zukav's first book is \"The Dancing Wu Li Masters\" and it was written with the help of Jack Sarfatti and other physicists.\n",+ "\n",+ "Search Query: What award did \"The Dancing Wu Li Masters\" receive?\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «John du Pont | John du Pont John Eleuthère du Pont (November 22, 1938 – December 9, 2010) was an American philanthropist, heir to the Du Pont family fortune, and a convicted murderer. He had been a published ornithologist, philatelist, conchologist, sports enthusiast, and self-styled wrestling coach. He died in prison while serving a sentence of 30 years for the murder of Dave Schultz. In 1972, du Pont founded and directed the Delaware Museum of Natural History and contributed to Villanova University and other institutions. In the 1980s, he established a wrestling facility at his Foxcatcher Farm after becoming interested in the sport and»\n",+ "[2] «Du Pont family | with political and business ventures, as well as philanthropic efforts on behalf of various organizations such as The Jockey Club, the Philadelphia Orchestra, the Delaware Art Museum, and the Grand Opera House. Two family members were the subjects of well-publicized criminal cases. Heir John Eleuthère du Pont was convicted of murdering wrestling coach Dave Schultz in 1996 and heir Robert H. Richards IV was convicted of raping his 3-year-old daughter in 2009. The family is depicted in the 2014 biographical film \"Foxcatcher\" with Steve Carell playing John Eleuthère du Pont and Vanessa Redgrave playing Jean du Pont, the wife of»\n",+ "\n",+ "Question: The heir to the Du Pont family fortune sponsored what wrestling team?\n",+ "\n",+ "Rationale: Let's think step by step. We now know that John du Pont was an American philanthropist and heir to the Du Pont family fortune. He was also a published ornithologist, philatelist, conchologist, sports enthusiast, and self-styled wrestling coach.\n",+ "\n",+ "Search Query: What wrestling team did John du Pont sponsor?\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Peter Outerbridge | Peter Outerbridge Peter Outerbridge (born June 30, 1966) is a Canadian actor, best known for his role as Ari Tasarov in the CW action series \"Nikita\", Dr. David Sandström in the TMN series \"ReGenesis\", Henrik \"Hank\" Johanssen in \"Orphan Black\", Bob Corbett in \"Bomb Girls\", William Easton in \"Saw VI\" and George Brown in the television film \"\". He also played the lead role of Detective William Murdoch in a three-episode mini-series, \"The Murdoch Mysteries\", in its initial run on Canadian television, with two episodes shown in 2004 and a third in 2005. Outerbridge was born and raised in Toronto,»\n",+ "[2] «Except the Dying | 2008. On March 3, 2015, Acorn Media announced a re-release for all three movies, set for May 26, 2015. Except the Dying Except the Dying is a 2004 made-for-TV film starring Peter Outerbridge, Colm Meaney, Keeley Hawes, William B. Davis and Flora Montgomery. It was adapted by Janet MacLean from the novel of the same name by Maureen Jennings. In this film, William Murdoch is introduced, as a man of strong principles, who uses his unique abilities to solve crimes, sometimes using advanced science for his time. On the street of Toronto, in the 1890s, the naked body of a»\n",+ "\n",+ "Question: Who was the director of the 2009 movie featuring Peter Outerbridge as William Easton?\n",+ "\n",+ "Rationale: Let's think step by step. We now know that Peter Outerbridge played William Easton in the 2009 movie.\n",+ "\n",+ "Search Query: Who was the director of the 2009 movie featuring Peter Outerbridge as William Easton?\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Julia Peterkin | Julia Peterkin Julia Peterkin (October 31, 1880 – August 10, 1961) was an American author from South Carolina. In 1929 she won the Pulitzer Prize for Novel/Literature, for her novel \"Scarlet Sister Mary.\" She wrote several novels about the plantation South, especially the Gullah people of the Low Country. She was one of the few white authors who wrote about the African-American experience. Julia Mood was born in Laurens County, South Carolina. Her father was a physician, and she was the third of his four children. Her mother died soon after her birth, and her father later married Janie Brogdon.»\n",+ "[2] «E. L. Doctorow | E. L. Doctorow Edgar Lawrence Doctorow (January 6, 1931 – July 21, 2015) was an American novelist, editor, and professor, best known internationally for his works of historical fiction. He has been described as one of the most important American novelists of the 20th century. He wrote twelve novels, three volumes of short fiction and a stage drama. They included the award-winning novels \"Ragtime\" (1975), \"Billy Bathgate\" (1989), and \"The March\" (2005). These, like many of his other works, placed fictional characters in recognizable historical contexts, with known historical figures, and often used different narrative styles. His stories were recognized»\n",+ "[3] «Julia Peterkin | Janie was the mother of Henry Ashleigh Mood, Julia's half-brother and her father's fourth child. He became a doctor. In 1896, at age 16, Julia Mood graduated from Converse College in Spartanburg, South Carolina; she earned her master's degree there a year later. She taught at the public school in Fort Motte, South Carolina, for a few years before she married William George Peterkin in 1903. He was a planter who owned Lang Syne, a cotton plantation near Fort Motte. Julia began writing short stories, inspired by the everyday life and management of the plantation. She was described as audacious»\n",+ "[4] «Julia Peterkin | such conventions. Hurston wrote that she had met Peterkin and would begin a correspondence, but no letters between them have ever been found. Julia Peterkin Julia Peterkin (October 31, 1880 – August 10, 1961) was an American author from South Carolina. In 1929 she won the Pulitzer Prize for Novel/Literature, for her novel \"Scarlet Sister Mary.\" She wrote several novels about the plantation South, especially the Gullah people of the Low Country. She was one of the few white authors who wrote about the African-American experience. Julia Mood was born in Laurens County, South Carolina. Her father was a physician,»\n",+ "[5] «E. L. Doctorow | Fe-Bornstein . He died of lung cancer on July 21, 2015, aged 84, in Manhattan. He is interred in Woodlawn Cemetery in the Bronx. E. L. Doctorow Edgar Lawrence Doctorow (January 6, 1931 – July 21, 2015) was an American novelist, editor, and professor, best known internationally for his works of historical fiction. He has been described as one of the most important American novelists of the 20th century. He wrote twelve novels, three volumes of short fiction and a stage drama. They included the award-winning novels \"Ragtime\" (1975), \"Billy Bathgate\" (1989), and \"The March\" (2005). These, like many of»\n",+ "\n",+ "Question: Who has a broader scope of profession: E. L. Doctorow or Julia Peterkin?\n",+ "\n",+ "Rationale: Let's think step by step. We now know that\u001b[32m Julia Peterkin was an American author from South Carolina and E. L. Doctorow was an American novelist, editor, and professor.\n",+ "\n",+ "Search Query: Who has a broader scope of profession: E. L. Doctorow or Julia Peterkin?\u001b[0m\u001b[31m \t (and 9 other completions)\u001b[0m\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Answer questions with short factoid answers.\n",+ "\n",+ "---\n",+ "\n",+ "Question: Who produced the album that included a re-recording of \"Lithium\"?\n",+ "Final Answer: Butch Vig\n",+ "\n",+ "Question: What city was the victim of Joseph Druces working in?\n",+ "Final Answer: Boston, Massachusetts\n",+ "\n",+ "Question: In what year was the star of To Hell and Back born?\n",+ "Final Answer: 1925\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context:\n",+ "${one or more sources, some of which may contain relevant content or clues}\n",+ "\n",+ "Question: ${the question to be answered}\n",+ "\n",+ "Rationale: Let's think step by step. ${a step-by-step deduction that identifies the correct response, which will be provided below}\n",+ "\n",+ "Final Answer: ${a short factoid answer, often between 1 and 5 words}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Gary Zukav | Gary Zukav Gary Zukav (born October 17, 1942) is an American spiritual teacher and the author of four consecutive New York Times Best Sellers. Beginning in 1998, he appeared more than 30 times on \"The Oprah Winfrey Show\" to discuss transformation in human consciousness concepts presented in his book \"The Seat of the Soul\". His first book, \"The Dancing Wu Li Masters\" (1979), won a U.S. National Book Award. Gary Zukav was born in Port Arthur, Texas, and spent his early childhood in San Antonio and Houston. His family moved to Pittsburg, Kansas, while he was in fourth grade. In»\n",+ "[2] «Gary Zukav | experience that led to his first book, \"Dancing Wu Li Masters\", written with extensive help from Jack Sarfatti and other physicists he met through Sarfatti as described in David Kaiser's book \"How the Hippies Saved Physics\". He later described this book as his \"first gift to Life\". In 1987 he moved to Mount Shasta, California, where he lived in a cabin as a self-described \"secular monk\" and spent extensive time in the surrounding wilderness. In 1993 he met Linda Francis. They co-founded the Seat of the Soul Institute in 1998 and moved to Ashland, Oregon, in 2000. Christopher Lehmann-Haupt reviewed»\n",+ "[3] «The Dancing Wu Li Masters | The Dancing Wu Li Masters The Dancing Wu Li Masters is a 1979 book by Gary Zukav, a popular science work exploring modern physics, and quantum phenomena in particular. It was awarded a 1980 U.S. National Book Award in category of Science. Although it explores empirical topics in modern physics research, \"The Dancing Wu Li Masters\" gained attention for leveraging metaphors taken from eastern spiritual movements, in particular the Huayen school of Buddhism with the monk Fazang's treatise on The Golden Lion, to explain quantum phenomena and has been regarded by some reviewers as a New Age work, although the»\n",+ "[4] «The Dancing Wu Li Masters | a 1976 physics conference of eastern and western scientists at Esalen Institute, California; and he used the occasion as material for his book. At the conference, it was said that the Chinese term for physics is 'Wu Li', or \"patterns of organic energy.\" Zukav, among others, conceptualized 'physics' as the dance of the Wu Li Masters – teachers of physical essence. Zukav explains the concept further: The Dancing Wu Li Masters The Dancing Wu Li Masters is a 1979 book by Gary Zukav, a popular science work exploring modern physics, and quantum phenomena in particular. It was awarded a 1980»\n",+ "\n",+ "Question: Which award did the first book of Gary Zukav receive?\n",+ "\n",+ "Rationale: Let's think step by step. The first book is \"The Dancing Wu Li Masters\" and it received a U.S. National Book Award.\n",+ "\n",+ "Final Answer: U.S. National Book Award\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «John du Pont | John du Pont John Eleuthère du Pont (November 22, 1938 – December 9, 2010) was an American philanthropist, heir to the Du Pont family fortune, and a convicted murderer. He had been a published ornithologist, philatelist, conchologist, sports enthusiast, and self-styled wrestling coach. He died in prison while serving a sentence of 30 years for the murder of Dave Schultz. In 1972, du Pont founded and directed the Delaware Museum of Natural History and contributed to Villanova University and other institutions. In the 1980s, he established a wrestling facility at his Foxcatcher Farm after becoming interested in the sport and»\n",+ "[2] «Du Pont family | with political and business ventures, as well as philanthropic efforts on behalf of various organizations such as The Jockey Club, the Philadelphia Orchestra, the Delaware Art Museum, and the Grand Opera House. Two family members were the subjects of well-publicized criminal cases. Heir John Eleuthère du Pont was convicted of murdering wrestling coach Dave Schultz in 1996 and heir Robert H. Richards IV was convicted of raping his 3-year-old daughter in 2009. The family is depicted in the 2014 biographical film \"Foxcatcher\" with Steve Carell playing John Eleuthère du Pont and Vanessa Redgrave playing Jean du Pont, the wife of»\n",+ "[3] «John du Pont | Foxcatcher team. Du Pont became a sponsor in wrestling, swimming, track, and the modern pentathlon. He was also involved in promoting a subset of the modern pentathlon (run, swim, shoot) as a separate event. He took up athletics and became a competitive wrestler in his 50s. His only prior wrestling experience was as a freshman in high school. He began competing again at the age of 55 in the 1992 Veteran's World Championships in Cali, Colombia; following that in 1993 in Toronto, Ontario; in 1994 in Rome, Italy; and in 1995 in Sofia, Bulgaria. In August 1988, a problem-plagued wrestling»\n",+ "[4] «John du Pont | stamps while in prison, he was not allowed to bring them there. After his mother's death, du Pont developed the Liseter Hall Farm in Newtown Square as a high-quality wrestling facility for amateur wrestlers. He called the private group \"Team Foxcatcher,\" after his father's noted racing stable. Du Pont established an Olympic swimming and wrestling training center and sponsored competitive events at the estate. He also allowed some people, such as Olympic champion wrestlers Mark Schultz and later his older brother Dave Schultz and his wife, to live in houses on the grounds for years. Dave Schultz also coached the»\n",+ "\n",+ "Question: The heir to the Du Pont family fortune sponsored what wrestling team?\n",+ "\n",+ "Rationale: Let's think step by step. The heir to the Du Pont family fortune was John du Pont. He was convicted of murdering wrestling coach Dave Schultz in 1996. He was also involved in promoting a subset of the modern pentathlon (run, swim, shoot) as a separate event. He took up athletics and became a competitive wrestler in his 50s. He sponsored competitive events at the estate.\n",+ "\n",+ "Final Answer: Team Foxcatcher\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Peter Outerbridge | Peter Outerbridge Peter Outerbridge (born June 30, 1966) is a Canadian actor, best known for his role as Ari Tasarov in the CW action series \"Nikita\", Dr. David Sandström in the TMN series \"ReGenesis\", Henrik \"Hank\" Johanssen in \"Orphan Black\", Bob Corbett in \"Bomb Girls\", William Easton in \"Saw VI\" and George Brown in the television film \"\". He also played the lead role of Detective William Murdoch in a three-episode mini-series, \"The Murdoch Mysteries\", in its initial run on Canadian television, with two episodes shown in 2004 and a third in 2005. Outerbridge was born and raised in Toronto,»\n",+ "[2] «Except the Dying | 2008. On March 3, 2015, Acorn Media announced a re-release for all three movies, set for May 26, 2015. Except the Dying Except the Dying is a 2004 made-for-TV film starring Peter Outerbridge, Colm Meaney, Keeley Hawes, William B. Davis and Flora Montgomery. It was adapted by Janet MacLean from the novel of the same name by Maureen Jennings. In this film, William Murdoch is introduced, as a man of strong principles, who uses his unique abilities to solve crimes, sometimes using advanced science for his time. On the street of Toronto, in the 1890s, the naked body of a»\n",+ "[3] «Saw VI | Saw VI Saw VI is a 2009 American horror film directed by Kevin Greutert from a screenplay written by Patrick Melton and Marcus Dunstan. It is the sixth installment in the \"Saw\" franchise and stars Tobin Bell, Costas Mandylor, Betsy Russell, Mark Rolston, Peter Outerbridge, and Shawnee Smith. It was produced by Mark Burg and Oren Koules of Twisted Pictures and distributed by Lionsgate. \"Saw VI\" continues the franchise's focus on the posthumous effects of the Jigsaw Killer and the progression of his successor, Mark Hoffman. In this film, Hoffman sets a series of traps for an insurance executive, William»\n",+ "\n",+ "Question: Who was the director of the 2009 movie featuring Peter Outerbridge as William Easton?\n",+ "\n",+ "Rationale: Let's think step by step. We know that Peter Outerbridge was in the movie \"Saw VI\" and that he played the character William Easton. We also know that the movie was released in 2009. From this, we can deduce that the director of the movie was Kevin Greutert.\n",+ "\n",+ "Final Answer: Kevin Greutert\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Julia Peterkin was an American author from South Carolina and E. L. Doctorow was an American novelist, editor, and professor.»\n",+ "[2] «Julia Peterkin | Julia Peterkin Julia Peterkin (October 31, 1880 – August 10, 1961) was an American author from South Carolina. In 1929 she won the Pulitzer Prize for Novel/Literature, for her novel \"Scarlet Sister Mary.\" She wrote several novels about the plantation South, especially the Gullah people of the Low Country. She was one of the few white authors who wrote about the African-American experience. Julia Mood was born in Laurens County, South Carolina. Her father was a physician, and she was the third of his four children. Her mother died soon after her birth, and her father later married Janie Brogdon.»\n",+ "[3] «E. L. Doctorow | E. L. Doctorow Edgar Lawrence Doctorow (January 6, 1931 – July 21, 2015) was an American novelist, editor, and professor, best known internationally for his works of historical fiction. He has been described as one of the most important American novelists of the 20th century. He wrote twelve novels, three volumes of short fiction and a stage drama. They included the award-winning novels \"Ragtime\" (1975), \"Billy Bathgate\" (1989), and \"The March\" (2005). These, like many of his other works, placed fictional characters in recognizable historical contexts, with known historical figures, and often used different narrative styles. His stories were recognized»\n",+ "[4] «Physician | scope of practice for a nurse practitioner in the United States is defined by regulatory boards of nursing, as opposed to boards of medicine that regulate physicians. Physician A physician, medical practitioner, medical doctor, or simply doctor is a professional who practises medicine, which is concerned with promoting, maintaining, or restoring health through the study, diagnosis, and treatment of disease, injury, and other physical and mental impairments. Physicians may focus their practice on certain disease categories, types of patients and methods of treatment—known as specialities—or they may assume responsibility for the provision of continuing and comprehensive medical care to individuals,»\n",+ "[5] «E. L. Doctorow | Utah, the University of California, Irvine, and Princeton University. He was the Loretta and Lewis Glucksman Professor of English and American Letters at New York University. In 2001 he donated his papers to the Fales Library of New York University. The library's director, Marvin Taylor, said Doctorow was \"one of the most important American novelists of the 20th century\". In 1954, Doctorow married fellow Columbia University student Helen Esther Setzer while serving in the U.S. Army in West Germany. The couple had three children: television writer and producer Richard Doctorow, singer-songwriter and record producer Caroline Doctorow Gatewood and Jenny Doctorow»\n",+ "[6] «Julia Peterkin | to Alfred Knopf, who published her first book, \"Green Thursday,\" in 1924. In addition to a number of subsequent novels, her short stories were published in magazines and newspaper throughout her career. Peterkin was among the few white authors to specialize in the African-American experience. She won a Pulitzer Prize in 1929 for her novel, \"Scarlet Sister Mary.\" Dr. Richard S. Burton, the chairperson of Pulitzer's fiction-literature jury, recommended that the first prize go to the novel \"Victim and Victor\" by John Rathbone Oliver. The School of Journalism chose Peterkin's book. Burton resigned from the jury. The book aroused some»\n",+ "\n",+ "Question: Who has a broader scope of profession: E. L. Doctorow or Julia Peterkin?\n",+ "\n",+ "Rationale: Let's think step by step.\u001b[32m We are given information about the professions of E. L. Doctorow and Julia Peterkin. E. L. Doctorow was an American novelist, editor, and professor. Julia Peterkin was an American author. Based on this, we can deduce that the scope of profession for E. L. Doctorow is broader than that of Julia Peterkin.\u001b[0m\u001b[31m \t (and 19 other completions)\u001b[0m\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "\n",+ "Answer questions with short factoid answers.\n",+ "\n",+ "---\n",+ "\n",+ "Question: Who produced the album that included a re-recording of \"Lithium\"?\n",+ "Final Answer: Butch Vig\n",+ "\n",+ "Question: What city was the victim of Joseph Druces working in?\n",+ "Final Answer: Boston, Massachusetts\n",+ "\n",+ "Question: In what year was the star of To Hell and Back born?\n",+ "Final Answer: 1925\n",+ "\n",+ "---\n",+ "\n",+ "Follow the following format.\n",+ "\n",+ "Context:\n",+ "${one or more sources, some of which may contain relevant content or clues}\n",+ "\n",+ "Question: ${the question to be answered}\n",+ "\n",+ "Rationale: Let's think step by step. ${a step-by-step deduction that identifies the correct response, which will be provided below}\n",+ "\n",+ "Final Answer: ${a short factoid answer, often between 1 and 5 words}\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Gary Zukav | Gary Zukav Gary Zukav (born October 17, 1942) is an American spiritual teacher and the author of four consecutive New York Times Best Sellers. Beginning in 1998, he appeared more than 30 times on \"The Oprah Winfrey Show\" to discuss transformation in human consciousness concepts presented in his book \"The Seat of the Soul\". His first book, \"The Dancing Wu Li Masters\" (1979), won a U.S. National Book Award. Gary Zukav was born in Port Arthur, Texas, and spent his early childhood in San Antonio and Houston. His family moved to Pittsburg, Kansas, while he was in fourth grade. In»\n",+ "[2] «Gary Zukav | experience that led to his first book, \"Dancing Wu Li Masters\", written with extensive help from Jack Sarfatti and other physicists he met through Sarfatti as described in David Kaiser's book \"How the Hippies Saved Physics\". He later described this book as his \"first gift to Life\". In 1987 he moved to Mount Shasta, California, where he lived in a cabin as a self-described \"secular monk\" and spent extensive time in the surrounding wilderness. In 1993 he met Linda Francis. They co-founded the Seat of the Soul Institute in 1998 and moved to Ashland, Oregon, in 2000. Christopher Lehmann-Haupt reviewed»\n",+ "[3] «The Dancing Wu Li Masters | The Dancing Wu Li Masters The Dancing Wu Li Masters is a 1979 book by Gary Zukav, a popular science work exploring modern physics, and quantum phenomena in particular. It was awarded a 1980 U.S. National Book Award in category of Science. Although it explores empirical topics in modern physics research, \"The Dancing Wu Li Masters\" gained attention for leveraging metaphors taken from eastern spiritual movements, in particular the Huayen school of Buddhism with the monk Fazang's treatise on The Golden Lion, to explain quantum phenomena and has been regarded by some reviewers as a New Age work, although the»\n",+ "[4] «The Dancing Wu Li Masters | a 1976 physics conference of eastern and western scientists at Esalen Institute, California; and he used the occasion as material for his book. At the conference, it was said that the Chinese term for physics is 'Wu Li', or \"patterns of organic energy.\" Zukav, among others, conceptualized 'physics' as the dance of the Wu Li Masters – teachers of physical essence. Zukav explains the concept further: The Dancing Wu Li Masters The Dancing Wu Li Masters is a 1979 book by Gary Zukav, a popular science work exploring modern physics, and quantum phenomena in particular. It was awarded a 1980»\n",+ "\n",+ "Question: Which award did the first book of Gary Zukav receive?\n",+ "\n",+ "Rationale: Let's think step by step. The first book is \"The Dancing Wu Li Masters\" and it received a U.S. National Book Award.\n",+ "\n",+ "Final Answer: U.S. National Book Award\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «John du Pont | John du Pont John Eleuthère du Pont (November 22, 1938 – December 9, 2010) was an American philanthropist, heir to the Du Pont family fortune, and a convicted murderer. He had been a published ornithologist, philatelist, conchologist, sports enthusiast, and self-styled wrestling coach. He died in prison while serving a sentence of 30 years for the murder of Dave Schultz. In 1972, du Pont founded and directed the Delaware Museum of Natural History and contributed to Villanova University and other institutions. In the 1980s, he established a wrestling facility at his Foxcatcher Farm after becoming interested in the sport and»\n",+ "[2] «Du Pont family | with political and business ventures, as well as philanthropic efforts on behalf of various organizations such as The Jockey Club, the Philadelphia Orchestra, the Delaware Art Museum, and the Grand Opera House. Two family members were the subjects of well-publicized criminal cases. Heir John Eleuthère du Pont was convicted of murdering wrestling coach Dave Schultz in 1996 and heir Robert H. Richards IV was convicted of raping his 3-year-old daughter in 2009. The family is depicted in the 2014 biographical film \"Foxcatcher\" with Steve Carell playing John Eleuthère du Pont and Vanessa Redgrave playing Jean du Pont, the wife of»\n",+ "[3] «John du Pont | Foxcatcher team. Du Pont became a sponsor in wrestling, swimming, track, and the modern pentathlon. He was also involved in promoting a subset of the modern pentathlon (run, swim, shoot) as a separate event. He took up athletics and became a competitive wrestler in his 50s. His only prior wrestling experience was as a freshman in high school. He began competing again at the age of 55 in the 1992 Veteran's World Championships in Cali, Colombia; following that in 1993 in Toronto, Ontario; in 1994 in Rome, Italy; and in 1995 in Sofia, Bulgaria. In August 1988, a problem-plagued wrestling»\n",+ "[4] «John du Pont | stamps while in prison, he was not allowed to bring them there. After his mother's death, du Pont developed the Liseter Hall Farm in Newtown Square as a high-quality wrestling facility for amateur wrestlers. He called the private group \"Team Foxcatcher,\" after his father's noted racing stable. Du Pont established an Olympic swimming and wrestling training center and sponsored competitive events at the estate. He also allowed some people, such as Olympic champion wrestlers Mark Schultz and later his older brother Dave Schultz and his wife, to live in houses on the grounds for years. Dave Schultz also coached the»\n",+ "\n",+ "Question: The heir to the Du Pont family fortune sponsored what wrestling team?\n",+ "\n",+ "Rationale: Let's think step by step. The heir to the Du Pont family fortune was John du Pont. He was convicted of murdering wrestling coach Dave Schultz in 1996. He was also involved in promoting a subset of the modern pentathlon (run, swim, shoot) as a separate event. He took up athletics and became a competitive wrestler in his 50s. He sponsored competitive events at the estate.\n",+ "\n",+ "Final Answer: Team Foxcatcher\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Peter Outerbridge | Peter Outerbridge Peter Outerbridge (born June 30, 1966) is a Canadian actor, best known for his role as Ari Tasarov in the CW action series \"Nikita\", Dr. David Sandström in the TMN series \"ReGenesis\", Henrik \"Hank\" Johanssen in \"Orphan Black\", Bob Corbett in \"Bomb Girls\", William Easton in \"Saw VI\" and George Brown in the television film \"\". He also played the lead role of Detective William Murdoch in a three-episode mini-series, \"The Murdoch Mysteries\", in its initial run on Canadian television, with two episodes shown in 2004 and a third in 2005. Outerbridge was born and raised in Toronto,»\n",+ "[2] «Except the Dying | 2008. On March 3, 2015, Acorn Media announced a re-release for all three movies, set for May 26, 2015. Except the Dying Except the Dying is a 2004 made-for-TV film starring Peter Outerbridge, Colm Meaney, Keeley Hawes, William B. Davis and Flora Montgomery. It was adapted by Janet MacLean from the novel of the same name by Maureen Jennings. In this film, William Murdoch is introduced, as a man of strong principles, who uses his unique abilities to solve crimes, sometimes using advanced science for his time. On the street of Toronto, in the 1890s, the naked body of a»\n",+ "[3] «Saw VI | Saw VI Saw VI is a 2009 American horror film directed by Kevin Greutert from a screenplay written by Patrick Melton and Marcus Dunstan. It is the sixth installment in the \"Saw\" franchise and stars Tobin Bell, Costas Mandylor, Betsy Russell, Mark Rolston, Peter Outerbridge, and Shawnee Smith. It was produced by Mark Burg and Oren Koules of Twisted Pictures and distributed by Lionsgate. \"Saw VI\" continues the franchise's focus on the posthumous effects of the Jigsaw Killer and the progression of his successor, Mark Hoffman. In this film, Hoffman sets a series of traps for an insurance executive, William»\n",+ "\n",+ "Question: Who was the director of the 2009 movie featuring Peter Outerbridge as William Easton?\n",+ "\n",+ "Rationale: Let's think step by step. We know that Peter Outerbridge was in the movie \"Saw VI\" and that he played the character William Easton. We also know that the movie was released in 2009. From this, we can deduce that the director of the movie was Kevin Greutert.\n",+ "\n",+ "Final Answer: Kevin Greutert\n",+ "\n",+ "---\n",+ "\n",+ "Context:\n",+ "[1] «Julia Peterkin was an American author from South Carolina and E. L. Doctorow was an American novelist, editor, and professor.»\n",+ "[2] «Julia Peterkin | Julia Peterkin Julia Peterkin (October 31, 1880 – August 10, 1961) was an American author from South Carolina. In 1929 she won the Pulitzer Prize for Novel/Literature, for her novel \"Scarlet Sister Mary.\" She wrote several novels about the plantation South, especially the Gullah people of the Low Country. She was one of the few white authors who wrote about the African-American experience. Julia Mood was born in Laurens County, South Carolina. Her father was a physician, and she was the third of his four children. Her mother died soon after her birth, and her father later married Janie Brogdon.»\n",+ "[3] «E. L. Doctorow | E. L. Doctorow Edgar Lawrence Doctorow (January 6, 1931 – July 21, 2015) was an American novelist, editor, and professor, best known internationally for his works of historical fiction. He has been described as one of the most important American novelists of the 20th century. He wrote twelve novels, three volumes of short fiction and a stage drama. They included the award-winning novels \"Ragtime\" (1975), \"Billy Bathgate\" (1989), and \"The March\" (2005). These, like many of his other works, placed fictional characters in recognizable historical contexts, with known historical figures, and often used different narrative styles. His stories were recognized»\n",+ "[4] «Physician | scope of practice for a nurse practitioner in the United States is defined by regulatory boards of nursing, as opposed to boards of medicine that regulate physicians. Physician A physician, medical practitioner, medical doctor, or simply doctor is a professional who practises medicine, which is concerned with promoting, maintaining, or restoring health through the study, diagnosis, and treatment of disease, injury, and other physical and mental impairments. Physicians may focus their practice on certain disease categories, types of patients and methods of treatment—known as specialities—or they may assume responsibility for the provision of continuing and comprehensive medical care to individuals,»\n",+ "[5] «E. L. Doctorow | Utah, the University of California, Irvine, and Princeton University. He was the Loretta and Lewis Glucksman Professor of English and American Letters at New York University. In 2001 he donated his papers to the Fales Library of New York University. The library's director, Marvin Taylor, said Doctorow was \"one of the most important American novelists of the 20th century\". In 1954, Doctorow married fellow Columbia University student Helen Esther Setzer while serving in the U.S. Army in West Germany. The couple had three children: television writer and producer Richard Doctorow, singer-songwriter and record producer Caroline Doctorow Gatewood and Jenny Doctorow»\n",+ "[6] «Julia Peterkin | to Alfred Knopf, who published her first book, \"Green Thursday,\" in 1924. In addition to a number of subsequent novels, her short stories were published in magazines and newspaper throughout her career. Peterkin was among the few white authors to specialize in the African-American experience. She won a Pulitzer Prize in 1929 for her novel, \"Scarlet Sister Mary.\" Dr. Richard S. Burton, the chairperson of Pulitzer's fiction-literature jury, recommended that the first prize go to the novel \"Victim and Victor\" by John Rathbone Oliver. The School of Journalism chose Peterkin's book. Burton resigned from the jury. The book aroused some»\n",+ "\n",+ "Question: Who has a broader scope of profession: E. L. Doctorow or Julia Peterkin?\n",+ "\n",+ "Rationale: Let's think step by step. We are given information about the professions of E. L. Doctorow and Julia Peterkin. E. L. Doctorow was an American novelist, editor, and professor. Julia Peterkin was an American author. Based on this, we can deduce that the scope of profession for E. L. Doctorow is broader than that of Julia Peterkin.\n",+ "\n",+ "Final Answer:\u001b[32m E. L. Doctorow\u001b[0m\n",+ "\n",+ "\n",+ "\n"+ ]+ },+ {+ "data": {+ "text/plain": [+ "('E. L. Doctorow', None)"+ ]+ },+ "execution_count": 24,+ "metadata": {},+ "output_type": "execute_result"+ }+ ],+ "source": [+ "multihop_QA_v2(dev[0].question), lm.inspect_history(n=3)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 25,+ "metadata": {},+ "outputs": [+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "100%|██████████| 13/13 [00:03<00:00, 4.09it/s]"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Answered 12 / 13 correctly.\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "\n"+ ]+ },+ {+ "data": {+ "text/html": [+ "<style type=\"text/css\">\n",+ "#T_7b33b_ th {\n",+ " text-align: left;\n",+ "}\n",+ "#T_7b33b_ td {\n",+ " text-align: left;\n",+ "}\n",+ "</style>\n",+ "<table id=\"T_7b33b_\">\n",+ " <thead>\n",+ " <tr>\n",+ " <th class=\"blank level0\" > </th>\n",+ " <th class=\"col_heading level0 col0\" >question</th>\n",+ " <th class=\"col_heading level0 col1\" >answer</th>\n",+ " <th class=\"col_heading level0 col2\" >prediction</th>\n",+ " <th class=\"col_heading level0 col3\" >correct</th>\n",+ " </tr>\n",+ " </thead>\n",+ " <tbody>\n",+ " <tr>\n",+ " <th id=\"T_7b33b_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",+ " <td id=\"T_7b33b_row0_col0\" class=\"data row0 col0\" >Who has a broader scope of profession: E. L. Doctorow or Julia Peterkin?</td>\n",+ " <td id=\"T_7b33b_row0_col1\" class=\"data row0 col1\" >['E. L. Doctorow', 'E.L. Doctorow', 'Doctorow']</td>\n",+ " <td id=\"T_7b33b_row0_col2\" class=\"data row0 col2\" >E. L. Doctorow</td>\n",+ " <td id=\"T_7b33b_row0_col3\" class=\"data row0 col3\" >✔️</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_7b33b_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",+ " <td id=\"T_7b33b_row1_col0\" class=\"data row1 col0\" >What documentary about the Gilgo Beach Killer debuted on A&E?</td>\n",+ " <td id=\"T_7b33b_row1_col1\" class=\"data row1 col1\" >['The Killing Season']</td>\n",+ " <td id=\"T_7b33b_row1_col2\" class=\"data row1 col2\" >The Killing Season</td>\n",+ " <td id=\"T_7b33b_row1_col3\" class=\"data row1 col3\" >✔️</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_7b33b_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",+ " <td id=\"T_7b33b_row2_col0\" class=\"data row2 col0\" >Right Back At It Again contains lyrics co-written by the singer born in what city?</td>\n",+ " <td id=\"T_7b33b_row2_col1\" class=\"data row2 col1\" >['Gainesville, Florida', 'Gainesville']</td>\n",+ " <td id=\"T_7b33b_row2_col2\" class=\"data row2 col2\" >Gainesville, Florida</td>\n",+ " <td id=\"T_7b33b_row2_col3\" class=\"data row2 col3\" >✔️</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_7b33b_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",+ " <td id=\"T_7b33b_row3_col0\" class=\"data row3 col0\" >What year was the party of the winner of the 1971 San Francisco mayoral election founded?</td>\n",+ " <td id=\"T_7b33b_row3_col1\" class=\"data row3 col1\" >['1828']</td>\n",+ " <td id=\"T_7b33b_row3_col2\" class=\"data row3 col2\" >1828</td>\n",+ " <td id=\"T_7b33b_row3_col3\" class=\"data row3 col3\" >✔️</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_7b33b_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",+ " <td id=\"T_7b33b_row4_col0\" class=\"data row4 col0\" >Which author is English: John Braine or Studs Terkel?</td>\n",+ " <td id=\"T_7b33b_row4_col1\" class=\"data row4 col1\" >['John Braine']</td>\n",+ " <td id=\"T_7b33b_row4_col2\" class=\"data row4 col2\" >John Braine</td>\n",+ " <td id=\"T_7b33b_row4_col3\" class=\"data row4 col3\" >✔️</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_7b33b_level0_row5\" class=\"row_heading level0 row5\" >5</th>\n",+ " <td id=\"T_7b33b_row5_col0\" class=\"data row5 col0\" >Anthony Dirrell is the brother of which super middleweight title holder?</td>\n",+ " <td id=\"T_7b33b_row5_col1\" class=\"data row5 col1\" >['Andre Dirrell']</td>\n",+ " <td id=\"T_7b33b_row5_col2\" class=\"data row5 col2\" >Andre Dirrell</td>\n",+ " <td id=\"T_7b33b_row5_col3\" class=\"data row5 col3\" >✔️</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_7b33b_level0_row6\" class=\"row_heading level0 row6\" >6</th>\n",+ " <td id=\"T_7b33b_row6_col0\" class=\"data row6 col0\" >In which city is the sports nutrition business established by Oliver Cookson based ?</td>\n",+ " <td id=\"T_7b33b_row6_col1\" class=\"data row6 col1\" >['Cheshire', 'Cheshire, UK']</td>\n",+ " <td id=\"T_7b33b_row6_col2\" class=\"data row6 col2\" >Cheshire</td>\n",+ " <td id=\"T_7b33b_row6_col3\" class=\"data row6 col3\" >✔️</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_7b33b_level0_row7\" class=\"row_heading level0 row7\" >7</th>\n",+ " <td id=\"T_7b33b_row7_col0\" class=\"data row7 col0\" >Find the birth date of the actor who played roles in First Wives Club and Searching for the Elephant.</td>\n",+ " <td id=\"T_7b33b_row7_col1\" class=\"data row7 col1\" >['February 13, 1980']</td>\n",+ " <td id=\"T_7b33b_row7_col2\" class=\"data row7 col2\" >1958</td>\n",+ " <td id=\"T_7b33b_row7_col3\" class=\"data row7 col3\" >❌</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_7b33b_level0_row8\" class=\"row_heading level0 row8\" >8</th>\n",+ " <td id=\"T_7b33b_row8_col0\" class=\"data row8 col0\" >Kyle Moran was born in the town on what river?</td>\n",+ " <td id=\"T_7b33b_row8_col1\" class=\"data row8 col1\" >['Castletown', 'Castletown River']</td>\n",+ " <td id=\"T_7b33b_row8_col2\" class=\"data row8 col2\" >Castletown River</td>\n",+ " <td id=\"T_7b33b_row8_col3\" class=\"data row8 col3\" >✔️</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_7b33b_level0_row9\" class=\"row_heading level0 row9\" >9</th>\n",+ " <td id=\"T_7b33b_row9_col0\" class=\"data row9 col0\" >What is the name of one branch of Robert D. Braun's speciality?</td>\n",+ " <td id=\"T_7b33b_row9_col1\" class=\"data row9 col1\" >['aeronautical engineering', 'astronautical engineering', 'aeronautics', 'astronautics']</td>\n",+ " <td id=\"T_7b33b_row9_col2\" class=\"data row9 col2\" >Aeronautics</td>\n",+ " <td id=\"T_7b33b_row9_col3\" class=\"data row9 col3\" >✔️</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_7b33b_level0_row10\" class=\"row_heading level0 row10\" >10</th>\n",+ " <td id=\"T_7b33b_row10_col0\" class=\"data row10 col0\" >Where was the actress who played the niece in the Priest film born?</td>\n",+ " <td id=\"T_7b33b_row10_col1\" class=\"data row10 col1\" >['Surrey', 'Guildford, Surrey']</td>\n",+ " <td id=\"T_7b33b_row10_col2\" class=\"data row10 col2\" >Surrey</td>\n",+ " <td id=\"T_7b33b_row10_col3\" class=\"data row10 col3\" >✔️</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_7b33b_level0_row11\" class=\"row_heading level0 row11\" >11</th>\n",+ " <td id=\"T_7b33b_row11_col0\" class=\"data row11 col0\" >Name the movie in which the daughter of Noel Harrison plays Violet Trefusis.</td>\n",+ " <td id=\"T_7b33b_row11_col1\" class=\"data row11 col1\" >['Portrait of a Marriage']</td>\n",+ " <td id=\"T_7b33b_row11_col2\" class=\"data row11 col2\" >Portrait of a Marriage</td>\n",+ " <td id=\"T_7b33b_row11_col3\" class=\"data row11 col3\" >✔️</td>\n",+ " </tr>\n",+ " <tr>\n",+ " <th id=\"T_7b33b_level0_row12\" class=\"row_heading level0 row12\" >12</th>\n",+ " <td id=\"T_7b33b_row12_col0\" class=\"data row12 col0\" >What year was the father of the Princes in the Tower born?</td>\n",+ " <td id=\"T_7b33b_row12_col1\" class=\"data row12 col1\" >['1442']</td>\n",+ " <td id=\"T_7b33b_row12_col2\" class=\"data row12 col2\" >1442</td>\n",+ " <td id=\"T_7b33b_row12_col3\" class=\"data row12 col3\" >✔️</td>\n",+ " </tr>\n",+ " </tbody>\n",+ "</table>\n"+ ],+ "text/plain": [+ "<pandas.io.formats.style.Styler at 0x7f090dbbff10>"+ ]+ },+ "metadata": {},+ "output_type": "display_data"+ }+ ],+ "source": [+ "evaluate(multihop_QA_v2, dev)"+ ]+ },+ {+ "attachments": {},+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "While Program 5 is undeniably better than, say, Program 1, it's far from perfect.\n",+ "\n",+ "Many different ideas can easily be implemented to make it more general or more powerful. Refer to the DSP paper for inspiration!"+ ]+ }+ ],+ "metadata": {+ "kernelspec": {+ "display_name": "colbert-api-001",+ "language": "python",+ "name": "python3"+ },+ "language_info": {+ "codemirror_mode": {+ "name": "ipython",+ "version": 3+ },+ "file_extension": ".py",+ "mimetype": "text/x-python",+ "name": "python",+ "nbconvert_exporter": "python",+ "pygments_lexer": "ipython3",+ "version": "3.7.11"+ },+ "orig_nbformat": 4,+ "vscode": {+ "interpreter": {+ "hash": "08fdf29897c5b8f3cfe29625f379bb8bd67a7135431cc64e36a1afcc33cadfae"+ }+ }+ },+ "nbformat": 4,+ "nbformat_minor": 2+}diff --git a/setup.py b/setup.pynew file mode 100644index 000000000..e42130388--- /dev/null+++ b/setup.py@@ -0,0 +1,31 @@+from setuptools import setup++setup(+ name="dsp",+ version="0.1.0",+ description="Demonstrate-Search-Predict",+ url="https://github.com/stanfordnlp/dsp",+ author="Omar Khattab",+ author_email="[email protected]",+ license="MIT License",+ packages=["dsp"],+ python_requires='>=3.8',+ install_requires=[+ "backoff",+ "joblib",+ "jupyter",+ "openai",+ "pandas",+ "spacy",+ "regex",+ ],+ classifiers=[+ "Development Status :: 3 - Alpha",+ "Intended Audience :: Science/Research",+ "License :: OSI Approved :: MIT License",+ "Operating System :: POSIX :: Linux",+ "Programming Language :: Python :: 3",+ "Programming Language :: Python :: 3.8",+ "Programming Language :: Python :: 3.9",+ ],+)