web search done right
update lambdamart
Mikkel Denker16e5953b51b8
diff --git a/configs/api.toml b/configs/api.tomlindex 16116991..90eeb0ff 100644--- a/configs/api.toml+++ b/configs/api.toml@@ -3,9 +3,9 @@ crossencoder_model_path = "data/cross_encoder"gossip_addr = "0.0.0.0:3005"gossip_seed_nodes = ["0.0.0.0:3006", "0.0.0.0:3007"]host = "0.0.0.0:3000"+lambda_model_path = "data/lambdamart.txt"prometheus_host = "0.0.0.0:3001"queries_csv_path = "data/queries_us.csv"-# lambda_model_path = "data/lambdamart.txt"# qa_model_path = "data/qa_model"bangs_path = "data/bangs.json"summarizer_path = "data/summarizer"diff --git a/configs/search_server.toml b/configs/search_server.tomlindex 35c9b30b..aee278bc 100644--- a/configs/search_server.toml+++ b/configs/search_server.toml@@ -1,14 +1,14 @@cluster_id = "dev_search"+entity_index_path = "data/entity"gossip_addr = "0.0.0.0:3006"gossip_seed_nodes = ["0.0.0.0:3005"]-shard_id = 0host = "0.0.0.0:3002"-index_path = "data/index"-entity_index_path = "data/entity"host_centrality_store_path = "data/centrality"+index_path = "data/index"+shard_id = 0# linear_model_path = "data/linear_model.json"# lambda_model_path = "data/lambdamart.txt"[snippet]max_considered_words = 10_000-num_words_for_lang_detection = 1_000\ No newline at end of file+num_words_for_lang_detection = 1_000diff --git a/core/src/entrypoint/configure.rs b/core/src/entrypoint/configure.rsindex 1216f2b2..abf06fc2 100644--- a/core/src/entrypoint/configure.rs+++ b/core/src/entrypoint/configure.rs@@ -43,6 +43,7 @@ fn download_files() {"bangs.json","content.rdf.u8.gz","english-wordnet-2022-subset.ttl",+ "lambdamart.txt",] {let path = Path::new(DATA_PATH).join(name);diff --git a/core/src/query/optic.rs b/core/src/query/optic.rsindex 55738d40..f621040c 100644--- a/core/src/query/optic.rs+++ b/core/src/query/optic.rs@@ -1644,4 +1644,83 @@ mod tests {assert_eq!(res.len(), 1);assert_eq!(res[0].domain, "example.com");}++ #[test]+ fn site_double_anchor() {+ let mut index = Index::temporary().expect("Unable to open index");++ let mut page = Webpage {+ html: Html::parse(+ r#"+ <html>+ <head>+ <title>This is an example site</title>+ </head>+ <body>+ test example+ </body>+ </html>+ "#,+ "https://example.com/test",+ )+ .unwrap(),+ fetch_time_ms: 500,+ ..Default::default()+ };++ page.html.set_clean_text("".to_string());++ index.insert(page).expect("failed to insert webpage");++ let mut page = Webpage {+ html: Html::parse(+ r#"+ <html>+ <head>+ <title>This is another sample website</title>+ </head>+ <body>+ example example example+ </body>+ </html>+ "#,+ "https://another-example.com/",+ )+ .unwrap(),+ fetch_time_ms: 500,+ ..Default::default()+ };++ page.html.set_clean_text("".to_string());++ index.insert(page).expect("failed to insert webpage");+ index.commit().expect("failed to commit index");++ let searcher = LocalSearcher::from(index);++ let res = searcher+ .search(&SearchQuery {+ query: "example".to_string(),+ ..Default::default()+ })+ .unwrap()+ .webpages;+ assert_eq!(res.len(), 2);++ let res = searcher+ .search(&SearchQuery {+ query: "example".to_string(),+ optic: Some(+ Optic::parse(+ "DiscardNonMatching; Rule { Matches { Site(\"|example.com|\") } }",+ )+ .unwrap(),+ ),+ ..Default::default()+ })+ .unwrap()+ .webpages;+ assert_eq!(res.len(), 1);+ assert_eq!(res[0].url, "https://example.com/test");+ }}diff --git a/core/src/query/pattern_query.rs b/core/src/query/pattern_query.rsindex 751b3051..5fcfe368 100644--- a/core/src/query/pattern_query.rs+++ b/core/src/query/pattern_query.rs@@ -191,7 +191,7 @@ fn can_optimize_site_domain(patterns: &[PatternPart], field: Field) -> bool {&& patterns[1..patterns.len() - 1].iter().all(|pattern| matches!(pattern, PatternPart::Raw(_)))- && (matches!(field, Field::Text(TextField::SiteWithout))+ && (matches!(field, Field::Text(TextField::UrlForSiteOperator))|| matches!(field, Field::Text(TextField::Domain)))}diff --git a/core/src/ranking/models/lambdamart.rs b/core/src/ranking/models/lambdamart.rsindex 2a8019ae..c2aeefc2 100644--- a/core/src/ranking/models/lambdamart.rs+++ b/core/src/ranking/models/lambdamart.rs@@ -275,7 +275,6 @@ mod tests {use super::*;#[test]- #[ignore = "train new model with updated signals"]fn simple() {let tree = include_str!("../../../testcases/lambdamart.txt");let tree = Tree::parse(tree).unwrap();@@ -300,6 +299,6 @@ mod tests {features.insert(Signal::HostCentrality, 0.031143193);features.insert(Signal::Bm25StemmedCleanBody, 52.1262779236);- assert_eq!((tree.predict(&features).unwrap() * 1000.0) as u64, 1016);+ assert_eq!((tree.predict(&features).unwrap() * 1000.0) as u64, 1038);}}diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rsindex 0ca2709f..3970de21 100644--- a/core/src/ranking/pipeline.rs+++ b/core/src/ranking/pipeline.rs@@ -349,7 +349,7 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {}}- pub fn ltr_for_query(+ pub fn initial_for_query(query: &mut SearchQuery,model: Option<Arc<LambdaMART>>,collector_config: CollectorConfig,@@ -374,6 +374,10 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {}pub fn apply(self, websites: Vec<T>) -> Vec<T> {+ if websites.len() <= 1 {+ return websites;+ }+self.stage.apply(websites,self.top_n,diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rsindex ea541634..f226cde6 100644--- a/core/src/searcher/local.rs+++ b/core/src/searcher/local.rs@@ -265,7 +265,7 @@ wherede_rank_similar: bool,) -> Result<InvertedIndexResult> {let mut query = query.clone();- let pipeline: RankingPipeline<RankingWebsite> = RankingPipeline::ltr_for_query(+ let pipeline: RankingPipeline<RankingWebsite> = RankingPipeline::initial_for_query(&mut query,self.lambda_model.clone(),self.collector_config.clone(),diff --git a/core/testcases/lambdamart.txt b/core/testcases/lambdamart.txtindex 04bd5a33..bb993695 100644--- a/core/testcases/lambdamart.txt+++ b/core/testcases/lambdamart.txt@@ -3,110 +3,54 @@ version=v3num_class=1num_tree_per_iteration=1label_index=0-max_feature_idx=35+max_feature_idx=28objective=lambdarank-feature_names=tracker_score query_centrality bm25_domain_no_tokenizer update_timestamp inbound_similarity bm25_stemmed_clean_body proximity_slop_8 bm25_title_if_homepage bm25_title bm25_clean_body_trigrams bm25_all_body bm25_clean_body lambda_mart bm25_domain_if_homepage proximity_slop_0 bm25_title_trigrams proximity_slop_4 cross_encoder bm25_description fetch_time_ms bm25_domain bm25_stemmed_title bm25_site_no_tokenizer region bm25 bm25_title_bigrams bm25_clean_body_bigrams bm25_site bm25_domain_name_if_homepage_no_tokenizer bm25_backlink_text bm25_url proximity_slop_2 proximity_slop_1 host_centrality page_centrality is_homepage-feature_infos=[0.012500000000000001:1] none none none none [0:70.5282745361] [0:68.376205444299998] none [0:73.018150329600005] [0:100.9539489746] [0:55.664962768599999] [0:60.530643463099999] none none [0:68.376205444299998] [0:70.553833007799994] [0:68.376205444299998] [9.6108999999999997e-06:0.99996737879999997] [0:78.642982482899995] [0:0.20000000000000001] [0:53.968997955299997] [0:71.416839599599996] none [0:0.72049163900000002] [0:723.68379974369998] [0:90.945877075200002] [0:111.3220901489] [0:52.891826629599997] none none [0:92.092498779300001] [0:68.376205444299998] [0:68.376205444299998] [0:0.034201912000000001] none none-tree_sizes=5468 5457 5457 5464+feature_names=bm25_clean_body_trigrams bm25_domain_name_if_homepage_no_tokenizer inbound_similarity fetch_time_ms bm25_title_bigrams bm25_domain_if_homepage_no_tokenizer is_homepage host_centrality bm25_stemmed_clean_body bm25_domain_if_homepage bm25_title bm25_title_trigrams bm25_domain_name_no_tokenizer tracker_score url_digits bm25_clean_body_bigrams page_centrality bm25_url bm25_domain bm25_site bm25_title_if_homepage url_slashes bm25_clean_body bm25_stemmed_title bm25_backlink_text update_timestamp bm25_site_no_tokenizer region bm25_domain_no_tokenizer+feature_infos=[0:66.744026184082031] none none [0:1] [0:80.374847412109375] none [0:1] [0:0.069144523999999999] [0:74.051651000976562] [0:13.650681495666504] [0:70.605201721191406] [0:42.40814208984375] none [0.018181818181818181:1] [0.041666666666666664:1] [0:85.190345764160156] [0:0.037276589999999998] [0:59.181015014648438] [0:42.313144683837891] [0:39.355148315429688] [0:28.333169937133789] [0.16666666666666666:0.5] [0:56.155899047851562] [0:75.045997619628906] [0:91.765708923339844] none none [0:0.70508371084865562] none+tree_sizes=871Tree=0-num_leaves=50+num_leaves=7num_cat=0-split_feature=33 26 33 26 17 11 24 19 21 17 19 0 5 20 0 18 10 25 8 10 10 17 9 26 33 33 0 30 30 18 33 33 33 8 23 10 19 0 26 24 18 11 33 5 19 19 0 0 17-split_gain=44.405 15.4883 10.4047 14.0192 8.21314 6.10102 6.05149 5.02747 4.90787 4.25664 3.71093 3.70335 3.67161 3.47685 7.62133 3.24401 3.54211 3.01026 4.34126 2.68312 2.68164 3.86668 2.62272 5.87262 2.48591 2.84619 4.35285 2.45874 2.39354 1.95176 1.63381 2.53615 1.45497 4.12841 1.34471 1.3104 0.960074 0.782643 2.10217 1.83263 0.708825 0.552426 0.770031 0.592349 1.17153 0.375368 0.365438 0.617484 0.277314-threshold=0.019958450000000006 28.131596565250003 0.010784120500000003 5.0413153171500005 0.0066878085500000012 26.930888175950006 116.52628231050001 0.0021141743500000007 32.370494842550009 0.039981298100000007 0.0056026365500000007 0.025320512800000004 42.425361633300007 25.618347167950002 0.18333333335000004 21.921377182000004 24.011768341100005 27.740101814250004 42.862270355200003 23.333234787000006 13.504986286150002 1.4367750000000001e-05 5.912924051300001 60.386631011950008 0.022674138000000003 0.021341419500000004 0.095454545450000025 25.740841865550003 1.0000000180025095e-35 23.481465339700005 0.023614218000000003 0.021798707500000004 0.016466209000000006 32.878534317050004 0.41275599895000009 23.831994056700001 0.012270400500000002 0.11805555555000001 60.787229537950004 579.16949462890011 24.733672142050004 30.347360610950002 0.013626538000000002 36.47881507875001 0.0029282639500000005 0.0011933599000000003 0.057189542500000003 0.040833333350000008 5.2856600000000005e-05-decision_type=2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2-left_child=2 6 4 5 11 8 30 -6 15 41 22 12 -1 24 -15 16 19 36 -19 20 21 -4 32 -24 25 26 -8 28 29 -26 31 -2 -5 -34 -23 -10 46 48 -39 -40 -17 -7 -43 -44 -45 -22 47 -13 -3-right_child=1 37 3 10 7 9 13 -9 35 -11 -12 17 -14 14 -16 40 -18 18 -20 -21 45 34 23 -25 27 -27 -28 -29 -30 -31 -32 -33 33 -35 -36 -37 -38 38 39 -41 -42 42 43 44 -46 -47 -48 -49 -50-leaf_value=0.0099999999999999378 -0.0091199608595756536 0.0062311489026760933 -0.0099999999999999083 0.01 0.009999999999999995 -0.0097878860439067922 -0.0085942871773466191 -0.0049836662301414826 0.0021769301778822117 0.0060698272247080541 -0.0081684232610071076 -0.010000000000000033 -0.0099999999999999742 -0.0099999999999999898 0.0099999999999999881 0.0099999999999999829 -0.0095811419893281225 0.010000000000000002 -0.0079474343913487171 0.00741525494488575 -0.0042878470540396816 -0.0014655660018141363 -0.0089738918116935434 0.0099999999999999881 0.0099999999999999898 -0.0099999999999999829 0.0099999999999999898 0.0099999999999999967 -0.0034467222464834037 -0.0003486167293733067 -0.0092395397883890465 0.0038055097313788372 -0.0065041014766525898 0.0099999999999999933 0.0070743822752000595 0.0099999999999999915 -0.0029580159551886899 0.0099999999999999985 -0.0062884611046340137 0.006733942969711396 0.0013650842744086682 -0.00063389996858715812 -0.0092480401123134128 0.0008833208489539662 -0.00843147895942843 -0.0098649380719365888 -0.0097211758890289497 -0.001785359774494242 0.01-leaf_weight=0.56641171313822525 0.65762367844581893 0.51561201317235728 0.71878887712956041 3.0031022378243497 1.1422825641930097 2.6949030765099442 0.47946248389780322 1.097903039306402 1.1777106970548632 0.58112015575170506 0.49766855500638474 0.53010737895965299 0.38576405495405197 1.2887303456664088 0.75562143325805653 0.49655948393046845 2.000880990177393 0.52365588769316562 0.94496165215969086 0.69511906802654255 0.40952638909220684 0.77647283300757419 0.81176451593637444 0.81951756402850151 1.2948529701679947 0.5298236459493636 0.91614221036434174 2.5986865982413292 0.90399648249149311 0.70297556370496739 1.310375604778528 0.89738045260310173 0.58617093041539181 1.0716468780301509 1.1343857888132329 0.98131821304559697 0.5984951891005047 1.9877550266683095 0.45562868891283848 0.66372755728662014 0.45583617314696312 0.44086324051022518 2.2359819705598061 0.51910728774964787 0.96516245789825916 1.1459277309477327 10.490613478934391 0.40243090211879462 9.141548893763682-leaf_count=24 22 24 18 42 14 60 16 16 16 10 16 12 6 32 6 8 40 16 14 8 20 8 16 30 20 8 32 40 14 12 32 14 26 22 24 12 8 18 24 12 12 14 56 16 20 38 316 14 62-internal_value=0 0.00523533 -0.0032127 -0.00111974 -0.00625122 -0.00349404 0.00124 0.00265658 -0.000941973 -0.00692299 0.0049751 -0.00763292 0.00189721 0.00316632 -0.00260772 -0.0027816 -0.00397865 -0.00830558 -0.00154803 -0.00168164 -0.00319258 -0.000114381 0.00601466 0.000558143 0.0047559 -0.000133712 0.0036119 0.00646749 0.003304 0.00635863 -0.00512664 -0.00166078 0.0079244 0.00416448 0.00360419 0.00573266 -0.00913111 0.0090965 0.00691377 0.00143323 0.00586715 -0.00802427 -0.00688208 -0.00762251 -0.00517373 -0.00839658 -0.00945454 -0.00645502 0.00979877-internal_weight=0 25.0999 40.9022 24.2195 16.6826 17.4297 12.3357 2.24019 9.99253 7.43714 6.78987 14.4424 0.952176 9.47029 2.04435 7.8335 6.8811 13.4903 1.46862 4.88022 4.1851 2.62965 6.2922 1.63128 7.42594 1.92543 1.3956 5.50051 2.90183 1.99783 2.86538 1.555 4.66092 1.65782 1.91086 2.15903 12.0216 12.7643 3.10711 1.11936 0.952396 6.85602 4.16111 3.72025 1.48427 1.55545 11.4232 0.932538 9.65716-internal_count=1360 388 972 532 440 380 248 30 204 176 152 410 30 180 38 176 156 380 30 116 108 50 136 46 142 56 48 86 46 32 68 36 90 48 32 28 350 140 54 36 20 166 106 92 36 58 342 26 86+split_feature=3 3 24 22 7 17+split_gain=0.123967 0.131918 0.158743 0.1777 0.116193 0.10828+threshold=0.018019480519480522 0.0058651530787753705 23.04527568817139 18.93778228759766 0.027130383000000004 11.632615089416506+decision_type=2 2 2 2 2 2+left_child=1 2 -1 -4 -5 -3+right_child=-2 5 3 4 -6 -7+leaf_value=0.010534121130993713 -0.014170690004583389 0.028615132328030597 -0.028104839097028086 0.0091416979972488207 -0.013600909116628418 0.0030381471291287114+leaf_weight=1.7825879901647566 1.2892666812986133 0.77476169914007165 0.82290455326437983 1.2607608251273634 1.0127553343772886 0.88816887885332108+leaf_count=33 37 15 18 37 30 27+internal_value=0 0.00279272 -0.00135243 -0.00819543 -0.000989172 0.0149545+internal_weight=0 6.54194 4.87901 3.09642 2.27352 1.66293+internal_count=197 160 118 85 67 42is_linear=0-shrinkage=0.005---Tree=1-num_leaves=50-num_cat=0-split_feature=33 8 33 26 5 21 18 24 17 19 26 17 19 33 0 5 11 25 8 9 26 10 11 24 17 10 24 0 5 33 8 17 8 0 11 19 5 8 19 11 23 33 33 17 33 0 0 8 18-split_gain=152.25 59.9795 44.9635 69.9036 34.8326 25.4545 25.4404 23.1446 16.9731 17.7931 16.0731 24.166 15.7759 15.4379 13.7787 14.686 13.1813 13.1641 14.2814 9.50549 22.6834 8.11811 14.5284 9.68489 7.75809 11.1195 7.47554 6.31758 10.3168 6.20151 16.6614 5.36193 4.99983 12.7044 4.33421 3.4442 3.9843 2.99276 2.83137 2.73824 4.71467 2.31028 2.77357 2.23162 1.89009 1.08313 2.676 1.07044 0.941669-threshold=0.019958450000000006 28.918019294700006 0.010153455000000002 1.5334161222000002 24.617109298700004 28.223515510550005 24.378225326500004 199.75513839720003 0.0066878085500000012 0.0021141743500000007 9.4430942535500026 0.0062548280500000003 0.0056026365500000007 0.015223103000000002 0.025320512800000004 42.425361633300007 26.930888175950006 27.740101814250004 42.862270355200003 5.912924051300001 60.386631011950008 24.193528175350004 20.459006309500001 78.93672296405002 0.00091614705000000009 24.557957649250003 124.34751701355002 0.22500000000000003 40.209836959850008 0.016466209000000006 32.878534317050004 5.7137000000000005e-05 24.108223915100002 0.087121212100000026 31.839186668400004 0.0021276981000000004 35.284496307350004 34.230171203600008 0.0014903262000000002 9.4340581894000017 0.41275599895000009 0.018645591500000006 0.014970756000000002 1.8400600000000005e-05 0.019852502000000005 0.057189542500000003 0.040833333350000008 25.625818252600002 44.153718948350004-decision_type=2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2-left_child=2 4 8 5 39 21 13 26 14 -10 11 16 19 37 15 -1 -3 45 -19 29 -21 22 41 -24 25 -15 31 -12 -29 -5 -31 -6 -28 -34 -27 -23 -37 -7 -25 40 -2 42 43 -4 -43 46 -16 -9 -18-right_child=1 10 3 12 7 6 -8 47 9 -11 27 -13 -14 24 17 -17 48 18 -20 20 -22 35 23 38 -26 34 32 28 -30 30 -32 -33 33 -35 -36 36 -38 -39 -40 -41 -42 44 -44 -45 -46 -47 -48 -49 -50-leaf_value=0.0099130311339063592 -0.009011865398763445 0.0056064980282179043 -0.0084946482773895788 0.0099062754703599393 -0.0066850502010743938 -0.0099332161041331638 -0.0088389029942627182 -0.004842649047076205 0.0099154155627031969 -0.007075919065934487 0.009742235542027541 0.0099057698944577197 -0.0085715932898408724 0.0078538848264257909 -0.0099139135924102163 -0.0099056211160224433 -0.0097398627305224309 0.0099056360027764811 -0.0068339042579592026 -0.0080001880272964603 0.0099046798605374863 -0.0093351332870926299 0.006779023677050881 0.00081884697706272839 0.0086611579307278135 -0.0099030809585950522 0.0099083742931500782 0.009935220996645483 -0.0064260958179860866 -0.0063336306371194785 0.0099052286125730759 0.005698006919842933 0.0099193622947462458 -0.004526655369067276 0.0019265029011451076 -0.0081735609371648074 -0.00067735680110681223 -0.00036224348856018384 -0.0095501624423914994 -0.0098304313006275755 0.0015933919902353011 0.0016324074209368353 -0.0099291147856710223 1.719381089374356e-05 -0.0076890664940039788 -0.0093019953939002843 -0.0014980253491691522 -0.0099576014002062795 -0.0045846494714091917-leaf_weight=2.2552217170596118 2.5878867954015732 2.6657359898090345 1.5182959735393544 14.155592277646074 1.8700705468654697 2.1442497372627325 5.7631907761096945 1.9339653477072705 3.0606446154415599 3.1027094051241875 44.616727475076914 6.7855033203959465 1.8968457430601109 3.2769667953252783 2.3642754554748531 1.5964577347040174 3.3677338659763336 1.9774547666311253 3.5825450420379639 3.3989775478839865 3.6884894529357553 16.415852703154091 4.2500208020210257 1.1155177652835848 8.2526729777455312 1.9529078155755999 7.8693827874958497 2.3271645009517665 1.6442525088787077 2.4532973393797866 4.4352577570825815 1.6415552049875257 3.666589193046093 2.6019746363162994 1.2829959169030187 4.5192292947322121 2.9165666252374649 1.3193091787397859 1.6064561903476713 6.5796297639608383 1.7611531913280467 1.2492604553699487 6.5586289912462234 1.5624984875321386 0.96303997933864582 39.427225884050131 1.5729702226817606 2.1712007597088814 1.2019770853221414-leaf_count=22 10 10 10 50 18 16 44 16 14 16 112 28 16 14 12 6 36 16 14 16 30 88 8 6 22 16 28 8 16 26 22 4 14 12 8 28 20 10 18 44 14 20 44 10 4 312 14 8 10-internal_value=0 0.00497737 -0.00304754 -0.000949354 -0.000512942 -0.0038025 0.000317978 0.00314559 -0.00648982 0.00136177 0.00784344 0.0031274 0.00538506 0.00321296 -0.00740676 0.00169853 -0.00322944 -0.00812359 -0.000880364 0.00632612 0.00131794 -0.00611905 -0.00366426 0.0020629 0.00544149 0.00136174 0.00563282 0.00920433 0.00316128 0.00801283 0.0041219 -0.000896429 0.00725457 0.00392307 -0.0052128 -0.00805638 -0.0052333 -0.00628752 -0.00530074 -0.00779565 -0.00471724 -0.00703336 -0.00809093 -0.00417766 -0.00242534 -0.00905228 -0.00655168 -0.00754792 -0.00838388-internal_weight=0 95.2925 155.636 96.6961 32.6834 66.6677 23.9923 21.7547 58.9395 6.16335 62.6091 14.021 30.0285 18.2291 52.7762 3.85168 7.23545 48.9245 5.56 28.1316 7.08747 42.6754 18.8237 6.97199 14.7655 6.51287 17.6496 48.5881 3.97142 21.0441 6.88856 3.51163 14.1379 6.26856 3.2359 23.8516 7.4358 3.46356 2.72197 10.9287 4.34904 11.8517 9.63942 3.08079 2.2123 43.3645 3.93725 4.10517 4.56971-internal_count=1360 388 972 546 168 386 130 100 426 30 220 84 160 86 396 28 56 368 30 144 46 256 120 32 60 38 76 136 24 98 48 22 54 26 24 136 48 26 24 68 24 88 64 20 24 338 26 24 46-is_linear=0-shrinkage=0.005---Tree=2-num_leaves=50-num_cat=0-split_feature=33 8 17 11 0 26 26 17 21 17 10 19 21 33 30 5 24 19 11 5 26 15 19 33 5 8 30 21 24 10 0 26 0 10 26 10 17 8 17 10 8 17 11 14 18 8 26 10 18-split_gain=123.56 48.7975 30.9068 24.6786 23.9449 34.8431 19.553 21.9259 27.3465 15.7557 14.4067 13.9334 15.1573 13.9901 26.016 13.4799 14.4816 12.2247 15.5713 12.1234 10.1412 9.37119 8.88257 8.59522 7.47114 7.23078 6.15149 9.43505 6.60996 5.10747 4.91394 4.89723 4.89465 4.70571 4.69164 4.48818 3.50872 2.9719 2.76934 5.14391 2.34614 2.08664 2.45537 1.89102 1.88417 1.59661 5.2212 1.90012 1.57823-threshold=0.019958450000000006 27.158587455750006 0.00033038530000000004 27.547188758850002 0.057189542500000003 66.052501678450014 7.1899642944500011 0.0062548280500000003 26.600150108350004 3.4804450000000005e-05 24.557957649250003 0.0056026365500000007 24.183298110950002 0.015295206000000002 34.031578064000009 30.177063941950006 180.53593158720003 0.0029542162000000005 20.176387786900005 19.430234909050004 5.5020630360000009 1.0000000180025095e-35 0.0010005005000000003 0.012058684500000002 20.663570404050002 39.69972801210001 28.999861717200002 44.565795898400005 95.25605201720002 1.0000000180025095e-35 0.15476190480000004 60.787229537950004 0.10555555555000003 35.239673614500006 8.0346117019500021 35.187662124650011 1.0899200000000001e-05 24.846534729000002 0.0076535505000000009 28.534720420800003 35.033393859850008 1.7003300000000004e-05 45.418594360350006 61.749555587750002 45.236038208000004 17.976120948750005 1.0000000180025095e-35 36.921825408950006 21.802299499550006-decision_type=2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2-left_child=2 3 4 37 9 10 7 8 -3 19 17 12 20 14 34 -5 32 18 29 -1 -4 36 24 -7 -20 -11 28 -28 -21 -6 -8 -32 -17 -33 -14 -13 40 -2 -15 -40 -12 44 45 -35 -38 46 -43 -48 -47-right_child=1 6 11 15 5 23 30 -9 -10 25 21 35 13 38 -16 16 -18 -19 22 26 -22 -23 -24 -25 -26 -27 27 -29 -30 -31 31 33 -34 43 -36 -37 41 -39 39 -41 -42 42 -44 -45 -46 48 47 -49 -50-leaf_value=-0.0098162312194386963 -0.007987589736032287 0.0098682937561520873 -0.0090556226748188685 0.0088065372491295344 0.0013655468532508131 -0.0041892132805463854 0.0098295506853026491 0.0098159723809600873 -0.0078885976861671862 -0.0098236432304169251 -0.0075042214938492524 -0.00037557177776654594 0.0017275916648638301 0.0098526870661669857 -0.0098430945075329863 -0.00013864876746703706 -0.0086106537390147984 -0.0083432055572098297 0.0057623928505988022 0.0010112247913873827 0.0033296840743482288 0.0038699212576462157 0.0083760976732490233 0.0098136184012235905 -0.0073820771174936467 0.0016618625640628904 -0.006492739460603472 0.009854076780269767 0.0098359088343056809 -0.0084536593322206786 0.0098170595323266042 -0.0066374597648179469 0.0098150053284256321 0.00083393074826519924 0.0098360701093015007 -0.0098542990949112771 -0.0096628621972514265 -0.0009219703639646987 0.00034700986553109218 0.0098370951485808999 0.00080657906025458385 -0.0079606000891040592 -0.0021429397028506795 0.0098647420344281662 -0.0040736334652674757 -0.0093657853358157881 0.0098905470837741712 -0.0026733236165661944 -0.0047255320161123538-leaf_weight=1.575511679053309 8.8357827514410037 3.2305430471897143 4.1529105007648495 6.2050460763275614 1.74238812923432 1.4567152857780445 35.826935090124607 6.4490259215235701 6.5938016995787621 4.2532508820295325 1.6178893595933908 1.9894462227821339 3.4240810871124285 7.2800140082836133 4.1033293902873993 2.5757095664739591 3.5400923937559128 7.455853819847106 1.965584009885786 3.1460566669702512 2.745403990149498 1.7009713947772969 4.823063800111413 4.4242249801754951 2.4022134616971016 2.0216841697692871 2.4229104295372959 1.3886024653911588 6.518871171399951 5.5193735957145691 3.0224805120378724 1.2530913427472117 2.3729385174810886 1.1494650691747665 3.7243828773498535 3.3547490686178207 25.345614746212963 1.7896835915744294 2.7908682078123093 2.9237627014517784 1.7873198762536047 4.809075675904757 2.1141694486141196 1.169391542673111 1.6032203286886204 12.545615587383507 0.60904508084058739 0.59486456215381622 2.1458504982292643-leaf_count=20 64 10 22 18 12 12 98 28 60 24 12 12 16 28 24 14 26 76 6 18 12 4 36 28 26 6 22 6 38 62 12 8 14 16 10 18 194 14 12 10 8 36 18 6 8 76 12 12 36-internal_value=0 0.0048093 -0.00283585 -0.000992561 -0.00420222 -0.005395 0.00731205 0.00265265 -0.00204959 0.000532859 -0.00627138 0.00113308 0.00241295 0.00427348 0.000191902 0.00320515 -0.000889322 -0.00303213 -0.00062531 0.00330767 -0.0041265 -0.00768272 0.00369841 0.0063451 -0.00146684 -0.0061232 0.00484197 -0.000537301 0.00696336 -0.00609764 0.00909946 0.00513295 0.00463425 0.00116938 0.00595215 -0.00632572 -0.00805229 -0.0067975 0.00780764 0.0052024 -0.00314206 -0.00838825 -0.00727564 0.00538814 -0.00933035 -0.00779974 -0.00562942 0.0036826 -0.00868803-internal_weight=0 84.014 142.479 25.3193 105.99 84.6631 58.6947 16.2734 9.82434 21.3269 78.7821 36.4889 31.1448 24.2464 11.2518 14.6938 8.48874 23.9085 16.4526 15.052 6.89831 54.8736 9.19086 5.88094 4.3678 6.27494 13.4764 3.81151 9.66493 7.26176 42.4214 6.59443 4.94865 3.57195 7.14846 5.3442 53.1727 10.6255 12.9946 5.71463 3.40521 49.7675 22.8186 2.31886 26.9488 20.7045 6.01299 1.20391 14.6915-internal_count=1360 388 972 150 808 674 238 98 70 134 634 164 134 100 50 72 54 218 142 104 34 416 68 40 32 30 84 28 56 74 140 42 28 30 26 30 412 78 50 22 20 392 190 22 202 172 60 24 112-is_linear=0-shrinkage=0.005---Tree=3-num_leaves=50-num_cat=0-split_feature=33 8 17 0 26 17 11 26 17 21 11 11 19 8 33 30 26 10 0 5 19 5 10 30 21 24 10 0 26 5 20 11 23 24 18 5 17 17 26 33 33 17 17 19 30 30 10 33 5-split_gain=122.03 47.1967 30.9872 19.548 30.0652 20.2562 18.4669 17.3936 19.2893 24.2543 15.4895 15.1185 19.8703 13.5899 22.0719 26.5849 15.3893 13.1669 14.5908 11.5188 9.76489 9.07555 6.71338 5.67539 7.46293 6.0919 4.57457 4.42589 4.52685 3.66486 3.38861 3.30437 4.90931 2.79385 2.05371 1.83061 1.58896 1.35776 1.25748 1.23288 7.54702 1.84498 5.5099 1.42987 1.17893 4.5426 1.0637 2.91643 0.999161-threshold=0.019958450000000006 27.158587455750006 0.00033038530000000004 0.057189542500000003 66.052501678450014 3.4804450000000005e-05 27.547188758850002 7.1899642944500011 0.0062548280500000003 26.600150108350004 27.467996597300004 20.176387786900005 0.0029542162000000005 27.462724685700007 0.015295206000000002 32.923410415650004 5.5020630360000009 27.969018936150004 0.10555555555000003 19.430234909050004 0.0010005005000000003 54.445077896150003 24.193528175350004 28.999861717200002 44.565795898400005 95.25605201720002 1.0000000180025095e-35 0.15476190480000004 60.787229537950004 40.209836959850008 43.487449645950008 9.4340581894000017 0.41275599895000009 182.36229705810004 20.416842460650006 25.280503273000004 1.0837500000000001e-05 2.3751900000000002e-05 27.136941909800004 0.010422712000000002 0.012167945000000001 1.9047900000000001e-05 2.5816900000000003e-05 0.012270400500000002 34.031578064000009 28.621761322050002 31.730455398550003 0.023952123000000002 28.991178512600005-decision_type=2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2-left_child=2 6 3 5 10 19 31 8 9 -3 11 26 20 16 15 38 -4 -8 37 -1 22 -6 -13 25 -25 -21 -5 -9 -29 -30 44 32 -2 -20 -28 -33 -12 -19 -15 43 -41 -42 -43 -38 45 -16 -11 -48 -44-right_child=1 7 13 4 21 -7 17 27 -10 46 36 12 -14 14 30 -17 -18 18 33 23 -22 -23 -24 24 -26 -27 34 28 29 -31 -32 35 -34 -35 -36 -37 39 -39 -40 40 41 42 48 -45 -46 -47 47 -49 -50-leaf_value=-0.009735484290709152 -0.0077997160604779993 0.0097892928315321519 -0.0090402090501182069 -6.7551674772469675e-06 0.0097686797150063863 -0.0084421212177286219 0.0097927814967422658 0.0097493300365762772 0.0097319240613288425 -0.009753310160297372 -0.0041787010613604869 0.0040024288708004367 -0.0094732796066520891 0.0042210158497068585 0.0097963418389761386 -0.0097600848441952732 0.0035022281440736311 -0.0090559226084526278 0.0097308144815816625 0.001034029365984943 0.0097890683830102482 -0.0056743049826746836 -0.0097669574812041318 -0.0058205312931484676 0.0097589541384304878 0.0097525410075975632 -0.0085788848013285644 0.0097369436335925184 0.0054683204088723505 -0.0052864222235926277 0.0013642457935270772 -0.0095848035561727704 0.0038651530812490106 0.0017788844547361432 -0.002146799162852557 -0.0027439546410395104 -0.0094365916208346053 -0.0030757633619894783 0.0097818753086814855 0.004305457652136766 -0.0094896984485105512 0.0028567343672873937 -0.0048401838041101774 -0.0048635013387279338 0.0097452906425091849 0.001352937347753492 0.0013759221098089832 -0.0097961901173875838 -0.0093999651687309334-leaf_weight=1.528825238347056 1.9284897595643997 2.8999190777540189 5.8166402131319028 2.180921062827113 3.9779065251350394 5.0623577311635009 4.4770334362983695 35.064194981008768 5.8378064930438986 3.3479171991348267 2.4687739424407473 2.454122960567473 3.3166270405054084 2.0150853358209151 4.2708440870046598 5.6151065528392792 4.2199865188449621 3.0195221751928312 2.170052669942379 2.987590476870535 4.6908980458974838 1.2504300549626348 1.3846900165081022 2.3127136006951328 1.1513491310179231 6.0832049399614334 11.054187595844267 2.8841942548751822 1.9780068797990682 1.3212530016899107 2.2421094905585042 5.2508369833230972 1.694563493132593 2.249711811542511 1.39795622229576 1.2017739973962305 21.761921420693401 1.3842904716730116 2.0517177432775497 1.2901765368878866 11.20223566144705 1.3514821175485838 1.6040376499295232 1.8549927920103062 5.5904756858944893 2.5405823923647404 0.96053120121359858 1.4907369762659071 4.786165863275528-leaf_count=20 10 10 30 20 26 30 10 98 28 30 18 22 34 14 20 34 16 24 14 18 38 14 16 22 6 38 112 12 14 16 12 44 14 14 14 10 182 10 6 18 86 18 16 10 22 10 12 18 30-internal_value=0 0.00490488 -0.00295229 -0.00439569 -0.00550609 0.000134362 -0.00105916 0.00740408 0.00270113 -0.00201711 -0.00633785 -0.00328746 0.000911542 0.00112863 0.00314837 -0.00270903 -0.00376663 0.00280879 -0.000734844 0.00322155 0.0049495 0.00607527 -0.000964297 0.00480187 -0.000642384 0.00688098 -0.00668681 0.00906154 0.00516134 0.00116137 0.00702099 -0.00616513 -0.00234386 0.00568319 -0.00785678 -0.00831072 -0.00808165 -0.00717612 0.00702649 -0.00830138 -0.00739563 -0.00819253 -0.00631552 -0.0090774 0.00804366 0.00664705 -0.00792097 -0.00541839 -0.00825539-internal_weight=0 79.1608 131.516 97.1536 78.0275 19.126 23.3763 55.7846 14.5369 8.6991 72.7992 26.4794 11.8463 34.3625 24.3259 9.68191 10.0366 13.3006 8.82358 14.0637 8.52971 5.22834 3.83881 12.5349 3.46406 9.0708 14.6331 41.2476 6.18345 3.29926 14.644 10.0757 3.62305 4.41976 12.4521 6.45261 46.3198 4.40381 4.0668 43.851 20.2341 18.9439 7.74169 23.6169 12.4019 6.81143 5.79919 2.45127 6.3902-internal_count=1360 388 972 808 674 134 150 238 98 70 634 256 110 164 118 54 46 72 62 104 76 40 38 84 28 56 146 140 42 30 64 78 24 28 126 54 378 34 20 360 168 150 64 192 52 30 60 30 46-is_linear=0-shrinkage=0.005+shrinkage=0.05end of treesfeature_importances:-host_centrality=23-cross_encoder=22-bm25_clean_body_bigrams=18-tracker_score=17-bm25_all_body=16-bm25_title=15-fetch_time_ms=15-bm25_stemmed_clean_body=14-bm25_clean_body=13-bm25=9-bm25_description=8-bm25_url=8-bm25_stemmed_title=7-region=3-bm25_clean_body_trigrams=2-bm25_domain=2-bm25_title_bigrams=2-proximity_slop_0=1-bm25_title_trigrams=1+bm25_stemmed_title=82+tracker_score=77+fetch_time_ms=75+page_centrality=69+bm25_clean_body=66+host_centrality=62+bm25_backlink_text=47+bm25_url=46+bm25_stemmed_clean_body=34+bm25_title_if_homepage=26+bm25_clean_body_bigrams=14+bm25_title_bigrams=13+bm25_title=12+bm25_site=12+url_slashes=12+bm25_clean_body_trigrams=8+bm25_domain=5+bm25_domain_if_homepage=4+bm25_title_trigrams=4+is_homepage=1parameters:[boosting: gbdt]@@ -117,15 +61,15 @@ parameters:[data: ][valid: ][num_iterations: 100]-[learning_rate: 0.005]-[num_leaves: 50]+[learning_rate: 0.05]+[num_leaves: 20][num_threads: -1][deterministic: 0][force_col_wise: 0][force_row_wise: 0][histogram_pool_size: -1]-[max_depth: -1]-[min_data_in_leaf: 10]+[max_depth: 10]+[min_data_in_leaf: 20][min_sum_hessian_in_leaf: 0.001][bagging_fraction: 1][pos_bagging_fraction: 1]@@ -137,7 +81,7 @@ parameters:[feature_fraction_seed: 2][extra_trees: 0][extra_seed: 6]-[early_stopping_round: 5]+[early_stopping_round: 0][first_metric_only: 0][max_delta_step: 0][lambda_l1: 0]@@ -206,8 +150,8 @@ parameters:[tweedie_variance_power: 1.5][lambdarank_truncation_level: 30][lambdarank_norm: 1]-[label_gain: ]-[eval_at: 1,3,5,10]+[label_gain: 0,1,2,3,4,5,6,7,8,9,10]+[eval_at: 1,2,3,5,10][multi_error_top_k: 1][auc_mu_weights: ][num_machines: 1]diff --git a/ltr/generate_data.py b/ltr/generate_data.pydeleted file mode 100644index 2ff0da53..00000000--- a/ltr/generate_data.py+++ /dev/null@@ -1,285 +0,0 @@-import requests-import random-import json-import pandas as pd-from tqdm import tqdm-import torch-import urllib.request-from rwkv.model import RWKV-from rwkv.utils import PIPELINE, PIPELINE_ARGS-import os---STRACT_ENDPOINT = "localhost:3000"-NUM_QUERIES = 100-TOP_N_RESULTS = 10--CENTRALITY_BOOST_OPTIC = 'Ranking(Signal("host_centrality"), 100000000);'--rwkv_model = "https://huggingface.co/BlinkDL/rwkv-4-raven/resolve/main/RWKV-4-Raven-7B-v9-Eng99%25-Other1%25-20230412-ctx8192.pth"-# download model-if not os.path.exists("data/rwkv_model.pth"):- print("Downloading model")- urllib.request.urlretrieve(rwkv_model, "data/rwkv_model.pth")--if not os.path.exists("data/rwkv_tokenizer.json"):- urllib.request.urlretrieve(- "https://raw.githubusercontent.com/BlinkDL/ChatRWKV/main/v2/20B_tokenizer.json",- "data/rwkv_tokenizer.json",- )--torch.set_num_threads(12)-model = RWKV(- model="data/rwkv_model.pth",- strategy="cpu fp32",-)-pipeline = PIPELINE(model, "data/rwkv_tokenizer.json")-print("model ready")---def search(q: str, top_n, optic=None):- if q == "":- return []-- url = f"http://{STRACT_ENDPOINT}/beta/api/search"- r = requests.post(- url,- json={- "query": q,- "page": 0,- "numResults": top_n,- "returnRankingSignals": True,- "optic": optic,- },- )- search_result = r.json()["webpages"]- res = []-- for r in search_result:- if "Normal" not in r["snippet"]:- continue-- snip = r["snippet"]["Normal"]["text"]- snip = snip.replace("<b>", "").replace("</b>", "")- res.append(- {- "domain": r["domain"],- "title": r["title"],- "url": r["url"],- "rankingSignals": r["rankingSignals"],- "snippet": snip,- "body": r["body"],- }- )-- return res---def score_prompt(query: str, res):- r = {k: v for (k, v) in res.items() if k in ["domain", "title", "snippet"]}- # r['domain_score'] = '{:f}'.format(- # res['rankingSignals']['host_centrality'])- encoded = json.dumps(r)-- inst = "You are a search engine evaluator. Evaluate the following result based on the query. A good search result is relevant to the query and comes from a trustworthy domain. The score for the search result should be between 0.0 and 1.0."- return f"""Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.--# Instruction:-{inst}--query: england-result: {{domain: "theguardian.com", title: "Hungary fans fight with police inside Wembley at start of England match | World Cup 2022 qualifiers | The Guardian",- snippet: "Tyrone Mings said that England's players were not fazed by Hungary's fans booing the knee. “We've faced criticism for taking the knee and we have collectively stood passionately together,” the <b>England</b> defender said."}}--# Response:-score: 0.8<|endoftext|>--# Instruction:-{inst}--query: vim plugins-result: {{domain: "revelry.co", title: "My VIM Setup - Coding Creativity At Revelry",- snippet: "Here are the plugins in my Vim setup that I really like: sessionman. im is a Vim session manager, meaning it will save your open buffers to a file for easy re-opening. It's useful in cases when you've been working on a ticket and you need to have one set of files open and then need to switch to another ticket that requires"}}--# Response:-score: 0.2<|endoftext|>--# Instruction:-{inst}--query: q learning algorithm-result: {{domain: "datascience.stackexchange.com", title: "Q-learning why do we subtract the Q(s, a) term during update? - Data Science Stack Exchange",- snippet: "The q-learning algorithm is an off-policy algorithm , unlike SARSA . The Bellman equation describes q-learning as follows: "The q value for action $a$ taken in state $s$ at time $t$ becomes equal to: that same q-value plus small amount of: currently"}}--# Response:-score: 1.0<|endoftext|>--# Instruction:-{inst}-query: mutton curry variations-result: {{domain: "earthspice.in", title: "NADAN KOZHI CHICKEN CURRY - EarthSpice",- snippet: "VARIATIONS AND SUBSTITUTES You can make this dish without coconut milk for a thinner gravy and different taste. Roasted chicken can also be used Chicken stock can be added along with the coconut milk Earthspice Organic Spice Provide Company in all India, Likes, Garam Masala, Tea Masala, Batthi Masala and Etc."}}--# Response:-score: 0.0<|endoftext|>--# Instruction:-{inst}--query: {query}-result: {encoded}--# Response:-score:"""---def run_model(prompt: str) -> str:- print("running model")- print(prompt)-- all_tokens = []- occurrence = {}- state = None- max_token_count = 10- args = PIPELINE_ARGS(- temperature=0.2,- top_p=0.0,- alpha_frequency=0.4,- alpha_presence=0.4,- token_ban=[], # ban the generation of some tokens- token_stop=[0],- ) # stop generation whenever you see any token here-- for i in range(max_token_count):- out, state = model.forward(- pipeline.encode(prompt) if i == 0 else [token], state- )- for n in occurrence:- out[n] -= args.alpha_presence + occurrence[n] * args.alpha_frequency-- token = pipeline.sample_logits(- out, temperature=args.temperature, top_p=args.top_p- )- if token in args.token_stop:- break- all_tokens += [token]- if token not in occurrence:- occurrence[token] = 1- else:- occurrence[token] += 1-- output = pipeline.decode(all_tokens).split("\n")[0].strip()- print("START OF OUTPUT")- print(output)- print("END OF OUTPUT")- return output---def score(query, results):- res = [None for _ in range(len(results))]- for i, r in enumerate(results):- p = score_prompt(query, r)- res[i] = float(run_model(p))-- return res---def query_prompt(res):- r = {k: v for (k, v) in res.items() if k in ["domain", "title", "body"]}- r["body"] = " ".join(r["body"].split(" ")[:200])[:1000]-- inst = "Generate a keyword based search query such that the following result would be considered a good result. The query should be at most 4 keywords long. You should only output a single query and nothing else"-- return f"""Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.--# Instruction:-{inst}--domain: wikipedia.org-body: On this Wikipedia the language links are at the top of the page across from the article title. Go to top . Map of Colts Neck Township in Monmouth County. Inset: Location of Monmouth County highlighted in the State of New Jersey. Census Bureau map of Colts Neck Township, New Jersey The township has been ranked as one of the state's highest-income communities. Based on data from the American Community Survey for 2013–2017, Colts Neck residents had a median household income of $167,480, ranked fifth in the state among municipalities with more than 10,000 residents, more than double the statewide median of $76,475.[24] [25] The township has a Farmland Preservation Committee which to date has preserved nearly 1,000 acres (400 ha) of land, providing one way in which Colts Neck has been able to prevent large-scale development. The township has strict zoning regulations, and because there is no public water or sewage service, most homes must be built on lots covering a minimum of 2, 5 and 10 a--# Response:-query:"colts neck wikipedia"<|endoftext|>--# Instruction:-{inst}--domain: slideshare.net-body: 1. - C herkasy region is not only the geographical cent- er of Ukraine; it is its spiritual center: rich nature, generous fertile black soils, unique historical and cultural heritage. In Cherkasy region, there were formed the first Cossack republic and Ukrainian statehood. Its first hetman capital was here, in Chyhyryn, during the times of our great countryman Bohdan Khmelnytsky. Many European countries’ ambassadors were given a reception here. The Ukrainian Cossack phenomenon be- gan and the Cossack rebellions and XVII century national liberation revolution started on these lands. Cherkasy region is a place of a great deed of the na- tion during the Second World War. Hundreds of monu- ments and memorial steles are on the paces of Korsun- Shevchenkivsky battle, Uman-Batoshanska military operation, and the battle for the Dnipro River. DuringtheKyivRustimesitsSouthernborderswerepro- tected by city-fortresses of poroska, posulska, and posupi- yska defense lines: Korsun, Zheln, Rymiv, Piso--# Response:-query:"cherkasy ukraine"<|endoftext|>--# Instruction:-{inst}--domain: {r['domain']}-body: {r['body']}--# Response:-query:"""---def good_query(res):- p = query_prompt(res)- return run_model(p).replace('"', "").lower().split(",")[0]---queries = requests.get(- "http://s3.trystract.com/public/queries_us_big.csv"-).text.splitlines()-random.shuffle(queries)--ranking_signals = {}-scores = {"query": [], "url": [], "score": []}--queries_taken = 0--with tqdm(total=NUM_QUERIES) as pbar:- for query in queries:- if queries_taken >= NUM_QUERIES:- break-- res = search(query, 1, optic=CENTRALITY_BOOST_OPTIC)-- if len(res) < 1:- continue-- query = good_query(res[0])-- res = search(query, TOP_N_RESULTS)- if len(res) < TOP_N_RESULTS:- continue-- try:- for i, s in enumerate(score(query, res)):- if s is None:- continue-- url = res[i]["url"]- signals = res[i]["rankingSignals"]-- scores["query"].append(query)- scores["url"].append(url)- scores["score"].append(s)- ranking_signals[url] = signals- except Exception as e:- print("Error", e)- continue-- queries_taken += 1- pbar.update(1)--df = pd.DataFrame(scores)-df = df.groupby(["query", "url"])-df = df.mean().sort_values(by="score", ascending=False).reset_index()-df["rank"] = df.groupby("query")["score"].rank(ascending=False)-df["rankingSignals"] = df["url"].map(ranking_signals)--# convert to object-res = json.loads(df.to_json(orient="records"))---# save res in ltr_scores.json prettified-with open("ltr_scores.json", "w") as f:- json.dump(res, f, indent=2)diff --git a/ltr/lambdamart.py b/ltr/lambdamart.pyindex aec2ec3f..9f727c87 100644--- a/ltr/lambdamart.py+++ b/ltr/lambdamart.py@@ -2,24 +2,70 @@ import lightgbm as lgbimport numpy as npimport jsonfrom pprint import pprint+import sqlite3+import sys-with open("ltr_scores.json") as f:- scores = json.load(f)+con = sqlite3.connect("data/ranking-annotation.sqlite")+cur = con.cursor()-feature2id = {k: i for i, (k, _) in enumerate(- scores[0]['ranking_signals'].items())}-id2feature = {v: k for k, v in feature2id.items()}+res = cur.execute(+ """+ SELECT qid, query+ FROM queries+ WHERE EXISTS (+ SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.annotated_rank IS NOT NULL+ )+"""+)+queries = {qid: {"query": query} for qid, query in res.fetchall()}++for qid in queries:+ res = cur.execute(+ """+ SELECT qid, url, annotated_rank, webpage_json+ FROM search_results+ WHERE qid = ?+ """,+ (qid,),+ )+ urls = {+ url: {"rank": rank, "signals": json.loads(page)["rankingSignals"]}+ for _, url, rank, page in res.fetchall()+ }+ urls = [+ (url, w["rank"], w["signals"])+ for url, w in urls.items()+ if w["rank"] is not None+ ]++ urls = sorted(urls, key=lambda x: x[1], reverse=False)+ urls = [(url, i, signals) for (url, _, signals), i in zip(urls, range(len(urls)))]+ queries[qid]["urls"] = urls+++feature2id = {}+id2feature = {}-queries = {} # query -> {url: {score, features}-for item in scores:- query = item["query"]- if query not in queries:- queries[query] = {}- masked_features = {k: v for k, v in item["ranking_signals"].items() if k not in set([- "CrossEncoder"])}- queries[query][item["url"]] = {- "score": item["score"], "features": item["ranking_signals"], "masked_features": masked_features}+for qid, data in queries.items():+ for url, score, signals in data["urls"]:+ for feature, value in signals.items():+ if feature not in feature2id:+ id = len(feature2id)+ feature2id[feature] = id+ id2feature[id] = feature+# convert to qid -> {url: {score, features}+new_queries = {}++for qid, data in queries.items():+ new_queries[qid] = {}+ for url, score, signals in data["urls"]:+ new_queries[qid][url] = {+ "score": score,+ "features": {feature2id[k]: v for k, v in signals.items()},+ }++queries = new_queries# Create datasetitems = list(queries.items())@@ -28,30 +74,18 @@ X_train = []y_train = []q_train = []for query, urls in items[:train_size]:- # we have 2 instances per query- # one normal, one masked- q_train.append(query)q_train.append(query)for url, data in urls.items():- X_train.append([data["features"][k] for k in feature2id])- y_train.append(data["score"])- X_train.append([data["masked_features"].get(k, 0.0)- for k in feature2id])+ X_train.append([data["features"].get(k, 0) for k in id2feature])y_train.append(data["score"])X_test = []y_test = []q_test = []for query, urls in items[train_size:]:- # we have 2 instances per query- # one normal, one masked- q_test.append(query)q_test.append(query)for url, data in urls.items():- X_test.append([data["features"][k] for k in feature2id])- y_test.append(data["score"])- X_test.append([data["masked_features"].get(k, None)- for k in feature2id])+ X_test.append([data["features"].get(k, 0) for k in id2feature])y_test.append(data["score"])X_train = np.array(X_train)@@ -60,28 +94,54 @@ y_train = np.array(y_train)X_test = np.array(X_test)y_test = np.array(y_test)-# Convert score from 0.0-1.0 to 0-3-y_train = (y_train * 3).astype(int)-y_test = (y_test * 3).astype(int)-# Create groupq_train = np.array([len(queries[qid]) for qid in q_train])q_test = np.array([len(queries[qid]) for qid in q_test])+print("Train size:", len(X_train))+print("Test size:", len(X_test))+# Train model+n_estimators = 100model = lgb.LGBMRanker(objective="lambdarank",metric="ndcg",- num_leaves=50,- n_estimators=100,- min_data_in_leaf=10,- max_depth=None,- early_stopping_rounds=5,- learning_rate=0.005,+ importance_type="gain",+ num_leaves=20,+ n_estimators=n_estimators,+ max_depth=10,+ learning_rate=0.05,+ label_gain=[i for i in range(max(y_train.max(), y_test.max()) + 1)],+)+model.fit(+ X_train,+ y_train,+ group=q_train,+ feature_name=[k for k in feature2id],+ eval_set=[(X_test, y_test)],+ eval_group=[q_test],+ eval_at=[1, 2, 3, 5, 10],+ eval_metric="ndcg",+)++best_iteration = (+ n_estimators - 1 if model.best_iteration_ is None else model.best_iteration_)-model.fit(X_train, y_train, group=q_train, feature_name=[k for k in feature2id], eval_set=[(X_test, y_test)], eval_group=[- q_test], eval_at=[1, 3, 5, 10], eval_metric="ndcg")++print("Best iteration:", best_iteration)# dump model-model.booster_.save_model("data/lambdamart.txt",- num_iteration=model.best_iteration_)+model.booster_.save_model(+ "data/lambdamart.txt",+ start_iteration=best_iteration,+ num_iteration=best_iteration,+)++# print feature importance+pprint(+ sorted(+ [(id2feature[i], v) for i, v in enumerate(model.feature_importances_) if v > 0],+ key=lambda x: x[1],+ reverse=True,+ )+)diff --git a/optics/testcases/samples b/optics/testcases/samplesindex 9bd299a2..bf1f8bee 160000--- a/optics/testcases/samples+++ b/optics/testcases/samples@@ -1 +1 @@-Subproject commit 9bd299a2f4bfd7803f2549d322279abe97fd3bea+Subproject commit bf1f8bee48797f7c5c72a7ffe94c9ed935047031