web search done right
actually skip disambiguation pages for entity index. turns out that none of the usual disambiguation elements from the online wiki are present in the .zim dump. instead, disambiguation pages seem to have a "<meta property='mw:PageProp/disambiguation'>" element which we can use.
Mikkel Denkerf4e7d1972c7c
diff --git a/crates/core/src/entrypoint/entity.rs b/crates/core/src/entrypoint/entity.rsindex 413311d6..366796ed 100644--- a/crates/core/src/entrypoint/entity.rs+++ b/crates/core/src/entrypoint/entity.rs@@ -38,6 +38,12 @@ impl<'a> EntityIterator<'a> {}}+impl From<Article> for Entity {+ fn from(article: Article) -> Self {+ article_to_entity(article)+ }+}+fn article_to_entity(article: Article) -> Entity {let root = kuchiki::parse_html().one(article.content);@@ -99,20 +105,18 @@ fn article_to_entity(article: Article) -> Entity {.map(|n| node_into_span(n.as_node())).unwrap_or_default();- let mut is_disambiguation = false;-- if let Some(catlinks) = root.select_first("#catlinks") {- for link in catlinks.as_node().select("a").unwrap() {- if link- .text_contents()- .to_ascii_lowercase()- .contains("disambiguation")- {- is_disambiguation = true;- break;- }- }- }+ let is_disambiguation = root+ .select("meta")+ .map(|mut metas| {+ metas.any(|meta| {+ meta.attributes+ .borrow()+ .get("property")+ .map(|prop| prop == "mw:PageProp/disambiguation")+ .unwrap_or(false)+ })+ })+ .unwrap_or(false);Entity {is_disambiguation,@@ -479,4 +483,29 @@ mod tests {));}++ #[test]+ fn disambig() {+ let content = include_str!("../../testcases/entity/disambig.html");+ let article = Article {+ url: "disambig".to_string(),+ title: "disambig".to_string(),+ content: content.to_string(),+ };++ let entity = Entity::from(article);++ assert!(entity.is_disambiguation);++ let content = include_str!("../../testcases/entity/aristotle.html");+ let article = Article {+ url: "disambig".to_string(),+ title: "disambig".to_string(),+ content: content.to_string(),+ };++ let entity = Entity::from(article);++ assert!(!entity.is_disambiguation);+ }}diff --git a/crates/core/src/ranking/mod.rs b/crates/core/src/ranking/mod.rsindex e5f2f72a..e241cb9e 100644--- a/crates/core/src/ranking/mod.rs+++ b/crates/core/src/ranking/mod.rs@@ -295,7 +295,7 @@ mod tests {).unwrap(),host_centrality: 1.0,- fetch_time_ms: 499,+ fetch_time_ms: 4999,..Default::default()}).expect("failed to insert webpage");@@ -320,7 +320,7 @@ mod tests {).unwrap(),host_centrality: 1.0,- fetch_time_ms: 500,+ fetch_time_ms: 5000,..Default::default()}).expect("failed to insert webpage");diff --git a/crates/core/testcases/entity/disambig.html b/crates/core/testcases/entity/disambig.htmlnew file mode 100644index 00000000..4f4abf7a--- /dev/null+++ b/crates/core/testcases/entity/disambig.html@@ -0,0 +1,124 @@+<!DOCTYPE html>+<html class="client-js"><head>+ <meta charset="UTF-8">+ <title>Test</title>+ <meta name="viewport" content="width=device-width, initial-scale=1.0">+ <link rel="canonical" href="https://en.wikipedia.org/wiki/Test">+ <link href="../-/mw/skins.minerva.base.reset|skins.minerva.content.styles|ext.cite.style|site.styles|mobile.app.pagestyles.android|mediawiki.page.gallery.styles|mediawiki.skinning.content.parsoid.css" rel="stylesheet" type="text/css">+ <link href="../-/style.css" rel="stylesheet" type="text/css"><link href="../-/content.parsoid.css" rel="stylesheet" type="text/css"><link href="../-/inserted_style.css" rel="stylesheet" type="text/css">+ <script data-article-id="Test" id="script-js" src="../-/script.js"></script><script src="../-/masonry.min.js"></script><script src="../-/article_list_home.js"></script><script src="../-/images_loaded.min.js"></script><script src="../-/node_module/details-element-polyfill/dist/details-element-polyfill.js"></script>+</head>++<body class="mediawiki mw-hide-empty-elt ns-0 ns-subject stable skin-minerva action-view animations">+ <div id="mw-mf-viewport" class="feature-header-v2">+ <div id="mw-mf-page-center">+ <div id="content" class="mw-body">+ <a id="top"></a>+ <div id="bodyContent" class="content mw-parser-output">++ <div id="mw-content-text" style="direction: ltr;"><section data-mw-section-id="0" id="mwAQ"><p id="mwAg"><meta id="mwBA">+</p>+<style data-mw-deduplicate="TemplateStyles:r1134653256" id="mwBw">.mw-parser-output .side-box{margin:4px 0;box-sizing:border-box;border:1px solid #aaa;font-size:88%;line-height:1.25em;background-color:#f9f9f9;display:flow-root}.mw-parser-output .side-box-abovebelow,.mw-parser-output .side-box-text{padding:0.25em 0.9em}.mw-parser-output .side-box-image{padding:2px 0 2px 0.9em;text-align:center}.mw-parser-output .side-box-imageright{padding:2px 0.9em 2px 0;text-align:center}@media(min-width:500px){.mw-parser-output .side-box-flex{display:flex;align-items:center}.mw-parser-output .side-box-text{flex:1}}@media(min-width:720px){.mw-parser-output .side-box{width:238px}.mw-parser-output .side-box-right{clear:right;float:right;margin-left:1em}.mw-parser-output .side-box-left{margin-right:1em}}</style><div class="side-box side-box-right sistersitebox" id="mwCA"><style data-mw-deduplicate="TemplateStyles:r1126788409">.mw-parser-output .plainlist ol,.mw-parser-output .plainlist ul{line-height:inherit;list-style:none;margin:0;padding:0}.mw-parser-output .plainlist ol li,.mw-parser-output .plainlist ul li{margin-bottom:0}</style>+<div class="side-box-flex">+<div class="side-box-image"><span class="noviewer"><span><img alt="" src="../I/Wiktionary-logo-en-v2.svg.png.webp" decoding="async" data-file-width="512" data-file-height="512" data-file-type="drawing" height="40" width="40" class="mw-file-element" loading="lazy"></span></span></div>+<div class="side-box-text plainlist">Look up <i><b><a href="https://en.wiktionary.org/wiki/test" title="wiktionary:test" class="extiw external">test</a></b></i>, <i><b><a href="https://en.wiktionary.org/wiki/testing" title="wiktionary:testing" class="extiw external">testing</a></b></i>, <i><b><a href="https://en.wiktionary.org/wiki/Test" title="wiktionary:Test" class="extiw external">Test</a></b></i>, or <i><b><a href="https://en.wiktionary.org/wiki/TEST" title="wiktionary:TEST" class="extiw external">TEST</a></b></i> in Wiktionary, the free dictionary.</div></div>+</div>+<!--+*********************************************************************+Attention ALL USERS (registered or not):+This is **NOT** a test page!+This is a disambiguation page for the term "test".+DO NOT practice here.+If you do, you may be blocked+for disruptive editing or vandalism. If you want to practice,+please use https://en.wikipedia.org/wiki/Wikipedia:Sandbox+Thanks...+*********************************************************************+-->++<p id="mwCQ"><b id="mwCg">Test(s)</b>, <b id="mwCw">testing</b>, or <b id="mwDA">TEST</b> may refer to:</p>++<ul id="mwDQ"><li id="mwDg"><a href="Test_(assessment)" title="Test (assessment)" class="mw-redirect" id="mwDw">Test (assessment)</a>, an educational assessment intended to measure the respondents' knowledge or other abilities</li></ul>++<style data-mw-deduplicate="TemplateStyles:r1044870489" id="mwEA">@media all and (max-width:720px){body.skin-minerva .mw-parser-output .tocright{display:none}.mw-parser-output .tocright{width:100%!important}}@media all and (min-width:720px){.mw-parser-output .tocright{float:right;clear:right;width:auto;margin:0 0 0.5em 1em}.mw-parser-output .tocright-clear-left{clear:left}.mw-parser-output .tocright-clear-both{clear:both}.mw-parser-output .tocright-clear-none{clear:none}}</style><div class="tocright " id="mwEQ"><meta property="mw:PageProp/toc"></div>++</section><section data-mw-section-id="1" id="mwEg"><h2 id="Arts_and_entertainment">Arts and entertainment</h2>+<ul id="mwEw"><li id="mwFA"><a href="Test_(2013_film)" title="Test (2013 film)" id="mwFQ"><i id="mwFg">Test</i> (2013 film)</a>, an American film</li>+<li id="mwFw"><a href="Test_(2014_film)" title="Test (2014 film)" id="mwGA"><i id="mwGQ">Test</i> (2014 film)</a>, a Russian film</li>+<li id="mwGg"><a href="Test_(group)" title="Test (group)" id="mwGw"><i id="mwHA">Test</i> (group)</a>, a jazz collective</li>+<li id="mwHQ"><a href="Tests_(album)" title="Tests (album)" id="mwHg"><i id="mwHw">Tests</i> (album)</a>, a 1998 album by The Microphones</li>+<li id="mwIA"><a href="Testing_(album)" title="Testing (album)" id="mwIQ"><i id="mwIg">Testing</i> (album)</a>, an album by ASAP Rocky</li></ul>++</section><section data-mw-section-id="2" id="mwIw"><h2 id="Computing">Computing</h2>+<ul id="mwJA"><li id="mwJQ"><a href=".test" title=".test" id="mwJg">.test</a>, a reserved top-level domain</li>+<li id="mwJw"><a href="Software_testing" title="Software testing" id="mwKA">Software testing</a></li>+<li id="mwKQ"><a href="Test_(Unix)" title="Test (Unix)" id="mwKg">test (Unix)</a>, a Unix command for evaluating conditional expressions</li>+<li id="mwKw"><a href="TEST_(x86_instruction)" title="TEST (x86 instruction)" id="mwLA">TEST (x86 instruction)</a>, an x86 assembly language instruction</li></ul>++</section><section data-mw-section-id="3" id="mwLQ"><h2 id="People">People</h2>+<ul id="mwLg"><li id="mwLw"><a href="Test_(wrestler)" title="Test (wrestler)" id="mwMA">Test (wrestler)</a>, ring name for Andrew Martin (1975–2009), Canadian professional wrestler</li>+<li id="mwMQ"><a href="John_Test" title="John Test" id="mwMg">John Test</a> (1771–1849), American politician</li>+<li id="mwMw"><a href="Zack_Test" title="Zack Test" id="mwNA">Zack Test</a> (born 1989), American rugby union player</li></ul>++</section><section data-mw-section-id="4" id="mwNQ"><h2 id="Science_and_technology">Science and technology</h2>+<ul id="mwNg"><li id="mwNw"><a href="Experiment" title="Experiment" id="mwOA">Experiment</a>, a procedure carried out in order to test (support, refute, or validate) a hypothesis</li>+<li id="mwOQ"><a href="Statistical_hypothesis_testing" title="Statistical hypothesis testing" id="mwOg">Statistical hypothesis testing</a>, techniques to reach conclusions about probabilistic behavior</li>+<li id="mwOw"><a href="Product_testing" title="Product testing" id="mwPA">Product testing</a></li>+<li id="mwPQ"><a href="System_testing" title="System testing" id="mwPg">System testing</a></li>+<li id="mwPw"><a href="Mechanical_testing" title="Mechanical testing" id="mwQA">Mechanical testing</a></li>+<li id="mwQQ"><a href="Test_equipment" title="Test equipment" id="mwQg">Test equipment</a></li>+<li id="mwQw"><a href="Stress_testing" title="Stress testing" id="mwRA">Stress testing</a>+<ul id="mwRQ"><li id="mwRg"><a href="Proof_test" title="Proof test" id="mwRw">Proof test</a>, a stress test to demonstrate the fitness of a load-bearing structure</li></ul></li>+<li id="mwSA"><a href="Test_(biology)" title="Test (biology)" id="mwSQ">Test (biology)</a>, the shell of sea urchins and certain microorganisms</li></ul>++</section><section data-mw-section-id="5" id="mwSg"><h2 id="Sports">Sports</h2>+<ul id="mwSw"><li id="mwTA"><a href="Test_cricket" title="Test cricket" id="mwTQ">Test cricket</a>, a series of matches played by two national representative teams</li>+<li id="mwTg"><a href="Test_match_(rugby_league)" title="Test match (rugby league)" id="mwTw">Test match (rugby league)</a>, a match between teams of the Rugby League International Federation</li>+<li id="mwUA"><a href="Test_match_(rugby_union)" title="Test match (rugby union)" id="mwUQ">Test match (rugby union)</a>, an international match usually played between two senior national teams</li>+<li id="mwUg"><i id="mwUw">The</i> <a href="Test_(greyhound_competition)" title="Test (greyhound competition)" id="mwVA"><i id="mwVQ">Test</i> (greyhound competition)</a>, a greyhound race run between 1941 and 2008</li></ul>++</section><section data-mw-section-id="6" id="mwVg"><h2 id="Other_uses">Other uses</h2>+<ul id="mwVw"><li id="mwWA"><a href="River_Test" title="River Test" id="mwWQ">River Test</a>, a river in England</li>+<li id="mwWg"><a href="Test_(law)" title="Test (law)" id="mwWw">Test (law)</a>, an applied method of evaluation used to resolve matters of jurisprudence</li></ul>++</section><section data-mw-section-id="7" id="mwXA"><h2 id="See_also">See also</h2>+<div class="side-box side-box-right sistersitebox selfreference" id="mwXg">+<div class="side-box-flex">+<div class="side-box-image"><span><span><img src="../I/Vector_search_icon.svg.png.webp" decoding="async" data-file-width="12" data-file-height="13" data-file-type="drawing" height="33" width="30" class="mw-file-element" loading="lazy"></span></span></div>+<div class="side-box-text plainlist">Search for <i><b><span class=""><a href="https://en.wikipedia.org/w/index.php?title=Special:Search&search=%22Test%22&ns0=1&fulltext=Search" class="external text external">"Test"</a></span></b></i> on Wikipedia.</div></div>+</div>+<ul id="mwXw"><li id="mwYA"><a href="Tester_(disambiguation)" title="Tester (disambiguation)" class="mw-redirect mw-disambig" id="mwYQ">Tester (disambiguation)</a>, person or device that tests or measures</li>+<li id="mwYg"><a href="The_Test_(disambiguation)" title="The Test (disambiguation)" class="mw-redirect mw-disambig" id="mwYw">The Test (disambiguation)</a></li>+<li id="mwZA"><span class="selfreference" id="mwZQ">All pages with titles beginning with <i>Test</i></span></li>+<li id="mwZg"><span class="selfreference" id="mwZw">All pages with titles containing <i>Test</i></span></li>+<li id="mwaA"><a href="Examination_(disambiguation)" title="Examination (disambiguation)" class="mw-redirect mw-disambig" id="mwaQ">Examination (disambiguation)</a></li>+<li id="mwag"><a href="Trial_(disambiguation)" title="Trial (disambiguation)" class="mw-disambig" id="mwaw">Trial (disambiguation)</a></li>+<li id="mwbA"><a href="Validation_(disambiguation)" title="Validation (disambiguation)" class="mw-redirect mw-disambig" id="mwbQ">Validation (disambiguation)</a></li>+<li id="mwbg"><a href="Verification_(disambiguation)" title="Verification (disambiguation)" class="mw-redirect mw-disambig" id="mwbw">Verification (disambiguation)</a></li></ul>++<span>+</span><style data-mw-deduplicate="TemplateStyles:r1097304697">.mw-parser-output .dmbox{display:flex;align-items:center;clear:both;margin:0.9em 1em;border-top:1px solid #ccc;border-bottom:1px solid #ccc;padding:0.25em 0.35em;font-style:italic}.mw-parser-output .dmbox>*{flex-shrink:0;margin:0 0.25em}.mw-parser-output .dmbox-body{flex-grow:1;flex-shrink:1;padding:0.1em 0}.mw-parser-output .dmbox-invalid-type{text-align:center}</style><span>+</span><meta property="mw:PageProp/disambiguation"></section><!--htdig_noindex--><div><div style="clear:both; background-image:linear-gradient(180deg, #E8E8E8, white); border-top: dashed 2px #AAAAAA; padding: 0.5em 0.5em 0.5em 0.5em; margin-top: 1em; direction: ltr;">+ This article is issued from <a class="external text" title="Last edited on 2024-01-06" href="https://en.wikipedia.org/wiki/?title=Test&oldid=1194049492">Wikipedia</a>. The text is licensed under <a class="external text" href="https://creativecommons.org/licenses/by-sa/4.0/">Creative Commons - Attribution - Sharealike</a>. Additional terms may apply for the media files.+</div>+</div><!--/htdig_noindex--></div>+ </div>+ </div>+ </div>+ </div>+ <script src="../-/mw/jsConfigVars.js"></script>+ <script src="../-/mw/startup.js"></script>+<script src="../-/mw/jquery,mediawiki.js"></script>+<script src="../-/mw/mediawiki.base.js"></script>+<script src="../-/mw/mediawiki.util.js"></script>+<script src="../-/mw/site.js"></script>+<script src="../-/mw/mediawiki.page.ready.js"></script>+<script src="../-/mw/jquery.cookie.js"></script>+<script src="../-/mw/mediawiki.cookie.js"></script>+<script src="../-/mw/mediawiki.storage.js"></script>+<script src="../-/mw/ext.gadget.VisibilityToggles.js"></script>+<script src="../-/mw/ext.gadget.defaultVisibilityToggles.js"></script>++++</body></html>diff --git a/scripts/article_html.py b/scripts/article_html.pyindex d1ea4576..62045dcc 100644--- a/scripts/article_html.py+++ b/scripts/article_html.py@@ -9,18 +9,23 @@ except ImportError:parser = argparse.ArgumentParser(description="Dump a specific article from a ZIM file")-parser.add_argument("--zim_file", help="Path to the ZIM file", type=str, default="data/test.zim")+parser.add_argument("--zim-file", help="Path to the ZIM file", type=str, default="data/test.zim")+parser.add_argument("--url", help="If the query should be treated as a direct URL", action="store_true", default=False)parser.add_argument("query", help="Query to search for", type=str)args = parser.parse_args()zim = Archive(args.zim_file)-query = Query().set_query(args.query)-searcher = Searcher(zim)-search = searcher.search(query)-search_count = search.getEstimatedMatches()-res = list(search.getResults(0, 1))+if args.url:+ path = f"A/{args.query}"+else:+ query = Query().set_query(args.query)+ searcher = Searcher(zim)+ search = searcher.search(query)+ search_count = search.getEstimatedMatches()+ res = list(search.getResults(0, 1))+ path = res[0]-article = zim.get_entry_by_path(res[0])+article = zim.get_entry_by_path(path)print(bytes(article.get_item().content).decode("UTF-8"))