web search done right
rust update and new git screenshot
Mikkel Denker330c02a96f70
diff --git a/Cargo.toml b/Cargo.tomlindex 89aae053..6b5d5f23 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -4,6 +4,7 @@ members = ["optics","kuchiki"]+resolver = "2"[profile.release]debug = truediff --git a/core/src/alice/generate.rs b/core/src/alice/generate.rsindex f69d777c..f82547a4 100644--- a/core/src/alice/generate.rs+++ b/core/src/alice/generate.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::sync::Arc;+use std::{rc::Rc, sync::Arc};use aes_gcm::{Aes256Gcm, Key};use tch::{IndexOp, Kind, Tensor};@@ -43,7 +43,7 @@ pub enum TokenGeneratorState {}pub struct AliceTokenGenerator {- model: Arc<RawModel>,+ model: Rc<RawModel>,// state needs to be a queue of size 2 to have the ability// to generate a new token if the current one is bannedstates: LeakyQueue<ModelState>,@@ -60,7 +60,7 @@ pub struct AliceTokenGenerator {impl AliceTokenGenerator {pub fn new(- model: Arc<RawModel>,+ model: Rc<RawModel>,end_tokens: Vec<i64>,state: Tensor,tokens: &[i64],diff --git a/core/src/alice/mod.rs b/core/src/alice/mod.rsindex f9fb7c15..5141d6fd 100644--- a/core/src/alice/mod.rs+++ b/core/src/alice/mod.rs@@ -27,6 +27,7 @@use std::{io::{Read, Write},path::Path,+ rc::Rc,sync::Arc,};@@ -230,7 +231,7 @@ impl Searcher {}pub struct Alice {- inner: Arc<RawModel>,+ inner: Rc<RawModel>,tokenizer: Arc<Tokenizer>,summarizer: Arc<ExtractiveSummarizer>,end_tokens: Vec<i64>,@@ -305,7 +306,7 @@ impl Alice {let summarizer = Arc::new(summarizer);- let inner = Arc::new(model);+ let inner = Rc::new(model);let tokenizer = Arc::new(Tokenizer::open(folder.as_ref().join("tokenizer.json"))?);let end_tokens = vec![tokenizer.tokenizer.token_to_id("<|endoftext|>").unwrap() as i64];diff --git a/core/src/crawler/worker.rs b/core/src/crawler/worker.rsindex 7177f810..8dbc8b1a 100644--- a/core/src/crawler/worker.rs+++ b/core/src/crawler/worker.rs@@ -192,7 +192,7 @@ impl Worker {if let Ok(Some(sitemap)) = self.robotstxt.sitemap(&retryable_url.url).await {let sitemap_urls = self.urls_from_sitemap(sitemap, 0, 5).await;- discovered_urls.extend(sitemap_urls.into_iter());+ discovered_urls.extend(sitemap_urls);}}@@ -223,7 +223,7 @@ impl Worker {}}- discovered_urls.extend(res.new_urls.into_iter());+ discovered_urls.extend(res.new_urls);url_responses.push(res.response);}diff --git a/core/src/entrypoint/search_server.rs b/core/src/entrypoint/search_server.rsindex b6ee2dbd..83255c63 100644--- a/core/src/entrypoint/search_server.rs+++ b/core/src/entrypoint/search_server.rs@@ -122,7 +122,6 @@ pub async fn run(config: SearchServerConfig) -> Result<()> {if let Some(homepage) = local_searcher.get_homepage(url) {if let Some(desc) = homepage.description() {result.insert(Url::from(url.clone()), desc.clone());- } else {}}}diff --git a/core/src/query/pattern_query.rs b/core/src/query/pattern_query.rsindex 60be9654..029ef073 100644--- a/core/src/query/pattern_query.rs+++ b/core/src/query/pattern_query.rs@@ -369,8 +369,6 @@ impl PatternWeight {let fieldnorm_reader = self.fieldnorm_reader(reader)?;- if can_optimize_site_domain(&self.patterns, ALL_FIELDS[self.field.field_id() as usize]) {}-let mut term_postings_list = Vec::with_capacity(self.raw_terms.len());for term in &self.raw_terms {if let Some(postings) = readerdiff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rsindex a11c2030..78093885 100644--- a/core/src/ranking/pipeline.rs+++ b/core/src/ranking/pipeline.rs@@ -137,7 +137,7 @@ impl<M: CrossEncoder> ReRanker<M> {let query = &self.query.as_ref().unwrap().query;let scores = self.crossencoder.run(query, &bodies);- for (website, score) in websites.iter_mut().zip(scores.into_iter()) {+ for (website, score) in websites.iter_mut().zip(scores) {let website = website.as_mut_ranking();website.signals.insert(Signal::CrossEncoder, score);}diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rsindex 13744360..242a2215 100644--- a/core/src/searcher/distributed.rs+++ b/core/src/searcher/distributed.rs@@ -293,7 +293,7 @@ impl DistributedSearcher {.unzip();if let Ok(websites) = shard.retrieve_websites(&pointers, query).await {- for (index, website) in indexes.into_iter().zip(websites.into_iter()) {+ for (index, website) in indexes.into_iter().zip(websites) {retrieved_webpages[index] = Some(website);}}diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rsindex 3c89e904..659efff6 100644--- a/core/src/searcher/local.rs+++ b/core/src/searcher/local.rs@@ -328,7 +328,7 @@ impl LocalSearcher {.map(DisplayedWebpage::from).collect();- for (webpage, ranking) in webpages.iter_mut().zip(top_websites.into_iter()) {+ for (webpage, ranking) in webpages.iter_mut().zip(top_websites) {let mut ranking_signals = HashMap::new();for signal in ALL_SIGNALS {diff --git a/core/src/subdomain_count.rs b/core/src/subdomain_count.rsindex 0a945087..66bc008e 100644--- a/core/src/subdomain_count.rs+++ b/core/src/subdomain_count.rs@@ -50,7 +50,7 @@ impl SubdomainCounter {pub fn merge(&mut self, other: Self) {for (key, val) in other.inner.iter() {let mut current = self.inner.get(&key).unwrap_or_default();- current.extend(val.into_iter());+ current.extend(val);self.inner.insert(key, current);}}diff --git a/core/src/tokenizer.rs b/core/src/tokenizer.rsindex 4929eb89..2b2f31ea 100644--- a/core/src/tokenizer.rs+++ b/core/src/tokenizer.rs@@ -481,22 +481,22 @@ fn flatten(val: serde_json::Value) -> Vec<String> {while let Some(elem) = stack.pop() {match elem.val {- serde_json::Value::Null => res.push(- itertools::intersperse(elem.parent_keys.into_iter(), ".".to_string()).collect(),- ),+ serde_json::Value::Null => {+ res.push(itertools::intersperse(elem.parent_keys, ".".to_string()).collect())+ }serde_json::Value::Bool(b) => {let key: String =- itertools::intersperse(elem.parent_keys.into_iter(), ".".to_string()).collect();+ itertools::intersperse(elem.parent_keys, ".".to_string()).collect();res.push(format!("{key}=\"{b}\""))}serde_json::Value::Number(n) => {let key: String =- itertools::intersperse(elem.parent_keys.into_iter(), ".".to_string()).collect();+ itertools::intersperse(elem.parent_keys, ".".to_string()).collect();res.push(format!("{key}=\"{n}\""))}serde_json::Value::String(s) => {let key: String =- itertools::intersperse(elem.parent_keys.into_iter(), ".".to_string()).collect();+ itertools::intersperse(elem.parent_keys, ".".to_string()).collect();res.push(format!("{key}=\"{}\"", s.replace('"', "\\\"")))}serde_json::Value::Array(arr) => {@@ -531,8 +531,7 @@ impl FlattenedJson {let json = serde_json::to_string(value)?;let val: serde_json::Value = serde_json::from_str(&json)?;- let flattened_json =- itertools::intersperse(flatten(val).into_iter(), "\n".to_string()).collect();+ let flattened_json = itertools::intersperse(flatten(val), "\n".to_string()).collect();Ok(Self { flattened_json })}diff --git a/core/src/webpage/just_text.rs b/core/src/webpage/just_text.rsindex 84a1e6fe..0aebd97c 100644--- a/core/src/webpage/just_text.rs+++ b/core/src/webpage/just_text.rs@@ -489,7 +489,7 @@ impl JustText {}}- for (paragraph, classification) in paragraphs.iter_mut().zip(new_classes.into_iter()) {+ for (paragraph, classification) in paragraphs.iter_mut().zip(new_classes) {paragraph.classification = classification;}diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rsindex 162ae1ca..bea44cbe 100644--- a/core/src/webpage/mod.rs+++ b/core/src/webpage/mod.rs@@ -42,7 +42,7 @@ use self::{};pub static URL_REGEX: once_cell::sync::Lazy<Regex> = once_cell::sync::Lazy::new(|| {- Regex::new(r#"(((http|ftp|https):/{2})+(([0-9a-z_-]+\.)+(aero|asia|biz|cat|com|coop|edu|gov|info|int|jobs|mil|mobi|museum|name|net|org|pro|tel|travel|ac|ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|ax|az|ba|bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|br|bs|bt|bv|bw|by|bz|ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cu|cv|cx|cy|cz|cz|de|dj|dk|dm|do|dz|ec|ee|eg|er|es|et|eu|fi|fj|fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|is|it|je|jm|jo|jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly|ma|mc|md|me|mg|mh|mk|ml|mn|mn|mo|mp|mr|ms|mt|mu|mv|mw|mx|my|mz|na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz|nom|pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py|qa|re|ra|rs|ru|rw|sa|sb|sc|sd|se|sg|sh|si|sj|sj|sk|sl|sm|sn|so|sr|st|su|sv|sy|sz|tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz|ua|ug|uk|us|uy|uz|va|vc|ve|vg|vi|vn|vu|wf|ws|ye|yt|yu|za|zm|zw|arpa)(:[0-9]+)?((/([~0-9a-zA-Z\#\+%@\./_-]+))?(\?[0-9a-zA-Z\+%@/&\[\];=_-]+)?)?))\b"#).unwrap()+ Regex::new(r"(((http|ftp|https):/{2})+(([0-9a-z_-]+\.)+(aero|asia|biz|cat|com|coop|edu|gov|info|int|jobs|mil|mobi|museum|name|net|org|pro|tel|travel|ac|ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|ax|az|ba|bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|br|bs|bt|bv|bw|by|bz|ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cu|cv|cx|cy|cz|cz|de|dj|dk|dm|do|dz|ec|ee|eg|er|es|et|eu|fi|fj|fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|is|it|je|jm|jo|jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly|ma|mc|md|me|mg|mh|mk|ml|mn|mn|mo|mp|mr|ms|mt|mu|mv|mw|mx|my|mz|na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz|nom|pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py|qa|re|ra|rs|ru|rw|sa|sb|sc|sd|se|sg|sh|si|sj|sj|sk|sl|sm|sn|so|sr|st|su|sv|sy|sz|tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz|ua|ug|uk|us|uy|uz|va|vc|ve|vg|vi|vn|vu|wf|ws|ye|yt|yu|za|zm|zw|arpa)(:[0-9]+)?((/([~0-9a-zA-Z\#\+%@\./_-]+))?(\?[0-9a-zA-Z\+%@/&\[\];=_-]+)?)?))\b").unwrap()});#[derive(PartialEq, Eq, Debug)]@@ -456,8 +456,8 @@ impl Html {}}));- links.extend(self.links_tag().into_iter());- links.extend(self.metadata_links().into_iter());+ links.extend(self.links_tag());+ links.extend(self.metadata_links());links}diff --git a/core/src/webpage/schema_org/microdata.rs b/core/src/webpage/schema_org/microdata.rsindex 597999a1..ae7a4c3d 100644--- a/core/src/webpage/schema_org/microdata.rs+++ b/core/src/webpage/schema_org/microdata.rs@@ -606,7 +606,7 @@ mod tests {#[test]fn schema_image_object_example() {let root = kuchiki::parse_html().one(- r##"+ r#"<html><div itemscope itemtype="https://schema.org/ImageObject"><h2 itemprop="name">Beach in Mexico</h2>@@ -623,7 +623,7 @@ mod tests {<span itemprop="description">I took this picture while on vacation last year.</span></div></html>- "##,+ "#,);let res = parse_schema(root);@@ -647,7 +647,7 @@ mod tests {#[test]fn schema_person_example() {let root = kuchiki::parse_html().one(- r##"+ r#"<div itemscope itemtype="https://schema.org/Person"><span itemprop="name">Jane Doe</span><img src="janedoe.jpg" itemprop="image" alt="Photo of Jane Doe"/>@@ -675,7 +675,7 @@ mod tests {<a href="http://www.xyz.edu/students/bobsmith.html" itemprop="colleague">Bob Smith</a></div>- "##,+ "#,);let expected_json = r#"{diff --git a/frontend/src/pages/explore.astro b/frontend/src/pages/explore.astroindex 4f0ef457..0637fc7b 100644--- a/frontend/src/pages/explore.astro+++ b/frontend/src/pages/explore.astro@@ -206,7 +206,6 @@ import { askama } from "../askama";}).then(response => response.json()).then(data => {- console.log(data);resultContainer.classList.remove("hidden");result.innerHTML = "";diff --git a/git-assets/screenshot.png b/git-assets/screenshot.pngindex 0428e1f1..c281b9c0 100644Binary files a/git-assets/screenshot.png and b/git-assets/screenshot.png differ