Skip to content

web search done right

rust update and new git screenshot

Mikkel Denker330c02a96f70
diff --git a/Cargo.toml b/Cargo.toml
index 89aae053..6b5d5f23 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -4,6 +4,7 @@ members = [
"optics",
"kuchiki"
]
+resolver = "2"
[profile.release]
debug = true
diff --git a/core/src/alice/generate.rs b/core/src/alice/generate.rs
index f69d777c..f82547a4 100644
--- a/core/src/alice/generate.rs
+++ b/core/src/alice/generate.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::sync::Arc;
+use std::{rc::Rc, sync::Arc};
use aes_gcm::{Aes256Gcm, Key};
use tch::{IndexOp, Kind, Tensor};
@@ -43,7 +43,7 @@ pub enum TokenGeneratorState {
}
pub struct AliceTokenGenerator {
- model: Arc<RawModel>,
+ model: Rc<RawModel>,
// state needs to be a queue of size 2 to have the ability
// to generate a new token if the current one is banned
states: LeakyQueue<ModelState>,
@@ -60,7 +60,7 @@ pub struct AliceTokenGenerator {
impl AliceTokenGenerator {
pub fn new(
- model: Arc<RawModel>,
+ model: Rc<RawModel>,
end_tokens: Vec<i64>,
state: Tensor,
tokens: &[i64],
diff --git a/core/src/alice/mod.rs b/core/src/alice/mod.rs
index f9fb7c15..5141d6fd 100644
--- a/core/src/alice/mod.rs
+++ b/core/src/alice/mod.rs
@@ -27,6 +27,7 @@
use std::{
io::{Read, Write},
path::Path,
+ rc::Rc,
sync::Arc,
};
@@ -230,7 +231,7 @@ impl Searcher {
}
pub struct Alice {
- inner: Arc<RawModel>,
+ inner: Rc<RawModel>,
tokenizer: Arc<Tokenizer>,
summarizer: Arc<ExtractiveSummarizer>,
end_tokens: Vec<i64>,
@@ -305,7 +306,7 @@ impl Alice {
let summarizer = Arc::new(summarizer);
- let inner = Arc::new(model);
+ let inner = Rc::new(model);
let tokenizer = Arc::new(Tokenizer::open(folder.as_ref().join("tokenizer.json"))?);
let end_tokens = vec![tokenizer.tokenizer.token_to_id("<|endoftext|>").unwrap() as i64];
diff --git a/core/src/crawler/worker.rs b/core/src/crawler/worker.rs
index 7177f810..8dbc8b1a 100644
--- a/core/src/crawler/worker.rs
+++ b/core/src/crawler/worker.rs
@@ -192,7 +192,7 @@ impl Worker {
if let Ok(Some(sitemap)) = self.robotstxt.sitemap(&retryable_url.url).await {
let sitemap_urls = self.urls_from_sitemap(sitemap, 0, 5).await;
- discovered_urls.extend(sitemap_urls.into_iter());
+ discovered_urls.extend(sitemap_urls);
}
}
@@ -223,7 +223,7 @@ impl Worker {
}
}
- discovered_urls.extend(res.new_urls.into_iter());
+ discovered_urls.extend(res.new_urls);
url_responses.push(res.response);
}
diff --git a/core/src/entrypoint/search_server.rs b/core/src/entrypoint/search_server.rs
index b6ee2dbd..83255c63 100644
--- a/core/src/entrypoint/search_server.rs
+++ b/core/src/entrypoint/search_server.rs
@@ -122,7 +122,6 @@ pub async fn run(config: SearchServerConfig) -> Result<()> {
if let Some(homepage) = local_searcher.get_homepage(url) {
if let Some(desc) = homepage.description() {
result.insert(Url::from(url.clone()), desc.clone());
- } else {
}
}
}
diff --git a/core/src/query/pattern_query.rs b/core/src/query/pattern_query.rs
index 60be9654..029ef073 100644
--- a/core/src/query/pattern_query.rs
+++ b/core/src/query/pattern_query.rs
@@ -369,8 +369,6 @@ impl PatternWeight {
let fieldnorm_reader = self.fieldnorm_reader(reader)?;
- if can_optimize_site_domain(&self.patterns, ALL_FIELDS[self.field.field_id() as usize]) {}
-
let mut term_postings_list = Vec::with_capacity(self.raw_terms.len());
for term in &self.raw_terms {
if let Some(postings) = reader
diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rs
index a11c2030..78093885 100644
--- a/core/src/ranking/pipeline.rs
+++ b/core/src/ranking/pipeline.rs
@@ -137,7 +137,7 @@ impl<M: CrossEncoder> ReRanker<M> {
let query = &self.query.as_ref().unwrap().query;
let scores = self.crossencoder.run(query, &bodies);
- for (website, score) in websites.iter_mut().zip(scores.into_iter()) {
+ for (website, score) in websites.iter_mut().zip(scores) {
let website = website.as_mut_ranking();
website.signals.insert(Signal::CrossEncoder, score);
}
diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rs
index 13744360..242a2215 100644
--- a/core/src/searcher/distributed.rs
+++ b/core/src/searcher/distributed.rs
@@ -293,7 +293,7 @@ impl DistributedSearcher {
.unzip();
if let Ok(websites) = shard.retrieve_websites(&pointers, query).await {
- for (index, website) in indexes.into_iter().zip(websites.into_iter()) {
+ for (index, website) in indexes.into_iter().zip(websites) {
retrieved_webpages[index] = Some(website);
}
}
diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rs
index 3c89e904..659efff6 100644
--- a/core/src/searcher/local.rs
+++ b/core/src/searcher/local.rs
@@ -328,7 +328,7 @@ impl LocalSearcher {
.map(DisplayedWebpage::from)
.collect();
- for (webpage, ranking) in webpages.iter_mut().zip(top_websites.into_iter()) {
+ for (webpage, ranking) in webpages.iter_mut().zip(top_websites) {
let mut ranking_signals = HashMap::new();
for signal in ALL_SIGNALS {
diff --git a/core/src/subdomain_count.rs b/core/src/subdomain_count.rs
index 0a945087..66bc008e 100644
--- a/core/src/subdomain_count.rs
+++ b/core/src/subdomain_count.rs
@@ -50,7 +50,7 @@ impl SubdomainCounter {
pub fn merge(&mut self, other: Self) {
for (key, val) in other.inner.iter() {
let mut current = self.inner.get(&key).unwrap_or_default();
- current.extend(val.into_iter());
+ current.extend(val);
self.inner.insert(key, current);
}
}
diff --git a/core/src/tokenizer.rs b/core/src/tokenizer.rs
index 4929eb89..2b2f31ea 100644
--- a/core/src/tokenizer.rs
+++ b/core/src/tokenizer.rs
@@ -481,22 +481,22 @@ fn flatten(val: serde_json::Value) -> Vec<String> {
while let Some(elem) = stack.pop() {
match elem.val {
- serde_json::Value::Null => res.push(
- itertools::intersperse(elem.parent_keys.into_iter(), ".".to_string()).collect(),
- ),
+ serde_json::Value::Null => {
+ res.push(itertools::intersperse(elem.parent_keys, ".".to_string()).collect())
+ }
serde_json::Value::Bool(b) => {
let key: String =
- itertools::intersperse(elem.parent_keys.into_iter(), ".".to_string()).collect();
+ itertools::intersperse(elem.parent_keys, ".".to_string()).collect();
res.push(format!("{key}=\"{b}\""))
}
serde_json::Value::Number(n) => {
let key: String =
- itertools::intersperse(elem.parent_keys.into_iter(), ".".to_string()).collect();
+ itertools::intersperse(elem.parent_keys, ".".to_string()).collect();
res.push(format!("{key}=\"{n}\""))
}
serde_json::Value::String(s) => {
let key: String =
- itertools::intersperse(elem.parent_keys.into_iter(), ".".to_string()).collect();
+ itertools::intersperse(elem.parent_keys, ".".to_string()).collect();
res.push(format!("{key}=\"{}\"", s.replace('"', "\\\"")))
}
serde_json::Value::Array(arr) => {
@@ -531,8 +531,7 @@ impl FlattenedJson {
let json = serde_json::to_string(value)?;
let val: serde_json::Value = serde_json::from_str(&json)?;
- let flattened_json =
- itertools::intersperse(flatten(val).into_iter(), "\n".to_string()).collect();
+ let flattened_json = itertools::intersperse(flatten(val), "\n".to_string()).collect();
Ok(Self { flattened_json })
}
diff --git a/core/src/webpage/just_text.rs b/core/src/webpage/just_text.rs
index 84a1e6fe..0aebd97c 100644
--- a/core/src/webpage/just_text.rs
+++ b/core/src/webpage/just_text.rs
@@ -489,7 +489,7 @@ impl JustText {
}
}
- for (paragraph, classification) in paragraphs.iter_mut().zip(new_classes.into_iter()) {
+ for (paragraph, classification) in paragraphs.iter_mut().zip(new_classes) {
paragraph.classification = classification;
}
diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rs
index 162ae1ca..bea44cbe 100644
--- a/core/src/webpage/mod.rs
+++ b/core/src/webpage/mod.rs
@@ -42,7 +42,7 @@ use self::{
};
pub static URL_REGEX: once_cell::sync::Lazy<Regex> = once_cell::sync::Lazy::new(|| {
- Regex::new(r#"(((http|ftp|https):/{2})+(([0-9a-z_-]+\.)+(aero|asia|biz|cat|com|coop|edu|gov|info|int|jobs|mil|mobi|museum|name|net|org|pro|tel|travel|ac|ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|ax|az|ba|bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|br|bs|bt|bv|bw|by|bz|ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cu|cv|cx|cy|cz|cz|de|dj|dk|dm|do|dz|ec|ee|eg|er|es|et|eu|fi|fj|fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|is|it|je|jm|jo|jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly|ma|mc|md|me|mg|mh|mk|ml|mn|mn|mo|mp|mr|ms|mt|mu|mv|mw|mx|my|mz|na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz|nom|pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py|qa|re|ra|rs|ru|rw|sa|sb|sc|sd|se|sg|sh|si|sj|sj|sk|sl|sm|sn|so|sr|st|su|sv|sy|sz|tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz|ua|ug|uk|us|uy|uz|va|vc|ve|vg|vi|vn|vu|wf|ws|ye|yt|yu|za|zm|zw|arpa)(:[0-9]+)?((/([~0-9a-zA-Z\#\+%@\./_-]+))?(\?[0-9a-zA-Z\+%@/&\[\];=_-]+)?)?))\b"#).unwrap()
+ Regex::new(r"(((http|ftp|https):/{2})+(([0-9a-z_-]+\.)+(aero|asia|biz|cat|com|coop|edu|gov|info|int|jobs|mil|mobi|museum|name|net|org|pro|tel|travel|ac|ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|ax|az|ba|bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|br|bs|bt|bv|bw|by|bz|ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cu|cv|cx|cy|cz|cz|de|dj|dk|dm|do|dz|ec|ee|eg|er|es|et|eu|fi|fj|fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|is|it|je|jm|jo|jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly|ma|mc|md|me|mg|mh|mk|ml|mn|mn|mo|mp|mr|ms|mt|mu|mv|mw|mx|my|mz|na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz|nom|pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py|qa|re|ra|rs|ru|rw|sa|sb|sc|sd|se|sg|sh|si|sj|sj|sk|sl|sm|sn|so|sr|st|su|sv|sy|sz|tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz|ua|ug|uk|us|uy|uz|va|vc|ve|vg|vi|vn|vu|wf|ws|ye|yt|yu|za|zm|zw|arpa)(:[0-9]+)?((/([~0-9a-zA-Z\#\+%@\./_-]+))?(\?[0-9a-zA-Z\+%@/&\[\];=_-]+)?)?))\b").unwrap()
});
#[derive(PartialEq, Eq, Debug)]
@@ -456,8 +456,8 @@ impl Html {
}
}));
- links.extend(self.links_tag().into_iter());
- links.extend(self.metadata_links().into_iter());
+ links.extend(self.links_tag());
+ links.extend(self.metadata_links());
links
}
diff --git a/core/src/webpage/schema_org/microdata.rs b/core/src/webpage/schema_org/microdata.rs
index 597999a1..ae7a4c3d 100644
--- a/core/src/webpage/schema_org/microdata.rs
+++ b/core/src/webpage/schema_org/microdata.rs
@@ -606,7 +606,7 @@ mod tests {
#[test]
fn schema_image_object_example() {
let root = kuchiki::parse_html().one(
- r##"
+ r#"
<html>
<div itemscope itemtype="https://schema.org/ImageObject">
<h2 itemprop="name">Beach in Mexico</h2>
@@ -623,7 +623,7 @@ mod tests {
<span itemprop="description">I took this picture while on vacation last year.</span>
</div>
</html>
- "##,
+ "#,
);
let res = parse_schema(root);
@@ -647,7 +647,7 @@ mod tests {
#[test]
fn schema_person_example() {
let root = kuchiki::parse_html().one(
- r##"
+ r#"
<div itemscope itemtype="https://schema.org/Person">
<span itemprop="name">Jane Doe</span>
<img src="janedoe.jpg" itemprop="image" alt="Photo of Jane Doe"/>
@@ -675,7 +675,7 @@ mod tests {
<a href="http://www.xyz.edu/students/bobsmith.html" itemprop="colleague">
Bob Smith</a>
</div>
- "##,
+ "#,
);
let expected_json = r#"
{
diff --git a/frontend/src/pages/explore.astro b/frontend/src/pages/explore.astro
index 4f0ef457..0637fc7b 100644
--- a/frontend/src/pages/explore.astro
+++ b/frontend/src/pages/explore.astro
@@ -206,7 +206,6 @@ import { askama } from "../askama";
})
.then(response => response.json())
.then(data => {
- console.log(data);
resultContainer.classList.remove("hidden");
result.innerHTML = "";
diff --git a/git-assets/screenshot.png b/git-assets/screenshot.png
index 0428e1f1..c281b9c0 100644
Binary files a/git-assets/screenshot.png and b/git-assets/screenshot.png differ