web search done right
possible fix to be able to open indexes that doesn't contain all the necesarry fields. this is usefull to be able to update the code continously in production and not having to wait for next indexing (#118)
Mikkel Denker92f933e5fe9b
diff --git a/core/src/crawler/site_graph.rs b/core/src/crawler/site_graph.rsindex b13a1cff..56aef931 100644--- a/core/src/crawler/site_graph.rs+++ b/core/src/crawler/site_graph.rs@@ -16,12 +16,13 @@//! In-memory graph that the worker constructs for the site during crawl.-use hashbrown::{HashMap, HashSet};+use std::collections::{BTreeMap, BTreeSet};+use url::Url;use crate::{hyperloglog::HyperLogLog, kahan_sum::KahanSum};-#[derive(Clone, Debug, PartialEq, Eq, Hash)]+#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]pub struct Node {url: Url,}@@ -38,13 +39,13 @@ impl From<Node> for Url {}}-#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]pub struct NodeRef(usize);#[derive(Default)]pub struct SiteGraph {nodes: Vec<Node>,- node_refs: HashMap<Node, NodeRef>,+ node_refs: BTreeMap<Node, NodeRef>,edges: Vec<(NodeRef, NodeRef)>,}@@ -73,28 +74,28 @@ impl SiteGraph {self.nodes.get(node_ref.0)}- pub fn compute_centralities(&self) -> HashMap<NodeRef, f64> {- let mut counters: HashMap<_, _> = self+ pub fn compute_centralities(&self) -> BTreeMap<NodeRef, f64> {+ let mut counters: BTreeMap<_, _> = self.node_refs- .iter()- .map(|(_, node_ref)| {+ .values()+ .map(|node_ref| {let mut counter: HyperLogLog<64> = HyperLogLog::default();counter.add(node_ref.0 as u64);(*node_ref, counter)}).collect();- let mut centralities: HashMap<_, _> = self+ let mut centralities: BTreeMap<_, _> = self.node_refs- .iter()- .map(|(_, node_ref)| {+ .values()+ .map(|node_ref| {let kahan = KahanSum::default();(*node_ref, kahan)}).collect();- let mut changed_nodes: HashSet<_> = self.node_refs.values().copied().collect();+ let mut changed_nodes: BTreeSet<_> = self.node_refs.values().copied().collect();let mut t = 0;loop {@@ -103,7 +104,7 @@ impl SiteGraph {}let mut new_counters = counters.clone();- let mut new_changed_nodes = HashSet::new();+ let mut new_changed_nodes = BTreeSet::new();for (from, to) in &self.edges {if changed_nodes.contains(from) {diff --git a/core/src/fastfield_reader.rs b/core/src/fastfield_reader.rsindex 0fa273e1..311804dc 100644--- a/core/src/fastfield_reader.rs+++ b/core/src/fastfield_reader.rs@@ -20,7 +20,7 @@ use tantivy::{columnar::ColumnValues, DocId, SegmentId};use crate::{enum_map::EnumMap,- schema::{DataType, FastField, ALL_FIELDS},+ schema::{DataType, FastField, FieldMapping},};#[derive(Default, Clone)]@@ -48,7 +48,7 @@ impl FastFieldReader {let mut field_readers = Vec::new();- for field in ALL_FIELDS.iter().filter_map(|field| field.as_fast()) {+ for field in FieldMapping::all().filter_map(|field| field.as_fast()) {let field_reader = match field.data_type() {DataType::U64 => {let reader = fastfield_readers.u64(field.name()).unwrap();diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rsindex 7820a432..5b55d8f9 100644--- a/core/src/inverted_index.rs+++ b/core/src/inverted_index.rs@@ -43,7 +43,7 @@ use crate::query::Query;use crate::ranking::initial::Score;use crate::ranking::pipeline::RankingWebsite;use crate::ranking::SignalAggregator;-use crate::schema::{FastField, Field, TextField, ALL_FIELDS};+use crate::schema::{FastField, Field, FieldMapping, TextField};use crate::search_ctx::Ctx;use crate::snippet::TextSnippet;use crate::tokenizer::{@@ -605,8 +605,8 @@ impl From<TantivyDocument> for RetrievedWebpage {let mut webpage = RetrievedWebpage::default();for value in doc.field_values() {- match ALL_FIELDS[value.field.field_id() as usize] {- Field::Text(TextField::Title) => {+ match FieldMapping::get(value.field.field_id() as usize).copied() {+ Some(Field::Text(TextField::Title)) => {webpage.title = value.value().as_value()@@ -614,7 +614,7 @@ impl From<TantivyDocument> for RetrievedWebpage {.expect("Title field should be text").to_string();}- Field::Text(TextField::StemmedCleanBody) => {+ Some(Field::Text(TextField::StemmedCleanBody)) => {webpage.body = value.value().as_value()@@ -622,7 +622,7 @@ impl From<TantivyDocument> for RetrievedWebpage {.expect("Body field should be text").to_string();}- Field::Text(TextField::Description) => {+ Some(Field::Text(TextField::Description)) => {let desc = value.value().as_value()@@ -632,7 +632,7 @@ impl From<TantivyDocument> for RetrievedWebpage {webpage.description = if desc.is_empty() { None } else { Some(desc) }}- Field::Text(TextField::Url) => {+ Some(Field::Text(TextField::Url)) => {webpage.url = value.value().as_value()@@ -640,7 +640,7 @@ impl From<TantivyDocument> for RetrievedWebpage {.expect("Url field should be text").to_string();}- Field::Fast(FastField::LastUpdated) => {+ Some(Field::Fast(FastField::LastUpdated)) => {webpage.updated_time = {let timestamp = value.value().as_value().as_u64().unwrap() as i64;if timestamp == 0 {@@ -650,7 +650,7 @@ impl From<TantivyDocument> for RetrievedWebpage {}}}- Field::Text(TextField::AllBody) => {+ Some(Field::Text(TextField::AllBody)) => {webpage.dirty_body = value.value().as_value()@@ -658,13 +658,13 @@ impl From<TantivyDocument> for RetrievedWebpage {.expect("All body field should be text").to_string();}- Field::Fast(FastField::Region) => {+ Some(Field::Fast(FastField::Region)) => {webpage.region = {let id = value.value().as_value().as_u64().unwrap();Region::from_id(id)}}- Field::Text(TextField::DmozDescription) => {+ Some(Field::Text(TextField::DmozDescription)) => {let desc = value.value().as_value()@@ -674,7 +674,7 @@ impl From<TantivyDocument> for RetrievedWebpage {webpage.dmoz_description = if desc.is_empty() { None } else { Some(desc) }}- Field::Text(TextField::SchemaOrgJson) => {+ Some(Field::Text(TextField::SchemaOrgJson)) => {let json = value.value().as_value()@@ -684,15 +684,15 @@ impl From<TantivyDocument> for RetrievedWebpage {webpage.schema_org = serde_json::from_str(&json).unwrap_or_default();}- Field::Fast(FastField::LikelyHasAds) => {+ Some(Field::Fast(FastField::LikelyHasAds)) => {webpage.likely_has_ads =value.value().as_value().as_u64().unwrap_or_default() != 0;}- Field::Fast(FastField::LikelyHasPaywall) => {+ Some(Field::Fast(FastField::LikelyHasPaywall)) => {webpage.likely_has_paywall =value.value().as_value().as_u64().unwrap_or_default() != 0;}- Field::Text(TextField::RecipeFirstIngredientTagId) => {+ Some(Field::Text(TextField::RecipeFirstIngredientTagId)) => {let tag_id = value.value().as_value()diff --git a/core/src/query/parser.rs b/core/src/query/parser.rsindex 18b70777..7da75358 100644--- a/core/src/query/parser.rs+++ b/core/src/query/parser.rs@@ -22,7 +22,7 @@ use tantivy::{use crate::{bangs::BANG_PREFIXES,floor_char_boundary,- schema::{Field, TextField, ALL_FIELDS},+ schema::{Field, FieldMapping, TextField},};#[derive(Debug, Clone)]@@ -103,10 +103,10 @@ fn simple_into_tantivy(.iter().filter(|field| {matches!(- ALL_FIELDS[field.field_id() as usize],- Field::Text(TextField::AllBody)- | Field::Text(TextField::Title)- | Field::Text(TextField::Url)+ FieldMapping::get(field.field_id() as usize),+ Some(Field::Text(TextField::AllBody))+ | Some(Field::Text(TextField::Title))+ | Some(Field::Text(TextField::Url)))}).copied()@@ -150,22 +150,23 @@ impl Term {Term::Phrase(phrase) => {let mut phrases = Vec::with_capacity(fields.len());- for field in fields+ for (field, tv_field) in fields.iter()- .filter(|field| ALL_FIELDS[field.field_id() as usize].is_searchable())- .filter(|field| ALL_FIELDS[field.field_id() as usize].has_pos())+ .filter_map(|tv_field| {+ FieldMapping::get(tv_field.field_id() as usize)+ .map(|mapped| (mapped, *tv_field))+ })+ .filter(|(field, _)| field.is_searchable())+ .filter(|(field, _)| field.has_pos()){- let mut processed_terms = Term::process_tantivy_term(phrase, *field);+ let mut processed_terms = Term::process_tantivy_term(phrase, tv_field);if processed_terms.is_empty() {continue;}if processed_terms.len() == 1 {- let options = ALL_FIELDS[field.field_id() as usize]- .as_text()- .unwrap()- .index_option();+ let options = field.as_text().unwrap().index_option();phrases.push((Occur::Should,@@ -196,8 +197,8 @@ impl Term {.iter().find(|field| {matches!(- ALL_FIELDS[field.field_id() as usize],- Field::Text(TextField::Title)+ FieldMapping::get(field.field_id() as usize),+ Some(Field::Text(TextField::Title)))}).unwrap();@@ -209,8 +210,8 @@ impl Term {.iter().find(|field| {matches!(- ALL_FIELDS[field.field_id() as usize],- Field::Text(TextField::AllBody)+ FieldMapping::get(field.field_id() as usize),+ Some(Field::Text(TextField::AllBody)))}).unwrap();@@ -222,8 +223,8 @@ impl Term {.iter().find(|field| {matches!(- ALL_FIELDS[field.field_id() as usize],- Field::Text(TextField::Url)+ FieldMapping::get(field.field_id() as usize),+ Some(Field::Text(TextField::Url)))}).unwrap();@@ -247,7 +248,11 @@ impl Term {) -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {fields.iter()- .filter(|field| ALL_FIELDS[field.field_id() as usize].is_searchable())+ .filter_map(|tv_field| {+ FieldMapping::get(tv_field.field_id() as usize)+ .filter(|field| field.is_searchable())+ .map(|_| tv_field)+ }).map(|field| (Occur::Should, Term::tantivy_text_query(field, &term.0))).collect()}@@ -260,8 +265,8 @@ impl Term {.iter().filter(|field| {matches!(- ALL_FIELDS[field.field_id() as usize],- Field::Text(TextField::UrlForSiteOperator)+ FieldMapping::get(field.field_id() as usize),+ Some(Field::Text(TextField::UrlForSiteOperator)))}).map(|field| {@@ -293,7 +298,8 @@ impl Term {) -> Box<dyn tantivy::query::Query + 'static> {let mut processed_terms = Term::process_tantivy_term(term, *field);- let option = ALL_FIELDS[field.field_id() as usize]+ let option = FieldMapping::get(field.field_id() as usize)+ .unwrap().as_text().unwrap().index_option();@@ -323,9 +329,9 @@ impl Term {term: &str,tantivy_field: tantivy::schema::Field,) -> Vec<tantivy::Term> {- match ALL_FIELDS[tantivy_field.field_id() as usize] {- Field::Fast(_) => vec![tantivy::Term::from_field_text(tantivy_field, term)],- Field::Text(text_field) => {+ match FieldMapping::get(tantivy_field.field_id() as usize) {+ Some(Field::Fast(_)) => vec![tantivy::Term::from_field_text(tantivy_field, term)],+ Some(Field::Text(text_field)) => {let mut terms: Vec<tantivy::Term> = Vec::new();let mut tokenizer = text_field.query_tokenizer();let mut token_stream = tokenizer.token_stream(term);@@ -336,6 +342,7 @@ impl Term {terms}+ None => vec![],}}}diff --git a/core/src/query/pattern_query/weight.rs b/core/src/query/pattern_query/weight.rsindex 722da0a5..90b9d08a 100644--- a/core/src/query/pattern_query/weight.rs+++ b/core/src/query/pattern_query/weight.rs@@ -25,7 +25,7 @@ use tantivy::{use crate::{fastfield_reader::FastFieldReader,ranking::bm25::Bm25Weight,- schema::{FastField, Field, TextField, ALL_FIELDS},+ schema::{FastField, Field, FieldMapping, TextField},};use super::scorer::{@@ -64,9 +64,11 @@ impl FastSiteDomainPatternWeight {let fieldnorm_reader = self.fieldnorm_reader(reader)?;- let field_no_tokenizer = match ALL_FIELDS[self.field.field_id() as usize] {- Field::Text(TextField::UrlForSiteOperator) => Field::Text(TextField::SiteNoTokenizer),- Field::Text(TextField::Domain) => Field::Text(TextField::DomainNoTokenizer),+ let field_no_tokenizer = match FieldMapping::get(self.field.field_id() as usize) {+ Some(Field::Text(TextField::UrlForSiteOperator)) => {+ Field::Text(TextField::SiteNoTokenizer)+ }+ Some(Field::Text(TextField::Domain)) => Field::Text(TextField::DomainNoTokenizer),_ => unreachable!(),};@@ -168,23 +170,29 @@ impl PatternWeight {return Ok(None);}- let num_tokens_fastfield = match &ALL_FIELDS[self.field.field_id() as usize] {- Field::Text(TextField::Title) => Ok(FastField::NumTitleTokens),- Field::Text(TextField::CleanBody) => Ok(FastField::NumCleanBodyTokens),- Field::Text(TextField::Url) => Ok(FastField::NumUrlTokens),- Field::Text(TextField::Domain) => Ok(FastField::NumDomainTokens),- Field::Text(TextField::UrlForSiteOperator) => {+ let num_tokens_fastfield = match FieldMapping::get(self.field.field_id() as usize) {+ Some(Field::Text(TextField::Title)) => Ok(FastField::NumTitleTokens),+ Some(Field::Text(TextField::CleanBody)) => Ok(FastField::NumCleanBodyTokens),+ Some(Field::Text(TextField::Url)) => Ok(FastField::NumUrlTokens),+ Some(Field::Text(TextField::Domain)) => Ok(FastField::NumDomainTokens),+ Some(Field::Text(TextField::UrlForSiteOperator)) => {Ok(FastField::NumUrlForSiteOperatorTokens)}- Field::Text(TextField::Description) => Ok(FastField::NumDescriptionTokens),- Field::Text(TextField::MicroformatTags) => Ok(FastField::NumMicroformatTagsTokens),- Field::Text(TextField::FlattenedSchemaOrgJson) => {+ Some(Field::Text(TextField::Description)) => Ok(FastField::NumDescriptionTokens),+ Some(Field::Text(TextField::MicroformatTags)) => {+ Ok(FastField::NumMicroformatTagsTokens)+ }+ Some(Field::Text(TextField::FlattenedSchemaOrgJson)) => {Ok(FastField::NumFlattenedSchemaTokens)}- field => Err(TantivyError::InvalidArgument(format!(+ Some(field) => Err(TantivyError::InvalidArgument(format!("{} is not supported in pattern query",field.name()))),+ None => Err(TantivyError::InvalidArgument(format!(+ "Field with id {} is not supported in pattern query",+ self.field.field_id()+ ))),}?;// "*" matches everythingdiff --git a/core/src/schema.rs b/core/src/schema.rsindex 63b87caa..a714799b 100644--- a/core/src/schema.rs+++ b/core/src/schema.rs@@ -292,7 +292,7 @@ pub enum Field {Text(TextField),}-pub static ALL_FIELDS: [Field; 65] = [+static ALL_FIELDS: [Field; 65] = [Field::Text(TextField::Title),Field::Text(TextField::CleanBody),Field::Text(TextField::StemmedTitle),@@ -361,6 +361,20 @@ pub static ALL_FIELDS: [Field; 65] = [Field::Fast(FastField::LikelyHasPaywall),];+pub struct FieldMapping;++impl FieldMapping {+ #[inline]+ pub fn get(field_id: usize) -> Option<&'static Field> {+ ALL_FIELDS.get(field_id)+ }++ #[inline]+ pub fn all() -> impl Iterator<Item = &'static Field> {+ ALL_FIELDS.iter()+ }+}+impl Field {fn default_text_options(&self) -> tantivy::schema::TextOptions {let tokenizer = self.as_text().unwrap().indexing_tokenizer();diff --git a/core/src/similar_hosts.rs b/core/src/similar_hosts.rsindex c34a0042..54797472 100644--- a/core/src/similar_hosts.rs+++ b/core/src/similar_hosts.rs@@ -156,6 +156,7 @@ impl SimilarHostsFinder {scored_nodes.into_iter().filter(|ScoredNodeID { node_id, score: _ }| {+ // remove dead links (nodes without outgoing edges might be dead links)!self.webgraph.raw_outgoing_edges(node_id).is_empty()}).filter_map(|ScoredNodeID { node_id, score }| {diff --git a/core/src/webpage/html/into_tantivy.rs b/core/src/webpage/html/into_tantivy.rsindex 0ef4f5e7..9c09c57f 100644--- a/core/src/webpage/html/into_tantivy.rs+++ b/core/src/webpage/html/into_tantivy.rs@@ -17,7 +17,7 @@use crate::{ceil_char_boundary,prehashed::hash,- schema::{FastField, TextField},+ schema::{FastField, FieldMapping, TextField},simhash, split_u128, tokenizer,webpage::url_ext::UrlExt,Error, Result,@@ -30,7 +30,7 @@ use whatlang::Lang;use super::{find_recipe_first_ingredient_tag_id, schema_org, Html};-use crate::schema::{Field, ALL_FIELDS, FLOAT_SCALING};+use crate::schema::{Field, FLOAT_SCALING};impl Html {fn pretokenize_title(&self) -> Result<PreTokenizedString> {@@ -197,7 +197,10 @@ impl Html {let domain_hash = split_u128(hash(self.url().root_domain().unwrap_or_default()).0);let title_hash = split_u128(hash(self.title().unwrap_or_default()).0);- for field in &ALL_FIELDS {+ for field in schema+ .fields()+ .filter_map(|(field, _)| FieldMapping::get(field.field_id() as usize))+ {let tantivy_field = schema.get_field(field.name()).unwrap_or_else(|_| panic!("Unknown field: {}", field.name()));diff --git a/frontend/src/lib/components/Article.svelte b/frontend/src/lib/components/Article.svelteindex 051b2de2..2a118791 100644--- a/frontend/src/lib/components/Article.svelte+++ b/frontend/src/lib/components/Article.svelte@@ -1,5 +1,5 @@<article- class="prose prose-sm mx-auto py-20 leading-6 prose-headings:text-center prose-headings:font-medium prose-h4:float-left prose-h4:my-0 prose-h4:mr-3 prose-h4:leading-6"+ class="px-5 prose prose-sm mx-auto py-20 leading-6 prose-headings:text-center prose-headings:font-medium prose-h4:float-left prose-h4:my-0 prose-h4:mr-3 prose-h4:leading-6"><slot /></article>diff --git a/frontend/tsconfig.json b/frontend/tsconfig.jsonindex 794b95b6..12eb87b8 100644--- a/frontend/tsconfig.json+++ b/frontend/tsconfig.json@@ -1,6 +1,7 @@{"extends": "./.svelte-kit/tsconfig.json","compilerOptions": {+ "outDir": "build","allowJs": true,"checkJs": true,"esModuleInterop": true,@@ -14,4 +15,4 @@//// If you want to overwrite includes/excludes, make sure to copy over the relevant includes/excludes// from the referenced tsconfig.json - TypeScript does not merge them in-}+}\ No newline at end of file