web search done right
update tantivy
Mikkel Denkera0b48e2b2b31
diff --git a/Cargo.lock b/Cargo.lockindex 9cd8bbed..a9e764aa 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -1445,9 +1445,9 @@ dependencies = [[[package]]name = "fs4"-version = "0.6.6"+version = "0.7.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "2eeb4ed9e12f43b7fa0baae3f9cdda28352770132ef2e09a23760c29cae8bd47"+checksum = "29f9df8a11882c4e3335eb2d18a0137c505d9ca927470b0cac9c6f0ae07d28f7"dependencies = ["rustix","windows-sys 0.48.0",@@ -2292,9 +2292,9 @@ dependencies = [[[package]]name = "lru"-version = "0.11.1"+version = "0.12.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a4a83fb7698b3643a0e34f9ae6f2e8f0178c0fd42f8b59d493aa271ff3a5bf21"+checksum = "1efa59af2ddfad1854ae27d75009d538d0998b4b2fd47083e743ac1a10e46c60"dependencies = ["hashbrown 0.14.0",]@@ -2460,9 +2460,9 @@ dependencies = [[[package]]name = "memmap2"-version = "0.7.1"+version = "0.9.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "f49388d20533534cd19360ad3d6a7dadc885944aa802ba3995040c5ec11288c6"+checksum = "deaba38d7abf1d4cca21cc89e932e542ba2b9258664d2a9ef0e61512039c9375"dependencies = ["libc",]@@ -2868,8 +2868,7 @@ checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"[[package]]name = "ownedbytes"version = "0.6.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "6e8a72b918ae8198abb3a18c190288123e1d442b6b9a7d709305fd194688b4b7"+source = "git+https://github.com/quickwit-oss/tantivy?rev=182f58cea#182f58cea6bb012fc81b20a92317484e861e0e80"dependencies = ["stable_deref_trait",]@@ -4317,7 +4316,6 @@ dependencies = ["num_cpus","once_cell","optics",- "parking_lot","parse_wiki_text","proptest","proptest-derive",@@ -4460,8 +4458,7 @@ checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"[[package]]name = "tantivy"version = "0.21.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "c1d4675fed6fe2218ce11445374e181e864a8ffd0f28e7e0591ccfc38cd000ae"+source = "git+https://github.com/quickwit-oss/tantivy?rev=182f58cea#182f58cea6bb012fc81b20a92317484e861e0e80"dependencies = ["aho-corasick 1.0.4","arc-swap",@@ -4474,6 +4471,7 @@ dependencies = ["crossbeam-channel","downcast-rs","fastdivide",+ "fnv","fs4","htmlescape","itertools 0.11.0",@@ -4482,7 +4480,7 @@ dependencies = ["lru","lz4_flex 0.11.1","measure_time",- "memmap2 0.7.1",+ "memmap2 0.9.0","murmurhash32","num_cpus","once_cell",@@ -4512,8 +4510,7 @@ dependencies = [[[package]]name = "tantivy-bitpacker"version = "0.5.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "cecb164321482301f514dd582264fa67f70da2d7eb01872ccd71e35e0d96655a"+source = "git+https://github.com/quickwit-oss/tantivy?rev=182f58cea#182f58cea6bb012fc81b20a92317484e861e0e80"dependencies = ["bitpacking",]@@ -4521,8 +4518,7 @@ dependencies = [[[package]]name = "tantivy-columnar"version = "0.2.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "8d85f8019af9a78b3118c11298b36ffd21c2314bd76bbcd9d12e00124cbb7e70"+source = "git+https://github.com/quickwit-oss/tantivy?rev=182f58cea#182f58cea6bb012fc81b20a92317484e861e0e80"dependencies = ["fastdivide","fnv",@@ -4537,8 +4533,7 @@ dependencies = [[[package]]name = "tantivy-common"version = "0.6.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "af4a3a975e604a2aba6b1106a04505e1e7a025e6def477fab6e410b4126471e1"+source = "git+https://github.com/quickwit-oss/tantivy?rev=182f58cea#182f58cea6bb012fc81b20a92317484e861e0e80"dependencies = ["async-trait","byteorder",@@ -4561,8 +4556,7 @@ dependencies = [[[package]]name = "tantivy-query-grammar"version = "0.21.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1d39c5a03100ac10c96e0c8b07538e2ab8b17da56434ab348309b31f23fada77"+source = "git+https://github.com/quickwit-oss/tantivy?rev=182f58cea#182f58cea6bb012fc81b20a92317484e861e0e80"dependencies = ["nom",]@@ -4570,19 +4564,17 @@ dependencies = [[[package]]name = "tantivy-sstable"version = "0.2.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "fc0c1bb43e5e8b8e05eb8009610344dbf285f06066c844032fbb3e546b3c71df"+source = "git+https://github.com/quickwit-oss/tantivy?rev=182f58cea#182f58cea6bb012fc81b20a92317484e861e0e80"dependencies = ["tantivy-common","tantivy-fst",- "zstd 0.12.4",+ "zstd 0.13.0",][[package]]name = "tantivy-stacker"version = "0.2.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b2c078595413f13f218cf6f97b23dcfd48936838f1d3d13a1016e05acd64ed6c"+source = "git+https://github.com/quickwit-oss/tantivy?rev=182f58cea#182f58cea6bb012fc81b20a92317484e861e0e80"dependencies = ["murmurhash32","tantivy-common",@@ -4591,8 +4583,7 @@ dependencies = [[[package]]name = "tantivy-tokenizer-api"version = "0.2.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "347b6fb212b26d3505d224f438e3c4b827ab8bd847fe9953ad5ac6b8f9443b66"+source = "git+https://github.com/quickwit-oss/tantivy?rev=182f58cea#182f58cea6bb012fc81b20a92317484e861e0e80"dependencies = ["serde",]@@ -5705,11 +5696,11 @@ dependencies = [[[package]]name = "zstd"-version = "0.12.4"+version = "0.13.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1a27595e173641171fc74a1232b7b1c7a7cb6e18222c11e9dfb9888fa424c53c"+checksum = "bffb3309596d527cfcba7dfc6ed6052f1d39dfbd7c867aa2e865e4a449c10110"dependencies = [- "zstd-safe 6.0.6",+ "zstd-safe 7.0.0",][[package]]@@ -5724,11 +5715,10 @@ dependencies = [[[package]]name = "zstd-safe"-version = "6.0.6"+version = "7.0.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "ee98ffd0b48ee95e6c5168188e44a54550b1564d9d530ee21d5f0eaed1069581"+checksum = "43747c7422e2924c11144d5229878b98180ef8b06cca4ab5af37afc8a8d8ea3e"dependencies = [- "libc","zstd-sys",]diff --git a/Cargo.toml b/Cargo.tomlindex 3db2bda9..ce29614f 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -58,7 +58,6 @@ mime = "0.3.17"min-max-heap = "1.3.0"num_cpus = "1.15.0"once_cell = "1.13.1"-parking_lot = "0.12.1"parse_wiki_text = "0.1.5"proptest = "1.2.0"proptest-derive = "0.3.0"@@ -91,7 +90,7 @@ scylla = "0.7.0"serde = {version = "1.0.137", features = ["rc", "derive"]}serde_json = "1.0.81"serde_urlencoded = "0.7.1"-tantivy = "0.21.0"+tantivy = {git = "https://github.com/quickwit-oss/tantivy", rev = "182f58cea"}tch = "0.13.0"thiserror = "1.0.31"tikv-jemallocator = "0.5"diff --git a/configs/indexer/local.toml b/configs/indexer/local.tomlindex 4636c551..0a80ea5d 100644--- a/configs/indexer/local.toml+++ b/configs/indexer/local.toml@@ -1,11 +1,17 @@-centrality_store_path = "./data/centrality"+host_centrality_store_path = "./data/centrality"limit_warc_files = 10-minimum_clean_words = 40+# minimum_clean_words = 40[warc_source]# type = "Local"# folder = "./data"# names = ["sample.warc.gz"]-type = "HTTP"-base_url = "https://data.commoncrawl.org/"-warc_paths_file = "data/warc.paths"+# type = "HTTP"+# base_url = "https://data.commoncrawl.org/"+# warc_paths_file = "data/warc.paths"+access_key = "crawl_nOf7tywztpjg2"+bucket = "crawl"+endpoint = "http://s3.trystract.com"+folder = "400m_try11"+secret_key = "4Kzo0CPdQYnlFHRgwKpUA7mACzSLsA6OpG6qDFl4"+type = "S3"diff --git a/core/Cargo.toml b/core/Cargo.tomlindex 40ebdc5b..926df329 100644--- a/core/Cargo.toml+++ b/core/Cargo.toml@@ -66,7 +66,6 @@ min-max-heap = {workspace = true}num_cpus = {workspace = true}once_cell = {workspace = true}optics = {path = "../optics"}-parking_lot = {workspace = true}parse_wiki_text = {workspace = true}publicsuffix = {workspace = true}quick-xml = {workspace = true}diff --git a/core/src/api/user_count.rs b/core/src/api/user_count.rsindex c602c7f0..6635b2e6 100644--- a/core/src/api/user_count.rs+++ b/core/src/api/user_count.rs@@ -18,9 +18,9 @@ use std::sync::Arc;use anyhow::{anyhow, Result};use chrono::{NaiveDateTime, Utc};-use parking_lot::Mutex;use ring::rand::SecureRandom;use ring::{digest, pbkdf2, rand};+use std::sync::Mutex;use crate::hyperloglog::HyperLogLog;use crate::metrics::Counter;@@ -131,10 +131,16 @@ impl<F: Frequency> UserCount<F> {}pub fn inc<T: serde::Serialize>(&self, user_id: &T) -> Result<()> {- self.inner.lock().inc(user_id)+ self.inner+ .lock()+ .unwrap_or_else(|e| e.into_inner())+ .inc(user_id)}pub fn metric(&self) -> Counter {- self.inner.lock().metric()+ self.inner+ .lock()+ .unwrap_or_else(|e| e.into_inner())+ .metric()}}diff --git a/core/src/crawler/router.rs b/core/src/crawler/router.rsindex 75005ee4..2f9cb68f 100644--- a/core/src/crawler/router.rs+++ b/core/src/crawler/router.rs@@ -22,7 +22,7 @@ impl RemoteCoordinator {Ok(sonic::service::ResilientConnection::create_with_timeout(self.addr,- Duration::from_secs(3600),+ Duration::from_secs(60),retry,).await?)@@ -32,7 +32,7 @@ impl RemoteCoordinator {let conn = self.conn().await?;let response = conn- .send_with_timeout(&GetJobs { num_jobs }, Duration::from_secs(60 * 60))+ .send_with_timeout(&GetJobs { num_jobs }, Duration::from_secs(90)).await?;Ok(response)@@ -41,7 +41,7 @@ impl RemoteCoordinator {async fn insert_urls(&self, urls: HashMap<Domain, Vec<UrlToInsert>>) -> Result<()> {let conn = self.conn().await?;- conn.send_with_timeout(&InsertUrls { urls }, Duration::from_secs(60 * 60))+ conn.send_with_timeout(&InsertUrls { urls }, Duration::from_secs(90)).await?;Ok(())@@ -50,7 +50,7 @@ impl RemoteCoordinator {async fn mark_jobs_complete(&self, domains: Vec<DomainCrawled>) -> Result<()> {let conn = self.conn().await?;- conn.send_with_timeout(&MarkJobsComplete { domains }, Duration::from_secs(60 * 60))+ conn.send_with_timeout(&MarkJobsComplete { domains }, Duration::from_secs(90)).await?;Ok(())diff --git a/core/src/crawler/worker.rs b/core/src/crawler/worker.rsindex da6f235a..332524ba 100644--- a/core/src/crawler/worker.rs+++ b/core/src/crawler/worker.rs@@ -115,7 +115,7 @@ impl WorkerThread {Ok(sonic::service::ResilientConnection::create_with_timeout(router,- Duration::from_secs(3600),+ Duration::from_secs(90),retry,).await?)@@ -153,7 +153,7 @@ impl WorkerThread {responses: results,num_jobs: self.num_jobs_per_fetch,},- Duration::from_secs(60 * 60),+ Duration::from_secs(90),).await;diff --git a/core/src/entity_index/mod.rs b/core/src/entity_index/mod.rsindex 9f37a8a8..85aacad8 100644--- a/core/src/entity_index/mod.rs+++ b/core/src/entity_index/mod.rs@@ -28,7 +28,7 @@ use tantivy::{query::{BooleanQuery, MoreLikeThisQuery, Occur, QueryClone, TermQuery},schema::{BytesOptions, IndexRecordOption, Schema, TextFieldIndexing, TextOptions},tokenizer::Tokenizer,- DocAddress, IndexReader, IndexWriter, Searcher, Term,+ DocAddress, IndexReader, IndexWriter, Searcher, TantivyDocument, Term,};use tracing::info;use url::Url;@@ -92,8 +92,8 @@ fn schema() -> Schema {builder.build()}-fn entity_to_tantivy(entity: Entity, schema: &tantivy::schema::Schema) -> tantivy::Document {- let mut doc = tantivy::Document::new();+fn entity_to_tantivy(entity: Entity, schema: &tantivy::schema::Schema) -> TantivyDocument {+ let mut doc = TantivyDocument::new();doc.add_text(schema.get_field("title").unwrap(), entity.title);doc.add_text(@@ -373,11 +373,11 @@ impl EntityIndex {let info = self.schema.get_field("info").unwrap();let links = self.schema.get_field("links").unwrap();- let doc = searcher.doc(doc_address).unwrap();+ let doc: TantivyDocument = searcher.doc(doc_address).unwrap();let title = doc.get_first(title).and_then(|val| match val {- tantivy::schema::Value::Str(string) => Some(string.clone()),+ tantivy::schema::OwnedValue::Str(string) => Some(string.clone()),_ => None,}).unwrap();@@ -385,7 +385,7 @@ impl EntityIndex {let entity_abstract = doc.get_first(entity_abstract).and_then(|val| match val {- tantivy::schema::Value::Str(string) => Some(string.clone()),+ tantivy::schema::OwnedValue::Str(string) => Some(string.clone()),_ => None,}).unwrap();@@ -394,7 +394,7 @@ impl EntityIndex {bincode::deserialize(doc.get_first(info).and_then(|val| match val {- tantivy::schema::Value::Bytes(bytes) => Some(bytes),+ tantivy::schema::OwnedValue::Bytes(bytes) => Some(bytes),_ => None,}).unwrap(),@@ -420,7 +420,7 @@ impl EntityIndex {bincode::deserialize(doc.get_first(links).and_then(|val| match val {- tantivy::schema::Value::Bytes(bytes) => Some(bytes),+ tantivy::schema::OwnedValue::Bytes(bytes) => Some(bytes),_ => None,}).unwrap(),diff --git a/core/src/fastfield_reader.rs b/core/src/fastfield_reader.rsindex 523cd9ef..0fa273e1 100644--- a/core/src/fastfield_reader.rs+++ b/core/src/fastfield_reader.rs@@ -54,10 +54,6 @@ impl FastFieldReader {let reader = fastfield_readers.u64(field.name()).unwrap();FieldReader::U64(reader.values)}- DataType::F64 => {- let reader = fastfield_readers.f64(field.name()).unwrap();- FieldReader::F64(reader.values)- }};field_readers.push((field, field_reader));@@ -82,7 +78,6 @@ impl FastFieldReader {pub enum FieldValue {U64(u64),U64s(Vec<u64>),- F64(f64),}impl From<FieldValue> for Option<Vec<u64>> {@@ -105,14 +100,12 @@ impl From<FieldValue> for Option<u64> {pub enum FieldReader {U64(Arc<dyn ColumnValues<u64>>),- F64(Arc<dyn ColumnValues<f64>>),}impl FieldReader {pub fn get(&self, doc: &DocId) -> FieldValue {match self {FieldReader::U64(reader) => FieldValue::U64(reader.get_val(*doc)),- FieldReader::F64(reader) => FieldValue::F64(reader.get_val(*doc)),}}}diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rsindex 05680f33..9c1450c6 100644--- a/core/src/inverted_index.rs+++ b/core/src/inverted_index.rs@@ -30,9 +30,9 @@ use serde::{Deserialize, Serialize};use tantivy::collector::Count;use tantivy::directory::MmapDirectory;use tantivy::merge_policy::NoMergePolicy;-use tantivy::schema::Schema;+use tantivy::schema::{Schema, Value};use tantivy::tokenizer::TokenizerManager;-use tantivy::{Document, IndexReader, IndexWriter, SegmentMeta};+use tantivy::{IndexReader, IndexWriter, SegmentMeta, TantivyDocument};use url::Url;use crate::collector::{Hashes, MainCollector};@@ -303,7 +303,7 @@ impl InvertedIndex {pub fn website_host_node(&self, website: &WebsitePointer) -> Result<Option<NodeID>> {let searcher = self.reader.searcher();- let doc = searcher.doc(website.address.into())?;+ let doc: TantivyDocument = searcher.doc(website.address.into())?;let field1 = self.schema()@@ -414,7 +414,7 @@ impl InvertedIndex {doc_address: DocAddress,searcher: &tantivy::Searcher,) -> Result<RetrievedWebpage> {- let doc = searcher.doc(doc_address.into())?;+ let doc: TantivyDocument = searcher.doc(doc_address.into())?;Ok(RetrievedWebpage::from(doc))}@@ -560,30 +560,33 @@ impl RetrievedWebpage {}}-impl From<Document> for RetrievedWebpage {- fn from(doc: Document) -> Self {+impl From<TantivyDocument> for RetrievedWebpage {+ fn from(doc: TantivyDocument) -> Self {let mut webpage = RetrievedWebpage::default();for value in doc.field_values() {match ALL_FIELDS[value.field.field_id() as usize] {Field::Text(TextField::Title) => {webpage.title = value- .value- .as_text()+ .value()+ .as_value()+ .as_str().expect("Title field should be text").to_string();}Field::Text(TextField::StemmedCleanBody) => {webpage.body = value- .value- .as_text()+ .value()+ .as_value()+ .as_str().expect("Body field should be text").to_string();}Field::Text(TextField::Description) => {let desc = value- .value- .as_text()+ .value()+ .as_value()+ .as_str().expect("Description field should be text").to_string();@@ -591,14 +594,15 @@ impl From<Document> for RetrievedWebpage {}Field::Text(TextField::Url) => {webpage.url = value- .value- .as_text()+ .value()+ .as_value()+ .as_str().expect("Url field should be text").to_string();}Field::Fast(FastField::LastUpdated) => {webpage.updated_time = {- let timestamp = value.value.as_u64().unwrap() as i64;+ let timestamp = value.value().as_value().as_u64().unwrap() as i64;if timestamp == 0 {None} else {@@ -608,21 +612,23 @@ impl From<Document> for RetrievedWebpage {}Field::Text(TextField::AllBody) => {webpage.dirty_body = value- .value- .as_text()+ .value()+ .as_value()+ .as_str().expect("All body field should be text").to_string();}Field::Fast(FastField::Region) => {webpage.region = {- let id = value.value.as_u64().unwrap();+ let id = value.value().as_value().as_u64().unwrap();Region::from_id(id)}}Field::Text(TextField::DmozDescription) => {let desc = value- .value- .as_text()+ .value()+ .as_value()+ .as_str().expect("Dmoz description field should be text").to_string();@@ -630,8 +636,9 @@ impl From<Document> for RetrievedWebpage {}Field::Text(TextField::SchemaOrgJson) => {let json = value- .value- .as_text()+ .value()+ .as_value()+ .as_str().expect("Schema.org json field should be stored as text").to_string();diff --git a/core/src/schema.rs b/core/src/schema.rsindex 82027f54..7aefebdd 100644--- a/core/src/schema.rs+++ b/core/src/schema.rs@@ -521,7 +521,7 @@ impl Field {Field::Fast(FastField::TitleHash2) => {IndexingOption::Integer(NumericOptions::default().set_fast())}- Field::Fast(FastField::PreComputedScore) => IndexingOption::Float(+ Field::Fast(FastField::PreComputedScore) => IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed()@@ -619,7 +619,6 @@ pub fn create_schema() -> tantivy::schema::Schema {match field.options() {IndexingOption::Text(options) => builder.add_text_field(field.name(), options),IndexingOption::Integer(options) => builder.add_u64_field(field.name(), options),- IndexingOption::Float(options) => builder.add_f64_field(field.name(), options),};}@@ -629,12 +628,10 @@ pub fn create_schema() -> tantivy::schema::Schema {pub enum IndexingOption {Text(tantivy::schema::TextOptions),Integer(tantivy::schema::NumericOptions),- Float(tantivy::schema::NumericOptions),}pub enum DataType {U64,- F64,}impl FastField {@@ -665,7 +662,7 @@ impl FastField {FastField::UrlHash2 => DataType::U64,FastField::DomainHash1 => DataType::U64,FastField::DomainHash2 => DataType::U64,- FastField::PreComputedScore => DataType::F64,+ FastField::PreComputedScore => DataType::U64,FastField::HostNodeID1 => DataType::U64,FastField::HostNodeID2 => DataType::U64,FastField::SimHash => DataType::U64,diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rsindex edf41aa5..a7977af0 100644--- a/core/src/searcher/local.rs+++ b/core/src/searcher/local.rs@@ -17,6 +17,8 @@use std::collections::HashMap;use std::sync::Arc;+use tantivy::schema::Value;+use tantivy::TantivyDocument;use url::Url;use crate::config::{CollectorConfig, SnippetConfig};@@ -239,10 +241,10 @@ impl LocalSearcher {let schema = self.index.schema();for website in &mut ranking_websites {- let doc = ctx.tv_searcher.doc(website.pointer.address.into())?;+ let doc: TantivyDocument = ctx.tv_searcher.doc(website.pointer.address.into())?;website.title = Some(doc.get_first(schema.get_field(TextField::Title.name()).unwrap())- .map(|text| text.as_text().unwrap().to_string())+ .map(|text| text.as_value().as_str().unwrap().to_string()).unwrap_or_default(),);website.clean_body = Some(@@ -251,7 +253,7 @@ impl LocalSearcher {.get_field(TextField::StemmedCleanBody.name()).unwrap(),)- .map(|text| text.as_text().unwrap().to_string())+ .map(|text| text.as_value().as_str().unwrap().to_string()).unwrap_or_default(),);}diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rsindex a17be861..9487df6b 100644--- a/core/src/webpage/mod.rs+++ b/core/src/webpage/mod.rs@@ -27,7 +27,10 @@ use itertools::Itertools;use kuchiki::{iter::NodeEdge, traits::TendrilSink, NodeRef};use regex::Regex;use std::{collections::HashMap, panic, str::FromStr};-use tantivy::tokenizer::{PreTokenizedString, Tokenizer};+use tantivy::{+ tokenizer::{PreTokenizedString, Tokenizer},+ TantivyDocument,+};use url::Url;use whatlang::Lang;@@ -160,7 +163,7 @@ impl Webpage {})}- pub fn into_tantivy(self, schema: &tantivy::schema::Schema) -> Result<tantivy::Document> {+ pub fn into_tantivy(self, schema: &tantivy::schema::Schema) -> Result<TantivyDocument> {let region = Region::guess_from(&self);let dmoz_description = self.dmoz_description();@@ -226,11 +229,11 @@ impl Webpage {self.fetch_time_ms,);- doc.add_f64(+ doc.add_u64(schema.get_field(Field::Fast(FastField::PreComputedScore).name()).expect("failed to get pre_computed_score field"),- self.pre_computed_score,+ (self.pre_computed_score * FLOAT_SCALING as f64) as u64,);match &self.node_id {@@ -930,8 +933,8 @@ impl Html {PreTokenizedString { text, tokens }}- pub fn into_tantivy(self, schema: &tantivy::schema::Schema) -> Result<tantivy::Document> {- let mut doc = tantivy::Document::new();+ pub fn into_tantivy(self, schema: &tantivy::schema::Schema) -> Result<TantivyDocument> {+ let mut doc = TantivyDocument::new();let title = self.pretokenize_title()?;let all_text = self.pretokenize_all_text()?;diff --git a/core/src/webpage/schema_org/mod.rs b/core/src/webpage/schema_org/mod.rsindex bc3b27b9..036a4aa7 100644--- a/core/src/webpage/schema_org/mod.rs+++ b/core/src/webpage/schema_org/mod.rs@@ -224,6 +224,14 @@ pub fn parse(root: NodeRef) -> Vec<Item> {res.into_iter().map(Item::from).collect()}+pub(crate) fn flattened_json(schemas: Vec<Item>) -> Result<FlattenedJson> {+ let single_maps: Vec<_> = schemas+ .into_iter()+ .map(|item| item.into_single_map())+ .collect();+ FlattenedJson::new(&single_maps)+}+#[cfg(test)]mod tests {use kuchiki::traits::TendrilSink;@@ -519,11 +527,3 @@ mod tests {assert_eq!(&instructions, "Helt enkelt som navnet antyder, så kom alle ingredienserne i en stor gryde på én gang. Kog retten op, rør godt rundt i gryden og skru ned for varmen. Lad det simrekoge under låg i 10-12 minutter, til spaghettien er perfekt kogt – al dente med lidt bid i. Server med revet parmesan og basilikum.");}}--pub(crate) fn flattened_json(schemas: Vec<Item>) -> Result<FlattenedJson> {- let single_maps: Vec<_> = schemas- .into_iter()- .map(|item| item.into_single_map())- .collect();- FlattenedJson::new(&single_maps)-}diff --git a/kuchiki/src/lib.rs b/kuchiki/src/lib.rsindex a037f173..bbe14e5f 100644--- a/kuchiki/src/lib.rs+++ b/kuchiki/src/lib.rs@@ -18,6 +18,7 @@ mod node_data_ref;mod parser;mod select;mod serializer;+#[cfg(test)]mod tests;mod tree;