Skip to content

web search done right

update tantivy

Mikkel Denkera0b48e2b2b31
diff --git a/Cargo.lock b/Cargo.lock
index 9cd8bbed..a9e764aa 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1445,9 +1445,9 @@ dependencies = [
[[package]]
name = "fs4"
-version = "0.6.6"
+version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2eeb4ed9e12f43b7fa0baae3f9cdda28352770132ef2e09a23760c29cae8bd47"
+checksum = "29f9df8a11882c4e3335eb2d18a0137c505d9ca927470b0cac9c6f0ae07d28f7"
dependencies = [
"rustix",
"windows-sys 0.48.0",
@@ -2292,9 +2292,9 @@ dependencies = [
[[package]]
name = "lru"
-version = "0.11.1"
+version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4a83fb7698b3643a0e34f9ae6f2e8f0178c0fd42f8b59d493aa271ff3a5bf21"
+checksum = "1efa59af2ddfad1854ae27d75009d538d0998b4b2fd47083e743ac1a10e46c60"
dependencies = [
"hashbrown 0.14.0",
]
@@ -2460,9 +2460,9 @@ dependencies = [
[[package]]
name = "memmap2"
-version = "0.7.1"
+version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f49388d20533534cd19360ad3d6a7dadc885944aa802ba3995040c5ec11288c6"
+checksum = "deaba38d7abf1d4cca21cc89e932e542ba2b9258664d2a9ef0e61512039c9375"
dependencies = [
"libc",
]
@@ -2868,8 +2868,7 @@ checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
[[package]]
name = "ownedbytes"
version = "0.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6e8a72b918ae8198abb3a18c190288123e1d442b6b9a7d709305fd194688b4b7"
+source = "git+https://github.com/quickwit-oss/tantivy?rev=182f58cea#182f58cea6bb012fc81b20a92317484e861e0e80"
dependencies = [
"stable_deref_trait",
]
@@ -4317,7 +4316,6 @@ dependencies = [
"num_cpus",
"once_cell",
"optics",
- "parking_lot",
"parse_wiki_text",
"proptest",
"proptest-derive",
@@ -4460,8 +4458,7 @@ checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"
[[package]]
name = "tantivy"
version = "0.21.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c1d4675fed6fe2218ce11445374e181e864a8ffd0f28e7e0591ccfc38cd000ae"
+source = "git+https://github.com/quickwit-oss/tantivy?rev=182f58cea#182f58cea6bb012fc81b20a92317484e861e0e80"
dependencies = [
"aho-corasick 1.0.4",
"arc-swap",
@@ -4474,6 +4471,7 @@ dependencies = [
"crossbeam-channel",
"downcast-rs",
"fastdivide",
+ "fnv",
"fs4",
"htmlescape",
"itertools 0.11.0",
@@ -4482,7 +4480,7 @@ dependencies = [
"lru",
"lz4_flex 0.11.1",
"measure_time",
- "memmap2 0.7.1",
+ "memmap2 0.9.0",
"murmurhash32",
"num_cpus",
"once_cell",
@@ -4512,8 +4510,7 @@ dependencies = [
[[package]]
name = "tantivy-bitpacker"
version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cecb164321482301f514dd582264fa67f70da2d7eb01872ccd71e35e0d96655a"
+source = "git+https://github.com/quickwit-oss/tantivy?rev=182f58cea#182f58cea6bb012fc81b20a92317484e861e0e80"
dependencies = [
"bitpacking",
]
@@ -4521,8 +4518,7 @@ dependencies = [
[[package]]
name = "tantivy-columnar"
version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d85f8019af9a78b3118c11298b36ffd21c2314bd76bbcd9d12e00124cbb7e70"
+source = "git+https://github.com/quickwit-oss/tantivy?rev=182f58cea#182f58cea6bb012fc81b20a92317484e861e0e80"
dependencies = [
"fastdivide",
"fnv",
@@ -4537,8 +4533,7 @@ dependencies = [
[[package]]
name = "tantivy-common"
version = "0.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af4a3a975e604a2aba6b1106a04505e1e7a025e6def477fab6e410b4126471e1"
+source = "git+https://github.com/quickwit-oss/tantivy?rev=182f58cea#182f58cea6bb012fc81b20a92317484e861e0e80"
dependencies = [
"async-trait",
"byteorder",
@@ -4561,8 +4556,7 @@ dependencies = [
[[package]]
name = "tantivy-query-grammar"
version = "0.21.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d39c5a03100ac10c96e0c8b07538e2ab8b17da56434ab348309b31f23fada77"
+source = "git+https://github.com/quickwit-oss/tantivy?rev=182f58cea#182f58cea6bb012fc81b20a92317484e861e0e80"
dependencies = [
"nom",
]
@@ -4570,19 +4564,17 @@ dependencies = [
[[package]]
name = "tantivy-sstable"
version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fc0c1bb43e5e8b8e05eb8009610344dbf285f06066c844032fbb3e546b3c71df"
+source = "git+https://github.com/quickwit-oss/tantivy?rev=182f58cea#182f58cea6bb012fc81b20a92317484e861e0e80"
dependencies = [
"tantivy-common",
"tantivy-fst",
- "zstd 0.12.4",
+ "zstd 0.13.0",
]
[[package]]
name = "tantivy-stacker"
version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b2c078595413f13f218cf6f97b23dcfd48936838f1d3d13a1016e05acd64ed6c"
+source = "git+https://github.com/quickwit-oss/tantivy?rev=182f58cea#182f58cea6bb012fc81b20a92317484e861e0e80"
dependencies = [
"murmurhash32",
"tantivy-common",
@@ -4591,8 +4583,7 @@ dependencies = [
[[package]]
name = "tantivy-tokenizer-api"
version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "347b6fb212b26d3505d224f438e3c4b827ab8bd847fe9953ad5ac6b8f9443b66"
+source = "git+https://github.com/quickwit-oss/tantivy?rev=182f58cea#182f58cea6bb012fc81b20a92317484e861e0e80"
dependencies = [
"serde",
]
@@ -5705,11 +5696,11 @@ dependencies = [
[[package]]
name = "zstd"
-version = "0.12.4"
+version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a27595e173641171fc74a1232b7b1c7a7cb6e18222c11e9dfb9888fa424c53c"
+checksum = "bffb3309596d527cfcba7dfc6ed6052f1d39dfbd7c867aa2e865e4a449c10110"
dependencies = [
- "zstd-safe 6.0.6",
+ "zstd-safe 7.0.0",
]
[[package]]
@@ -5724,11 +5715,10 @@ dependencies = [
[[package]]
name = "zstd-safe"
-version = "6.0.6"
+version = "7.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ee98ffd0b48ee95e6c5168188e44a54550b1564d9d530ee21d5f0eaed1069581"
+checksum = "43747c7422e2924c11144d5229878b98180ef8b06cca4ab5af37afc8a8d8ea3e"
dependencies = [
- "libc",
"zstd-sys",
]
diff --git a/Cargo.toml b/Cargo.toml
index 3db2bda9..ce29614f 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -58,7 +58,6 @@ mime = "0.3.17"
min-max-heap = "1.3.0"
num_cpus = "1.15.0"
once_cell = "1.13.1"
-parking_lot = "0.12.1"
parse_wiki_text = "0.1.5"
proptest = "1.2.0"
proptest-derive = "0.3.0"
@@ -91,7 +90,7 @@ scylla = "0.7.0"
serde = {version = "1.0.137", features = ["rc", "derive"]}
serde_json = "1.0.81"
serde_urlencoded = "0.7.1"
-tantivy = "0.21.0"
+tantivy = {git = "https://github.com/quickwit-oss/tantivy", rev = "182f58cea"}
tch = "0.13.0"
thiserror = "1.0.31"
tikv-jemallocator = "0.5"
diff --git a/configs/indexer/local.toml b/configs/indexer/local.toml
index 4636c551..0a80ea5d 100644
--- a/configs/indexer/local.toml
+++ b/configs/indexer/local.toml
@@ -1,11 +1,17 @@
-centrality_store_path = "./data/centrality"
+host_centrality_store_path = "./data/centrality"
limit_warc_files = 10
-minimum_clean_words = 40
+# minimum_clean_words = 40
[warc_source]
# type = "Local"
# folder = "./data"
# names = ["sample.warc.gz"]
-type = "HTTP"
-base_url = "https://data.commoncrawl.org/"
-warc_paths_file = "data/warc.paths"
+# type = "HTTP"
+# base_url = "https://data.commoncrawl.org/"
+# warc_paths_file = "data/warc.paths"
+access_key = "crawl_nOf7tywztpjg2"
+bucket = "crawl"
+endpoint = "http://s3.trystract.com"
+folder = "400m_try11"
+secret_key = "4Kzo0CPdQYnlFHRgwKpUA7mACzSLsA6OpG6qDFl4"
+type = "S3"
diff --git a/core/Cargo.toml b/core/Cargo.toml
index 40ebdc5b..926df329 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -66,7 +66,6 @@ min-max-heap = {workspace = true}
num_cpus = {workspace = true}
once_cell = {workspace = true}
optics = {path = "../optics"}
-parking_lot = {workspace = true}
parse_wiki_text = {workspace = true}
publicsuffix = {workspace = true}
quick-xml = {workspace = true}
diff --git a/core/src/api/user_count.rs b/core/src/api/user_count.rs
index c602c7f0..6635b2e6 100644
--- a/core/src/api/user_count.rs
+++ b/core/src/api/user_count.rs
@@ -18,9 +18,9 @@ use std::sync::Arc;
use anyhow::{anyhow, Result};
use chrono::{NaiveDateTime, Utc};
-use parking_lot::Mutex;
use ring::rand::SecureRandom;
use ring::{digest, pbkdf2, rand};
+use std::sync::Mutex;
use crate::hyperloglog::HyperLogLog;
use crate::metrics::Counter;
@@ -131,10 +131,16 @@ impl<F: Frequency> UserCount<F> {
}
pub fn inc<T: serde::Serialize>(&self, user_id: &T) -> Result<()> {
- self.inner.lock().inc(user_id)
+ self.inner
+ .lock()
+ .unwrap_or_else(|e| e.into_inner())
+ .inc(user_id)
}
pub fn metric(&self) -> Counter {
- self.inner.lock().metric()
+ self.inner
+ .lock()
+ .unwrap_or_else(|e| e.into_inner())
+ .metric()
}
}
diff --git a/core/src/crawler/router.rs b/core/src/crawler/router.rs
index 75005ee4..2f9cb68f 100644
--- a/core/src/crawler/router.rs
+++ b/core/src/crawler/router.rs
@@ -22,7 +22,7 @@ impl RemoteCoordinator {
Ok(sonic::service::ResilientConnection::create_with_timeout(
self.addr,
- Duration::from_secs(3600),
+ Duration::from_secs(60),
retry,
)
.await?)
@@ -32,7 +32,7 @@ impl RemoteCoordinator {
let conn = self.conn().await?;
let response = conn
- .send_with_timeout(&GetJobs { num_jobs }, Duration::from_secs(60 * 60))
+ .send_with_timeout(&GetJobs { num_jobs }, Duration::from_secs(90))
.await?;
Ok(response)
@@ -41,7 +41,7 @@ impl RemoteCoordinator {
async fn insert_urls(&self, urls: HashMap<Domain, Vec<UrlToInsert>>) -> Result<()> {
let conn = self.conn().await?;
- conn.send_with_timeout(&InsertUrls { urls }, Duration::from_secs(60 * 60))
+ conn.send_with_timeout(&InsertUrls { urls }, Duration::from_secs(90))
.await?;
Ok(())
@@ -50,7 +50,7 @@ impl RemoteCoordinator {
async fn mark_jobs_complete(&self, domains: Vec<DomainCrawled>) -> Result<()> {
let conn = self.conn().await?;
- conn.send_with_timeout(&MarkJobsComplete { domains }, Duration::from_secs(60 * 60))
+ conn.send_with_timeout(&MarkJobsComplete { domains }, Duration::from_secs(90))
.await?;
Ok(())
diff --git a/core/src/crawler/worker.rs b/core/src/crawler/worker.rs
index da6f235a..332524ba 100644
--- a/core/src/crawler/worker.rs
+++ b/core/src/crawler/worker.rs
@@ -115,7 +115,7 @@ impl WorkerThread {
Ok(sonic::service::ResilientConnection::create_with_timeout(
router,
- Duration::from_secs(3600),
+ Duration::from_secs(90),
retry,
)
.await?)
@@ -153,7 +153,7 @@ impl WorkerThread {
responses: results,
num_jobs: self.num_jobs_per_fetch,
},
- Duration::from_secs(60 * 60),
+ Duration::from_secs(90),
)
.await;
diff --git a/core/src/entity_index/mod.rs b/core/src/entity_index/mod.rs
index 9f37a8a8..85aacad8 100644
--- a/core/src/entity_index/mod.rs
+++ b/core/src/entity_index/mod.rs
@@ -28,7 +28,7 @@ use tantivy::{
query::{BooleanQuery, MoreLikeThisQuery, Occur, QueryClone, TermQuery},
schema::{BytesOptions, IndexRecordOption, Schema, TextFieldIndexing, TextOptions},
tokenizer::Tokenizer,
- DocAddress, IndexReader, IndexWriter, Searcher, Term,
+ DocAddress, IndexReader, IndexWriter, Searcher, TantivyDocument, Term,
};
use tracing::info;
use url::Url;
@@ -92,8 +92,8 @@ fn schema() -> Schema {
builder.build()
}
-fn entity_to_tantivy(entity: Entity, schema: &tantivy::schema::Schema) -> tantivy::Document {
- let mut doc = tantivy::Document::new();
+fn entity_to_tantivy(entity: Entity, schema: &tantivy::schema::Schema) -> TantivyDocument {
+ let mut doc = TantivyDocument::new();
doc.add_text(schema.get_field("title").unwrap(), entity.title);
doc.add_text(
@@ -373,11 +373,11 @@ impl EntityIndex {
let info = self.schema.get_field("info").unwrap();
let links = self.schema.get_field("links").unwrap();
- let doc = searcher.doc(doc_address).unwrap();
+ let doc: TantivyDocument = searcher.doc(doc_address).unwrap();
let title = doc
.get_first(title)
.and_then(|val| match val {
- tantivy::schema::Value::Str(string) => Some(string.clone()),
+ tantivy::schema::OwnedValue::Str(string) => Some(string.clone()),
_ => None,
})
.unwrap();
@@ -385,7 +385,7 @@ impl EntityIndex {
let entity_abstract = doc
.get_first(entity_abstract)
.and_then(|val| match val {
- tantivy::schema::Value::Str(string) => Some(string.clone()),
+ tantivy::schema::OwnedValue::Str(string) => Some(string.clone()),
_ => None,
})
.unwrap();
@@ -394,7 +394,7 @@ impl EntityIndex {
bincode::deserialize(
doc.get_first(info)
.and_then(|val| match val {
- tantivy::schema::Value::Bytes(bytes) => Some(bytes),
+ tantivy::schema::OwnedValue::Bytes(bytes) => Some(bytes),
_ => None,
})
.unwrap(),
@@ -420,7 +420,7 @@ impl EntityIndex {
bincode::deserialize(
doc.get_first(links)
.and_then(|val| match val {
- tantivy::schema::Value::Bytes(bytes) => Some(bytes),
+ tantivy::schema::OwnedValue::Bytes(bytes) => Some(bytes),
_ => None,
})
.unwrap(),
diff --git a/core/src/fastfield_reader.rs b/core/src/fastfield_reader.rs
index 523cd9ef..0fa273e1 100644
--- a/core/src/fastfield_reader.rs
+++ b/core/src/fastfield_reader.rs
@@ -54,10 +54,6 @@ impl FastFieldReader {
let reader = fastfield_readers.u64(field.name()).unwrap();
FieldReader::U64(reader.values)
}
- DataType::F64 => {
- let reader = fastfield_readers.f64(field.name()).unwrap();
- FieldReader::F64(reader.values)
- }
};
field_readers.push((field, field_reader));
@@ -82,7 +78,6 @@ impl FastFieldReader {
pub enum FieldValue {
U64(u64),
U64s(Vec<u64>),
- F64(f64),
}
impl From<FieldValue> for Option<Vec<u64>> {
@@ -105,14 +100,12 @@ impl From<FieldValue> for Option<u64> {
pub enum FieldReader {
U64(Arc<dyn ColumnValues<u64>>),
- F64(Arc<dyn ColumnValues<f64>>),
}
impl FieldReader {
pub fn get(&self, doc: &DocId) -> FieldValue {
match self {
FieldReader::U64(reader) => FieldValue::U64(reader.get_val(*doc)),
- FieldReader::F64(reader) => FieldValue::F64(reader.get_val(*doc)),
}
}
}
diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rs
index 05680f33..9c1450c6 100644
--- a/core/src/inverted_index.rs
+++ b/core/src/inverted_index.rs
@@ -30,9 +30,9 @@ use serde::{Deserialize, Serialize};
use tantivy::collector::Count;
use tantivy::directory::MmapDirectory;
use tantivy::merge_policy::NoMergePolicy;
-use tantivy::schema::Schema;
+use tantivy::schema::{Schema, Value};
use tantivy::tokenizer::TokenizerManager;
-use tantivy::{Document, IndexReader, IndexWriter, SegmentMeta};
+use tantivy::{IndexReader, IndexWriter, SegmentMeta, TantivyDocument};
use url::Url;
use crate::collector::{Hashes, MainCollector};
@@ -303,7 +303,7 @@ impl InvertedIndex {
pub fn website_host_node(&self, website: &WebsitePointer) -> Result<Option<NodeID>> {
let searcher = self.reader.searcher();
- let doc = searcher.doc(website.address.into())?;
+ let doc: TantivyDocument = searcher.doc(website.address.into())?;
let field1 = self
.schema()
@@ -414,7 +414,7 @@ impl InvertedIndex {
doc_address: DocAddress,
searcher: &tantivy::Searcher,
) -> Result<RetrievedWebpage> {
- let doc = searcher.doc(doc_address.into())?;
+ let doc: TantivyDocument = searcher.doc(doc_address.into())?;
Ok(RetrievedWebpage::from(doc))
}
@@ -560,30 +560,33 @@ impl RetrievedWebpage {
}
}
-impl From<Document> for RetrievedWebpage {
- fn from(doc: Document) -> Self {
+impl From<TantivyDocument> for RetrievedWebpage {
+ fn from(doc: TantivyDocument) -> Self {
let mut webpage = RetrievedWebpage::default();
for value in doc.field_values() {
match ALL_FIELDS[value.field.field_id() as usize] {
Field::Text(TextField::Title) => {
webpage.title = value
- .value
- .as_text()
+ .value()
+ .as_value()
+ .as_str()
.expect("Title field should be text")
.to_string();
}
Field::Text(TextField::StemmedCleanBody) => {
webpage.body = value
- .value
- .as_text()
+ .value()
+ .as_value()
+ .as_str()
.expect("Body field should be text")
.to_string();
}
Field::Text(TextField::Description) => {
let desc = value
- .value
- .as_text()
+ .value()
+ .as_value()
+ .as_str()
.expect("Description field should be text")
.to_string();
@@ -591,14 +594,15 @@ impl From<Document> for RetrievedWebpage {
}
Field::Text(TextField::Url) => {
webpage.url = value
- .value
- .as_text()
+ .value()
+ .as_value()
+ .as_str()
.expect("Url field should be text")
.to_string();
}
Field::Fast(FastField::LastUpdated) => {
webpage.updated_time = {
- let timestamp = value.value.as_u64().unwrap() as i64;
+ let timestamp = value.value().as_value().as_u64().unwrap() as i64;
if timestamp == 0 {
None
} else {
@@ -608,21 +612,23 @@ impl From<Document> for RetrievedWebpage {
}
Field::Text(TextField::AllBody) => {
webpage.dirty_body = value
- .value
- .as_text()
+ .value()
+ .as_value()
+ .as_str()
.expect("All body field should be text")
.to_string();
}
Field::Fast(FastField::Region) => {
webpage.region = {
- let id = value.value.as_u64().unwrap();
+ let id = value.value().as_value().as_u64().unwrap();
Region::from_id(id)
}
}
Field::Text(TextField::DmozDescription) => {
let desc = value
- .value
- .as_text()
+ .value()
+ .as_value()
+ .as_str()
.expect("Dmoz description field should be text")
.to_string();
@@ -630,8 +636,9 @@ impl From<Document> for RetrievedWebpage {
}
Field::Text(TextField::SchemaOrgJson) => {
let json = value
- .value
- .as_text()
+ .value()
+ .as_value()
+ .as_str()
.expect("Schema.org json field should be stored as text")
.to_string();
diff --git a/core/src/schema.rs b/core/src/schema.rs
index 82027f54..7aefebdd 100644
--- a/core/src/schema.rs
+++ b/core/src/schema.rs
@@ -521,7 +521,7 @@ impl Field {
Field::Fast(FastField::TitleHash2) => {
IndexingOption::Integer(NumericOptions::default().set_fast())
}
- Field::Fast(FastField::PreComputedScore) => IndexingOption::Float(
+ Field::Fast(FastField::PreComputedScore) => IndexingOption::Integer(
NumericOptions::default()
.set_fast()
.set_indexed()
@@ -619,7 +619,6 @@ pub fn create_schema() -> tantivy::schema::Schema {
match field.options() {
IndexingOption::Text(options) => builder.add_text_field(field.name(), options),
IndexingOption::Integer(options) => builder.add_u64_field(field.name(), options),
- IndexingOption::Float(options) => builder.add_f64_field(field.name(), options),
};
}
@@ -629,12 +628,10 @@ pub fn create_schema() -> tantivy::schema::Schema {
pub enum IndexingOption {
Text(tantivy::schema::TextOptions),
Integer(tantivy::schema::NumericOptions),
- Float(tantivy::schema::NumericOptions),
}
pub enum DataType {
U64,
- F64,
}
impl FastField {
@@ -665,7 +662,7 @@ impl FastField {
FastField::UrlHash2 => DataType::U64,
FastField::DomainHash1 => DataType::U64,
FastField::DomainHash2 => DataType::U64,
- FastField::PreComputedScore => DataType::F64,
+ FastField::PreComputedScore => DataType::U64,
FastField::HostNodeID1 => DataType::U64,
FastField::HostNodeID2 => DataType::U64,
FastField::SimHash => DataType::U64,
diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rs
index edf41aa5..a7977af0 100644
--- a/core/src/searcher/local.rs
+++ b/core/src/searcher/local.rs
@@ -17,6 +17,8 @@
use std::collections::HashMap;
use std::sync::Arc;
+use tantivy::schema::Value;
+use tantivy::TantivyDocument;
use url::Url;
use crate::config::{CollectorConfig, SnippetConfig};
@@ -239,10 +241,10 @@ impl LocalSearcher {
let schema = self.index.schema();
for website in &mut ranking_websites {
- let doc = ctx.tv_searcher.doc(website.pointer.address.into())?;
+ let doc: TantivyDocument = ctx.tv_searcher.doc(website.pointer.address.into())?;
website.title = Some(
doc.get_first(schema.get_field(TextField::Title.name()).unwrap())
- .map(|text| text.as_text().unwrap().to_string())
+ .map(|text| text.as_value().as_str().unwrap().to_string())
.unwrap_or_default(),
);
website.clean_body = Some(
@@ -251,7 +253,7 @@ impl LocalSearcher {
.get_field(TextField::StemmedCleanBody.name())
.unwrap(),
)
- .map(|text| text.as_text().unwrap().to_string())
+ .map(|text| text.as_value().as_str().unwrap().to_string())
.unwrap_or_default(),
);
}
diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rs
index a17be861..9487df6b 100644
--- a/core/src/webpage/mod.rs
+++ b/core/src/webpage/mod.rs
@@ -27,7 +27,10 @@ use itertools::Itertools;
use kuchiki::{iter::NodeEdge, traits::TendrilSink, NodeRef};
use regex::Regex;
use std::{collections::HashMap, panic, str::FromStr};
-use tantivy::tokenizer::{PreTokenizedString, Tokenizer};
+use tantivy::{
+ tokenizer::{PreTokenizedString, Tokenizer},
+ TantivyDocument,
+};
use url::Url;
use whatlang::Lang;
@@ -160,7 +163,7 @@ impl Webpage {
})
}
- pub fn into_tantivy(self, schema: &tantivy::schema::Schema) -> Result<tantivy::Document> {
+ pub fn into_tantivy(self, schema: &tantivy::schema::Schema) -> Result<TantivyDocument> {
let region = Region::guess_from(&self);
let dmoz_description = self.dmoz_description();
@@ -226,11 +229,11 @@ impl Webpage {
self.fetch_time_ms,
);
- doc.add_f64(
+ doc.add_u64(
schema
.get_field(Field::Fast(FastField::PreComputedScore).name())
.expect("failed to get pre_computed_score field"),
- self.pre_computed_score,
+ (self.pre_computed_score * FLOAT_SCALING as f64) as u64,
);
match &self.node_id {
@@ -930,8 +933,8 @@ impl Html {
PreTokenizedString { text, tokens }
}
- pub fn into_tantivy(self, schema: &tantivy::schema::Schema) -> Result<tantivy::Document> {
- let mut doc = tantivy::Document::new();
+ pub fn into_tantivy(self, schema: &tantivy::schema::Schema) -> Result<TantivyDocument> {
+ let mut doc = TantivyDocument::new();
let title = self.pretokenize_title()?;
let all_text = self.pretokenize_all_text()?;
diff --git a/core/src/webpage/schema_org/mod.rs b/core/src/webpage/schema_org/mod.rs
index bc3b27b9..036a4aa7 100644
--- a/core/src/webpage/schema_org/mod.rs
+++ b/core/src/webpage/schema_org/mod.rs
@@ -224,6 +224,14 @@ pub fn parse(root: NodeRef) -> Vec<Item> {
res.into_iter().map(Item::from).collect()
}
+pub(crate) fn flattened_json(schemas: Vec<Item>) -> Result<FlattenedJson> {
+ let single_maps: Vec<_> = schemas
+ .into_iter()
+ .map(|item| item.into_single_map())
+ .collect();
+ FlattenedJson::new(&single_maps)
+}
+
#[cfg(test)]
mod tests {
use kuchiki::traits::TendrilSink;
@@ -519,11 +527,3 @@ mod tests {
assert_eq!(&instructions, "Helt enkelt som navnet antyder, så kom alle ingredienserne i en stor gryde på én gang. Kog retten op, rør godt rundt i gryden og skru ned for varmen. Lad det simrekoge under låg i 10-12 minutter, til spaghettien er perfekt kogt – al dente med lidt bid i. Server med revet parmesan og basilikum.");
}
}
-
-pub(crate) fn flattened_json(schemas: Vec<Item>) -> Result<FlattenedJson> {
- let single_maps: Vec<_> = schemas
- .into_iter()
- .map(|item| item.into_single_map())
- .collect();
- FlattenedJson::new(&single_maps)
-}
diff --git a/kuchiki/src/lib.rs b/kuchiki/src/lib.rs
index a037f173..bbe14e5f 100644
--- a/kuchiki/src/lib.rs
+++ b/kuchiki/src/lib.rs
@@ -18,6 +18,7 @@ mod node_data_ref;
mod parser;
mod select;
mod serializer;
+
#[cfg(test)]
mod tests;
mod tree;