Skip to content

web search done right

Parse DMOZ data (#66)

Mikkel Denkerbbca94c37e14
diff --git a/configs/webgraph/local.toml b/configs/webgraph/local.toml
index 51e0fb33..d0474276 100644
--- a/configs/webgraph/local.toml
+++ b/configs/webgraph/local.toml
@@ -1,11 +1,18 @@
-limit_warc_files = 4
+limit_warc_files = 10
graph_base_path = "data/webgraph"
-batch_size = 2
+batch_size = 1
[warc_source]
-type = "HTTP"
-base_url = "https://data.commoncrawl.org/"
-warc_paths_file = "warc.paths"
-# type = "Local"
-# folder = "./data/warc_files"
-# names = ["CC-MAIN-20220116093137-20220116123137-00049.warc.gz"]
\ No newline at end of file
+# type = "HTTP"
+# base_url = "https://data.commoncrawl.org/"
+# warc_paths_file = "warc.paths"
+type = "Local"
+folder = "./data/warc_files"
+names = [
+ "CC-MAIN-20220116093137-20220116123137-00044.warc.gz",
+ "CC-MAIN-20220116093137-20220116123137-00045.warc.gz",
+ "CC-MAIN-20220116093137-20220116123137-00046.warc.gz",
+ "CC-MAIN-20220116093137-20220116123137-00047.warc.gz",
+ "CC-MAIN-20220116093137-20220116123137-00048.warc.gz",
+ "CC-MAIN-20220116093137-20220116123137-00049.warc.gz",
+ ]
\ No newline at end of file
diff --git a/frontend/src/components/Header.astro b/frontend/src/components/Header.astro
index b3110d49..8c9fe8c9 100644
--- a/frontend/src/components/Header.astro
+++ b/frontend/src/components/Header.astro
@@ -11,7 +11,7 @@ const { icon = true, searchbar } = Astro.props;
---
<div
- class="grid items-center h-24 grid-cols-[auto_1fr_auto] sm:grid-cols-[3rem_1fr_auto] sm:gap-4 md:gap-5 p-4 lg:gap-10 lg:px-8"
+ class="grid items-center h-24 grid-cols-[auto_1fr_auto] sm:grid-cols-[3rem_1fr_auto] sm:gap-4 md:gap-5 p-4 lg:gap-10 lg:pr-8 lg:pl-14"
>
{
icon && (
diff --git a/frontend/src/pages/search.astro b/frontend/src/pages/search.astro
index dc15744e..a4b614b7 100644
--- a/frontend/src/pages/search.astro
+++ b/frontend/src/pages/search.astro
@@ -16,7 +16,7 @@ import { askama } from "../askama";
</div>
<div
- class="grid sm:px-5 md:pl-20 lg:px-32 mx-1 lg:mx-0 md:gap-x-12 md:grid-cols-[minmax(50ch,48rem)_1fr] md:grid-rows-[auto_1fr] gap-y-6 pt-3 search-content"
+ class="grid sm:px-5 md:pl-20 lg:px-36 mx-1 lg:mx-0 md:gap-x-12 md:grid-cols-[minmax(50ch,48rem)_1fr] md:grid-rows-[auto_1fr] gap-y-6 pt-3 search-content"
>
<!-- Stats and settings -->
<div class="flex w-full justify-between mx-auto">
@@ -79,17 +79,10 @@ import { askama } from "../askama";
{
askama.for_("item in search_result", () => (
<div class="flex w-full">
- <div class="flex grow min-w-0 flex-col space-y-1.5">
+ <div class="flex grow min-w-0 flex-col space-y-1">
<div class="flex min-w-0">
- <div class="flex flex-col space-y-1.5 grow min-w-0">
- <div class="flex items-center space-x-2 text-sm">
- <img
- src={askama.fake(
- "data:image/png;base64, {{ item.favicon_base64 }}",
- "{{image.avatar}}"
- )}
- width="13px"
- />
+ <div class="flex flex-col space-y-1 grow min-w-0">
+ <div class="flex items-center text-sm">
<a
class="text-gray-600 hover:no-underline truncate url"
href="{{ item.url }}"
@@ -97,7 +90,7 @@ import { askama } from "../askama";
{askama`item.pretty_url $ {{internet.url}}`}
</a>
</div>
- <a class="title-link truncate text-lg font-medium" title="{{ item.title }}" href="{{ item.url }}">
+ <a class="title-link truncate text-xl font-medium" title="{{ item.title }}" href="{{ item.url }}">
{askama`item.title $ {{lorem.sentence}}`}
</a>
</div>
@@ -105,7 +98,7 @@ import { askama } from "../askama";
<Hero class="w-6" icon="adjustments" />
</div>
</div>
- <div class="text-sm">
+ <div class="text-sm text-snippet">
{askama.if_("let Some(val) = item.last_updated", () => (
<span class="text-gray-500">
{askama`val $ 2. May 2022`}
diff --git a/frontend/tailwind.config.cjs b/frontend/tailwind.config.cjs
index b284f291..3e331733 100644
--- a/frontend/tailwind.config.cjs
+++ b/frontend/tailwind.config.cjs
@@ -7,11 +7,15 @@ module.exports = {
extend: {
colors: {
brand: "rgb(11 123 255 / <alpha-value>)",
+ snippet: "#4d5156",
},
fontFamily: {
sans: [
- "MontserratVariable",
- "Montserrat",
+ // "MontserratVariable",
+ // "Montserrat",
+ "Helvetica",
+ "Arial",
+ "sans-serif",
...defaultTheme.fontFamily.sans,
],
},
diff --git a/justfile b/justfile
index b5b75f01..220f8d9d 100644
--- a/justfile
+++ b/justfile
@@ -20,9 +20,9 @@
@astro:
cd frontend; npm run dev
-@configure:
+@configure *ARGS:
cd frontend; npm install; npm run build
- cargo run --release --all-features -- configure
+ cargo run --release --all-features -- configure {{ARGS}}
@entity:
rm -rf data/entity
diff --git a/nohup.out b/nohup.out
deleted file mode 100644
index 0da8ba7d..00000000
--- a/nohup.out
+++ /dev/null
@@ -1,20 +0,0 @@
- Compiling cuely v0.1.0 (/Users/mikkeldenker/cuely/cuely)
- Compiling cuely v0.1.0 (/Users/mikkeldenker/cuely/cuely)
- Finished dev [unoptimized + debuginfo] target(s) in 4.40s
- Running `target/debug/cuely frontend data/index data/queries_us.csv data/entity data/bangs.json`
-2022-09-28T09:43:26.587009Z  INFO cuely::entrypoint::frontend: listening on 0.0.0.0:3000
- Blocking waiting for file lock on package cache
- Blocking waiting for file lock on package cache
- Blocking waiting for file lock on package cache
- Blocking waiting for file lock on build directory
- Finished dev [unoptimized + debuginfo] target(s) in 4.31s
- Running `target/debug/cuely frontend data/index data/queries_us.csv data/entity data/bangs.json`
-thread 'main' panicked at 'unable to open rocks db: Error { message: "IO error: While lock file: data/entity/attribute_occurrences/LOCK: Resource temporarily unavailable" }', src/kv/rocksdb_store.rs:39:41
-note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
- Blocking waiting for file lock on package cache
- Blocking waiting for file lock on package cache
- Blocking waiting for file lock on package cache
- Compiling cuely v0.1.0 (/Users/mikkeldenker/cuely/cuely)
- Finished dev [unoptimized + debuginfo] target(s) in 4.22s
- Running `target/debug/cuely frontend data/index data/queries_us.csv data/entity data/bangs.json`
-2022-09-28T09:44:05.809083Z  INFO cuely::entrypoint::frontend: listening on 0.0.0.0:3000
diff --git a/src/entrypoint/configure.rs b/src/entrypoint/configure.rs
index 95556fea..401357d9 100644
--- a/src/entrypoint/configure.rs
+++ b/src/entrypoint/configure.rs
@@ -21,7 +21,7 @@ use tokio::fs::File;
use tokio::io;
use tracing::{debug, info};
-use crate::entrypoint::indexer;
+use crate::entrypoint::{dmoz_parser, indexer};
use crate::Result;
use std::fs::{self};
use std::path::Path;
@@ -49,6 +49,7 @@ fn download_files() {
"queries_us.csv",
"sample.warc.gz",
"bangs.json",
+ "content.rdf.u8.gz",
] {
info!("Downloading {}", name);
let mut object = client
@@ -138,6 +139,14 @@ fn create_inverted_index() -> Result<()> {
let worker = indexer::IndexingWorker::new(
centrality_path.to_str().unwrap().to_string(),
Some(webgraph_path.to_str().unwrap().to_string()),
+ None,
+ Some(
+ Path::new(DATA_PATH)
+ .join("human_annotations")
+ .to_str()
+ .unwrap()
+ .to_string(),
+ ),
);
let index = indexer::process_job(&job, &worker);
@@ -163,23 +172,72 @@ fn create_entity_index() -> Result<()> {
Ok(())
}
+fn parse_topics() -> Result<()> {
+ let dmoz_path = Path::new(DATA_PATH).join("content.rdf.u8.gz");
+
+ let topics = dmoz_parser::parse(dmoz_path)?;
+ topics.save(Path::new(DATA_PATH).join("human_annotations"))
+}
+
+fn create_topic_centrality() {
+ let index_path = Path::new(DATA_PATH)
+ .join("index")
+ .to_str()
+ .unwrap()
+ .to_string();
+ let topics_path = Path::new(DATA_PATH)
+ .join("human_annotations")
+ .to_str()
+ .unwrap()
+ .to_string();
+ let webgraph_path = Path::new(DATA_PATH)
+ .join("webgraph")
+ .to_str()
+ .unwrap()
+ .to_string();
+ let approximate_harmonic_path = Path::new(DATA_PATH)
+ .join("centrality")
+ .join("approx_harmonic")
+ .to_str()
+ .unwrap()
+ .to_string();
+ let output_path = Path::new(DATA_PATH)
+ .join("topic_centrality")
+ .to_str()
+ .unwrap()
+ .to_string();
+
+ super::topic_centrality::run(
+ index_path,
+ topics_path,
+ webgraph_path,
+ approximate_harmonic_path,
+ output_path,
+ );
+}
+
fn index_files() -> Result<()> {
create_webgraph()?;
calculate_centrality();
+ parse_topics()?;
create_inverted_index()?;
+ create_topic_centrality();
create_entity_index()?;
Ok(())
}
-pub fn run() -> Result<()> {
+pub fn run(skip_download: bool) -> Result<()> {
let p = Path::new(DATA_PATH);
if !p.exists() {
fs::create_dir_all(p)?;
}
- download_files();
+ if !skip_download {
+ download_files();
+ }
+
index_files()?;
Ok(())
diff --git a/src/entrypoint/dmoz_parser.rs b/src/entrypoint/dmoz_parser.rs
new file mode 100644
index 00000000..632ddfee
--- /dev/null
+++ b/src/entrypoint/dmoz_parser.rs
@@ -0,0 +1,158 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use flate2::read::MultiGzDecoder;
+use quick_xml::events::Event;
+
+use crate::{
+ human_website_annotations::{Info, Topic},
+ webpage::Url,
+ Result,
+};
+use std::{
+ collections::HashMap,
+ fs::File,
+ io::{BufRead, BufReader},
+ path::Path,
+};
+
+use crate::human_website_annotations;
+
+struct PageIterator<R: BufRead> {
+ reader: quick_xml::Reader<R>,
+ buf: Vec<u8>,
+}
+
+impl<R: BufRead> From<R> for PageIterator<R> {
+ fn from(reader: R) -> Self {
+ Self {
+ reader: quick_xml::Reader::from_reader(reader),
+ buf: Vec::new(),
+ }
+ }
+}
+
+impl<R: BufRead> Iterator for PageIterator<R> {
+ type Item = Page;
+
+ fn next(&mut self) -> Option<Self::Item> {
+ let mut current_page = None;
+ let mut inside_topic = false;
+ let mut inside_desc = false;
+
+ while let Ok(event) = self.reader.read_event(&mut self.buf) {
+ match event {
+ Event::Start(ref e) if e.name() == b"ExternalPage" => {
+ let url_attr = e
+ .attributes()
+ .filter(|attr| attr.is_ok())
+ .find(|attr| attr.as_ref().unwrap().key == b"about")
+ .unwrap()
+ .unwrap();
+ let url = self.reader.decode(&url_attr.value).unwrap().to_string();
+ current_page = Some(Page::new(url));
+ inside_desc = false;
+ inside_topic = false;
+ }
+ Event::End(ref e) if e.name() == b"ExternalPage" => break,
+ Event::Start(ref e) if e.name() == b"topic" && current_page.is_some() => {
+ inside_topic = true;
+ }
+ Event::End(ref e) if e.name() == b"topic" && current_page.is_some() => {
+ inside_topic = false
+ }
+ Event::Start(ref e) if e.name() == b"d:Description" && current_page.is_some() => {
+ inside_desc = true;
+ }
+ Event::End(ref e) if e.name() == b"d:Description" && current_page.is_some() => {
+ inside_desc = false
+ }
+ Event::Text(ref e) => {
+ if inside_topic {
+ if let Some(page) = &mut current_page {
+ let bytes = e.unescaped().unwrap();
+ let topic = self.reader.decode(&bytes).unwrap();
+ page.topic.push_str(topic);
+ }
+ } else if inside_desc {
+ if let Some(page) = &mut current_page {
+ let bytes = e.unescaped().unwrap();
+ let desc = self.reader.decode(&bytes).unwrap();
+ page.description.push_str(desc);
+ }
+ }
+ }
+ Event::Eof => break,
+ _ => (),
+ }
+ }
+
+ current_page
+ }
+}
+
+#[derive(Debug, Default)]
+struct Page {
+ url: String,
+ description: String,
+ topic: String,
+}
+
+impl Page {
+ fn new(url: String) -> Self {
+ Self {
+ url,
+ ..Default::default()
+ }
+ }
+}
+
+pub fn parse<P: AsRef<Path>>(dmoz_file: P) -> Result<human_website_annotations::Mapper> {
+ let file = File::open(dmoz_file)?;
+ let reader = BufReader::new(file);
+ let reader = BufReader::new(MultiGzDecoder::new(reader));
+ let mut map = HashMap::new();
+
+ for page in PageIterator::from(reader) {
+ let url = Url::from(page.url.clone());
+
+ if !url.is_homepage() {
+ continue;
+ }
+
+ if page.topic.contains("World") {
+ continue;
+ }
+
+ let topic = Topic::from_string("/".to_string() + page.topic.as_str());
+
+ let info = Info {
+ description: page.description,
+ topic,
+ };
+
+ map.insert(url.site().to_string(), info);
+ }
+
+ Ok(map.into())
+}
+
+pub fn run<P: AsRef<Path>>(dmoz_file: P, output_path: P) -> Result<()> {
+ let mapper = parse(dmoz_file)?;
+ mapper.save(output_path)?;
+
+ Ok(())
+}
diff --git a/src/entrypoint/entity.rs b/src/entrypoint/entity.rs
index 6ec057e5..af3334bd 100644
--- a/src/entrypoint/entity.rs
+++ b/src/entrypoint/entity.rs
@@ -30,14 +30,14 @@ use crate::{
use bzip2::bufread::MultiBzDecoder;
use parse_wiki_text::Node;
-use quick_xml::{events::Event, Reader};
+use quick_xml::events::Event;
struct EntityIterator<R: BufRead> {
inside_title: bool,
inside_text: bool,
buf: Vec<u8>,
current_entity: Option<EntityBuilder>,
- reader: Reader<R>,
+ reader: quick_xml::Reader<R>,
}
impl<R: BufRead> From<R> for EntityIterator<R> {
@@ -47,7 +47,7 @@ impl<R: BufRead> From<R> for EntityIterator<R> {
inside_text: false,
buf: Vec::new(),
current_entity: None,
- reader: Reader::from_reader(reader),
+ reader: quick_xml::Reader::from_reader(reader),
}
}
}
diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rs
index f65ec2fb..848c5d6a 100644
--- a/src/entrypoint/indexer.rs
+++ b/src/entrypoint/indexer.rs
@@ -32,7 +32,8 @@ use crate::warc::WarcFile;
use crate::webgraph::{Node, Webgraph, WebgraphBuilder};
use crate::webpage::{Html, Link, Webpage};
use crate::{
- HttpConfig, IndexingLocalConfig, IndexingMasterConfig, LocalConfig, Result, WarcSource,
+ human_website_annotations, HttpConfig, IndexingLocalConfig, IndexingMasterConfig, LocalConfig,
+ Result, WarcSource,
};
use super::crawl_stability::CrawlStability;
@@ -59,6 +60,7 @@ pub struct IndexingWorker {
centrality_store: CentralityStore,
webgraph: Option<Webgraph>,
crawl_stabilty: Option<CrawlStability>,
+ topics: Option<human_website_annotations::Mapper>,
}
impl IndexingWorker {
@@ -66,6 +68,7 @@ impl IndexingWorker {
centrality_store_path: String,
webgraph_path: Option<String>,
crawl_stability_path: Option<String>,
+ topics_path: Option<String>,
) -> Self {
Self {
centrality_store: CentralityStore::open(centrality_store_path),
@@ -77,6 +80,7 @@ impl IndexingWorker {
.open()
}),
crawl_stabilty: crawl_stability_path.map(CrawlStability::open),
+ topics: topics_path.map(|path| human_website_annotations::Mapper::open(path).unwrap()),
}
}
}
@@ -177,6 +181,16 @@ async fn async_process_job(job: &Job, worker: &IndexingWorker) -> Index {
.get(&Node::from_url(html.url()).into_host())
.copied();
+ let mut host_topic = None;
+ let mut dmoz_description = None;
+
+ if let Some(mapper) = worker.topics.as_ref() {
+ if let Some(info) = mapper.get(&html.url().site().to_string()) {
+ host_topic = Some(info.topic.clone());
+ dmoz_description = Some(info.description.clone())
+ }
+ }
+
let mut webpage = Webpage {
html,
backlinks,
@@ -187,6 +201,8 @@ async fn async_process_job(job: &Job, worker: &IndexingWorker) -> Index {
pre_computed_score: 0.0,
node_id,
crawl_stability,
+ host_topic,
+ dmoz_description,
};
webpage.pre_computed_score =
@@ -343,20 +359,26 @@ impl Indexer {
centrality_store_path: String,
webgraph_path: Option<String>,
crawl_stability_path: Option<String>,
+ topics_path: Option<String>,
) -> Result<()> {
tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap()
.block_on(async {
- IndexingWorker::new(centrality_store_path, webgraph_path, crawl_stability_path)
- .run::<Job, FrozenIndex>(
- worker_addr
- .parse::<SocketAddr>()
- .expect("Could not parse worker address"),
- )
- .await
- .unwrap();
+ IndexingWorker::new(
+ centrality_store_path,
+ webgraph_path,
+ crawl_stability_path,
+ topics_path,
+ )
+ .run::<Job, FrozenIndex>(
+ worker_addr
+ .parse::<SocketAddr>()
+ .expect("Could not parse worker address"),
+ )
+ .await
+ .unwrap();
});
Ok(())
}
@@ -373,6 +395,7 @@ impl Indexer {
config.centrality_store_path.clone(),
config.webgraph_path.clone(),
config.crawl_stability_path.clone(),
+ config.topics_path.clone(),
);
let indexes: Vec<_> = warc_paths
diff --git a/src/entrypoint/mod.rs b/src/entrypoint/mod.rs
index eb8d8982..036cc66f 100644
--- a/src/entrypoint/mod.rs
+++ b/src/entrypoint/mod.rs
@@ -18,10 +18,12 @@ mod centrality;
#[cfg(feature = "dev")]
pub mod configure;
pub mod crawl_stability;
+pub mod dmoz_parser;
mod entity;
pub mod frontend;
pub mod indexer;
pub mod search_server;
+pub mod topic_centrality;
mod webgraph;
use std::{fs::File, path::Path};
diff --git a/src/entrypoint/topic_centrality.rs b/src/entrypoint/topic_centrality.rs
new file mode 100644
index 00000000..b925dcb4
--- /dev/null
+++ b/src/entrypoint/topic_centrality.rs
@@ -0,0 +1,43 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::{
+ human_website_annotations,
+ index::Index,
+ webgraph::{
+ centrality::{
+ approximate_harmonic::ApproximatedHarmonicCentrality, topic::TopicCentrality,
+ },
+ WebgraphBuilder,
+ },
+};
+
+pub fn run(
+ index_path: String,
+ topics_path: String,
+ webgraph_path: String,
+ approximate_harmonic_path: String,
+ output_path: String,
+) {
+ let index = Index::open(index_path).unwrap();
+ let topics = human_website_annotations::Mapper::open(topics_path).unwrap();
+ let webgraph = WebgraphBuilder::new(webgraph_path).with_host_graph().open();
+ let approx = ApproximatedHarmonicCentrality::open(approximate_harmonic_path).unwrap();
+
+ let centrality: TopicCentrality = TopicCentrality::build(&index, topics, webgraph, approx);
+
+ centrality.save(output_path).unwrap();
+}
diff --git a/src/human_website_annotations.rs b/src/human_website_annotations.rs
new file mode 100644
index 00000000..64eca81f
--- /dev/null
+++ b/src/human_website_annotations.rs
@@ -0,0 +1,125 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use itertools::Itertools;
+use serde::{Deserialize, Serialize};
+use tantivy::schema::Facet;
+
+use crate::Result;
+use std::{
+ collections::{HashMap, HashSet},
+ fs::File,
+ io::{BufReader, Read, Write},
+ path::Path,
+};
+
+#[derive(Debug, Hash, PartialEq, Eq, Clone, Serialize, Deserialize)]
+pub struct Topic<const N: usize = 2> {
+ detailed_topics: Vec<String>,
+}
+
+impl<const N: usize> From<Facet> for Topic<N> {
+ fn from(facet: Facet) -> Self {
+ Self {
+ detailed_topics: facet
+ .to_path()
+ .into_iter()
+ .take(N)
+ .map(String::from)
+ .collect(),
+ }
+ }
+}
+
+impl<const N: usize> Topic<N> {
+ pub fn from_string(s: String) -> Self {
+ Self {
+ detailed_topics: s
+ .split('/')
+ .skip(2)
+ .take(N)
+ .map(String::from)
+ .collect::<Vec<_>>(),
+ }
+ }
+
+ pub(crate) fn as_facet(&self) -> Facet {
+ Facet::from_path(self.detailed_topics.clone().into_iter())
+ }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct Info {
+ pub description: String,
+ pub topic: Topic,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct Mapper(HashMap<String, Info>);
+
+impl From<HashMap<String, Info>> for Mapper {
+ fn from(map: HashMap<String, Info>) -> Self {
+ Self(map)
+ }
+}
+
+impl Mapper {
+ pub fn save<P: AsRef<Path>>(self, path: P) -> Result<()> {
+ let mut file = File::options()
+ .create(true)
+ .truncate(true)
+ .write(true)
+ .open(path)?;
+
+ let bytes = bincode::serialize(&self)?;
+ file.write_all(&bytes)?;
+
+ Ok(())
+ }
+
+ pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
+ let mut reader = BufReader::new(File::open(path)?);
+
+ let mut bytes = Vec::new();
+ reader.read_to_end(&mut bytes)?;
+
+ Ok(bincode::deserialize(&bytes)?)
+ }
+
+ pub fn get(&self, host: &String) -> Option<&Info> {
+ self.0.get(host)
+ }
+
+ pub fn all_topics(&self) -> HashSet<Topic> {
+ self.0.iter().map(|(_, info)| info.topic.clone()).collect()
+ }
+
+ pub fn top_topics(&self, top_n: usize) -> Vec<Topic> {
+ let mut topics: HashMap<Topic, usize> = HashMap::new();
+
+ for topic in self.0.iter().map(|(_, info)| info.topic.clone()) {
+ *topics.entry(topic).or_default() += 1;
+ }
+
+ topics
+ .into_iter()
+ .sorted_by_key(|(_, count)| *count)
+ .rev()
+ .map(|(topic, _)| topic)
+ .take(top_n)
+ .collect()
+ }
+}
diff --git a/src/inverted_index.rs b/src/inverted_index.rs
index 688fefd0..d93b020e 100644
--- a/src/inverted_index.rs
+++ b/src/inverted_index.rs
@@ -25,6 +25,7 @@ use tantivy::{Document, IndexReader, IndexWriter, SegmentMeta};
use crate::collector::Hashes;
use crate::fastfield_cache::FastFieldCache;
+use crate::human_website_annotations::Topic;
use crate::image_store::Image;
use crate::query::Query;
use crate::schema::{FastField, Field, TextField, ALL_FIELDS};
@@ -85,8 +86,8 @@ pub struct InvertedIndex {
pub path: String,
tantivy_index: tantivy::Index,
writer: IndexWriter,
- reader: IndexReader,
- fastfield_cache: Arc<FastFieldCache>,
+ pub(crate) reader: IndexReader,
+ pub(crate) fastfield_cache: Arc<FastFieldCache>,
schema: Arc<Schema>,
}
@@ -224,6 +225,7 @@ impl InvertedIndex {
&page.body,
&page.dirty_body,
&page.description,
+ &page.dmoz_description,
&page.region,
&searcher,
)?;
@@ -392,9 +394,11 @@ pub struct RetrievedWebpage {
pub body: String,
pub dirty_body: String,
pub description: Option<String>,
+ pub dmoz_description: Option<String>,
pub favicon: Option<Image>,
pub primary_image: Option<StoredPrimaryImage>,
pub updated_time: Option<NaiveDateTime>,
+ pub host_topic: Option<Topic>,
pub region: Region,
}
@@ -467,6 +471,22 @@ impl From<Document> for RetrievedWebpage {
Region::from_id(id)
}
}
+ Field::Text(TextField::HostTopic) => {
+ let facet = value.value.as_facet().unwrap();
+
+ if !facet.is_root() {
+ webpage.host_topic = Some(facet.clone().into())
+ }
+ }
+ Field::Text(TextField::DmozDescription) => {
+ let desc = value
+ .value
+ .as_text()
+ .expect("Dmoz description field should be text")
+ .to_string();
+
+ webpage.dmoz_description = if desc.is_empty() { None } else { Some(desc) }
+ }
_ => {}
}
}
@@ -723,7 +743,9 @@ mod tests {
pre_computed_score: 0.0,
primary_image: None,
node_id: None,
+ host_topic: None,
crawl_stability: 0.0,
+ dmoz_description: None,
})
.expect("failed to insert webpage");
diff --git a/src/lib.rs b/src/lib.rs
index 1a48fd9f..d4e26185 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -40,6 +40,7 @@ mod entity_index;
mod exponential_backoff;
mod fastfield_cache;
mod frontend;
+mod human_website_annotations;
mod image_downloader;
mod image_store;
pub mod index;
@@ -86,6 +87,7 @@ pub struct IndexingLocalConfig {
output_path: Option<String>,
download_images: Option<bool>,
host_centrality_threshold: Option<f64>,
+ topics_path: Option<String>,
centrality_store_path: String,
}
diff --git a/src/main.rs b/src/main.rs
index 14557cde..36c679c9 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -49,6 +49,10 @@ enum Commands {
output_path: String,
host_rank_paths: Vec<String>,
},
+ DmozParser {
+ dmoz_file: String,
+ output_path: String,
+ },
Webgraph {
#[clap(subcommand)]
options: WebgraphOptions,
@@ -65,8 +69,18 @@ enum Commands {
ms_sleep_between_req: u64,
output_dir: String,
},
+ TopicCentrality {
+ index_path: String,
+ topics_path: String,
+ webgraph_path: String,
+ approximate_harmonic_path: String,
+ output_path: String,
+ },
#[cfg(feature = "dev")]
- Configure,
+ Configure {
+ #[clap(long, takes_value = false)]
+ skip_download: bool,
+ },
}
#[derive(Subcommand)]
@@ -86,6 +100,7 @@ enum IndexingOptions {
centrality_store_path: String,
webgraph_path: Option<String>,
crawl_stability_path: Option<String>,
+ topics_path: Option<String>,
},
Local {
config_path: String,
@@ -124,12 +139,14 @@ fn main() -> Result<()> {
centrality_store_path,
webgraph_path,
crawl_stability_path,
+ topics_path,
} => {
entrypoint::Indexer::run_worker(
address,
centrality_store_path,
webgraph_path,
crawl_stability_path,
+ topics_path,
)?;
}
IndexingOptions::Local { config_path } => {
@@ -194,7 +211,7 @@ fn main() -> Result<()> {
autosuggest_scrape::run(queries_to_scrape, gl, ms_sleep_between_req, output_dir)?;
}
#[cfg(feature = "dev")]
- Commands::Configure => configure::run()?,
+ Commands::Configure { skip_download } => configure::run(skip_download)?,
Commands::CrawlStability {
output_path,
host_rank_paths,
@@ -202,6 +219,23 @@ fn main() -> Result<()> {
entrypoint::crawl_stability::CrawlStability::build(host_rank_paths, output_path)
.unwrap();
}
+ Commands::DmozParser {
+ dmoz_file,
+ output_path,
+ } => entrypoint::dmoz_parser::run(dmoz_file, output_path).unwrap(),
+ Commands::TopicCentrality {
+ index_path,
+ topics_path,
+ webgraph_path,
+ approximate_harmonic_path,
+ output_path,
+ } => entrypoint::topic_centrality::run(
+ index_path,
+ topics_path,
+ webgraph_path,
+ approximate_harmonic_path,
+ output_path,
+ ),
}
Ok(())
diff --git a/src/ranking/centrality_store.rs b/src/ranking/centrality_store.rs
index b7b2a776..d6079f7f 100644
--- a/src/ranking/centrality_store.rs
+++ b/src/ranking/centrality_store.rs
@@ -55,7 +55,7 @@ impl CentralityStore {
pub fn open<P: AsRef<Path>>(path: P) -> Self {
Self {
harmonic: HarmonicCentralityStore::open(path.as_ref().join("harmonic")),
- approx_harmonic: ApproximatedHarmonicCentrality::load(
+ approx_harmonic: ApproximatedHarmonicCentrality::open(
path.as_ref().join("approx_harmonic"),
)
.ok()
diff --git a/src/ranking/goggles/mod.rs b/src/ranking/goggles/mod.rs
index 62adc9fb..d4e22c5e 100644
--- a/src/ranking/goggles/mod.rs
+++ b/src/ranking/goggles/mod.rs
@@ -444,6 +444,8 @@ mod tests {
crawl_stability: 0.0,
primary_image: None,
node_id: None,
+ host_topic: None,
+ dmoz_description: None,
})
.expect("failed to insert webpage");
index
@@ -472,6 +474,8 @@ mod tests {
crawl_stability: 0.0,
fetch_time_ms: 500,
node_id: None,
+ host_topic: None,
+ dmoz_description: None,
})
.expect("failed to insert webpage");
@@ -579,6 +583,8 @@ mod tests {
crawl_stability: 0.0,
primary_image: None,
node_id: None,
+ host_topic: None,
+ dmoz_description: None,
})
.expect("failed to insert webpage");
index
@@ -606,6 +612,8 @@ mod tests {
crawl_stability: 0.0,
fetch_time_ms: 500,
node_id: None,
+ host_topic: None,
+ dmoz_description: None,
})
.expect("failed to insert webpage");
@@ -692,6 +700,8 @@ mod tests {
crawl_stability: 0.0,
primary_image: None,
node_id: None,
+ host_topic: None,
+ dmoz_description: None,
})
.expect("failed to insert webpage");
index
@@ -720,6 +730,8 @@ mod tests {
crawl_stability: 0.0,
fetch_time_ms: 500,
node_id: None,
+ dmoz_description: None,
+ host_topic: None,
})
.expect("failed to insert webpage");
index
@@ -748,6 +760,8 @@ mod tests {
crawl_stability: 0.0,
fetch_time_ms: 500,
node_id: None,
+ host_topic: None,
+ dmoz_description: None,
})
.expect("failed to insert webpage");
@@ -830,6 +844,8 @@ mod tests {
pre_computed_score: 0.0,
crawl_stability: 0.0,
primary_image: None,
+ host_topic: None,
+ dmoz_description: None,
node_id: Some(
*centrality_store
.approx_harmonic
@@ -864,6 +880,8 @@ mod tests {
pre_computed_score: 0.0,
crawl_stability: 0.0,
fetch_time_ms: 500,
+ dmoz_description: None,
+ host_topic: None,
node_id: Some(
*centrality_store
.approx_harmonic
@@ -898,6 +916,8 @@ mod tests {
pre_computed_score: 0.0,
crawl_stability: 0.0,
fetch_time_ms: 500,
+ host_topic: None,
+ dmoz_description: None,
node_id: Some(
*centrality_store
.approx_harmonic
diff --git a/src/ranking/mod.rs b/src/ranking/mod.rs
index 1b289256..2d205d36 100644
--- a/src/ranking/mod.rs
+++ b/src/ranking/mod.rs
@@ -17,7 +17,7 @@
mod bm25;
pub mod centrality_store;
pub mod goggles;
-mod initial;
+pub mod initial;
pub mod signal;
pub mod site_rankings;
@@ -31,6 +31,7 @@ use crate::{
fastfield_cache::FastFieldCache,
inverted_index,
searcher::NUM_RESULTS_PER_PAGE,
+ webgraph::centrality::topic,
webpage::region::{Region, RegionCount},
};
@@ -44,6 +45,7 @@ pub struct Ranker {
aggregator: SignalAggregator,
fastfield_cache: Arc<FastFieldCache>,
de_rank_similar: bool,
+ topic_scorer: Option<topic::Scorer>,
}
impl Ranker {
@@ -60,6 +62,7 @@ impl Ranker {
max_docs: None,
de_rank_similar: true,
fastfield_cache,
+ topic_scorer: None,
}
}
@@ -86,10 +89,16 @@ impl Ranker {
}
pub fn collector(&self) -> impl Collector<Fruit = Vec<inverted_index::WebsitePointer>> {
+ let mut aggregator = self.aggregator.clone();
+
+ if let Some(topic_scorer) = self.topic_scorer.clone() {
+ aggregator.register_topic_scorer(topic_scorer);
+ }
+
let score_tweaker = InitialScoreTweaker::new(
Arc::clone(&self.region_count),
self.selected_region,
- self.aggregator.clone(),
+ aggregator,
Arc::clone(&self.fastfield_cache),
);
@@ -110,17 +119,31 @@ impl Ranker {
collector.tweak_score(score_tweaker)
}
+
+ pub(crate) fn set_topic_scorer(&mut self, topic_scorer: topic::Scorer) {
+ self.topic_scorer = Some(topic_scorer);
+ }
}
#[cfg(test)]
mod tests {
+ use std::collections::HashMap;
+
use crate::{
+ human_website_annotations::{self, Info, Topic},
index::Index,
searcher::{LocalSearcher, SearchQuery},
+ webgraph::{
+ centrality::{
+ approximate_harmonic::ApproximatedHarmonicCentrality, topic::TopicCentrality,
+ },
+ Node, WebgraphBuilder,
+ },
webpage::{Html, Link, Webpage},
};
const CONTENT: &str = "this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever";
+ const CONTENT_2: &str = "what should i write in this text what should i write in this text what should i write in this text what should i write in this text what should i write in this text what should i write in this text what should i write in this text";
#[test]
fn host_centrality_ranking() {
@@ -152,7 +175,9 @@ mod tests {
pre_computed_score: 0.0,
crawl_stability: 0.0,
primary_image: None,
+ host_topic: None,
node_id: None,
+ dmoz_description: None,
})
.expect("failed to insert webpage");
index
@@ -180,7 +205,9 @@ mod tests {
pre_computed_score: 0.0,
crawl_stability: 0.0,
primary_image: None,
+ host_topic: None,
node_id: None,
+ dmoz_description: None,
})
.expect("failed to insert webpage");
@@ -232,7 +259,9 @@ mod tests {
pre_computed_score: 0.0,
crawl_stability: 0.0,
primary_image: None,
+ host_topic: None,
node_id: None,
+ dmoz_description: None,
})
.expect("failed to insert webpage");
index
@@ -260,7 +289,9 @@ mod tests {
pre_computed_score: 0.0,
crawl_stability: 0.0,
primary_image: None,
+ host_topic: None,
node_id: None,
+ dmoz_description: None,
})
.expect("failed to insert webpage");
@@ -311,7 +342,9 @@ mod tests {
pre_computed_score: 0.0,
crawl_stability: 0.0,
primary_image: None,
+ host_topic: None,
node_id: None,
+ dmoz_description: None,
})
.expect("failed to insert webpage");
index
@@ -339,7 +372,9 @@ mod tests {
pre_computed_score: 0.0,
crawl_stability: 0.0,
primary_image: None,
+ host_topic: None,
node_id: None,
+ dmoz_description: None,
})
.expect("failed to insert webpage");
index
@@ -365,7 +400,9 @@ mod tests {
pre_computed_score: 0.0,
crawl_stability: 0.0,
primary_image: None,
+ host_topic: None,
node_id: None,
+ dmoz_description: None,
})
.expect("failed to insert webpage");
@@ -416,7 +453,9 @@ mod tests {
pre_computed_score: 0.0,
crawl_stability: 0.0,
primary_image: None,
+ host_topic: None,
node_id: None,
+ dmoz_description: None,
})
.expect("failed to insert webpage");
index
@@ -444,7 +483,9 @@ mod tests {
crawl_stability: 0.0,
page_centrality: 0.0,
primary_image: None,
+ host_topic: None,
node_id: None,
+ dmoz_description: None,
})
.expect("failed to insert webpage");
@@ -491,7 +532,9 @@ mod tests {
pre_computed_score: 0.0,
crawl_stability: 0.0,
primary_image: None,
+ host_topic: None,
node_id: None,
+ dmoz_description: None,
})
.expect("failed to insert webpage");
index
@@ -528,8 +571,10 @@ mod tests {
pre_computed_score: 0.0,
crawl_stability: 0.0,
primary_image: None,
+ host_topic: None,
node_id: None,
- })
+ dmoz_description: None,
+ })
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
@@ -580,7 +625,9 @@ mod tests {
pre_computed_score: 0.0,
crawl_stability: 0.0,
primary_image: None,
+ host_topic: None,
node_id: None,
+ dmoz_description: None,
})
.expect("failed to insert webpage");
index
@@ -605,7 +652,9 @@ mod tests {
crawl_stability: 0.0,
page_centrality: 0.0,
primary_image: None,
+ host_topic: None,
node_id: None,
+ dmoz_description: None,
})
.expect("failed to insert webpage");
@@ -653,7 +702,9 @@ mod tests {
crawl_stability: 0.0,
page_centrality: 0.0,
primary_image: None,
+ host_topic: None,
node_id: None,
+ dmoz_description: None,
})
.expect("failed to insert webpage");
@@ -679,7 +730,9 @@ mod tests {
pre_computed_score: 0.0,
crawl_stability: 0.0,
primary_image: None,
+ host_topic: None,
node_id: None,
+ dmoz_description: None,
})
.expect("failed to insert webpage");
@@ -706,7 +759,9 @@ mod tests {
pre_computed_score: 0.0,
crawl_stability: 0.0,
primary_image: None,
+ host_topic: None,
node_id: None,
+ dmoz_description: None,
})
.expect("failed to insert webpage");
@@ -805,7 +860,9 @@ mod tests {
fetch_time_ms: 500,
page_centrality: 0.0,
primary_image: None,
+ host_topic: None,
node_id: None,
+ dmoz_description: None,
})
.expect("failed to insert webpage");
index
@@ -833,7 +890,9 @@ mod tests {
crawl_stability: 0.0,
page_centrality: 0.0,
primary_image: None,
+ host_topic: None,
node_id: None,
+ dmoz_description: None,
})
.expect("failed to insert webpage");
index
@@ -861,7 +920,9 @@ mod tests {
crawl_stability: 0.0,
page_centrality: 0.0,
primary_image: None,
+ host_topic: None,
node_id: None,
+ dmoz_description: None,
})
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
@@ -916,7 +977,9 @@ mod tests {
crawl_stability: 0.0,
page_centrality: 0.0,
primary_image: None,
+ host_topic: None,
node_id: None,
+ dmoz_description: None,
})
.expect("failed to insert webpage");
index
@@ -944,7 +1007,9 @@ mod tests {
pre_computed_score: 0.0,
crawl_stability: 0.0,
primary_image: None,
+ host_topic: None,
node_id: None,
+ dmoz_description: None,
})
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
@@ -998,7 +1063,9 @@ mod tests {
crawl_stability: 1.0,
page_centrality: 0.0,
primary_image: None,
+ host_topic: None,
node_id: None,
+ dmoz_description: None,
})
.expect("failed to insert webpage");
index
@@ -1026,7 +1093,9 @@ mod tests {
pre_computed_score: 0.0,
crawl_stability: 0.0,
primary_image: None,
+ host_topic: None,
node_id: None,
+ dmoz_description: None,
})
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
@@ -1050,4 +1119,195 @@ mod tests {
assert_eq!(result.documents[0].url, "https://www.first.com");
assert_eq!(result.documents[1].url, "https://www.second.com");
}
+
+ #[test]
+ fn topic_score() {
+ let mut index = Index::temporary().expect("Unable to open index");
+ let topic_a = Topic::from_string("/root/topic_a".to_string());
+ let topic_b = Topic::from_string("/root/topic_b".to_string());
+
+ index
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ <body>
+ <div>
+ {CONTENT} topic_a
+ </div>
+ <div>
+ {}
+ </div>
+ </body>
+ </html>
+ "#,
+ crate::rand_words(100)
+ ),
+ "https://www.a.com",
+ ),
+ backlinks: vec![],
+ host_centrality: 1.0,
+ fetch_time_ms: 1000,
+ pre_computed_score: 0.0,
+ crawl_stability: 1.0,
+ page_centrality: 0.0,
+ primary_image: None,
+ host_topic: Some(topic_a.clone()),
+ node_id: Some(1),
+ dmoz_description: None,
+ })
+ .expect("failed to insert webpage");
+ index
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ <body>
+ <div>
+ {CONTENT} topic_b
+ </div>
+ <div>
+ {}
+ </div>
+ </body>
+ </html>
+ "#,
+ crate::rand_words(100)
+ ),
+ "https://www.b.com",
+ ),
+ backlinks: vec![],
+ host_centrality: 1.0,
+ fetch_time_ms: 1000,
+ page_centrality: 0.0,
+ pre_computed_score: 0.0,
+ crawl_stability: 0.0,
+ primary_image: None,
+ host_topic: Some(topic_b.clone()),
+ node_id: Some(2),
+ dmoz_description: None,
+ })
+ .expect("failed to insert webpage");
+ index
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ <body>
+ <div>
+ {CONTENT_2} topic_a topic_b
+ </div>
+ <div>
+ {}
+ </div>
+ </body>
+ </html>
+ "#,
+ crate::rand_words(100)
+ ),
+ "https://www.wrong.com",
+ ),
+ backlinks: vec![],
+ host_centrality: 1.01,
+ fetch_time_ms: 0,
+ page_centrality: 0.0,
+ pre_computed_score: 0.0,
+ crawl_stability: 0.0,
+ primary_image: None,
+ host_topic: None,
+ node_id: Some(0),
+ dmoz_description: None,
+ })
+ .expect("failed to insert webpage");
+ index.commit().expect("failed to commit index");
+
+ let mut mapper = HashMap::new();
+ mapper.insert(
+ "www.a.com".to_string(),
+ Info {
+ description: String::new(),
+ topic: topic_a,
+ },
+ );
+ mapper.insert(
+ "www.b.com".to_string(),
+ Info {
+ description: String::new(),
+ topic: topic_b,
+ },
+ );
+ let topics = human_website_annotations::Mapper::from(mapper);
+
+ let mut webgraph = WebgraphBuilder::new(crate::gen_temp_path())
+ .with_host_graph()
+ .open();
+
+ webgraph.insert(
+ Node::from("www.wrong.com"),
+ Node::from("www.a.com"),
+ String::new(),
+ );
+ webgraph.insert(
+ Node::from("www.wrong.com"),
+ Node::from("www.b.com"),
+ String::new(),
+ );
+
+ webgraph.flush();
+
+ let approx = ApproximatedHarmonicCentrality::new(&webgraph);
+
+ let topic_centrality = TopicCentrality::build(&index, topics, webgraph, approx);
+
+ let mut searcher = LocalSearcher::new(index);
+ searcher.set_topic_centrality(topic_centrality);
+
+ let result = searcher
+ .search(&SearchQuery {
+ original: "topic_a".to_string(),
+ selected_region: None,
+ goggle_program: None,
+ skip_pages: None,
+ site_rankings: None,
+ })
+ .expect("Search failed")
+ .into_websites()
+ .unwrap()
+ .webpages;
+
+ assert_eq!(result.num_docs, 2);
+ assert_eq!(result.documents.len(), 2);
+ assert_eq!(result.documents[0].url, "https://www.a.com");
+ assert_eq!(result.documents[1].url, "https://www.wrong.com");
+
+ let result = searcher
+ .search(&SearchQuery {
+ original: "topic_b".to_string(),
+ selected_region: None,
+ goggle_program: None,
+ skip_pages: None,
+ site_rankings: None,
+ })
+ .expect("Search failed")
+ .into_websites()
+ .unwrap()
+ .webpages;
+
+ assert_eq!(result.num_docs, 2);
+ assert_eq!(result.documents.len(), 2);
+ assert_eq!(result.documents[0].url, "https://www.b.com");
+ assert_eq!(result.documents[1].url, "https://www.wrong.com");
+ }
}
diff --git a/src/ranking/signal.rs b/src/ranking/signal.rs
index f3a5a87d..8dfa2041 100644
--- a/src/ranking/signal.rs
+++ b/src/ranking/signal.rs
@@ -17,7 +17,7 @@
use crate::{
fastfield_cache,
schema::{FastField, TextField},
- webgraph::centrality::approximate_harmonic,
+ webgraph::centrality::{approximate_harmonic, topic},
webpage::Webpage,
Result,
};
@@ -45,9 +45,10 @@ pub enum Signal {
Region,
PersonalCentrality,
CrawlStability,
+ TopicCentrality,
}
-pub const ALL_SIGNALS: [Signal; 10] = [
+pub const ALL_SIGNALS: [Signal; 11] = [
Signal::Bm25,
Signal::HostCentrality,
Signal::PageCentrality,
@@ -58,6 +59,7 @@ pub const ALL_SIGNALS: [Signal; 10] = [
Signal::Region,
Signal::PersonalCentrality,
Signal::CrawlStability,
+ Signal::TopicCentrality,
];
struct SignalValue {
@@ -66,6 +68,7 @@ struct SignalValue {
current_timestamp: usize,
selected_region: Option<Region>,
personal_centrality: f64,
+ topic_score: Option<f64>,
}
impl Signal {
@@ -129,20 +132,22 @@ impl Signal {
boost + region_count.score(&webpage_region)
}
Signal::CrawlStability => value.fastfield_value.unwrap() as f64 / FLOAT_SCALING as f64,
+ Signal::TopicCentrality => value.topic_score.unwrap_or_default(),
}
}
fn default_coefficient(&self) -> f64 {
match self {
Signal::Bm25 => 1.0,
- Signal::HostCentrality => 2048.0,
- Signal::PageCentrality => 4096.0,
+ Signal::HostCentrality => 2500.0,
+ Signal::PageCentrality => 4500.0,
+ Signal::TopicCentrality => 2500.0,
Signal::IsHomepage => 0.1,
Signal::FetchTimeMs => 0.01,
Signal::UpdateTimestamp => 80.0,
Signal::NumTrackers => 20.0,
Signal::Region => 60.0,
- Signal::PersonalCentrality => 2048.0,
+ Signal::PersonalCentrality => 2500.0,
Signal::CrawlStability => 20.0,
}
}
@@ -158,6 +163,7 @@ impl Signal {
"num_trackers" => Some(Signal::NumTrackers),
"region" => Some(Signal::Region),
"personal_centrality" => Some(Signal::PersonalCentrality),
+ "topic_centrality" => Some(Signal::TopicCentrality),
_ => None,
}
}
@@ -174,6 +180,7 @@ impl Signal {
Signal::Region => Some(FastField::Region),
Signal::CrawlStability => Some(FastField::CrawlStability),
Signal::PersonalCentrality => None,
+ Signal::TopicCentrality => None,
}
}
}
@@ -245,6 +252,7 @@ pub struct SignalAggregator {
field_boost: FieldBoost,
fetch_time_ms_cache: [f64; 1000],
update_time_cache: Vec<f64>,
+ topic_scorer: Option<topic::Scorer>,
}
impl std::fmt::Debug for SignalAggregator {
@@ -283,6 +291,7 @@ impl SignalAggregator {
field_boost,
fetch_time_ms_cache,
update_time_cache,
+ topic_scorer: None,
}
}
@@ -290,6 +299,10 @@ impl SignalAggregator {
self.fastfield_cache = Some(cache);
}
+ pub fn register_topic_scorer(&mut self, topic_scorer: topic::Scorer) {
+ self.topic_scorer = Some(topic_scorer);
+ }
+
pub fn score(
&self,
doc: DocId,
@@ -298,6 +311,21 @@ impl SignalAggregator {
current_timestamp: usize,
selected_region: Option<Region>,
) -> f64 {
+ let topic_score = self.topic_scorer.as_ref().and_then(|scorer| {
+ self.fastfield_cache.as_ref().and_then(|cache| {
+ cache
+ .get_doc_cache(&FastField::HostNodeID)
+ .get_u64(&doc)
+ .and_then(|host| {
+ if host == u64::MAX {
+ None
+ } else {
+ Some(scorer.score(host))
+ }
+ })
+ })
+ });
+
ALL_SIGNALS
.into_iter()
.map(|signal| {
@@ -323,6 +351,7 @@ impl SignalAggregator {
current_timestamp,
selected_region,
personal_centrality,
+ topic_score,
},
)
})
@@ -350,7 +379,12 @@ impl SignalAggregator {
.unwrap_or(0),
Signal::NumTrackers => webpage.html.trackers().len() as u64,
Signal::Region => Region::guess_from(webpage).unwrap_or(Region::All).id(),
- _ => panic!("signal cannot be determined from webpage"),
+ Signal::CrawlStability => {
+ (webpage.crawl_stability * (FLOAT_SCALING as f64)) as u64
+ }
+ Signal::Bm25 | Signal::PersonalCentrality | Signal::TopicCentrality => {
+ panic!("signal cannot be determined from webpage")
+ }
};
let current_timestamp = Utc::now().timestamp() as usize;
@@ -365,6 +399,7 @@ impl SignalAggregator {
current_timestamp,
selected_region: None,
personal_centrality: 0.0,
+ topic_score: None,
},
)
})
diff --git a/src/ranking/site_rankings.rs b/src/ranking/site_rankings.rs
index 52da2aa2..ac2cba72 100644
--- a/src/ranking/site_rankings.rs
+++ b/src/ranking/site_rankings.rs
@@ -152,6 +152,8 @@ mod tests {
page_centrality: 0.0,
primary_image: None,
crawl_stability: 0.0,
+ dmoz_description: None,
+ host_topic: None,
node_id: Some(
graph
.host
@@ -187,6 +189,8 @@ mod tests {
pre_computed_score: 0.0,
primary_image: None,
crawl_stability: 0.0,
+ host_topic: None,
+ dmoz_description: None,
node_id: Some(
graph
.host
@@ -222,6 +226,8 @@ mod tests {
pre_computed_score: 0.0,
primary_image: None,
crawl_stability: 0.0,
+ dmoz_description: None,
+ host_topic: None,
node_id: Some(
graph
.host
diff --git a/src/schema.rs b/src/schema.rs
index fe040981..a65b6da2 100644
--- a/src/schema.rs
+++ b/src/schema.rs
@@ -43,6 +43,8 @@ pub enum TextField {
BacklinkText,
PrimaryImage,
Description,
+ HostTopic,
+ DmozDescription,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
@@ -75,7 +77,7 @@ pub enum Field {
Text(TextField),
}
-pub static ALL_FIELDS: [Field; 36] = [
+pub static ALL_FIELDS: [Field; 38] = [
Field::Text(TextField::Title),
Field::Text(TextField::CleanBody),
Field::Text(TextField::StemmedTitle),
@@ -92,6 +94,8 @@ pub static ALL_FIELDS: [Field; 36] = [
Field::Text(TextField::BacklinkText),
Field::Text(TextField::PrimaryImage),
Field::Text(TextField::Description),
+ Field::Text(TextField::HostTopic),
+ Field::Text(TextField::DmozDescription),
// FAST FIELDS
Field::Fast(FastField::IsHomepage),
Field::Fast(FastField::HostCentrality),
@@ -176,6 +180,12 @@ impl Field {
Field::Text(TextField::Description) => {
IndexingOption::Text(self.default_text_options().set_stored())
}
+ Field::Text(TextField::HostTopic) => {
+ IndexingOption::Facet(tantivy::schema::FacetOptions::default())
+ }
+ Field::Text(TextField::DmozDescription) => {
+ IndexingOption::Text(self.default_text_options().set_stored())
+ }
Field::Fast(FastField::IsHomepage) => IndexingOption::Integer(
NumericOptions::default()
.set_fast(Cardinality::SingleValue)
@@ -292,6 +302,8 @@ impl Field {
Field::Text(TextField::PrimaryImage) => "primary_image_uuid",
Field::Text(TextField::TitleIfHomepage) => "title_if_homepage",
Field::Text(TextField::AllBody) => "all_body",
+ Field::Text(TextField::HostTopic) => "host_topic",
+ Field::Text(TextField::DmozDescription) => "dmoz_description",
Field::Fast(FastField::HostCentrality) => "host_centrality",
Field::Fast(FastField::PageCentrality) => "page_centrality",
Field::Fast(FastField::IsHomepage) => "is_homepage",
@@ -318,17 +330,19 @@ impl Field {
pub fn boost(&self) -> Option<f32> {
match self {
Field::Text(TextField::Site) => Some(3.0),
- Field::Text(TextField::TitleIfHomepage) => Some(2.0),
- Field::Text(TextField::DomainIfHomepage) => Some(10.0),
+ Field::Text(TextField::TitleIfHomepage) => Some(3.0),
+ Field::Text(TextField::DomainIfHomepage) => Some(6.0),
Field::Text(TextField::DomainNameIfHomepageNoTokenizer) => Some(30.0),
Field::Text(TextField::StemmedCleanBody) | Field::Text(TextField::StemmedTitle) => {
- Some(0.1)
+ Some(0.5)
}
Field::Text(TextField::CleanBody) => Some(4.0),
+ Field::Text(TextField::DmozDescription) => Some(3.0),
Field::Text(TextField::Title) => Some(10.0),
Field::Text(TextField::Url) => Some(1.0),
Field::Text(TextField::Domain) => Some(1.0),
Field::Text(TextField::AllBody) => Some(0.01),
+ Field::Text(TextField::HostTopic) => Some(1.0),
Field::Text(TextField::BacklinkText) => Some(4.0),
Field::Text(TextField::SiteNoTokenizer)
| Field::Text(TextField::DomainNoTokenizer)
@@ -341,7 +355,9 @@ impl Field {
pub fn is_searchable(&self) -> bool {
!matches!(
self,
- Field::Text(TextField::PrimaryImage) | Field::Text(TextField::BacklinkText)
+ Field::Text(TextField::PrimaryImage)
+ | Field::Text(TextField::BacklinkText)
+ | Field::Text(TextField::HostTopic)
) && !self.is_fast()
}
@@ -367,6 +383,8 @@ impl Field {
"description" => Some(Field::Text(TextField::Description)),
"all_body" => Some(Field::Text(TextField::AllBody)),
"title_if_homepage" => Some(Field::Text(TextField::TitleIfHomepage)),
+ "host_topic" => Some(Field::Text(TextField::HostTopic)),
+ "dmoz_description" => Some(Field::Text(TextField::DmozDescription)),
"host_centrality" => Some(Field::Fast(FastField::HostCentrality)),
"page_centrality" => Some(Field::Fast(FastField::PageCentrality)),
"is_homepage" => Some(Field::Fast(FastField::IsHomepage)),
@@ -411,6 +429,7 @@ pub fn create_schema() -> tantivy::schema::Schema {
IndexingOption::Integer(options) => builder.add_u64_field(field.name(), options),
IndexingOption::Float(options) => builder.add_f64_field(field.name(), options),
IndexingOption::Bytes(options) => builder.add_bytes_field(field.name(), options),
+ IndexingOption::Facet(options) => builder.add_facet_field(field.name(), options),
};
}
@@ -422,6 +441,7 @@ pub enum IndexingOption {
Integer(tantivy::schema::NumericOptions),
Float(tantivy::schema::NumericOptions),
Bytes(tantivy::schema::BytesOptions),
+ Facet(tantivy::schema::FacetOptions),
}
pub enum DataType {
diff --git a/src/searcher/local.rs b/src/searcher/local.rs
index 6380b354..74e2f439 100644
--- a/src/searcher/local.rs
+++ b/src/searcher/local.rs
@@ -29,6 +29,7 @@ use crate::ranking::centrality_store::CentralityStore;
use crate::ranking::goggles::{self, Goggle};
use crate::ranking::{Ranker, SignalAggregator};
use crate::spell::Correction;
+use crate::webgraph::centrality::topic::TopicCentrality;
use crate::webpage::region::Region;
use crate::webpage::Url;
use crate::{inverted_index, Error, Result};
@@ -40,6 +41,7 @@ pub struct LocalSearcher {
entity_index: Option<EntityIndex>,
bangs: Option<Bangs>,
centrality_store: Option<CentralityStore>,
+ topic_centrality: Option<TopicCentrality>,
}
impl From<Index> for LocalSearcher {
@@ -55,6 +57,7 @@ impl LocalSearcher {
entity_index: None,
bangs: None,
centrality_store: None,
+ topic_centrality: None,
}
}
@@ -70,6 +73,10 @@ impl LocalSearcher {
self.centrality_store = Some(centrality_store);
}
+ pub fn set_topic_centrality(&mut self, topic_centrality: TopicCentrality) {
+ self.topic_centrality = Some(topic_centrality);
+ }
+
pub fn search_initial(
&self,
query: &SearchQuery,
@@ -145,6 +152,11 @@ impl LocalSearcher {
}
}
+ if let Some(topic_centrality) = self.topic_centrality.as_ref() {
+ let topic_scorer = topic_centrality.scorer(&parsed_query);
+ ranker.set_topic_scorer(topic_scorer);
+ }
+
ranker = ranker.with_max_docs(10_000_000, self.index.num_segments());
ranker.de_rank_similar(de_rank_similar);
@@ -287,7 +299,9 @@ mod tests {
pre_computed_score: 0.0,
primary_image: None,
node_id: None,
+ host_topic: None,
crawl_stability: 0.0,
+ dmoz_description: None,
})
.expect("failed to insert webpage");
}
diff --git a/src/snippet.rs b/src/snippet.rs
index 0d04f1b4..c7ba2c77 100644
--- a/src/snippet.rs
+++ b/src/snippet.rs
@@ -31,7 +31,7 @@ use whatlang::Lang;
/// This will require us to store each paragraph of the webpage separately to get adequate performance.
/// Implementing SnippetIL will also allow us to correctly add "..." to the snippet.
-const DEFAULT_MAX_NUM_CHARS: usize = 280;
+const DEFAULT_MAX_NUM_CHARS: usize = 250;
#[derive(Debug)]
struct FragmentCandidate {
@@ -79,7 +79,7 @@ struct Snippet {
highlighted: Vec<Range<usize>>,
}
-const HIGHLIGHTEN_PREFIX: &str = "<b style=\"font-weight: 500;\">";
+const HIGHLIGHTEN_PREFIX: &str = "<b style=\"font-weight: 700;\">";
const HIGHLIGHTEN_POSTFIX: &str = "</b>";
impl Snippet {
@@ -262,6 +262,7 @@ pub fn generate(
text: &str,
dirty_text: &str,
description: &Option<String>,
+ dmoz_description: &Option<String>,
region: &Region,
searcher: &tantivy::Searcher,
) -> Result<String> {
@@ -277,7 +278,7 @@ pub fn generate(
if snippet.fragment.is_empty() {
if text.is_empty() {
- match description {
+ match dmoz_description {
Some(desc) => {
let lang = match region.lang() {
Some(lang) => lang,
@@ -292,21 +293,38 @@ pub fn generate(
snippet.fragment = desc.chars().take(DEFAULT_MAX_NUM_CHARS).collect();
}
}
- None => {
- let lang = match region.lang() {
- Some(lang) => lang,
- None => whatlang::detect_lang(dirty_text).unwrap_or(Lang::Eng),
- };
+ None => match description {
+ Some(desc) => {
+ let lang = match region.lang() {
+ Some(lang) => lang,
+ None => whatlang::detect_lang(desc).unwrap_or(Lang::Eng),
+ };
+
+ let tokenizer = Stemmed::with_forced_language(lang).into();
+ let generator = SnippetGenerator::create(searcher, query, tokenizer)?;
+
+ snippet = generator.snippet(desc);
+ if snippet.fragment.is_empty() {
+ snippet.fragment = desc.chars().take(DEFAULT_MAX_NUM_CHARS).collect();
+ }
+ }
+ None => {
+ let lang = match region.lang() {
+ Some(lang) => lang,
+ None => whatlang::detect_lang(dirty_text).unwrap_or(Lang::Eng),
+ };
- let tokenizer = Stemmed::with_forced_language(lang).into();
- let generator = SnippetGenerator::create(searcher, query, tokenizer)?;
+ let tokenizer = Stemmed::with_forced_language(lang).into();
+ let generator = SnippetGenerator::create(searcher, query, tokenizer)?;
- snippet = generator.snippet(dirty_text);
+ snippet = generator.snippet(dirty_text);
- if snippet.fragment.is_empty() {
- snippet.fragment = dirty_text.chars().take(DEFAULT_MAX_NUM_CHARS).collect();
+ if snippet.fragment.is_empty() {
+ snippet.fragment =
+ dirty_text.chars().take(DEFAULT_MAX_NUM_CHARS).collect();
+ }
}
- }
+ },
}
} else {
snippet.fragment = text.chars().take(DEFAULT_MAX_NUM_CHARS).collect();
@@ -381,7 +399,7 @@ Survey in 2016, 2017, and 2018."#;
assert_eq!(result.webpages.num_docs, 1);
assert_eq!(result.webpages.documents.len(), 1);
- assert_eq!(result.webpages.documents[0].snippet, format!("{HIGHLIGHTEN_PREFIX}Rust{HIGHLIGHTEN_POSTFIX} is a systems programming {HIGHLIGHTEN_PREFIX}language{HIGHLIGHTEN_POSTFIX} sponsored by Mozilla which describes it as a \"safe, concurrent, practical {HIGHLIGHTEN_PREFIX}language{HIGHLIGHTEN_POSTFIX}\", supporting functional and imperative-procedural paradigms. {HIGHLIGHTEN_PREFIX}Rust{HIGHLIGHTEN_POSTFIX} is syntactically similar to C++[according to whom?], but its designers intend it to provide..."));
+ assert_eq!(result.webpages.documents[0].snippet, format!("{HIGHLIGHTEN_PREFIX}Rust{HIGHLIGHTEN_POSTFIX} is a systems programming {HIGHLIGHTEN_PREFIX}language{HIGHLIGHTEN_POSTFIX} sponsored by Mozilla which describes it as a \"safe, concurrent, practical {HIGHLIGHTEN_PREFIX}language{HIGHLIGHTEN_POSTFIX}\", supporting functional and imperative-procedural paradigms. {HIGHLIGHTEN_PREFIX}Rust{HIGHLIGHTEN_POSTFIX} is syntactically similar to C++[according to whom?], but its..."));
}
#[test]
@@ -424,7 +442,7 @@ Survey in 2016, 2017, and 2018."#;
assert_eq!(result.webpages.num_docs, 1);
assert_eq!(result.webpages.documents.len(), 1);
- assert_eq!(result.webpages.documents[0].snippet, format!("Rust is a systems programming language sponsored by Mozilla which {HIGHLIGHTEN_PREFIX}describes{HIGHLIGHTEN_POSTFIX} it as a \"safe, concurrent, practical language\", supporting functional and imperative-procedural paradigms. Rust is syntactically similar to C++[according to whom?], but its designers intend it to provide..."));
+ assert_eq!(result.webpages.documents[0].snippet, format!("Rust is a systems programming language sponsored by Mozilla which {HIGHLIGHTEN_PREFIX}describes{HIGHLIGHTEN_POSTFIX} it as a \"safe, concurrent, practical language\", supporting functional and imperative-procedural paradigms. Rust is syntactically similar to C++[according to whom?], but its..."));
}
#[test]
@@ -639,7 +657,7 @@ Survey in 2016, 2017, and 2018."#;
assert_eq!(result.webpages.documents.len(), 1);
assert_eq!(
result.webpages.documents[0].snippet,
- format!("Rust is a systems programming language sponsored by Mozilla which describes it as a \"safe, concurrent, practical language\", supporting functional and imperative-procedural {HIGHLIGHTEN_PREFIX}paradigms{HIGHLIGHTEN_POSTFIX}. Rust is syntactically similar to C++[according to whom?], but its designers intend it to provide...")
+ format!("Rust is a systems programming language sponsored by Mozilla which describes it as a \"safe, concurrent, practical language\", supporting functional and imperative-procedural {HIGHLIGHTEN_PREFIX}paradigms{HIGHLIGHTEN_POSTFIX}. Rust is syntactically similar to C++[according to whom?], but its...")
);
}
}
diff --git a/src/webgraph/centrality/approximate_harmonic.rs b/src/webgraph/centrality/approximate_harmonic.rs
index 4eb302fb..c727d1bf 100644
--- a/src/webgraph/centrality/approximate_harmonic.rs
+++ b/src/webgraph/centrality/approximate_harmonic.rs
@@ -40,7 +40,8 @@ use crate::Result;
const NUM_PROXY_NODES: usize = 10_000;
const BEST_PROXY_NODES_PER_USER_NODE: usize = 3;
-const USER_NODES_LIMIT: usize = 20; // if the user specifies more than this number of nodes, the remaining nodes will be merged into existing
+const USER_NODES_LIMIT: usize = 100; // if the user specifies more than this number of nodes, the remaining nodes will be merged into existing
+pub const SHIFT: f64 = 1.0;
#[derive(Serialize, Deserialize)]
pub struct ProxyNode {
@@ -153,14 +154,14 @@ impl Scorer {
return *cached;
}
- let res = (1.0
+ let res = (SHIFT
+ (self
.liked_nodes
.iter()
.filter_map(|liked_node| {
liked_node.best_dist(&node).map(|dist| (dist, liked_node))
})
- .map(|(dist, liked_node)| liked_node.weight as f64 / dist as f64)
+ .map(|(dist, liked_node)| liked_node.weight as f64 / (dist as f64 + 1.0))
.sum::<f64>()
- self
.disliked_nodes
@@ -170,7 +171,9 @@ impl Scorer {
.best_dist(&node)
.map(|dist| (dist, disliked_node))
})
- .map(|(dist, disliked_node)| disliked_node.weight as f64 / dist as f64)
+ .map(|(dist, disliked_node)| {
+ disliked_node.weight as f64 / (dist as f64 + 1.0)
+ })
.sum::<f64>())
/ self.num_liked_nodes as f64)
.max(0.0);
@@ -324,6 +327,28 @@ impl ApproximatedHarmonicCentrality {
)
}
+ pub fn scorer_without_fixed(&self, liked_nodes: &[Node], disliked_nodes: &[Node]) -> Scorer {
+ let liked_nodes = liked_nodes
+ .iter()
+ .map(|node| node.clone().into_host())
+ .filter_map(|node| self.node2id.get(&node).copied())
+ .collect_vec();
+
+ let disliked_nodes = disliked_nodes
+ .iter()
+ .map(|node| node.clone().into_host())
+ .filter_map(|node| self.node2id.get(&node).copied())
+ .collect_vec();
+
+ Scorer::new(
+ &self.proxy_nodes,
+ &liked_nodes,
+ &disliked_nodes,
+ HashMap::new(),
+ &self.betweenness,
+ )
+ }
+
pub fn save<P: AsRef<Path>>(&self, path: P) -> Result<()> {
let mut file = File::options()
.create(true)
@@ -337,7 +362,7 @@ impl ApproximatedHarmonicCentrality {
Ok(())
}
- pub fn load<P: AsRef<Path>>(path: P) -> Result<Self> {
+ pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
let file = File::open(path)?;
let mut reader = BufReader::new(file);
@@ -546,4 +571,24 @@ mod tests {
)
);
}
+
+ #[test]
+ fn liked_nodes_centrality_without_fix() {
+ let graph = test_graph();
+ let centrality = ApproximatedHarmonicCentrality::new_with_num_proxy(&graph, 3);
+
+ let liked_nodes = vec![Node::from("D".to_string()), Node::from("E".to_string())];
+
+ let scorer = centrality.scorer_without_fixed(&liked_nodes, &[]);
+
+ assert_eq!(
+ scorer.score(*centrality.node2id.get(&liked_nodes[0]).unwrap()),
+ 1.625
+ );
+
+ assert_eq!(
+ scorer.score(*centrality.node2id.get(&liked_nodes[1]).unwrap()),
+ 1.75
+ );
+ }
}
diff --git a/src/webgraph/centrality/mod.rs b/src/webgraph/centrality/mod.rs
index 885a3ea5..15c13d7a 100644
--- a/src/webgraph/centrality/mod.rs
+++ b/src/webgraph/centrality/mod.rs
@@ -17,3 +17,4 @@
pub mod approximate_harmonic;
pub mod betweenness;
pub mod harmonic;
+pub mod topic;
diff --git a/src/webgraph/centrality/topic.rs b/src/webgraph/centrality/topic.rs
new file mode 100644
index 00000000..c6338a99
--- /dev/null
+++ b/src/webgraph/centrality/topic.rs
@@ -0,0 +1,231 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::{
+ collections::{HashMap, HashSet},
+ fs::File,
+ io::{BufReader, Read, Write},
+ path::Path,
+ sync::Arc,
+};
+
+use indicatif::ProgressIterator;
+use itertools::Itertools;
+use rayon::prelude::*;
+use serde::{Deserialize, Serialize};
+use tantivy::{
+ query::{BooleanQuery, Occur, QueryClone, TermQuery},
+ schema::IndexRecordOption,
+};
+
+use crate::{
+ human_website_annotations::Mapper,
+ index::Index,
+ ranking::{initial::InitialScoreTweaker, SignalAggregator},
+ schema::{FastField, Field, TextField},
+ webgraph::{centrality::approximate_harmonic::SHIFT, Webgraph},
+};
+use crate::{query::Query, Result};
+
+use super::approximate_harmonic::ApproximatedHarmonicCentrality;
+
+const TOP_TERMS: usize = 1_000_000;
+pub const NUM_TOPICS: usize = 50;
+
+#[derive(Clone)]
+pub struct Scorer {
+ term_weights: Vec<f64>,
+ host_centrality: Arc<Vec<Vec<f64>>>,
+}
+
+impl Scorer {
+ pub fn score(&self, host: u64) -> f64 {
+ match self.host_centrality.get(host as usize) {
+ Some(host_score) => host_score
+ .iter()
+ .zip_eq(self.term_weights.iter())
+ .map(|(h, t)| h * t)
+ .sum(),
+ None => 0.0,
+ }
+ }
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct TopicCentrality {
+ terms: HashMap<String, Vec<f64>>,
+ host_centrality: Arc<Vec<Vec<f64>>>, // host_idx => [f64; N]
+}
+
+impl TopicCentrality {
+ pub fn build(
+ index: &Index,
+ topics: Mapper,
+ webgraph: Webgraph,
+ approx: ApproximatedHarmonicCentrality,
+ ) -> Self {
+ let body_field = index
+ .schema()
+ .get_field(Field::Text(TextField::CleanBody).name())
+ .unwrap();
+
+ let searcher = index.inverted_index.reader.searcher();
+ let mut term_count: HashMap<String, u64> = HashMap::new();
+
+ for segment in searcher.segment_readers() {
+ let reader = segment.inverted_index(body_field).unwrap();
+
+ let mut stream = reader.terms().stream().unwrap();
+
+ while let Some((term, info)) = stream.next() {
+ let term = std::str::from_utf8(term)
+ .expect("Bytes are not unicode")
+ .to_string();
+ *term_count.entry(term).or_default() += info.doc_freq as u64;
+ }
+ }
+
+ let top_topics = topics.top_topics(NUM_TOPICS);
+ let topic_field = searcher
+ .schema()
+ .get_field(Field::Text(TextField::HostTopic).name())
+ .unwrap();
+ let collector = tantivy::collector::Count;
+
+ let term_scores: HashMap<String, Vec<f64>> = term_count
+ .into_iter()
+ .sorted_by_key(|(_, count)| *count)
+ .rev()
+ .take(TOP_TERMS)
+ .progress()
+ .par_bridge()
+ .map(|(term, total_doc_freq)| {
+ let mut scores = vec![0.0; NUM_TOPICS];
+ for (i, topic) in top_topics.iter().enumerate() {
+ let facet = topic.as_facet();
+ let topic = tantivy::Term::from_facet(topic_field, &facet);
+ let topic = TermQuery::new(topic, IndexRecordOption::Basic).box_clone();
+
+ let term = tantivy::Term::from_field_text(body_field, &term);
+ let term = TermQuery::new(term, IndexRecordOption::Basic).box_clone();
+
+ let query = BooleanQuery::new(vec![(Occur::Must, topic), (Occur::Must, term)]);
+
+ let topic_freq = searcher.search(&query, &collector).unwrap();
+ scores[i] = topic_freq as f64 / total_doc_freq as f64;
+ }
+
+ (term, scores)
+ })
+ .collect();
+
+ let score_tweaker = InitialScoreTweaker::new(
+ Arc::new(index.region_count.clone()),
+ None,
+ SignalAggregator::default(),
+ Arc::clone(&index.inverted_index.fastfield_cache),
+ );
+
+ let graph_node_id = index
+ .schema()
+ .get_field(Field::Fast(FastField::HostNodeID).name())
+ .unwrap();
+
+ let collector = tantivy::collector::TopDocs::with_limit(1000).tweak_score(score_tweaker);
+
+ let mut nodes: Vec<_> = webgraph.host.as_ref().unwrap().nodes().collect();
+ nodes.sort();
+
+ let mut node_scores = vec![vec![0.0; NUM_TOPICS]; nodes.len()];
+
+ for (i, topic) in top_topics.iter().enumerate() {
+ let facet = topic.as_facet();
+ let topic = tantivy::Term::from_facet(topic_field, &facet);
+ let query = TermQuery::new(topic, IndexRecordOption::Basic).box_clone();
+
+ let top_sites: HashSet<_> = searcher
+ .search(&query, &collector)
+ .unwrap()
+ .into_iter()
+ .filter_map(|(_, doc_address)| {
+ let doc = searcher.doc(doc_address).unwrap();
+ let id = doc.get_first(graph_node_id).unwrap().as_u64().unwrap();
+ if id == u64::MAX {
+ None
+ } else {
+ Some(webgraph.host.as_ref().unwrap().id2node(&id).unwrap())
+ }
+ })
+ .collect();
+
+ let top_sites: Vec<_> = top_sites.into_iter().collect();
+
+ assert!(!top_sites.is_empty());
+
+ let scorer = approx.scorer_without_fixed(&top_sites, &[]);
+
+ for node in &nodes {
+ node_scores[*node as usize][i] = scorer.score(*node) - SHIFT;
+ }
+ }
+
+ Self {
+ terms: term_scores,
+ host_centrality: Arc::new(node_scores),
+ }
+ }
+
+ pub fn save<P: AsRef<Path>>(self, path: P) -> Result<()> {
+ let mut file = File::options()
+ .create(true)
+ .truncate(true)
+ .write(true)
+ .open(path)?;
+
+ let bytes = bincode::serialize(&self)?;
+ file.write_all(&bytes)?;
+
+ Ok(())
+ }
+
+ pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
+ let mut reader = BufReader::new(File::open(path)?);
+
+ let mut bytes = Vec::new();
+ reader.read_to_end(&mut bytes)?;
+
+ Ok(bincode::deserialize(&bytes)?)
+ }
+
+ pub fn scorer(&self, query: &Query) -> Scorer {
+ let terms = query.simple_terms();
+ let mut term_weights = [0.0; NUM_TOPICS];
+
+ for term in terms {
+ if let Some(score) = self.terms.get(&term) {
+ term_weights
+ .iter_mut()
+ .zip_eq(score.iter())
+ .for_each(|(acc, score)| *acc += score)
+ }
+ }
+
+ Scorer {
+ term_weights: term_weights.to_vec(),
+ host_centrality: Arc::clone(&self.host_centrality),
+ }
+ }
+}
diff --git a/src/webgraph/graph_store.rs b/src/webgraph/graph_store.rs
index c3a2338a..6c8bacfd 100644
--- a/src/webgraph/graph_store.rs
+++ b/src/webgraph/graph_store.rs
@@ -84,8 +84,8 @@ where
K: Hash + Eq + Serialize + DeserializeOwned + Clone,
V: Serialize + DeserializeOwned + Clone,
{
- pub(crate) store: Box<dyn Kv<K, V> + Send + Sync>,
- pub(crate) cache: LruCache<K, V>,
+ store: Box<dyn Kv<K, V> + Send + Sync>,
+ cache: LruCache<K, V>,
}
impl<K, V> CachedTree<K, V>
@@ -121,6 +121,8 @@ where
}
fn insert(&mut self, key: K, value: V) {
+ self.cache.push(key, value);
+
if self.cache.len() == self.cache.cap() {
while self.cache.len() > self.cache.cap() / 2 {
if let Some((key, value)) = self.cache.pop_lru() {
@@ -130,19 +132,17 @@ where
self.store.flush();
}
-
- self.cache.push(key, value);
}
- fn flush(&mut self) {
- for (key, value) in self.cache.iter() {
- self.store.insert(key.clone(), value.clone());
+ pub(crate) fn flush(&mut self) {
+ while let Some((key, value)) = self.cache.pop_lru() {
+ self.store.insert(key, value);
}
self.store.flush();
}
- fn iter(&self) -> impl Iterator<Item = (K, V)> + '_ {
+ pub(crate) fn iter(&self) -> impl Iterator<Item = (K, V)> + '_ {
self.store.iter()
}
}
@@ -292,6 +292,8 @@ impl<S: Store> GraphStore<S> {
self.insert(from, to, edge.label);
}
+
+ self.flush();
}
}
diff --git a/src/webgraph/mod.rs b/src/webgraph/mod.rs
index d8ff7c58..6ee4ab42 100644
--- a/src/webgraph/mod.rs
+++ b/src/webgraph/mod.rs
@@ -97,68 +97,81 @@ pub struct EdgeIterator<'a> {
current_block_idx: usize,
blocks: Vec<u64>,
adjacency: &'a Mutex<Adjacency>,
- current_block: Option<Box<dyn Iterator<Item = Edge>>>,
+ current_block: Box<dyn Iterator<Item = Edge>>,
}
impl<'a> EdgeIterator<'a> {
fn new(adjacency: &'a Mutex<Adjacency>) -> EdgeIterator<'a> {
+ adjacency.lock().unwrap().tree.inner.flush();
+
let blocks: Vec<_> = adjacency
.lock()
.unwrap()
.tree
.inner
- .store
.iter()
.map(|(key, _)| key)
.collect();
+ let first_block = adjacency
+ .lock()
+ .unwrap()
+ .tree
+ .inner
+ .get(&blocks[0])
+ .unwrap()
+ .clone();
+
EdgeIterator {
current_block_idx: 0,
blocks,
adjacency,
- current_block: None,
- }
- }
-
- fn load_next_block(&mut self) {
- if self.current_block_idx < self.blocks.len() {
- let block_id = self.blocks[self.current_block_idx];
- let block = self
- .adjacency
- .lock()
- .unwrap()
- .tree
- .inner
- .get(&block_id)
- .unwrap()
- .clone();
-
- self.current_block = Some(Box::new(block.into_iter().flat_map(|(node_id, edges)| {
+ current_block: Box::new(first_block.into_iter().flat_map(|(node_id, edges)| {
edges.into_iter().map(move |edge| Edge {
from: node_id,
to: edge.other,
label: edge.label,
})
- })));
-
- self.current_block_idx += 1;
+ })),
}
}
+
+ fn load_next_block(&mut self) {
+ let block_id = self.blocks[self.current_block_idx];
+ let block = self
+ .adjacency
+ .lock()
+ .unwrap()
+ .tree
+ .inner
+ .get(&block_id)
+ .unwrap()
+ .clone();
+
+ self.current_block = Box::new(block.into_iter().flat_map(|(node_id, edges)| {
+ edges.into_iter().map(move |edge| Edge {
+ from: node_id,
+ to: edge.other,
+ label: edge.label,
+ })
+ }));
+
+ self.current_block_idx += 1;
+ }
}
impl<'a> Iterator for EdgeIterator<'a> {
type Item = Edge;
fn next(&mut self) -> Option<Self::Item> {
- if let Some(res) = self.current_block.as_mut().and_then(|it| it.next()) {
- return Some(res);
- }
+ let mut res = self.current_block.next();
- if self.current_block.is_none() {
+ while self.current_block_idx < self.blocks.len() && res.is_none() {
self.load_next_block();
+ res = self.current_block.next();
}
- self.current_block.as_mut().and_then(|it| it.next())
+ res
}
}
diff --git a/src/webpage/mod.rs b/src/webpage/mod.rs
index 953c5bfa..420c383d 100644
--- a/src/webpage/mod.rs
+++ b/src/webpage/mod.rs
@@ -14,6 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::{
+ human_website_annotations::Topic,
prehashed::{hash, split_u128},
schema::{FastField, TextField},
schema_org::SchemaOrg,
@@ -130,6 +131,8 @@ pub struct Webpage {
pub primary_image: Option<StoredPrimaryImage>,
pub node_id: Option<u64>,
pub crawl_stability: f64,
+ pub host_topic: Option<Topic>,
+ pub dmoz_description: Option<String>,
}
impl Webpage {
@@ -146,13 +149,33 @@ impl Webpage {
pre_computed_score: 0.0,
primary_image: None,
node_id: None,
+ host_topic: None,
crawl_stability: 0.0,
+ dmoz_description: None,
}
}
+ fn dmoz_description(&self) -> Option<String> {
+ self.dmoz_description.as_ref().and_then(|desc| {
+ if !self.html.metadata().iter().any(|metadata| {
+ if let Some(content) = metadata.get(&"content".to_string()) {
+ content.contains("noodp")
+ } else {
+ false
+ }
+ }) {
+ Some(desc.clone())
+ } else {
+ None
+ }
+ })
+ }
+
pub fn into_tantivy(self, schema: &tantivy::schema::Schema) -> Result<tantivy::Document> {
let region = Region::guess_from(&self);
+ let dmoz_description = self.dmoz_description();
+
let mut doc = self.html.into_tantivy(schema)?;
if let Ok(region) = region {
@@ -234,6 +257,25 @@ impl Webpage {
(self.crawl_stability * FLOAT_SCALING as f64) as u64,
);
+ let facet = self
+ .host_topic
+ .map(|topic| topic.as_facet())
+ .unwrap_or_default();
+
+ doc.add_facet(
+ schema
+ .get_field(Field::Text(TextField::HostTopic).name())
+ .expect("failed to get host_topic field"),
+ facet,
+ );
+
+ doc.add_text(
+ schema
+ .get_field(Field::Text(TextField::DmozDescription).name())
+ .expect("failed to get host_topic field"),
+ dmoz_description.unwrap_or_default(),
+ );
+
Ok(doc)
}
@@ -324,14 +366,8 @@ impl Html {
pub fn links(&self) -> Vec<Link> {
let mut links = Vec::new();
let mut open_links = Vec::new();
- let mut preprocessor = Preprocessor::new(["script", "style", "head", "noscript"]);
for edge in self.root.traverse() {
- preprocessor.update(&edge);
- if preprocessor.is_inside_removed() {
- continue;
- }
-
match edge {
NodeEdge::Start(node) => {
if let Some(element) = node.as_element() {
@@ -370,6 +406,16 @@ impl Html {
}
}
+ while let Some((text, attributes)) = open_links.pop() {
+ if let Some(dest) = attributes.borrow().get("href") {
+ links.push(Link {
+ source: self.url.clone(),
+ destination: dest.to_string().into(),
+ text: text.trim().to_string(),
+ });
+ }
+ }
+
links
}
@@ -718,6 +764,7 @@ impl Html {
doc.add_u64(tantivy_field, hash);
}
Field::Text(TextField::BacklinkText)
+ | Field::Text(TextField::HostTopic)
| Field::Fast(FastField::HostCentrality)
| Field::Fast(FastField::PageCentrality)
| Field::Fast(FastField::FetchTimeMs)
@@ -725,6 +772,7 @@ impl Html {
| Field::Fast(FastField::Region)
| Field::Fast(FastField::HostNodeID)
| Field::Fast(FastField::CrawlStability)
+ | Field::Text(TextField::DmozDescription)
| Field::Text(TextField::PrimaryImage) => {}
}
}
@@ -1645,4 +1693,73 @@ mod tests {
assert_eq!(html.title(), Some("Test site".to_string()));
assert_eq!(html.all_text(), Some("test".to_string()));
}
+
+ #[test]
+ fn dmoz_description() {
+ let html = Html::parse(
+ r#"
+ <html>
+ <head>
+ <title>Test site</title>
+ </head>
+ <body>
+ test
+ </body>
+ </html>
+ "#,
+ "example.com",
+ );
+
+ let webpage = Webpage {
+ html,
+ backlinks: Vec::new(),
+ host_centrality: 0.0,
+ page_centrality: 0.0,
+ fetch_time_ms: 500,
+ pre_computed_score: 0.0,
+ primary_image: None,
+ node_id: None,
+ crawl_stability: 0.0,
+ host_topic: None,
+ dmoz_description: Some("dmoz description".to_string()),
+ };
+
+ assert_eq!(
+ webpage.dmoz_description(),
+ Some("dmoz description".to_string())
+ )
+ }
+
+ #[test]
+ fn noodp_ignores_dmoz() {
+ let html = Html::parse(
+ r#"
+ <html>
+ <head>
+ <meta name="robots" content="noodp" />
+ <title>Test site</title>
+ </head>
+ <body>
+ test
+ </body>
+ </html>
+ "#,
+ "example.com",
+ );
+ let webpage = Webpage {
+ html,
+ backlinks: Vec::new(),
+ host_centrality: 0.0,
+ page_centrality: 0.0,
+ fetch_time_ms: 500,
+ pre_computed_score: 0.0,
+ primary_image: None,
+ node_id: None,
+ crawl_stability: 0.0,
+ host_topic: None,
+ dmoz_description: Some("dmoz description".to_string()),
+ };
+
+ assert_eq!(webpage.dmoz_description(), None)
+ }
}