Skip to content

web search done right

Disable webgraph compression. An insane amount of time was spent decompressing (about 80%) when fetching inbound links. This made it impractical to index backlink texts, which of course is very usefull information to have in the index.

Mikkel Denker3b8bdc22b653
diff --git a/Cargo.lock b/Cargo.lock
index 43ee9903..17847337 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -75,17 +75,6 @@ dependencies = [
"version_check",
]
-[[package]]
-name = "ahash"
-version = "0.8.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f"
-dependencies = [
- "cfg-if",
- "once_cell",
- "version_check",
-]
-
[[package]]
name = "aho-corasick"
version = "0.7.20"
@@ -1819,16 +1808,7 @@ version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
dependencies = [
- "ahash 0.7.6",
-]
-
-[[package]]
-name = "hashbrown"
-version = "0.13.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e"
-dependencies = [
- "ahash 0.8.3",
+ "ahash",
]
[[package]]
@@ -2439,15 +2419,6 @@ dependencies = [
"hashbrown 0.12.3",
]
-[[package]]
-name = "lru"
-version = "0.10.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "03f1160296536f10c833a82dca22267d5486734230d47bf00bf435885814ba1e"
-dependencies = [
- "hashbrown 0.13.2",
-]
-
[[package]]
name = "lz-str"
version = "0.2.1"
@@ -4360,7 +4331,6 @@ dependencies = [
"lalrpop-util",
"libc",
"logos",
- "lru 0.10.0",
"lz-str",
"maplit",
"md5",
@@ -4530,7 +4500,7 @@ dependencies = [
"itertools 0.10.5",
"levenshtein_automata",
"log",
- "lru 0.7.8",
+ "lru",
"lz4_flex",
"measure_time",
"memmap2",
diff --git a/Cargo.toml b/Cargo.toml
index 9bf3b0a3..dd136093 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -88,7 +88,6 @@ robotstxt-with-cache = "0.4.0"
rust-s3 = { version = "0.33.0", features = ["blocking", "tokio"] }
rusqlite = { version = "0.29.0", features = ["bundled", "modern-full", "buildtime_bindgen"] }
async-channel = "1.8.0"
-lru = "0.10.0"
url = { version = "2.4.0", features = ["serde"] }
anyhow = { version = "1.0.72", features = ["backtrace"] }
bytemuck = { version = "1.13.1", features = ["derive"] }
diff --git a/core/Cargo.toml b/core/Cargo.toml
index 6ade96bb..ca2c2721 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -88,7 +88,6 @@ reqwest-eventsource = { workspace = true }
robotstxt-with-cache = { workspace = true }
rust-s3 = { workspace = true }
async-channel = { workspace = true }
-lru = { workspace = true }
url = { workspace = true }
anyhow = { workspace = true }
bytemuck = { workspace = true }
diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rs
index 8477ea08..1188aa1e 100644
--- a/core/src/entrypoint/indexer.rs
+++ b/core/src/entrypoint/indexer.rs
@@ -17,7 +17,6 @@ use chrono::Utc;
use rayon::prelude::*;
use std::path::Path;
use std::thread;
-use url::Url;
use itertools::Itertools;
use serde::{Deserialize, Serialize};
@@ -31,7 +30,7 @@ use crate::mapreduce::{Map, Reduce, Worker};
use crate::ranking::centrality_store::IndexerCentralityStore;
use crate::ranking::SignalAggregator;
use crate::webgraph::{Node, Webgraph, WebgraphBuilder};
-use crate::webpage::{Html, Link, Webpage};
+use crate::webpage::{Html, Webpage};
use crate::{human_website_annotations, Result};
pub struct Indexer {}
@@ -168,22 +167,14 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
}
}
- let backlinks: Vec<Link> = worker
+ let backlink_labels: Vec<String> = worker
.webgraph
.as_ref()
.map(|webgraph| {
webgraph
- .ingoing_edges(Node::from(html.url()))
+ .raw_ingoing_edges_with_labels(&Node::from(html.url()).id())
.into_iter()
- .map(|edge| Link {
- source: Url::parse(&("http://".to_string() + edge.from.name.as_str()))
- .unwrap(),
- destination: Url::parse(
- &("http://".to_string() + edge.to.name.as_str()),
- )
- .unwrap(),
- text: edge.label,
- })
+ .map(|edge| edge.label.loaded().unwrap())
.collect()
})
.unwrap_or_else(Vec::new);
@@ -204,7 +195,7 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
has_page_centrality = true;
}
- if !backlinks.is_empty() {
+ if !backlink_labels.is_empty() {
has_backlinks = true;
}
@@ -227,7 +218,7 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
let mut webpage = Webpage {
html,
- backlinks,
+ backlink_labels,
page_centrality,
host_centrality,
fetch_time_ms,
diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rs
index 9cd236bd..6863bff6 100644
--- a/core/src/inverted_index.rs
+++ b/core/src/inverted_index.rs
@@ -616,7 +616,7 @@ mod tests {
use crate::{
ranking::{Ranker, SignalAggregator},
searcher::SearchQuery,
- webpage::{Html, Link},
+ webpage::Html,
};
use super::*;
@@ -891,11 +891,7 @@ mod tests {
"https://www.b.com",
)
.unwrap(),
- backlinks: vec![Link {
- source: Url::parse("https://www.a.com/").unwrap(),
- destination: Url::parse("https://www.b.com/").unwrap(),
- text: "B site is great".to_string(),
- }],
+ backlink_labels: vec!["B site is great".to_string()],
host_centrality: 1.0,
page_centrality: 0.0,
fetch_time_ms: 500,
diff --git a/core/src/query/optic.rs b/core/src/query/optic.rs
index e3a3ec8c..fc68c310 100644
--- a/core/src/query/optic.rs
+++ b/core/src/query/optic.rs
@@ -260,7 +260,7 @@ mod tests {
),
"https://www.a.com",
).unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: 0.0,
page_centrality: 0.0,
fetch_time_ms: 500,
@@ -288,7 +288,7 @@ mod tests {
),
"https://www.b.com",
).unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: 0.01,
page_centrality: 0.0,
@@ -386,7 +386,7 @@ mod tests {
),
"https://www.a.com/this/is/a/pattern",
).unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: 0.0,
page_centrality: 0.0,
fetch_time_ms: 500,
@@ -413,7 +413,7 @@ mod tests {
),
"https://www.b.com/this/is/b/pattern",
).unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: 0.0001,
page_centrality: 0.0,
@@ -489,7 +489,7 @@ mod tests {
),
"https://www.a.com/this/is/a/pattern",
).unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: 0.0,
page_centrality: 0.0,
fetch_time_ms: 500,
@@ -517,7 +517,7 @@ mod tests {
),
"https://www.b.com/this/is/b/pattern",
).unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: 0.0001,
page_centrality: 0.0,
@@ -545,7 +545,7 @@ mod tests {
),
"https://www.c.com/this/is/c/pattern",
).unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: 0.0001,
page_centrality: 0.0,
@@ -648,7 +648,7 @@ mod tests {
),
"https://www.a.com/this/is/a/pattern",
).unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: 0.0,
page_centrality: 0.0,
fetch_time_ms: 500,
@@ -676,7 +676,7 @@ mod tests {
),
"https://www.b.com/this/is/b/pattern",
).unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: 0.0001,
page_centrality: 0.0,
@@ -704,7 +704,7 @@ mod tests {
),
"https://www.c.com/this/is/c/pattern",
).unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: 0.0002,
page_centrality: 0.0,
@@ -779,7 +779,7 @@ mod tests {
),
"https://www.a.com/",
).unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: 0.0,
page_centrality: 0.0,
fetch_time_ms: 500,
@@ -822,7 +822,7 @@ mod tests {
),
"https://www.b.com/",
).unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: 0.0001,
page_centrality: 0.0,
@@ -956,7 +956,7 @@ mod tests {
),
"https://chat.stackoverflow.com",
).unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: 0.0,
page_centrality: 0.0,
fetch_time_ms: 500,
@@ -1030,7 +1030,7 @@ mod tests {
),
"https://a.com",
).unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: 0.0,
page_centrality: 0.0,
fetch_time_ms: 500,
@@ -1059,7 +1059,7 @@ mod tests {
),
"https://b.com/",
).unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: 0.0,
page_centrality: 0.0,
fetch_time_ms: 500,
@@ -1113,7 +1113,7 @@ mod tests {
include_str!("../../testcases/schema_org/infinity_war.html"),
"https://a.com",
).unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: 0.0,
page_centrality: 0.0,
fetch_time_ms: 500,
@@ -1171,7 +1171,7 @@ mod tests {
),
"https://example.com",
).unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: 0.0,
page_centrality: 0.0,
fetch_time_ms: 500,
@@ -1332,7 +1332,7 @@ mod tests {
),
"https://example.com",
).unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: 0.0,
page_centrality: 0.0,
fetch_time_ms: 500,
@@ -1405,7 +1405,7 @@ mod tests {
),
"https://example.com",
).unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: 0.0,
page_centrality: 0.0,
fetch_time_ms: 500,
@@ -1465,7 +1465,7 @@ mod tests {
),
"https://example.com",
).unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: 0.0,
page_centrality: 0.0,
fetch_time_ms: 500,
@@ -1494,7 +1494,7 @@ mod tests {
),
"https://example.com",
).unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: 0.0,
page_centrality: 0.0,
fetch_time_ms: 500,
@@ -1592,7 +1592,7 @@ mod tests {
"#,
"https://example.com/",
).unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: 0.0,
page_centrality: 0.0,
fetch_time_ms: 500,
diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rs
index 9f95bae7..77784573 100644
--- a/core/src/ranking/inbound_similarity.rs
+++ b/core/src/ranking/inbound_similarity.rs
@@ -364,7 +364,7 @@ mod tests {
"https://e.com",
)
.unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: 0.0,
fetch_time_ms: 500,
page_centrality: 0.0,
@@ -393,7 +393,7 @@ mod tests {
"https://d.com",
)
.unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: 0.01,
fetch_time_ms: 500,
page_centrality: 0.0,
diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rs
index 96a2fde6..042fb8f3 100644
--- a/core/src/ranking/mod.rs
+++ b/core/src/ranking/mod.rs
@@ -135,12 +135,11 @@ impl Ranker {
mod tests {
use optics::Optic;
- use url::Url;
use crate::{
index::Index,
searcher::{LocalSearcher, SearchQuery},
- webpage::{Html, Webpage, Link},
+ webpage::{Html, Webpage},
};
const CONTENT: &str = "this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever";
@@ -169,7 +168,7 @@ mod tests {
),
"https://www.a.com",
).unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: 0.0,
fetch_time_ms: 500,
page_centrality: 0.0,
@@ -197,7 +196,7 @@ mod tests {
),
"https://www.b.com",
).unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: 5.0,
fetch_time_ms: 500,
page_centrality: 0.0,
@@ -244,7 +243,7 @@ mod tests {
),
"https://www.a.com",
).unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: 0.0,
fetch_time_ms: 500,
page_centrality: 0.0,
@@ -272,7 +271,7 @@ mod tests {
),
"https://www.b.com",
).unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: 0.0,
fetch_time_ms: 500,
page_centrality: 5.0,
@@ -318,7 +317,7 @@ mod tests {
),
"https://www.old.com",
).unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: 1.0,
fetch_time_ms: 499,
page_centrality: 0.0,
@@ -346,7 +345,7 @@ mod tests {
),
"https://www.new.com",
).unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: 1.0,
fetch_time_ms: 500,
pre_computed_score: 0.0,
@@ -388,7 +387,7 @@ mod tests {
"#,
"https://www.first.com",
).unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: 0.0,
fetch_time_ms: 500,
page_centrality: 0.0,
@@ -425,7 +424,7 @@ mod tests {
</html>
"#,
"https://www.second.com").unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: 0.00003,
fetch_time_ms: 500,
page_centrality: 0.0,
@@ -468,11 +467,7 @@ mod tests {
"#,
"https://www.first.com",
).unwrap(),
- backlinks: vec![Link {
- source: Url::parse("https://www.second.com").unwrap(),
- destination: Url::parse("https://www.first.com").unwrap(),
- text: "test this is the best test site".to_string(),
- }],
+ backlink_labels: vec!["test this is the best test site".to_string()],
host_centrality: 0.0,
fetch_time_ms: 500,
page_centrality: 0.0,
@@ -497,7 +492,7 @@ mod tests {
"#,
"https://www.second.com",
).unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: 0.00003,
fetch_time_ms: 500,
pre_computed_score: 0.0,
@@ -540,7 +535,7 @@ mod tests {
"#,
"https://www.body.com",
).unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: 1.0,
fetch_time_ms: 20,
pre_computed_score: 0.0,
@@ -566,7 +561,7 @@ mod tests {
"#,
"https://www.title.com",
).unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: 1.0,
fetch_time_ms: 20,
page_centrality: 0.0,
@@ -593,7 +588,7 @@ mod tests {
"#,
"https://www.centrality.com",
).unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: 1.02,
fetch_time_ms: 500,
page_centrality: 0.0,
@@ -667,7 +662,7 @@ mod tests {
),
"https://www.first.com",
).unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: 1.0,
pre_computed_score: 0.0,
fetch_time_ms: 500,
@@ -699,7 +694,7 @@ mod tests {
),
"https://www.third.com",
).unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: 1.0,
fetch_time_ms: 500,
pre_computed_score: 0.0,
@@ -731,7 +726,7 @@ mod tests {
),
"https://www.second.com",
).unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: 1.0,
fetch_time_ms: 500,
pre_computed_score: 0.0,
@@ -785,7 +780,7 @@ mod tests {
),
"https://www.first.com",
).unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: 1.0,
fetch_time_ms: 0,
pre_computed_score: 0.0,
@@ -813,7 +808,7 @@ mod tests {
),
"https://www.second.com",
).unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: 1.0,
fetch_time_ms: 5000,
page_centrality: 0.0,
@@ -862,7 +857,7 @@ mod tests {
),
"https://www.first.com/one",
).unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: 1.0,
fetch_time_ms: 2,
pre_computed_score: 0.0,
@@ -890,7 +885,7 @@ mod tests {
),
"https://www.second.com/one/two",
).unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: 1.0,
fetch_time_ms: 1,
page_centrality: 0.0,
@@ -918,7 +913,7 @@ mod tests {
),
"https://www.third.com/one/two123",
).unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: 1.0,
fetch_time_ms: 0,
page_centrality: 0.0,
diff --git a/core/src/ranking/optics.rs b/core/src/ranking/optics.rs
index da791350..a8ec071f 100644
--- a/core/src/ranking/optics.rs
+++ b/core/src/ranking/optics.rs
@@ -96,7 +96,7 @@ mod tests {
"https://www.first.com",
)
.unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: 1.0,
fetch_time_ms: 500,
pre_computed_score: 0.0,
@@ -125,7 +125,7 @@ mod tests {
"https://www.second.com",
)
.unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: 1.0,
fetch_time_ms: 499,
page_centrality: 0.0,
@@ -154,7 +154,7 @@ mod tests {
"https://www.third.com",
)
.unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: 1.0,
fetch_time_ms: 0,
page_centrality: 0.0,
diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rs
index a0c64ced..fb5181c2 100644
--- a/core/src/searcher/local.rs
+++ b/core/src/searcher/local.rs
@@ -444,7 +444,7 @@ mod tests {
&format!("https://www.{i}.com"),
)
.unwrap(),
- backlinks: vec![],
+ backlink_labels: vec![],
host_centrality: (NUM_WEBSITES - i) as f64,
fetch_time_ms: 500,
page_centrality: 0.0,
diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rs
index e65b3df3..2e0688f9 100644
--- a/core/src/webgraph/mod.rs
+++ b/core/src/webgraph/mod.rs
@@ -19,10 +19,9 @@ use std::collections::{BTreeMap, BinaryHeap, HashSet};
use std::fs::File;
use std::io::{BufReader, BufWriter, Read, Write};
use std::path::Path;
-use std::sync::{Arc, Mutex};
+use std::sync::Arc;
use std::{cmp, fs};
-use lru::LruCache;
use rayon::prelude::*;
use serde::{Deserialize, Serialize};
use url::Url;
@@ -206,7 +205,7 @@ pub enum Loaded<T> {
NotYet,
}
impl<T> Loaded<T> {
- fn loaded(self) -> Option<T> {
+ pub fn loaded(self) -> Option<T> {
match self {
Loaded::Some(t) => Some(t),
Loaded::NotYet => None,
@@ -214,6 +213,20 @@ impl<T> Loaded<T> {
}
}
+impl<T> std::fmt::Display for Loaded<T>
+where
+ T: std::fmt::Display,
+{
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ match self {
+ Loaded::Some(val) => {
+ write!(f, "{}", val)
+ }
+ Loaded::NotYet => Ok(()),
+ }
+ }
+}
+
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct FullEdge {
pub from: Node,
@@ -346,7 +359,6 @@ type SegmentID = String;
#[derive(Serialize, Deserialize, Default)]
struct Meta {
comitted_segments: Vec<SegmentID>,
- next_node_id: u64,
}
struct SegmentMergeCandidate {
@@ -360,7 +372,6 @@ pub struct Webgraph {
segments: Vec<StoredSegment>,
executor: Arc<Executor>,
id2node: Store<NodeID, Node>,
- id2node_cache: Mutex<LruCache<NodeID, Node>>,
commit_mode: CommitMode,
meta: Meta,
}
@@ -417,7 +428,6 @@ impl Webgraph {
commit_mode,
executor: Arc::new(executor),
id2node: Store::open(path.as_ref().join("id2node")),
- id2node_cache: Mutex::new(LruCache::new(10_000.try_into().unwrap())),
meta,
}
}
@@ -495,6 +505,10 @@ impl Webgraph {
self.inner_ingoing_edges(node, false)
}
+ pub fn raw_ingoing_edges_with_labels(&self, node: &NodeID) -> Vec<Edge> {
+ self.inner_ingoing_edges(node, true)
+ }
+
fn inner_ingoing_edges(&self, node: &NodeID, load_labels: bool) -> Vec<Edge> {
self.executor
.map(
@@ -535,19 +549,7 @@ impl Webgraph {
}
pub fn id2node(&self, id: &NodeID) -> Option<Node> {
- let mut guard = self.id2node_cache.lock().unwrap();
-
- if let Some(node) = guard.get(id) {
- return Some(node.clone());
- }
-
- let node = self.id2node.get(id);
-
- if let Some(node) = &node {
- guard.put(*id, node.clone());
- }
-
- node
+ self.id2node.get(id)
}
pub fn nodes(&self) -> impl Iterator<Item = NodeID> + '_ {
diff --git a/core/src/webgraph/store.rs b/core/src/webgraph/store.rs
index 5f3bc323..ff81b224 100644
--- a/core/src/webgraph/store.rs
+++ b/core/src/webgraph/store.rs
@@ -47,6 +47,8 @@ where
options.set_block_based_table_factory(&block_options);
options.set_optimize_filters_for_hits(true);
+ options.set_compression_type(rocksdb::DBCompressionType::None);
+
let db = rocksdb::DB::open(&options, path.as_ref().to_str().unwrap()).unwrap();
Self {
diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rs
index 5fdb1d67..915edc73 100644
--- a/core/src/webpage/mod.rs
+++ b/core/src/webpage/mod.rs
@@ -114,7 +114,7 @@ impl<const N: usize> Preprocessor<N> {
pub struct Webpage {
pub html: Html,
- pub backlinks: Vec<Link>,
+ pub backlink_labels: Vec<String>,
pub host_centrality: f64,
pub page_centrality: f64,
pub fetch_time_ms: u64,
@@ -130,7 +130,7 @@ impl Webpage {
Ok(Self {
html,
- backlinks: Vec::new(),
+ backlink_labels: Vec::new(),
host_centrality: 0.0,
page_centrality: 0.0,
fetch_time_ms: 0,
@@ -180,7 +180,7 @@ impl Webpage {
}
let backlink_text: String = itertools::intersperse(
- self.backlinks.into_iter().map(|link| link.text),
+ self.backlink_labels.into_iter().map(|label| label),
"\n".to_string(),
)
.collect();
@@ -2034,7 +2034,7 @@ mod tests {
let webpage = Webpage {
html,
- backlinks: Vec::new(),
+ backlink_labels: Vec::new(),
host_centrality: 0.0,
page_centrality: 0.0,
fetch_time_ms: 500,
@@ -2068,7 +2068,7 @@ mod tests {
.unwrap();
let webpage = Webpage {
html,
- backlinks: Vec::new(),
+ backlink_labels: Vec::new(),
host_centrality: 0.0,
page_centrality: 0.0,
fetch_time_ms: 500,