Skip to content

web search done right

ability to easily export site rankings as an optic

Mikkel Denker7eb5387c90c2
diff --git a/core/src/api/mod.rs b/core/src/api/mod.rs
index c3786182..feb36918 100644
--- a/core/src/api/mod.rs
+++ b/core/src/api/mod.rs
@@ -191,6 +191,7 @@ pub async fn router(config: &FrontendConfig, counters: Counters) -> Result<Route
.route("/settings", get(optics::route))
.route("/settings/optics", get(optics::route))
.route("/settings/sites", get(sites::route))
+ .route("/settings/sites/export", get(sites::export))
.route("/settings/privacy", get(improvement::settings))
.route("/privacy-and-happy-lawyers", get(privacy::route))
.route("/webmasters", get(crawler::info_route))
diff --git a/core/src/api/sites.rs b/core/src/api/sites.rs
index 6f718374..8fe95dc8 100644
--- a/core/src/api/sites.rs
+++ b/core/src/api/sites.rs
@@ -19,6 +19,8 @@ use std::sync::Arc;
use super::{HtmlTemplate, State};
use askama::Template;
use axum::{extract, response::IntoResponse};
+use http::StatusCode;
+use optics::{Optic, SiteRankings};
#[allow(clippy::unused_async)]
pub async fn route(extract::State(state): extract::State<Arc<State>>) -> impl IntoResponse {
@@ -34,3 +36,42 @@ pub async fn route(extract::State(state): extract::State<Arc<State>>) -> impl In
struct SitesTemplate {
with_alice: Option<bool>,
}
+
+#[derive(serde::Deserialize)]
+pub struct ExportParams {
+ pub data: String,
+}
+
+#[allow(clippy::unused_async)]
+pub async fn export(
+ extract::Query(ExportParams { data }): extract::Query<ExportParams>,
+) -> Result<impl IntoResponse, StatusCode> {
+ let data = data.replace(' ', "+");
+ match lz_str::decompress_from_base64(&data) {
+ Some(bytes) => match String::from_utf16(&bytes) {
+ Ok(s) => match serde_json::from_str::<SiteRankings>(&s) {
+ Ok(site_rankings) => {
+ let optic = Optic {
+ site_rankings,
+ rules: vec![],
+ ..Default::default()
+ };
+
+ Ok(optic.to_string())
+ }
+ Err(err) => {
+ tracing::error!("Failed to parse data: {}", err);
+ Err(StatusCode::BAD_REQUEST)
+ }
+ },
+ Err(err) => {
+ tracing::error!("Failed to parse data: {}", err);
+ Err(StatusCode::BAD_REQUEST)
+ }
+ },
+ None => {
+ tracing::error!("Failed to de-compress data");
+ Err(StatusCode::BAD_REQUEST)
+ }
+ }
+}
diff --git a/core/src/config/mod.rs b/core/src/config/mod.rs
index 787a3a6d..188feb50 100644
--- a/core/src/config/mod.rs
+++ b/core/src/config/mod.rs
@@ -25,17 +25,17 @@ use std::net::SocketAddr;
#[derive(Debug, Deserialize, Clone)]
pub struct IndexingLocalConfig {
- pub(crate) limit_warc_files: Option<usize>,
- pub(crate) skip_warc_files: Option<usize>,
- pub(crate) warc_source: WarcSource,
- pub(crate) batch_size: Option<usize>,
- pub(crate) page_webgraph_path: Option<String>,
- pub(crate) output_path: Option<String>,
- pub(crate) host_centrality_threshold: Option<f64>,
- pub(crate) topics_path: Option<String>,
- pub(crate) host_centrality_store_path: String,
- pub(crate) page_centrality_store_path: Option<String>,
- pub(crate) minimum_clean_words: Option<usize>,
+ pub limit_warc_files: Option<usize>,
+ pub skip_warc_files: Option<usize>,
+ pub warc_source: WarcSource,
+ pub batch_size: Option<usize>,
+ pub page_webgraph_path: Option<String>,
+ pub output_path: Option<String>,
+ pub host_centrality_threshold: Option<f64>,
+ pub topics_path: Option<String>,
+ pub host_centrality_store_path: String,
+ pub page_centrality_store_path: Option<String>,
+ pub minimum_clean_words: Option<usize>,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
@@ -47,12 +47,12 @@ pub enum WebgraphLevel {
#[derive(Debug, Deserialize, Clone)]
pub struct WebgraphConstructConfig {
- pub(crate) level: WebgraphLevel,
- pub(crate) redirect_db_path: Option<String>,
- pub(crate) limit_warc_files: Option<usize>,
- pub(crate) warc_source: WarcSource,
- pub(crate) graph_base_path: Option<String>,
- pub(crate) batch_size: Option<usize>,
+ pub level: WebgraphLevel,
+ pub redirect_db_path: Option<String>,
+ pub limit_warc_files: Option<usize>,
+ pub warc_source: WarcSource,
+ pub graph_base_path: Option<String>,
+ pub batch_size: Option<usize>,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
@@ -130,8 +130,8 @@ pub struct LocalConfig {
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct HttpConfig {
- pub(crate) base_url: String,
- pub(crate) warc_paths_file: String,
+ pub base_url: String,
+ pub warc_paths_file: String,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rs
index 915edc73..72ff4944 100644
--- a/core/src/webpage/mod.rs
+++ b/core/src/webpage/mod.rs
@@ -179,11 +179,8 @@ impl Webpage {
);
}
- let backlink_text: String = itertools::intersperse(
- self.backlink_labels.into_iter().map(|label| label),
- "\n".to_string(),
- )
- .collect();
+ let backlink_text: String =
+ itertools::intersperse(self.backlink_labels, "\n".to_string()).collect();
doc.add_text(
schema
diff --git a/frontend/src/pages/settings/sites.astro b/frontend/src/pages/settings/sites.astro
index ea8fe308..5397402b 100644
--- a/frontend/src/pages/settings/sites.astro
+++ b/frontend/src/pages/settings/sites.astro
@@ -2,56 +2,68 @@
import SettingsLayout from "../../layouts/Settings.astro";
---
<SettingsLayout>
- <div>
- <h1 class="mb-2 text-2xl font-medium">Liked Sites</h1>
- <div class="text-sm">
- Sites that are similar to these sites receive a boost during search. Their results are more likely to appear in your search results.
- </div>
+ <div class="flex flex-col w-full h-full space-y-10">
+
<div>
- <div
- class="grid mb-10 w-full grid-cols-[auto_1fr] space-y-5"
- id="liked-list"
- >
- <div class="w-10"></div>
- <div class="font-medium">Site Name</div>
+ <h1 class="mb-2 text-2xl font-medium">Liked Sites</h1>
+ <div class="text-sm">
+ Sites that are similar to these sites receive a boost during search. Their results are more likely to appear in your search results.
+ </div>
+ <div>
+ <div
+ class="grid w-full grid-cols-[auto_1fr] space-y-5"
+ id="liked-list"
+ >
+ <div class="w-10"></div>
+ <div class="font-medium">Site Name</div>
+ </div>
</div>
</div>
- </div>
- <div>
- <h1 class="mb-2 text-2xl font-medium">Disliked Sites</h1>
- <div class="text-sm">
- Sites that are similar to these sites gets de-prioritized during search. Their results are less likely to appear in your search results.
- </div>
<div>
- <div
- class="grid mb-10 w-full grid-cols-[auto_1fr] space-y-5"
- id="disliked-list"
- >
- <div class="w-10"></div>
- <div class="font-medium">Site Name</div>
+ <h1 class="mb-2 text-2xl font-medium">Disliked Sites</h1>
+ <div class="text-sm">
+ Sites that are similar to these sites gets de-prioritized during search. Their results are less likely to appear in your search results.
+ </div>
+ <div>
+ <div
+ class="grid w-full grid-cols-[auto_1fr] space-y-5"
+ id="disliked-list"
+ >
+ <div class="w-10"></div>
+ <div class="font-medium">Site Name</div>
+ </div>
</div>
</div>
- </div>
- <div>
- <h1 class="mb-2 text-2xl font-medium">Blocked Sites</h1>
- <div class="text-sm">
- These are the sites you have blocked. They won't appear in any of
- your searches.
- </div>
<div>
- <div
- class="grid mb-10 w-full grid-cols-[auto_1fr] space-y-5"
- id="blocked-list"
- >
- <div class="w-10"></div>
- <div class="font-medium">Site Name</div>
+ <h1 class="mb-2 text-2xl font-medium">Blocked Sites</h1>
+ <div class="text-sm">
+ These are the sites you have blocked. They won't appear in any of
+ your searches.
+ </div>
+ <div>
+ <div
+ class="grid w-full grid-cols-[auto_1fr] space-y-5"
+ id="blocked-list"
+ >
+ <div class="w-10"></div>
+ <div class="font-medium">Site Name</div>
+ </div>
</div>
</div>
+
+ <div class="flex justify-center">
+ <a id="export-optic" href="#" download="exported.optic" class="w-fit bg-brand text-white opacity-75 hover:opacity-100 transition-colors duration-50 rounded-full py-2 px-5">
+ Clear all and export as optic
+ </a>
+ </div>
</div>
</SettingsLayout>
+
+<script is:inline src="/js/lz-string.min.js"></script>
+
<script is:inline>
let sites = JSON.parse(
window.localStorage.getItem("sites-ranking-adjustment") || "{}"
@@ -60,6 +72,7 @@ import SettingsLayout from "../../layouts/Settings.astro";
const likedList = document.getElementById("liked-list");
const dislikedList = document.getElementById("disliked-list");
const blockedList = document.getElementById("blocked-list");
+ const exportButton = document.getElementById("export-optic");
const BLOCKED = 0;
const LESS = 1;
@@ -134,5 +147,46 @@ import SettingsLayout from "../../layouts/Settings.astro";
updateList();
}
+ exportButton.addEventListener("click", () => {
+ sites = {}
+
+ window.localStorage.setItem(
+ "sites-ranking-adjustment",
+ JSON.stringify(sites)
+ );
+
+ updateList();
+ });
+
+ function updateExportLink() {
+ console.log(sites);
+ if (sites.length == 0) {
+ exportButton.href = "#";
+ return;
+ }
+
+ let data = {
+ liked: [],
+ disliked: [],
+ blocked: [],
+ };
+
+ for (const site in sites) {
+ if (sites[site] == MORE) {
+ data.liked.push(site);
+ } else if (sites[site] == LESS) {
+ data.disliked.push(site);
+ } else if (sites[site] == BLOCKED) {
+ data.blocked.push(site);
+ }
+ }
+
+ // @ts-ignore
+ const compressed = LZString.compressToBase64(JSON.stringify(data));
+ const url = window.location + "/export?data=" + compressed;
+ exportButton.href = url;
+ }
+
updateList();
+ updateExportLink();
</script>