web search done right
ability to easily export site rankings as an optic
Mikkel Denker7eb5387c90c2
diff --git a/core/src/api/mod.rs b/core/src/api/mod.rsindex c3786182..feb36918 100644--- a/core/src/api/mod.rs+++ b/core/src/api/mod.rs@@ -191,6 +191,7 @@ pub async fn router(config: &FrontendConfig, counters: Counters) -> Result<Route.route("/settings", get(optics::route)).route("/settings/optics", get(optics::route)).route("/settings/sites", get(sites::route))+ .route("/settings/sites/export", get(sites::export)).route("/settings/privacy", get(improvement::settings)).route("/privacy-and-happy-lawyers", get(privacy::route)).route("/webmasters", get(crawler::info_route))diff --git a/core/src/api/sites.rs b/core/src/api/sites.rsindex 6f718374..8fe95dc8 100644--- a/core/src/api/sites.rs+++ b/core/src/api/sites.rs@@ -19,6 +19,8 @@ use std::sync::Arc;use super::{HtmlTemplate, State};use askama::Template;use axum::{extract, response::IntoResponse};+use http::StatusCode;+use optics::{Optic, SiteRankings};#[allow(clippy::unused_async)]pub async fn route(extract::State(state): extract::State<Arc<State>>) -> impl IntoResponse {@@ -34,3 +36,42 @@ pub async fn route(extract::State(state): extract::State<Arc<State>>) -> impl Instruct SitesTemplate {with_alice: Option<bool>,}++#[derive(serde::Deserialize)]+pub struct ExportParams {+ pub data: String,+}++#[allow(clippy::unused_async)]+pub async fn export(+ extract::Query(ExportParams { data }): extract::Query<ExportParams>,+) -> Result<impl IntoResponse, StatusCode> {+ let data = data.replace(' ', "+");+ match lz_str::decompress_from_base64(&data) {+ Some(bytes) => match String::from_utf16(&bytes) {+ Ok(s) => match serde_json::from_str::<SiteRankings>(&s) {+ Ok(site_rankings) => {+ let optic = Optic {+ site_rankings,+ rules: vec![],+ ..Default::default()+ };++ Ok(optic.to_string())+ }+ Err(err) => {+ tracing::error!("Failed to parse data: {}", err);+ Err(StatusCode::BAD_REQUEST)+ }+ },+ Err(err) => {+ tracing::error!("Failed to parse data: {}", err);+ Err(StatusCode::BAD_REQUEST)+ }+ },+ None => {+ tracing::error!("Failed to de-compress data");+ Err(StatusCode::BAD_REQUEST)+ }+ }+}diff --git a/core/src/config/mod.rs b/core/src/config/mod.rsindex 787a3a6d..188feb50 100644--- a/core/src/config/mod.rs+++ b/core/src/config/mod.rs@@ -25,17 +25,17 @@ use std::net::SocketAddr;#[derive(Debug, Deserialize, Clone)]pub struct IndexingLocalConfig {- pub(crate) limit_warc_files: Option<usize>,- pub(crate) skip_warc_files: Option<usize>,- pub(crate) warc_source: WarcSource,- pub(crate) batch_size: Option<usize>,- pub(crate) page_webgraph_path: Option<String>,- pub(crate) output_path: Option<String>,- pub(crate) host_centrality_threshold: Option<f64>,- pub(crate) topics_path: Option<String>,- pub(crate) host_centrality_store_path: String,- pub(crate) page_centrality_store_path: Option<String>,- pub(crate) minimum_clean_words: Option<usize>,+ pub limit_warc_files: Option<usize>,+ pub skip_warc_files: Option<usize>,+ pub warc_source: WarcSource,+ pub batch_size: Option<usize>,+ pub page_webgraph_path: Option<String>,+ pub output_path: Option<String>,+ pub host_centrality_threshold: Option<f64>,+ pub topics_path: Option<String>,+ pub host_centrality_store_path: String,+ pub page_centrality_store_path: Option<String>,+ pub minimum_clean_words: Option<usize>,}#[derive(Debug, Serialize, Deserialize, Clone)]@@ -47,12 +47,12 @@ pub enum WebgraphLevel {#[derive(Debug, Deserialize, Clone)]pub struct WebgraphConstructConfig {- pub(crate) level: WebgraphLevel,- pub(crate) redirect_db_path: Option<String>,- pub(crate) limit_warc_files: Option<usize>,- pub(crate) warc_source: WarcSource,- pub(crate) graph_base_path: Option<String>,- pub(crate) batch_size: Option<usize>,+ pub level: WebgraphLevel,+ pub redirect_db_path: Option<String>,+ pub limit_warc_files: Option<usize>,+ pub warc_source: WarcSource,+ pub graph_base_path: Option<String>,+ pub batch_size: Option<usize>,}#[derive(Debug, Serialize, Deserialize, Clone)]@@ -130,8 +130,8 @@ pub struct LocalConfig {#[derive(Debug, Serialize, Deserialize, Clone)]pub struct HttpConfig {- pub(crate) base_url: String,- pub(crate) warc_paths_file: String,+ pub base_url: String,+ pub warc_paths_file: String,}#[derive(Debug, Serialize, Deserialize, Clone)]diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rsindex 915edc73..72ff4944 100644--- a/core/src/webpage/mod.rs+++ b/core/src/webpage/mod.rs@@ -179,11 +179,8 @@ impl Webpage {);}- let backlink_text: String = itertools::intersperse(- self.backlink_labels.into_iter().map(|label| label),- "\n".to_string(),- )- .collect();+ let backlink_text: String =+ itertools::intersperse(self.backlink_labels, "\n".to_string()).collect();doc.add_text(schemadiff --git a/frontend/src/pages/settings/sites.astro b/frontend/src/pages/settings/sites.astroindex ea8fe308..5397402b 100644--- a/frontend/src/pages/settings/sites.astro+++ b/frontend/src/pages/settings/sites.astro@@ -2,56 +2,68 @@import SettingsLayout from "../../layouts/Settings.astro";---<SettingsLayout>- <div>- <h1 class="mb-2 text-2xl font-medium">Liked Sites</h1>- <div class="text-sm">- Sites that are similar to these sites receive a boost during search. Their results are more likely to appear in your search results.- </div>+ <div class="flex flex-col w-full h-full space-y-10">+<div>- <div- class="grid mb-10 w-full grid-cols-[auto_1fr] space-y-5"- id="liked-list"- >- <div class="w-10"></div>- <div class="font-medium">Site Name</div>+ <h1 class="mb-2 text-2xl font-medium">Liked Sites</h1>+ <div class="text-sm">+ Sites that are similar to these sites receive a boost during search. Their results are more likely to appear in your search results.+ </div>+ <div>+ <div+ class="grid w-full grid-cols-[auto_1fr] space-y-5"+ id="liked-list"+ >+ <div class="w-10"></div>+ <div class="font-medium">Site Name</div>+ </div></div></div>- </div>- <div>- <h1 class="mb-2 text-2xl font-medium">Disliked Sites</h1>- <div class="text-sm">- Sites that are similar to these sites gets de-prioritized during search. Their results are less likely to appear in your search results.- </div><div>- <div- class="grid mb-10 w-full grid-cols-[auto_1fr] space-y-5"- id="disliked-list"- >- <div class="w-10"></div>- <div class="font-medium">Site Name</div>+ <h1 class="mb-2 text-2xl font-medium">Disliked Sites</h1>+ <div class="text-sm">+ Sites that are similar to these sites gets de-prioritized during search. Their results are less likely to appear in your search results.+ </div>+ <div>+ <div+ class="grid w-full grid-cols-[auto_1fr] space-y-5"+ id="disliked-list"+ >+ <div class="w-10"></div>+ <div class="font-medium">Site Name</div>+ </div></div></div>- </div>- <div>- <h1 class="mb-2 text-2xl font-medium">Blocked Sites</h1>- <div class="text-sm">- These are the sites you have blocked. They won't appear in any of- your searches.- </div><div>- <div- class="grid mb-10 w-full grid-cols-[auto_1fr] space-y-5"- id="blocked-list"- >- <div class="w-10"></div>- <div class="font-medium">Site Name</div>+ <h1 class="mb-2 text-2xl font-medium">Blocked Sites</h1>+ <div class="text-sm">+ These are the sites you have blocked. They won't appear in any of+ your searches.+ </div>+ <div>+ <div+ class="grid w-full grid-cols-[auto_1fr] space-y-5"+ id="blocked-list"+ >+ <div class="w-10"></div>+ <div class="font-medium">Site Name</div>+ </div></div></div>++ <div class="flex justify-center">+ <a id="export-optic" href="#" download="exported.optic" class="w-fit bg-brand text-white opacity-75 hover:opacity-100 transition-colors duration-50 rounded-full py-2 px-5">+ Clear all and export as optic+ </a>+ </div></div></SettingsLayout>++<script is:inline src="/js/lz-string.min.js"></script>+<script is:inline>let sites = JSON.parse(window.localStorage.getItem("sites-ranking-adjustment") || "{}"@@ -60,6 +72,7 @@ import SettingsLayout from "../../layouts/Settings.astro";const likedList = document.getElementById("liked-list");const dislikedList = document.getElementById("disliked-list");const blockedList = document.getElementById("blocked-list");+ const exportButton = document.getElementById("export-optic");const BLOCKED = 0;const LESS = 1;@@ -134,5 +147,46 @@ import SettingsLayout from "../../layouts/Settings.astro";updateList();}+ exportButton.addEventListener("click", () => {+ sites = {}++ window.localStorage.setItem(+ "sites-ranking-adjustment",+ JSON.stringify(sites)+ );++ updateList();+ });++ function updateExportLink() {+ console.log(sites);+ if (sites.length == 0) {+ exportButton.href = "#";+ return;+ }++ let data = {+ liked: [],+ disliked: [],+ blocked: [],+ };++ for (const site in sites) {+ if (sites[site] == MORE) {+ data.liked.push(site);+ } else if (sites[site] == LESS) {+ data.disliked.push(site);+ } else if (sites[site] == BLOCKED) {+ data.blocked.push(site);+ }+ }++ // @ts-ignore+ const compressed = LZString.compressToBase64(JSON.stringify(data));+ const url = window.location + "/export?data=" + compressed;+ exportButton.href = url;+ }+updateList();+ updateExportLink();</script>