Skip to content

web search done right

Ftr/site ranking adjustment (#61)

Mikkel Denker677f3a40b144
diff --git a/configs/indexer/local.toml b/configs/indexer/local.toml
index fd12a988..7c3fa1e9 100644
--- a/configs/indexer/local.toml
+++ b/configs/indexer/local.toml
@@ -4,9 +4,9 @@ limit_warc_files = 3
download_images = false
[warc_source]
-# type = "Local"
-# folder = "./data/warc_files"
-# names = ["CC-MAIN-20220116093137-20220116123137-00049.warc.gz"]
-type = "HTTP"
-base_url = "https://data.commoncrawl.org/"
-warc_paths_file = "warc.paths"
+type = "Local"
+folder = "./data/warc_files"
+names = ["CC-MAIN-20220116093137-20220116123137-00049.warc.gz"]
+# type = "HTTP"
+# base_url = "https://data.commoncrawl.org/"
+# warc_paths_file = "warc.paths"
diff --git a/frontend/public/opensearch.xml b/frontend/public/opensearch.xml
new file mode 100644
index 00000000..a57a1435
--- /dev/null
+++ b/frontend/public/opensearch.xml
@@ -0,0 +1,9 @@
+<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.1/" xmlns:moz="http://www.mozilla.org/2006/browser/search/">
+ <ShortName>Cuely</ShortName>
+ <Description>Open source search engine.</Description>
+ <InputEncoding>UTF-8</InputEncoding>
+ <Image width="32" height="32" type="image/x-icon">https://cuely.io/assets/favicon.ico</Image>
+ <Url rel="results" type="text/html" template="https://cuely.io/search?q={searchTerms}"/>
+ <Url rel="suggestions" type="application/x-suggestions+json" template="https://cuely.io/autosuggest/browser?q={searchTerms}"/>
+ <moz:SearchForm>https://cuely.io/</moz:SearchForm>
+</OpenSearchDescription>
\ No newline at end of file
diff --git a/frontend/src/components/Entity.astro b/frontend/src/components/Entity.astro
index 9cfa3ce6..3aa87a58 100644
--- a/frontend/src/components/Entity.astro
+++ b/frontend/src/components/Entity.astro
@@ -23,7 +23,7 @@ export interface Props {
</div>
))
}
- <div class="text-xl mb-2">
+ <div class="text-xl mb-5">
<a href="https://en.wikipedia.org/wiki/{{ entity.title }}"
>{askama`entity.title $ {{name.fullName}}`}
</a>
@@ -40,8 +40,8 @@ export interface Props {
</div>
{
askama.if_("!entity.info.is_empty()", () => (
- <div class="mt-5 flex w-full flex-col px-4 text-sm">
- <div class="grid border-separate border-spacing-x-4 border-spacing-y-1.5 grid-cols-[auto_1fr] gap-x-4">
+ <div class="mt-7 mb-2 flex w-full flex-col px-4 text-sm">
+ <div class="grid grid-cols-[auto_1fr] gap-x-4 gap-y-2">
{askama.for_("(key, value) in entity.info", () => (
<>
<div class="text-gray-500">{askama`key $ {{word.adjective}}`}</div>
diff --git a/frontend/src/components/Navbar.astro b/frontend/src/components/Navbar.astro
index e5c75fdf..a7f71ceb 100644
--- a/frontend/src/components/Navbar.astro
+++ b/frontend/src/components/Navbar.astro
@@ -3,7 +3,7 @@ import Hero from "./Hero.astro";
import { Icon } from "astro-icon";
const links = [
- { url: "/goggles", title: "Goggles" },
+ { url: "/settings", title: "Settings" },
{ url: "/about", title: "About" },
];
const icons = [
diff --git a/frontend/src/components/RankingModal.astro b/frontend/src/components/RankingModal.astro
new file mode 100644
index 00000000..2067a6c6
--- /dev/null
+++ b/frontend/src/components/RankingModal.astro
@@ -0,0 +1,106 @@
+<div class="flex text-sm flex-col py-5 px-2 items-center h-fit w-52 overflow-hidden absolute border rounded-lg drop-shadow-md bg-white origin-left modal-closed" id="ranking-modal">
+ <h2 class="w-fit text-center">
+ Do you like results from <span id="ranking-modal-site-label">EXAMPLE.com</span>?
+ </h2>
+ <div class="flex flex-col w-fit pt-4 space-y-3">
+ <button class="bg-white border rounded-full px-4 py-1 hover:bg-slate-600 hover:text-white" onclick="preferMore()" id="prefer-more">Prefer more</button>
+ <button class="bg-white border rounded-full px-4 py-1 hover:bg-slate-600 hover:text-white" onclick="preferLess()" id="prefer-less">Prefer less</button>
+ <button class="bg-white border rounded-full px-4 py-1 hover:bg-slate-600 hover:text-white" onclick="block()" id="prefer-block">Block</button>
+ </div>
+</div>
+
+<noscript>
+ <style>
+ #ranking-modal {
+ visibility: hidden;
+ }
+ </style>
+</noscript>
+
+<style>
+ #ranking-modal.modal-open {
+ transition: transform .15s ease-in-out,left .15s ease-in-out,top .15s ease-in-out;
+ transform: scale(1);
+ }
+
+ .selected {
+ @apply bg-slate-600 text-white;
+ }
+
+ #ranking-modal.modal-closed{
+ transition: none !important;
+ transform: scale(0);
+ }
+</style>
+
+<script is:inline>
+ var rankingModal = document.getElementById("ranking-modal");
+ var siteLabel = document.getElementById("ranking-modal-site-label");
+ var btnPreferMore = document.getElementById("prefer-more");
+ var btnPreferLess = document.getElementById("prefer-less");
+ var btnPreferBlock = document.getElementById("prefer-block");
+
+ var currentSite = "";
+
+ const BLOCKED = 0;
+ const LESS = 1;
+ const MORE = 2;
+
+ let rankings = JSON.parse(window.localStorage.getItem("sites-ranking-adjustment") || "{}");
+
+ rankingModal.addEventListener("click", (event) => {
+ event.stopPropagation();
+ });
+
+ function updateModal(site) {
+ siteLabel.innerHTML = site;
+ currentSite = site;
+
+ updateSelection();
+ }
+
+ function updateSelection() {
+ let rankings = JSON.parse(window.localStorage.getItem("sites-ranking-adjustment") || "{}");
+
+ btnPreferMore.classList.remove("selected");
+ btnPreferLess.classList.remove("selected");
+ btnPreferBlock.classList.remove("selected");
+
+ if (rankings[currentSite] != undefined) {
+ let pref = rankings[currentSite];
+
+ if (pref == MORE) {
+ btnPreferMore.classList.add("selected");
+ } else if (pref == LESS) {
+ btnPreferLess.classList.add("selected");
+ } else if (pref == BLOCKED) {
+ btnPreferBlock.classList.add("selected");
+ }
+ }
+ }
+
+ function setSitePreference(pref) {
+ if (rankings[currentSite] == pref) {
+ delete rankings[currentSite];
+ } else {
+ rankings[currentSite] = pref;
+ }
+
+ window.localStorage.setItem("sites-ranking-adjustment", JSON.stringify(rankings));
+
+ updateSelection();
+ document.getElementById('searchbar-form').submit();
+ }
+
+ function preferMore() {
+ setSitePreference(MORE);
+ }
+
+ function preferLess() {
+ setSitePreference(LESS);
+ }
+
+ function block() {
+ setSitePreference(BLOCKED);
+ }
+</script>
\ No newline at end of file
diff --git a/frontend/src/components/Searchbar.astro b/frontend/src/components/Searchbar.astro
index dd5b32dd..1e7632d5 100644
--- a/frontend/src/components/Searchbar.astro
+++ b/frontend/src/components/Searchbar.astro
@@ -73,12 +73,22 @@ const { autofocus, query } = Astro.props;
</div>
</div>
</div>
+
+ <input
+ type="hidden"
+ value=""
+ name="sr"
+ id="siteRankingsUuid"
+ />
</form>
<script>
const searchbar = document.querySelector("#searchbar")!;
+ const searchbarForm: HTMLFormElement = document.querySelector("#searchbar-form")!;
+ const siteRankingsUuid: HTMLInputElement = document.querySelector("#siteRankingsUuid")!;
const searchbarInput: HTMLInputElement =
document.querySelector(".searchbar-input")!;
+
let suggestionItems = () =>
Array.from(document.querySelectorAll(".suggestion-item")) as HTMLElement[];
@@ -251,4 +261,40 @@ const { autofocus, query } = Astro.props;
searchbar.classList.add("suggestions-visible");
fetchAndUpdateSuggestions();
});
+
+ HTMLFormElement.prototype._submit = HTMLFormElement.prototype.submit;
+
+ HTMLFormElement.prototype.submit = function (){
+ this.dispatchEvent(new SubmitEvent('submit', {
+ 'bubbles': true,
+ 'cancelable': true
+ }));
+
+ this._submit();
+ };
+
+ searchbarForm.addEventListener("submit", (event: Event) => {encodeSiteRankings()}, false);
+
+ function encodeSiteRankings() {
+ const BLOCKED = 0;
+ const LESS = 1;
+ const MORE = 2;
+
+ let sites = JSON.parse(window.localStorage.getItem("sites-ranking-adjustment") || "{}");
+
+ let rankings = {"preferred": [], "disliked": [], "blocked": []};
+
+ for (const [site, ranking] of Object.entries(sites)) {
+ if (ranking == MORE) {
+ rankings.preferred.push(site);
+ } else if (ranking == LESS) {
+ rankings.disliked.push(site);
+ } else if (ranking == BLOCKED) {
+ rankings.blocked.push(site);
+ }
+ }
+
+ let encoded = btoa(JSON.stringify(rankings));
+ siteRankingsUuid.value = encoded;
+ }
</script>
diff --git a/frontend/src/components/SettingsMenu.astro b/frontend/src/components/SettingsMenu.astro
new file mode 100644
index 00000000..2d74c066
--- /dev/null
+++ b/frontend/src/components/SettingsMenu.astro
@@ -0,0 +1,21 @@
+---
+const links = [
+ { url: "/settings", title: "Manage Goggles" },
+ { url: "/settings/sites", title: "Site Rankings" },
+];
+---
+
+<div class="relative flex flex-col right-20 space-y-3">
+ {
+ links.map((l) => (
+ <a class="link rounded-full px-2 py-1 text-center" href={l.url}>
+ {l.title}
+ </a>
+ ))
+ }
+</div>
+<style>
+ .link {
+ @apply transition hover:bg-brand/5 hover:text-brand hover:no-underline;
+ }
+</style>
diff --git a/frontend/src/layouts/Layout.astro b/frontend/src/layouts/Layout.astro
index b06f90fe..de2229bc 100644
--- a/frontend/src/layouts/Layout.astro
+++ b/frontend/src/layouts/Layout.astro
@@ -16,6 +16,7 @@ const { title } = Astro.props;
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<!-- <link rel="icon" type="image/svg+xml" href="/favicon.svg" /> -->
<link rel="icon" type="image/x-icon" href="/assets/favicon.ico" />
+ <link rel="search" type="application/opensearchdescription+xml" title="Cuely Search" href="/opensearch.xml">
<title>{title}</title>
</head>
<body class="font-light">
diff --git a/frontend/src/pages/about.mdx b/frontend/src/pages/about.mdx
index 4555cc5a..93e684a1 100644
--- a/frontend/src/pages/about.mdx
+++ b/frontend/src/pages/about.mdx
@@ -17,6 +17,6 @@ Oh, and if we ever become evil (e.g. by changing our motto) please take our code
## How do you make money?
-Well, currently we don't. We are bootstraped and trying to keep costs low. In the future we will have contextual ads based on your current search query. Just to re-iterate; we will only use your current search to match ads and will never track you across searches.
+Well, currently we don't. We are bootstraped and trying to keep costs low. In the future we will have contextual ads based on your current search query and a subscription option without ads. Just to re-iterate; we will only use your current search to match ads and will never track you across searches.
We will also have a paid api for developers.
diff --git a/frontend/src/pages/goggles.astro b/frontend/src/pages/goggles.astro
index da4bf0bd..8d478a69 100644
--- a/frontend/src/pages/goggles.astro
+++ b/frontend/src/pages/goggles.astro
@@ -2,6 +2,7 @@
import Footer from "../components/Footer.astro";
import Layout from "../layouts/Layout.astro";
import Header from "../components/Header.astro";
+import SettingsMenu from "../components/SettingsMenu.astro";
import { askama } from "../askama";
---
@@ -9,7 +10,8 @@ import { askama } from "../askama";
<div class="flex flex-col h-full w-full">
<Header/>
- <div class="flex flex-col w-full h-fit items-center pt-10">
+ <div class="flex w-full h-fit justify-center pt-10">
+ <SettingsMenu />
<div class="flex flex-col max-w-2xl">
<h1 class="font-medium text-2xl">Manage Goggles</h1>
<div class="text-sm mt-5">
diff --git a/frontend/src/pages/search.astro b/frontend/src/pages/search.astro
index bfe62e69..daa2494c 100644
--- a/frontend/src/pages/search.astro
+++ b/frontend/src/pages/search.astro
@@ -5,6 +5,7 @@ import Entity from "../components/Entity.astro";
import Hero from "../components/Hero.astro";
import GoggleSelector from "../components/GoggleSelector.astro";
import Header from "../components/Header.astro";
+import RankingModal from "../components/RankingModal.astro";
import { askama } from "../askama";
---
@@ -15,7 +16,7 @@ import { askama } from "../askama";
</div>
<div
- class="grid sm:px-5 md:pl-20 lg:px-32 mx-1 lg:mx-0 md:gap-x-12 md:grid-cols-[minmax(50ch,48rem)_1fr] md:grid-rows-[auto_1fr] gap-y-6 pt-3"
+ class="grid sm:px-5 md:pl-20 lg:px-32 mx-1 lg:mx-0 md:gap-x-12 md:grid-cols-[minmax(50ch,48rem)_1fr] md:grid-rows-[auto_1fr] gap-y-6 pt-3 search-content"
>
<!-- Stats and settings -->
<div class="flex w-full justify-between mx-auto">
@@ -60,7 +61,7 @@ import { askama } from "../askama";
</div>
<!-- Search results -->
- <div class="col-start-1 flex flex-col space-y-6">
+ <div class="col-start-1 flex flex-col max-w-4xl min-w-0 space-y-10">
{
askama.if_("let Some(correction) = spell_correction", () => (
<div>
@@ -78,25 +79,32 @@ import { askama } from "../askama";
{
askama.for_("item in search_result", () => (
<div class="flex w-full">
- <div class="flex w-full flex-col space-y-1">
- <div class="flex items-center space-x-2 text-sm">
- <img
- src={askama.fake(
- "data:image/png;base64, {{ item.favicon_base64 }}",
- "{{image.avatar}}"
- )}
- width="13px"
- />
- <a
- class="text-gray-600 hover:no-underline"
- href="{{ item.url }}"
- >
- {askama`item.pretty_url $ {{internet.url}}`}
- </a>
+ <div class="flex grow min-w-0 flex-col space-y-1.5">
+ <div class="flex min-w-0">
+ <div class="flex flex-col space-y-1.5 grow min-w-0">
+ <div class="flex items-center space-x-2 text-sm">
+ <img
+ src={askama.fake(
+ "data:image/png;base64, {{ item.favicon_base64 }}",
+ "{{image.avatar}}"
+ )}
+ width="13px"
+ />
+ <a
+ class="text-gray-600 hover:no-underline truncate url"
+ href="{{ item.url }}"
+ >
+ {askama`item.pretty_url $ {{internet.url}}`}
+ </a>
+ </div>
+ <a class="title-link truncate text-lg font-medium" title="{{ item.title }}" href="{{ item.url }}">
+ {askama`item.title $ {{lorem.sentence}}`}
+ </a>
+ </div>
+ <div class="adjust-btn min-w-fit hidden justify-center items-center md:flex" data-site="{{ item.site }}">
+ <Hero class="w-6" icon="adjustments" />
+ </div>
</div>
- <a class="title-link text-lg font-medium" href="{{ item.url }}">
- {askama`item.title $ {{lorem.sentence}}`}
- </a>
<div class="text-sm">
{askama.if_("let Some(val) = item.last_updated", () => (
<span class="text-gray-500">
@@ -108,7 +116,7 @@ import { askama } from "../askama";
</span>
</div>
</div>
- <div class="flex h-full items-center">
+ <div class="flex h-full w-20 items-center">
<div class="h-20 w-20 py-1 pl-2">
{askama.if_(
"let Some(val) = item.primary_image_uuid",
@@ -157,6 +165,9 @@ import { askama } from "../askama";
)
}
</div>
+
+ <!-- Ranking modal -->
+ <RankingModal />
</main>
<Footer />
@@ -166,11 +177,70 @@ import { askama } from "../askama";
.change-page-inactive {
@apply w-6 text-gray-500;
}
+
.change-page-active {
@apply w-6 text-brand/80 hover:text-brand;
}
.title-link {
@apply text-blue-800 visited:text-purple-800;
+ max-width: calc(100% - 30px);
+ }
+
+ .url {
+ max-width: calc(100% - 100px);
+ }
+
+ .adjust-btn {
+ @apply text-gray-500/25 hover:text-gray-500 w-5 hover:cursor-pointer;
+ }
+
+ .adjust-btn * {
+ pointer-events: none;
+ }
+
+ .search-content {
+ text-rendering: optimizeLegibility;
+ font-smooth: grayscale;
+ -moz-osx-font-smoothing: grayscale;
}
</style>
+
+<noscript>
+ <style>
+ .adjust-btn {
+ display: none;
+ }
+ </style>
+</noscript>
+
+<script is:inline>
+ var rankingAdjustButtons = document.querySelectorAll(".adjust-btn");
+ var rankingModal = document.getElementById("ranking-modal");
+
+ document.addEventListener("click", (elem) => {
+ rankingModal.classList.remove("modal-open");
+ rankingModal.classList.add("modal-closed");
+ })
+
+ var rankingModalHeight = rankingModal.clientHeight;
+
+ rankingAdjustButtons.forEach(btn => {
+ btn.addEventListener("click", (event) => {
+ event.stopPropagation();
+
+ const elem = event.target;
+
+ updateModal(elem.dataset.site);
+
+ const rect = elem.getBoundingClientRect();
+ rankingModal.style.left = rect.left + rect.width + 5 +"px";
+ rankingModal.style.top = (rect.top + document.documentElement.scrollTop - (rankingModalHeight/2)) + "px";
+
+ setTimeout(() => {
+ rankingModal.classList.remove("modal-closed");
+ rankingModal.classList.add("modal-open");
+ }, 0);
+ })
+ });
+</script>
\ No newline at end of file
diff --git a/frontend/src/pages/sites.astro b/frontend/src/pages/sites.astro
new file mode 100644
index 00000000..3aceecfa
--- /dev/null
+++ b/frontend/src/pages/sites.astro
@@ -0,0 +1,144 @@
+---
+import Footer from "../components/Footer.astro";
+import Layout from "../layouts/Layout.astro";
+import Header from "../components/Header.astro";
+import SettingsMenu from "../components/SettingsMenu.astro";
+---
+
+<Layout title="Cuely">
+ <div class="flex flex-col h-full w-full">
+ <Header />
+
+ <div class="flex w-full h-fit justify-center pt-10">
+ <SettingsMenu />
+ <div class="flex w-full flex-col max-w-2xl space-y-10">
+ <div>
+ <h1 class="font-medium text-2xl mb-2">Preffered Sites</h1>
+ <div class="text-sm">
+ These sites receive a boost during search.
+ Results from these sites are more likely to appear in your search results.
+ </div>
+ <div class="mt-5">
+ <div
+ class="grid grid-cols-[auto_1fr] w-full space-y-5"
+ id="preffered-list"
+ >
+ <div class="w-10"></div>
+ <div class="font-medium">Site Name</div>
+ </div>
+ </div>
+ </div>
+
+ <div>
+ <h1 class="font-medium text-2xl mb-2">Junk Sites</h1>
+ <div class="text-sm">
+ These sites get de-prioritized during search.
+ Results from these sites are less likely to appear in your search results.
+ </div>
+ <div class="mt-5">
+ <div
+ class="grid grid-cols-[auto_1fr] w-full space-y-5"
+ id="junk-list"
+ >
+ <div class="w-10"></div>
+ <div class="font-medium">Site Name</div>
+ </div>
+ </div>
+ </div>
+
+ <div>
+ <h1 class="font-medium text-2xl mb-2">Blocked Sites</h1>
+ <div class="text-sm">These are the sites you have blocked. They won't appear in any of your searches.</div>
+ <div class="mt-5">
+ <div
+ class="grid grid-cols-[auto_1fr] w-full space-y-5"
+ id="blocked-list"
+ >
+ <div class="w-10"></div>
+ <div class="font-medium">Site Name</div>
+ </div>
+ </div>
+ </div>
+ </div>
+ </div>
+ <div class="flex flex-grow"></div>
+ <Footer />
+ </div>
+</Layout>
+
+<script is:inline>
+ let sites = JSON.parse(window.localStorage.getItem("sites-ranking-adjustment") || "{}");
+
+ const prefferedList = document.getElementById("preffered-list");
+ const junkList = document.getElementById("junk-list");
+ const blockedList = document.getElementById("blocked-list");
+
+ const BLOCKED = 0;
+ const LESS = 1;
+ const MORE = 2;
+
+ function addMoreRow(site) {
+ addRow(site, prefferedList);
+ }
+
+ function addLessRow(site) {
+ addRow(site,junkList);
+ }
+
+ function addBlockedRow(site) {
+ addRow(site,blockedList);
+ }
+
+ function addRow(site, list) {
+ let img = document.createElement("div");
+ img.classList.add("site");
+
+ img.innerHTML =
+ '<img src="/assets/images/delete.svg" class="w-5 h-5 hover:cursor-pointer" onclick="deleteRow(this)" id="' + site + '"/>';
+
+ let name = document.createElement("div");
+ name.innerHTML = site;
+ name.classList.add("site");
+ name.classList.add("text-sm");
+
+ list.appendChild(img);
+ list.appendChild(name);
+ }
+
+
+ function updateList() {
+ sites = JSON.parse(window.localStorage.getItem("sites-ranking-adjustment") || "{}");
+
+ clearList();
+
+ for (const [site, ranking] of Object.entries(sites)) {
+ if (ranking == MORE) {
+ addMoreRow(site);
+ } else if (ranking == LESS) {
+ addLessRow(site);
+ } else if (ranking == BLOCKED) {
+ addBlockedRow(site);
+ }
+ }
+ }
+
+ function clearList() {
+ document
+ .querySelectorAll(".site")
+ .forEach((site) => site.remove());
+ }
+
+ function deleteRow(deleteBtn) {
+ let site = deleteBtn.id;
+ sites = JSON.parse(window.localStorage.getItem("sites-ranking-adjustment") || "{}");
+
+ if (sites[site] != undefined) {
+ delete sites[site];
+ window.localStorage.setItem("sites-ranking-adjustment", JSON.stringify(sites));
+ }
+
+ updateList();
+ }
+
+ updateList();
+</script>
\ No newline at end of file
diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rs
index 4619dceb..c4532da2 100644
--- a/src/entrypoint/indexer.rs
+++ b/src/entrypoint/indexer.rs
@@ -156,9 +156,10 @@ fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
}
}
+ index.commit().unwrap();
+
std::fs::remove_file(file).ok();
}
- index.commit().unwrap();
info!("{} done", name);
diff --git a/src/entrypoint/search_server.rs b/src/entrypoint/search_server.rs
index 08c8209f..97fb1a00 100644
--- a/src/entrypoint/search_server.rs
+++ b/src/entrypoint/search_server.rs
@@ -80,7 +80,13 @@ pub async fn run(config: SearchServerConfig) -> Result<()> {
.await
.ok();
}
- searcher::InitialSearchResult::Bang(_) => todo!(),
+ searcher::InitialSearchResult::Bang(bang) => {
+ req.respond(sonic::Response::Content(
+ searcher::InitialPrettifiedSearchResult::Bang(bang),
+ ))
+ .await
+ .ok();
+ }
},
Err(_) => {
req.respond::<inverted_index::SearchResult>(sonic::Response::Empty)
diff --git a/src/frontend/api.rs b/src/frontend/api.rs
index 2fe812f5..2eebc49b 100644
--- a/src/frontend/api.rs
+++ b/src/frontend/api.rs
@@ -44,6 +44,7 @@ pub async fn search(
original: query.to_string(),
selected_region,
goggle_program: None,
+ site_rankings: None,
skip_pages,
})
.await
diff --git a/src/frontend/autosuggest.rs b/src/frontend/autosuggest.rs
index 3617567d..9d59e5c5 100644
--- a/src/frontend/autosuggest.rs
+++ b/src/frontend/autosuggest.rs
@@ -63,6 +63,18 @@ pub async fn route(
}
}
+#[allow(clippy::unused_async)]
+pub async fn browser(
+ extract::Query(params): extract::Query<HashMap<String, String>>,
+ Extension(state): Extension<Arc<State>>,
+) -> impl IntoResponse {
+ if let Some(query) = params.get("q") {
+ Json((query.clone(), state.autosuggest.suggestions(query).unwrap()))
+ } else {
+ Json((String::new(), Vec::new()))
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;
diff --git a/src/frontend/mod.rs b/src/frontend/mod.rs
index a000196b..e788e457 100644
--- a/src/frontend/mod.rs
+++ b/src/frontend/mod.rs
@@ -37,8 +37,10 @@ mod api;
mod autosuggest;
mod goggles;
mod index;
+mod opensearch;
mod privacy;
pub mod search;
+mod sites;
pub struct HtmlTemplate<T>(T);
@@ -92,11 +94,15 @@ pub fn router(queries_csv_path: &str, shards: Vec<Vec<String>>) -> Result<Router
.route("/", get(index::route))
.route("/search", get(search::route))
.route("/autosuggest", get(autosuggest::route))
+ .route("/autosuggest/browser", get(autosuggest::browser))
.route("/favicon.ico", get(favicon))
.route("/about", get(about::route))
- .route("/goggles", get(goggles::route))
+ .route("/settings", get(goggles::route))
+ .route("/settings/goggles", get(goggles::route))
+ .route("/settings/sites", get(sites::route))
.route("/privacy-and-happy-lawyers", get(privacy::route))
.route("/api/beta/search", get(api::search))
+ .route("/opensearch.xml", get(opensearch::route))
.merge(SpaRouter::new("/assets", "frontend/dist/assets"))
.layer(Extension(state))
.layer(CompressionLayer::new()))
diff --git a/src/frontend/opensearch.rs b/src/frontend/opensearch.rs
new file mode 100644
index 00000000..6b222c07
--- /dev/null
+++ b/src/frontend/opensearch.rs
@@ -0,0 +1,24 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use axum::response::IntoResponse;
+
+#[allow(clippy::unused_async)]
+pub async fn route() -> impl IntoResponse {
+ (
+ [("content-type", "application/opensearchdescription+xml")],
+ include_str!("../../frontend/public/opensearch.xml"),
+ )
+}
diff --git a/src/frontend/search.rs b/src/frontend/search.rs
index c0f52557..a2a95a2f 100644
--- a/src/frontend/search.rs
+++ b/src/frontend/search.rs
@@ -19,6 +19,7 @@ use std::{collections::HashMap, sync::Arc};
use axum::Extension;
use crate::{
+ ranking::site_rankings::SiteRankings,
search_prettifier::{thousand_sep_number, DisplayedEntity, DisplayedWebpage},
searcher::{self, PrettifiedSearchResult, SearchQuery},
webpage::region::{Region, ALL_REGIONS},
@@ -89,6 +90,25 @@ pub async fn route(
}
});
+ let site_rankings: Option<SiteRankings> = match params.get("sr") {
+ Some(sr) => {
+ if !sr.is_empty() {
+ if let Ok(site_rankings) = base64::decode(sr) {
+ if let Ok(site_rankings) = std::str::from_utf8(&site_rankings) {
+ serde_json::from_str(site_rankings).ok()
+ } else {
+ None
+ }
+ } else {
+ None
+ }
+ } else {
+ None
+ }
+ }
+ None => None,
+ };
+
match state
.searcher
.search_prettified(&SearchQuery {
@@ -96,6 +116,7 @@ pub async fn route(
selected_region,
goggle_program: goggle,
skip_pages,
+ site_rankings,
})
.await
{
diff --git a/src/frontend/sites.rs b/src/frontend/sites.rs
new file mode 100644
index 00000000..5cfc3351
--- /dev/null
+++ b/src/frontend/sites.rs
@@ -0,0 +1,30 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use super::HtmlTemplate;
+use askama::Template;
+use axum::response::IntoResponse;
+
+#[allow(clippy::unused_async)]
+pub async fn route() -> impl IntoResponse {
+ let template = SitesTemplate {};
+
+ HtmlTemplate(template)
+}
+
+#[derive(Template)]
+#[template(path = "sites/index.html")]
+struct SitesTemplate {}
diff --git a/src/inverted_index.rs b/src/inverted_index.rs
index f9aac627..5b8846aa 100644
--- a/src/inverted_index.rs
+++ b/src/inverted_index.rs
@@ -28,6 +28,7 @@ use crate::prehashed::Prehashed;
use crate::query::Query;
use crate::schema::{Field, ALL_FIELDS};
use crate::snippet;
+use crate::tokenizer::Identity;
use crate::webpage::region::Region;
use crate::webpage::{StoredPrimaryImage, Webpage};
use crate::Result;
@@ -119,6 +120,11 @@ impl InvertedIndex {
.tokenizers()
.register(tokenizer.as_str(), tokenizer);
+ let tokenizer = Tokenizer::Identity(Identity::default());
+ tantivy_index
+ .tokenizers()
+ .register(tokenizer.as_str(), tokenizer);
+
let writer = tantivy_index.writer_with_num_threads(1, 1_000_000_000)?;
let merge_policy = NoMergePolicy::default();
@@ -453,6 +459,7 @@ impl From<Document> for RetrievedWebpage {
| Field::UrlWithoutQueryHash
| Field::TitleHash
| Field::UrlHash
+ | Field::DomainHash
| Field::NumTrackers
| Field::PreComputedScore
| Field::NumCleanBodyTokens
@@ -967,4 +974,49 @@ mod tests {
assert_eq!(result.documents[0].url, "https://www.example.com");
assert_eq!(result.documents[0].primary_image, None);
}
+
+ #[test]
+ fn id_links_removed_during_indexing() {
+ let mut index = InvertedIndex::temporary().expect("Unable to open index");
+ let query = Query::parse(
+ "website",
+ index.schema(),
+ index.tokenizers(),
+ &SignalAggregator::default(),
+ )
+ .expect("Failed to parse query");
+ let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());
+
+ let result = index
+ .search(&query, ranker.collector())
+ .expect("Search failed");
+ assert_eq!(result.documents.len(), 0);
+ assert_eq!(result.num_docs, 0);
+
+ index
+ .insert(Webpage::new(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ <body>
+ {CONTENT}
+ </body>
+ </html>
+ "#
+ ),
+ "https://www.example.com#tag",
+ ))
+ .expect("failed to insert webpage");
+ index.commit().expect("failed to commit index");
+
+ let result = index
+ .search(&query, ranker.collector())
+ .expect("Search failed");
+ assert_eq!(result.num_docs, 1);
+ assert_eq!(result.documents.len(), 1);
+ assert_eq!(result.documents[0].url, "https://www.example.com");
+ }
}
diff --git a/src/lib.rs b/src/lib.rs
index 193bf000..00a11250 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -54,6 +54,8 @@ mod snippet;
mod sonic;
mod spell;
mod tokenizer;
+#[allow(unused)]
+mod ttl_cache;
mod warc;
mod webgraph;
mod webpage;
diff --git a/src/query/mod.rs b/src/query/mod.rs
index 39e3f593..ea9520d9 100644
--- a/src/query/mod.rs
+++ b/src/query/mod.rs
@@ -142,10 +142,12 @@ impl Query {
})
}
- pub fn set_goggle(&mut self, goggle: &Goggle, schema: &Schema) {
+ pub fn set_goggles(&mut self, goggles: &[Goggle], schema: &Schema) {
let mut subqueries = vec![(Occur::Must, self.tantivy_query.box_clone())];
- subqueries.append(&mut goggle.as_tantivy(schema));
+ for goggle in goggles {
+ subqueries.append(&mut goggle.as_tantivy(schema));
+ }
self.tantivy_query = Box::new(BooleanQuery::new(subqueries))
}
@@ -532,8 +534,104 @@ mod tests {
&TokenizerManager::new(),
&SignalAggregator::default(),
)
- .expect("Failed to parse query");
+ .expect("failed to parse query");
assert!(query.is_empty())
}
+
+ #[test]
+ fn query_term_only_special_char() {
+ let index = InvertedIndex::temporary().expect("Unable to open index");
+
+ let query = Query::parse(
+ "&",
+ index.schema(),
+ index.tokenizers(),
+ &SignalAggregator::default(),
+ )
+ .expect("Failed to parse query");
+
+ assert!(!query.is_empty());
+ }
+
+ #[test]
+ fn site_query_split_domain() {
+ let mut index = InvertedIndex::temporary().expect("Unable to open index");
+
+ index
+ .insert(Webpage::new(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ <body>
+ This is a test website
+ </body>
+ </html>
+ "#,
+ "https://www.the-first.com",
+ ))
+ .expect("failed to insert webpage");
+ index
+ .insert(Webpage::new(
+ r#"
+ <html>
+ <head>
+ <title>Test test</title>
+ </head>
+ <body>
+ This test page does not contain the forbidden word
+ </body>
+ </html>
+ "#,
+ "https://www.second.com",
+ ))
+ .expect("failed to insert webpage");
+ index.commit().expect("failed to commit index");
+
+ let query = Query::parse(
+ "test site:first.com",
+ index.schema(),
+ index.tokenizers(),
+ &SignalAggregator::default(),
+ )
+ .expect("Failed to parse query");
+ let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());
+ let result = index
+ .search(&query, ranker.collector())
+ .expect("Search failed");
+ assert_eq!(result.num_docs, 0);
+ assert_eq!(result.documents.len(), 0);
+
+ let query = Query::parse(
+ "test site:the-first.com",
+ index.schema(),
+ index.tokenizers(),
+ &SignalAggregator::default(),
+ )
+ .expect("Failed to parse query");
+ let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());
+ let result = index
+ .search(&query, ranker.collector())
+ .expect("Search failed");
+ assert_eq!(result.num_docs, 1);
+ assert_eq!(result.documents.len(), 1);
+ assert_eq!(result.documents[0].url, "https://www.the-first.com");
+
+ let query = Query::parse(
+ "test site:www.the-first.com",
+ index.schema(),
+ index.tokenizers(),
+ &SignalAggregator::default(),
+ )
+ .expect("Failed to parse query");
+ let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());
+ let result = index
+ .search(&query, ranker.collector())
+ .expect("Search failed");
+ assert_eq!(result.num_docs, 1);
+ assert_eq!(result.documents.len(), 1);
+ assert_eq!(result.documents[0].url, "https://www.the-first.com");
+ }
}
diff --git a/src/query/parser.rs b/src/query/parser.rs
index 879f9fa1..56714e1f 100644
--- a/src/query/parser.rs
+++ b/src/query/parser.rs
@@ -185,7 +185,7 @@ impl Term {
.filter(|(field, _)| {
matches!(
ALL_FIELDS[field.field_id() as usize],
- Field::Domain | Field::Site
+ Field::DomainNoTokenizer | Field::SiteNoTokenizer
)
})
.into_iter()
diff --git a/src/ranking/goggles/mod.rs b/src/ranking/goggles/mod.rs
index 485267f7..d10cc10f 100644
--- a/src/ranking/goggles/mod.rs
+++ b/src/ranking/goggles/mod.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-const SCALE: f32 = 1000.0;
+const SCALE: f32 = 500.0;
pub mod ast;
mod const_query;
@@ -121,7 +121,7 @@ impl TryFrom<RawAction> for Action {
}
}
-#[derive(Debug, PartialEq, Eq)]
+#[derive(Debug, PartialEq, Eq, Clone)]
pub struct Instruction {
pub patterns: Vec<PatternPart>,
pub options: Vec<PatternOption>,
@@ -135,7 +135,7 @@ pub enum PatternPart {
Anchor,
}
-#[derive(Debug, PartialEq, Eq)]
+#[derive(Debug, PartialEq, Eq, Clone)]
pub enum PatternOption {
Site(String),
InUrl,
@@ -152,10 +152,10 @@ pub enum Action {
Discard,
}
-#[derive(Debug, Default)]
+#[derive(Debug, Default, Clone)]
pub struct Goggle {
pub aggregator: SignalAggregator,
- instructions: Vec<Instruction>,
+ pub instructions: Vec<Instruction>,
}
impl Goggle {
@@ -366,6 +366,7 @@ mod tests {
selected_region: None,
goggle_program: None,
skip_pages: None,
+ site_rankings: None,
})
.unwrap()
.into_websites()
@@ -388,6 +389,7 @@ mod tests {
.to_string(),
),
skip_pages: None,
+ site_rankings: None,
})
.unwrap()
.into_websites()
@@ -409,6 +411,7 @@ mod tests {
.to_string(),
),
skip_pages: None,
+ site_rankings: None,
})
.unwrap()
.into_websites()
@@ -495,6 +498,7 @@ mod tests {
include_str!("../../../testcases/goggles/quickstart.goggle").to_string(),
),
skip_pages: None,
+ site_rankings: None,
})
.unwrap()
.into_websites()
@@ -510,6 +514,7 @@ mod tests {
include_str!("../../../testcases/goggles/hacker_news.goggle").to_string(),
),
skip_pages: None,
+ site_rankings: None,
})
.unwrap()
.into_websites()
@@ -525,6 +530,7 @@ mod tests {
include_str!("../../../testcases/goggles/copycats_removal.goggle").to_string(),
),
skip_pages: None,
+ site_rankings: None,
})
.unwrap()
.into_websites()
@@ -630,6 +636,7 @@ mod tests {
.to_string(),
),
skip_pages: None,
+ site_rankings: None,
})
.unwrap()
.into_websites()
diff --git a/src/ranking/mod.rs b/src/ranking/mod.rs
index 8a5f5683..d34a5776 100644
--- a/src/ranking/mod.rs
+++ b/src/ranking/mod.rs
@@ -19,6 +19,7 @@ pub mod centrality_store;
pub mod goggles;
mod initial;
pub mod signal;
+pub mod site_rankings;
use std::sync::Arc;
@@ -176,6 +177,7 @@ mod tests {
selected_region: None,
goggle_program: None,
skip_pages: None,
+ site_rankings: None,
})
.expect("Search failed")
.into_websites()
@@ -249,6 +251,7 @@ mod tests {
selected_region: None,
goggle_program: None,
skip_pages: None,
+ site_rankings: None,
})
.expect("Search failed")
.into_websites()
@@ -344,6 +347,7 @@ mod tests {
selected_region: None,
goggle_program: None,
skip_pages: None,
+ site_rankings: None,
})
.expect("Search failed")
.into_websites()
@@ -418,6 +422,7 @@ mod tests {
selected_region: None,
goggle_program: None,
skip_pages: None,
+ site_rankings: None,
})
.expect("Search failed")
.into_websites()
@@ -497,6 +502,7 @@ mod tests {
selected_region: None,
goggle_program: None,
skip_pages: None,
+ site_rankings: None,
})
.expect("Search failed")
.into_websites()
@@ -569,6 +575,7 @@ mod tests {
selected_region: None,
goggle_program: None,
skip_pages: None,
+ site_rankings: None,
})
.expect("Search failed")
.into_websites()
@@ -671,6 +678,7 @@ mod tests {
.to_string(),
),
skip_pages: None,
+ site_rankings: None,
})
.unwrap()
.into_websites()
@@ -691,6 +699,7 @@ mod tests {
.to_string(),
),
skip_pages: None,
+ site_rankings: None,
})
.unwrap()
.into_websites()
@@ -710,6 +719,7 @@ mod tests {
.to_string(),
),
skip_pages: None,
+ site_rankings: None,
})
.unwrap()
.into_websites()
@@ -806,6 +816,7 @@ mod tests {
selected_region: None,
goggle_program: None,
skip_pages: None,
+ site_rankings: None,
})
.expect("Search failed")
.into_websites()
@@ -882,6 +893,7 @@ mod tests {
selected_region: None,
goggle_program: None,
skip_pages: None,
+ site_rankings: None,
})
.expect("Search failed")
.into_websites()
diff --git a/src/ranking/signal.rs b/src/ranking/signal.rs
index 28c2d069..6e0f2ddb 100644
--- a/src/ranking/signal.rs
+++ b/src/ranking/signal.rs
@@ -226,6 +226,7 @@ impl FieldBoost {
}
}
+#[derive(Clone)]
pub struct SignalAggregator {
readers: Vec<Option<DynamicFastFieldReader<u64>>>,
signal_coefficients: SignalCoefficient,
diff --git a/src/ranking/site_rankings.rs b/src/ranking/site_rankings.rs
new file mode 100644
index 00000000..b78ce7d6
--- /dev/null
+++ b/src/ranking/site_rankings.rs
@@ -0,0 +1,182 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use serde::{Deserialize, Serialize};
+
+use super::{
+ goggles::{Action, Goggle, Instruction, PatternOption},
+ SignalAggregator,
+};
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct SiteRankings {
+ pub preferred: Vec<String>,
+ pub disliked: Vec<String>,
+ pub blocked: Vec<String>,
+}
+
+impl SiteRankings {
+ pub fn into_goggle(self) -> Goggle {
+ let mut instructions = Vec::new();
+
+ for site in self.preferred {
+ instructions.push(Instruction {
+ patterns: Vec::new(),
+ options: vec![
+ PatternOption::Site(site),
+ PatternOption::Action(Action::Boost(5)),
+ ],
+ });
+ }
+
+ for site in self.disliked {
+ instructions.push(Instruction {
+ patterns: Vec::new(),
+ options: vec![
+ PatternOption::Site(site),
+ PatternOption::Action(Action::Downrank(5)),
+ ],
+ });
+ }
+
+ for site in self.blocked {
+ instructions.push(Instruction {
+ patterns: Vec::new(),
+ options: vec![
+ PatternOption::Site(site),
+ PatternOption::Action(Action::Discard),
+ ],
+ });
+ }
+
+ Goggle {
+ aggregator: SignalAggregator::default(),
+ instructions,
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use crate::{
+ index::Index,
+ ranking::site_rankings::SiteRankings,
+ searcher::{LocalSearcher, SearchQuery},
+ webpage::{Html, Webpage},
+ };
+ const CONTENT: &str = "this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever";
+
+ #[test]
+ fn site_rankings() {
+ let mut index = Index::temporary().expect("Unable to open index");
+
+ index
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ <body>
+ {CONTENT}
+ </body>
+ </html>
+ "#
+ ),
+ "https://www.first.com",
+ ),
+ backlinks: vec![],
+ host_centrality: 1.0,
+ fetch_time_ms: 5000,
+ pre_computed_score: 0.0,
+ page_centrality: 0.0,
+ primary_image: None,
+ })
+ .expect("failed to insert webpage");
+ index
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ <body>
+ {CONTENT}
+ </body>
+ </html>
+ "#
+ ),
+ "https://www.second.com",
+ ),
+ backlinks: vec![],
+ host_centrality: 1.0,
+ fetch_time_ms: 0,
+ page_centrality: 0.0,
+ pre_computed_score: 0.0,
+ primary_image: None,
+ })
+ .expect("failed to insert webpage");
+ index.commit().expect("failed to commit index");
+ let searcher = LocalSearcher::new(index, None, None);
+
+ let result = searcher
+ .search(&SearchQuery {
+ original: "test".to_string(),
+ selected_region: None,
+ goggle_program: None,
+ skip_pages: None,
+ site_rankings: Some(SiteRankings {
+ preferred: vec!["first.com".to_string()],
+ disliked: vec!["second.com".to_string()],
+ blocked: vec![],
+ }),
+ })
+ .expect("Search failed")
+ .into_websites()
+ .unwrap()
+ .webpages;
+
+ assert_eq!(result.num_docs, 2);
+ assert_eq!(result.documents.len(), 2);
+ assert_eq!(result.documents[0].url, "https://www.first.com");
+ assert_eq!(result.documents[1].url, "https://www.second.com");
+
+ let result = searcher
+ .search(&SearchQuery {
+ original: "test".to_string(),
+ selected_region: None,
+ goggle_program: None,
+ skip_pages: None,
+ site_rankings: Some(SiteRankings {
+ preferred: vec![],
+ disliked: vec!["second.com".to_string()],
+ blocked: vec!["first.com".to_string()],
+ }),
+ })
+ .expect("Search failed")
+ .into_websites()
+ .unwrap()
+ .webpages;
+
+ assert_eq!(result.num_docs, 1);
+ assert_eq!(result.documents.len(), 1);
+ assert_eq!(result.documents[0].url, "https://www.second.com");
+ }
+}
diff --git a/src/schema.rs b/src/schema.rs
index 2ae525e8..47c2a535 100644
--- a/src/schema.rs
+++ b/src/schema.rs
@@ -18,7 +18,7 @@ use tantivy::schema::{
BytesOptions, Cardinality, IndexRecordOption, NumericOptions, TextFieldIndexing, TextOptions,
};
-use crate::tokenizer::{Normal, Stemmed};
+use crate::tokenizer::{Identity, Normal, Stemmed};
pub const CENTRALITY_SCALING: u64 = 1_000_000_000;
@@ -58,10 +58,11 @@ pub enum Field {
UrlWithoutQueryHash,
TitleHash,
UrlHash,
+ DomainHash,
PreComputedScore,
}
-pub static ALL_FIELDS: [Field; 32] = [
+pub static ALL_FIELDS: [Field; 33] = [
Field::Title,
Field::CleanBody,
Field::StemmedTitle,
@@ -93,6 +94,7 @@ pub static ALL_FIELDS: [Field; 32] = [
Field::UrlWithoutQueryHash,
Field::TitleHash,
Field::UrlHash,
+ Field::DomainHash,
Field::PreComputedScore,
];
@@ -119,27 +121,18 @@ impl Field {
Field::Url => IndexingOption::Text(self.default_text_options().set_stored()),
Field::Site => IndexingOption::Text(self.default_text_options()),
Field::Domain => IndexingOption::Text(self.default_text_options()),
- Field::SiteNoTokenizer => IndexingOption::Text(
- TextOptions::default().set_indexing_options(
- TextFieldIndexing::default()
- .set_index_option(IndexRecordOption::WithFreqsAndPositions),
- ),
- ),
- Field::DomainNoTokenizer => IndexingOption::Text(
- TextOptions::default().set_indexing_options(
- TextFieldIndexing::default()
- .set_index_option(IndexRecordOption::WithFreqsAndPositions),
- ),
- ),
+ Field::SiteNoTokenizer => {
+ IndexingOption::Text(self.default_text_options_with_tokenizer(Identity::as_str()))
+ }
+ Field::DomainNoTokenizer => {
+ IndexingOption::Text(self.default_text_options_with_tokenizer(Identity::as_str()))
+ }
Field::AllBody => IndexingOption::Text(self.default_text_options().set_stored()),
Field::DomainIfHomepage => IndexingOption::Text(self.default_text_options()),
Field::TitleIfHomepage => IndexingOption::Text(self.default_text_options()),
- Field::DomainNameIfHomepageNoTokenizer => IndexingOption::Text(
- TextOptions::default().set_indexing_options(
- TextFieldIndexing::default()
- .set_index_option(IndexRecordOption::WithFreqsAndPositions),
- ),
- ),
+ Field::DomainNameIfHomepageNoTokenizer => {
+ IndexingOption::Text(self.default_text_options_with_tokenizer(Identity::as_str()))
+ }
Field::IsHomepage => IndexingOption::Integer(
NumericOptions::default()
.set_fast(Cardinality::SingleValue)
@@ -216,6 +209,9 @@ impl Field {
Field::UrlHash => IndexingOption::Integer(
NumericOptions::default().set_fast(Cardinality::MultiValues),
),
+ Field::DomainHash => IndexingOption::Integer(
+ NumericOptions::default().set_fast(Cardinality::MultiValues),
+ ),
Field::TitleHash => IndexingOption::Integer(
NumericOptions::default().set_fast(Cardinality::MultiValues),
),
@@ -262,6 +258,7 @@ impl Field {
Field::TitleIfHomepage => "title_if_homepage",
Field::TitleHash => "title_hash",
Field::UrlHash => "url_hash",
+ Field::DomainHash => "domain_hash",
}
}
@@ -292,6 +289,7 @@ impl Field {
| Field::Description
| Field::NumTrackers
| Field::NumUrlTokens
+ | Field::DomainHash
| Field::NumTitleTokens
| Field::NumCleanBodyTokens
| Field::NumDescriptionTokens
@@ -322,6 +320,7 @@ impl Field {
| Field::UrlWithoutQueryHash
| Field::TitleHash
| Field::UrlHash
+ | Field::DomainHash
| Field::PreComputedScore
)
}
@@ -353,6 +352,7 @@ impl Field {
"pre_computed_score" => Some(Field::PreComputedScore),
"title_if_homepage" => Some(Field::TitleIfHomepage),
"url_hash" => Some(Field::UrlHash),
+ "domain_hash" => Some(Field::DomainHash),
"title_hash" => Some(Field::TitleHash),
_ => None,
}
diff --git a/src/search_prettifier.rs b/src/search_prettifier.rs
index 1fd4c96a..67b0654d 100644
--- a/src/search_prettifier.rs
+++ b/src/search_prettifier.rs
@@ -83,6 +83,7 @@ pub fn html_escape(s: &str) -> String {
pub struct DisplayedWebpage {
pub title: String,
pub url: String,
+ pub site: String,
pub favicon_base64: String,
pub domain: String,
pub pretty_url: String,
@@ -92,9 +93,6 @@ pub struct DisplayedWebpage {
pub last_updated: Option<String>,
}
-const MAX_PRETTY_URL_LEN: usize = 50;
-const MAX_TITLE_LEN: usize = 70;
-
fn prettify_url(url: &Url) -> String {
let mut pretty_url = url.strip_query().to_string();
@@ -108,11 +106,6 @@ fn prettify_url(url: &Url) -> String {
.replace('/', " › ");
pretty_url = protocol + &pretty_url;
- if pretty_url.len() > MAX_PRETTY_URL_LEN {
- pretty_url = pretty_url.chars().take(MAX_PRETTY_URL_LEN).collect();
- pretty_url += "...";
- }
-
pretty_url
}
@@ -149,12 +142,7 @@ impl From<RetrievedWebpage> for DisplayedWebpage {
let domain = url.domain().to_string();
let pretty_url = prettify_url(&url);
- let mut title = html_escape(&webpage.title);
-
- if title.len() > MAX_TITLE_LEN {
- title = title.chars().take(MAX_TITLE_LEN).collect();
- title += "...";
- }
+ let title = html_escape(&webpage.title);
let favicon_bytes = webpage
.favicon
@@ -163,6 +151,7 @@ impl From<RetrievedWebpage> for DisplayedWebpage {
Self {
title,
+ site: url.site().to_string(),
url: webpage.url,
pretty_url,
domain,
diff --git a/src/searcher/local.rs b/src/searcher/local.rs
index f352423c..609ab31f 100644
--- a/src/searcher/local.rs
+++ b/src/searcher/local.rs
@@ -92,10 +92,18 @@ impl LocalSearcher {
}
}
+ let mut goggles = Vec::new();
+
if let Some(goggle) = &goggle {
- parsed_query.set_goggle(goggle, &self.index.schema());
+ goggles.push(goggle.clone());
+ }
+
+ if let Some(site_rankings) = &query.site_rankings {
+ goggles.push(site_rankings.clone().into_goggle())
}
+ parsed_query.set_goggles(&goggles, &self.index.schema());
+
let mut ranker = Ranker::new(
self.index.region_count.clone(),
goggle.map(|goggle| goggle.aggregator).unwrap_or_default(),
@@ -259,6 +267,7 @@ mod tests {
selected_region: None,
goggle_program: None,
skip_pages: Some(p),
+ site_rankings: None,
})
.unwrap()
.into_websites()
diff --git a/src/searcher/mod.rs b/src/searcher/mod.rs
index 61f0541a..5244134f 100644
--- a/src/searcher/mod.rs
+++ b/src/searcher/mod.rs
@@ -25,6 +25,7 @@ use crate::{
bangs::BangHit,
entity_index::StoredEntity,
inverted_index,
+ ranking::site_rankings::SiteRankings,
search_prettifier::{self, DisplayedEntity, DisplayedWebpage},
webpage::region::Region,
};
@@ -72,12 +73,13 @@ pub enum InitialPrettifiedSearchResult {
Bang(BangHit),
}
-#[derive(Serialize, Deserialize, Clone)]
+#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct SearchQuery {
pub original: String,
pub selected_region: Option<Region>,
pub goggle_program: Option<String>,
pub skip_pages: Option<usize>,
+ pub site_rankings: Option<SiteRankings>,
}
impl SearchQuery {
diff --git a/src/snippet.rs b/src/snippet.rs
index 462befd4..28027490 100644
--- a/src/snippet.rs
+++ b/src/snippet.rs
@@ -30,7 +30,7 @@ use whatlang::Lang;
/// This will require us to store each paragraph of the webpage separately to get adequate performance.
/// Implementing SnippetIL will also allow us to correctly add "..." to the snippet.
-const DEFAULT_MAX_NUM_CHARS: usize = 200;
+const DEFAULT_MAX_NUM_CHARS: usize = 280;
#[derive(Debug)]
struct FragmentCandidate {
@@ -372,6 +372,7 @@ Survey in 2016, 2017, and 2018."#;
selected_region: None,
goggle_program: None,
skip_pages: None,
+ site_rankings: None,
})
.expect("Search failed")
.into_websites()
@@ -379,7 +380,7 @@ Survey in 2016, 2017, and 2018."#;
assert_eq!(result.webpages.num_docs, 1);
assert_eq!(result.webpages.documents.len(), 1);
- assert_eq!(result.webpages.documents[0].snippet, "<b>Rust</b> is a systems programming <b>language</b> sponsored by Mozilla which describes it as a \"safe, concurrent, practical <b>language</b>\", supporting functional and imperative-procedural paradigms. <b>Rust</b> is...".to_string());
+ assert_eq!(result.webpages.documents[0].snippet, "<b>Rust</b> is a systems programming <b>language</b> sponsored by Mozilla which describes it as a \"safe, concurrent, practical <b>language</b>\", supporting functional and imperative-procedural paradigms. <b>Rust</b> is syntactically similar to C++[according to whom?], but its designers intend it to provide...".to_string());
}
#[test]
@@ -414,6 +415,7 @@ Survey in 2016, 2017, and 2018."#;
selected_region: None,
goggle_program: None,
skip_pages: None,
+ site_rankings: None,
})
.expect("Search failed")
.into_websites()
@@ -421,7 +423,7 @@ Survey in 2016, 2017, and 2018."#;
assert_eq!(result.webpages.num_docs, 1);
assert_eq!(result.webpages.documents.len(), 1);
- assert_eq!(result.webpages.documents[0].snippet, "Rust is a systems programming language sponsored by Mozilla which <b>describes</b> it as a \"safe, concurrent, practical language\", supporting functional and imperative-procedural paradigms. Rust is...".to_string());
+ assert_eq!(result.webpages.documents[0].snippet, "Rust is a systems programming language sponsored by Mozilla which <b>describes</b> it as a \"safe, concurrent, practical language\", supporting functional and imperative-procedural paradigms. Rust is syntactically similar to C++[according to whom?], but its designers intend it to provide...".to_string());
}
#[test]
@@ -611,6 +613,7 @@ Survey in 2016, 2017, and 2018."#;
selected_region: None,
goggle_program: None,
skip_pages: None,
+ site_rankings: None,
})
.expect("Search failed")
.into_websites()
@@ -620,7 +623,7 @@ Survey in 2016, 2017, and 2018."#;
assert_eq!(result.webpages.documents.len(), 1);
assert_eq!(
result.webpages.documents[0].snippet,
- "Rust is a systems programming language sponsored by Mozilla which describes it as a \"safe, concurrent, practical language\", supporting functional and imperative-procedural <b>paradigms</b>. Rust is...".to_string()
+ "Rust is a systems programming language sponsored by Mozilla which describes it as a \"safe, concurrent, practical language\", supporting functional and imperative-procedural <b>paradigms</b>. Rust is syntactically similar to C++[according to whom?], but its designers intend it to provide...".to_string()
);
}
}
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index 3c0bd193..6ae7bedf 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -48,25 +48,27 @@ impl From<Lang> for MyStemmer {
#[derive(Clone)]
pub enum Tokenizer {
- NormalTokenizer(Normal),
- StemmedTokenizer(Stemmed),
+ Normal(Normal),
+ Identity(Identity),
+ Stemmed(Stemmed),
}
impl Tokenizer {
pub fn new_stemmed() -> Self {
- Self::StemmedTokenizer(Stemmed::default())
+ Self::Stemmed(Stemmed::default())
}
pub fn as_str(&self) -> &'static str {
match self {
- Tokenizer::NormalTokenizer(_) => Normal::as_str(),
- Tokenizer::StemmedTokenizer(_) => Stemmed::as_str(),
+ Tokenizer::Normal(_) => Normal::as_str(),
+ Tokenizer::Stemmed(_) => Stemmed::as_str(),
+ Tokenizer::Identity(_) => Identity::as_str(),
}
}
}
impl Default for Tokenizer {
fn default() -> Self {
- Self::NormalTokenizer(Normal::default())
+ Self::Normal(Normal::default())
}
}
@@ -103,11 +105,21 @@ impl Stemmed {
}
}
+#[derive(Clone, Default)]
+pub struct Identity {}
+
+impl Identity {
+ pub fn as_str() -> &'static str {
+ "identity_tokenizer"
+ }
+}
+
impl tantivy::tokenizer::Tokenizer for Tokenizer {
fn token_stream<'a>(&self, text: &'a str) -> tantivy::tokenizer::BoxTokenStream<'a> {
match self {
- Tokenizer::NormalTokenizer(tokenizer) => tokenizer.token_stream(text),
- Tokenizer::StemmedTokenizer(tokenizer) => tokenizer.token_stream(text),
+ Tokenizer::Normal(tokenizer) => tokenizer.token_stream(text),
+ Tokenizer::Stemmed(tokenizer) => tokenizer.token_stream(text),
+ Tokenizer::Identity(tokenizer) => tokenizer.token_stream(text),
}
}
}
@@ -142,6 +154,53 @@ impl tantivy::tokenizer::Tokenizer for Stemmed {
}
}
+impl tantivy::tokenizer::Tokenizer for Identity {
+ fn token_stream<'a>(&self, text: &'a str) -> tantivy::tokenizer::BoxTokenStream<'a> {
+ BoxTokenStream::from(IdentityTokenStream::from(text.to_string()))
+ }
+}
+
+pub struct IdentityTokenStream {
+ num_advances: usize,
+ token: Option<tantivy::tokenizer::Token>,
+}
+
+impl From<String> for IdentityTokenStream {
+ fn from(text: String) -> Self {
+ Self {
+ num_advances: 0,
+ token: Some(tantivy::tokenizer::Token {
+ offset_from: 0,
+ offset_to: text.len(),
+ position: 0,
+ text,
+ ..Default::default()
+ }),
+ }
+ }
+}
+
+impl tantivy::tokenizer::TokenStream for IdentityTokenStream {
+ fn advance(&mut self) -> bool {
+ self.num_advances += 1;
+
+ if self.num_advances == 1 {
+ true
+ } else {
+ self.token = None;
+ false
+ }
+ }
+
+ fn token(&self) -> &tantivy::tokenizer::Token {
+ self.token.as_ref().unwrap()
+ }
+
+ fn token_mut(&mut self) -> &mut tantivy::tokenizer::Token {
+ self.token.as_mut().unwrap()
+ }
+}
+
#[derive(Logos, Debug, PartialEq)]
enum Token {
#[regex("[\\w|\\p{Han}|\\p{Hiragana}|\\p{Katakana}|\\p{Cyrillic}|\\p{Arabic}]+")]
diff --git a/src/ttl_cache.rs b/src/ttl_cache.rs
new file mode 100644
index 00000000..350e237d
--- /dev/null
+++ b/src/ttl_cache.rs
@@ -0,0 +1,139 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use std::{
+ collections::{HashMap, VecDeque},
+ hash::Hash,
+ time::{Duration, SystemTime},
+};
+
+pub struct TTLCache<K, V> {
+ ttl: Duration,
+ data: HashMap<K, V>,
+ insertion_order: VecDeque<K>,
+ insertion_times: HashMap<K, SystemTime>,
+ max_size: usize,
+}
+
+impl<K: Hash + Eq + Clone, V> TTLCache<K, V> {
+ pub fn with_ttl_and_max_size(ttl: Duration, max_size: usize) -> Self {
+ Self {
+ ttl,
+ data: HashMap::new(),
+ insertion_order: VecDeque::new(),
+ insertion_times: HashMap::new(),
+ max_size,
+ }
+ }
+
+ pub fn insert(&mut self, key: K, val: V) {
+ self.prune_old_entries();
+ let current_time = SystemTime::now();
+
+ if self.data.insert(key.clone(), val).is_some() {
+ let (idx, _) = self
+ .insertion_order
+ .iter()
+ .enumerate()
+ .find(|(_, k)| *k == &key)
+ .unwrap();
+
+ self.insertion_order.remove(idx);
+ }
+
+ self.insertion_times.insert(key.clone(), current_time);
+ self.insertion_order.push_back(key);
+ }
+
+ pub fn get(&self, key: &K) -> Option<&V> {
+ let current_time = SystemTime::now();
+
+ self.insertion_times.get(key).and_then(|insertion_time| {
+ if current_time.duration_since(*insertion_time).unwrap() < self.ttl {
+ self.data.get(key)
+ } else {
+ None
+ }
+ })
+ }
+
+ fn prune_old_entries(&mut self) {
+ let current_time = SystemTime::now();
+
+ while self.data.len() >= self.max_size {
+ let front = self.insertion_order.pop_front().unwrap();
+ self.insertion_times.remove(&front);
+ self.data.remove(&front);
+ }
+
+ while let Some(front) = self.insertion_order.front() {
+ if current_time
+ .duration_since(*self.insertion_times.get(front).unwrap())
+ .unwrap()
+ <= self.ttl
+ {
+ break;
+ }
+
+ let front = self.insertion_order.pop_front().unwrap();
+ self.insertion_times.remove(&front);
+ self.data.remove(&front);
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn simple() {
+ let mut cache = TTLCache::with_ttl_and_max_size(Duration::from_millis(5), 5);
+
+ cache.insert(0, 0);
+ std::thread::sleep(Duration::from_millis(3));
+ cache.insert(1, 1);
+
+ assert_eq!(cache.get(&0), Some(&0));
+ assert_eq!(cache.get(&1), Some(&1));
+
+ std::thread::sleep(Duration::from_millis(3));
+
+ assert_eq!(cache.get(&0), None);
+ assert_eq!(cache.get(&1), Some(&1));
+
+ std::thread::sleep(Duration::from_millis(3));
+
+ assert_eq!(cache.get(&0), None);
+ assert_eq!(cache.get(&1), None);
+
+ cache.insert(2, 2);
+
+ assert_eq!(cache.data.len(), 1);
+ assert_eq!(cache.insertion_order.len(), 1);
+ assert_eq!(cache.insertion_times.len(), 1);
+ }
+
+ #[test]
+ fn max_size() {
+ let mut cache = TTLCache::with_ttl_and_max_size(Duration::from_millis(5), 1);
+
+ cache.insert(0, 0);
+ cache.insert(1, 1);
+
+ assert_eq!(cache.get(&0), None);
+ assert_eq!(cache.get(&1), Some(&1));
+ }
+}
diff --git a/src/webpage/mod.rs b/src/webpage/mod.rs
index 089eceee..222d69f0 100644
--- a/src/webpage/mod.rs
+++ b/src/webpage/mod.rs
@@ -497,7 +497,7 @@ impl Html {
}
fn pretokenize_url(&self) -> PreTokenizedString {
- let url = self.url().full();
+ let url = self.url().full_without_id_tags();
self.pretokenize_string(url)
}
@@ -662,6 +662,12 @@ impl Html {
doc.add_u64(tantivy_field, u64s[0]);
doc.add_u64(tantivy_field, u64s[1]);
}
+ Field::DomainHash => {
+ let hash = hash(self.url().domain()).0;
+ let u64s = split_u128(hash);
+ doc.add_u64(tantivy_field, u64s[0]);
+ doc.add_u64(tantivy_field, u64s[1]);
+ }
Field::TitleHash => {
let hash = hash(self.title().unwrap_or_default()).0;
let u64s = split_u128(hash);
diff --git a/src/webpage/url.rs b/src/webpage/url.rs
index bdb1e191..a661400d 100644
--- a/src/webpage/url.rs
+++ b/src/webpage/url.rs
@@ -201,6 +201,15 @@ impl Url {
&self.0
}
}
+
+ pub fn full_without_id_tags(&self) -> String {
+ let full = self.full();
+ if let Some(id_begin) = full.find('#') {
+ full[..id_begin].to_string()
+ } else {
+ full
+ }
+ }
}
#[cfg(test)]
@@ -292,4 +301,11 @@ mod tests {
assert_eq!(url.without_query(), "https://test.example.com");
}
+
+ #[test]
+ fn url_without_id() {
+ let url: Url = "https://test.example.com#id".to_string().into();
+
+ assert_eq!(&url.full_without_id_tags(), "https://test.example.com");
+ }
}