web search done right
change paywall icon to text and add settings page to choose when results should be marked
Mikkel Denker45bec0f942cb
diff --git a/core/src/webpage/html/mod.rs b/core/src/webpage/html/mod.rsindex e0c60ccb..3f472e67 100644--- a/core/src/webpage/html/mod.rs+++ b/core/src/webpage/html/mod.rs@@ -303,22 +303,24 @@ impl Html {}pub fn likely_has_paywall(&self) -> bool {- self.schema_org()- .into_iter()- .filter(|item| {- item.types_contains("NewsArticle")- || item.types_contains("Article")- || item.types_contains("BlogPosting")- || item.types_contains("WebPage")- || item.types_contains("WebPageElement")- })- .any(|item| {- item.properties- .get("isAccessibleForFree")- .and_then(|value| value.clone().one().and_then(|v| v.try_into_string()))- .map(|value| value.parse().ok().unwrap_or(false))- .unwrap_or(false)- })+ for item in self.schema_org().into_iter().filter(|item| {+ item.types_contains("NewsArticle")+ || item.types_contains("Article")+ || item.types_contains("BlogPosting")+ || item.types_contains("WebPage")+ || item.types_contains("WebPageElement")+ }) {+ if let Some(free) = item+ .properties+ .get("isAccessibleForFree")+ .and_then(|value| value.clone().one().and_then(|v| v.try_into_string()))+ .map(|value| value.parse().ok().unwrap_or(false))+ {+ return !free;+ }+ }++ false}fn article_modified_time(&self) -> Option<DateTime<FixedOffset>> {@@ -1078,4 +1080,25 @@ mod tests {Some("ingredients".to_string()));}++ #[test]+ fn paywall() {+ let html = Html::parse(r##"+ <!DOCTYPE html>+<html lang="en" class="story nytapp-vi-article" xmlns:og="http://opengraphprotocol.org/schema/">+ <head>+ <meta charset="utf-8" />+ <title data-rh="true">Russian Ships Near Data Cables Are Too Close for U.S. Comfort - The New York Times</title>+ <meta data-rh="true" name="robots" content="noarchive, max-image-preview:large"/><meta data-rh="true" name="description" content="Russian submarines are aggressively operating near the undersea cables that carry almost all global Internet communications, raising concerns in the U.S."/><meta data-rh="true" property="og:url" content="https://www.nytimes.com/2015/10/26/world/europe/russian-presence-near-undersea-cables-concerns-us.html"/><meta data-rh="true" property="og:type" content="article"/><meta data-rh="true" property="og:title" content="Russian Ships Near Data Cables Are Too Close for U.S. Comfort (Published 2015)"/><meta data-rh="true" property="og:image" content="https://static01.nyt.com/images/2015/10/26/world/JP-SUBS/JP-SUBS-facebookJumbo.jpg?year=2015&h=550&w=1050&s=ceeb5555ae80dd2e2fe05553973d3948298bcba76c8d62a24907ae21e61140a7&k=ZQJBKqZ0VN"/><meta data-rh="true" property="og:image:alt" content="Adm. Mark Ferguson said the intensity of Russian submarine patrols had risen by almost 50 percent over the last year."/><meta data-rh="true" property="og:description" content="Russian submarines are aggressively operating near the undersea cables that carry almost all global Internet communications, raising concerns in the U.S."/><meta data-rh="true" property="twitter:url" content="https://www.nytimes.com/2015/10/26/world/europe/russian-presence-near-undersea-cables-concerns-us.html"/><meta data-rh="true" property="twitter:title" content="Russian Ships Near Data Cables Are Too Close for U.S. Comfort (Published 2015)"/><meta data-rh="true" property="twitter:description" content="Russian submarines are aggressively operating near the undersea cables that carry almost all global Internet communications, raising concerns in the U.S."/><meta data-rh="true" property="twitter:image" content="https://static01.nyt.com/images/2015/10/26/world/JP-SUBS/JP-SUBS-articleLarge.jpg?year=2015&h=400&w=600&s=1c25983f231db27a32cbc8063f067135d143de5f7c2d671b5798f0b76507c68f&k=ZQJBKqZ0VN&tw=1"/><meta data-rh="true" property="twitter:image:alt" content="Adm. Mark Ferguson said the intensity of Russian submarine patrols had risen by almost 50 percent over the last year."/><meta data-rh="true" property="twitter:card" content="summary_large_image"/> <link data-rh="true" rel="canonical" href="https://www.nytimes.com/2015/10/26/world/europe/russian-presence-near-undersea-cables-concerns-us.html"/><link data-rh="true" rel="alternate" href="nyt://article/a608af24-ea77-5f40-b5ee-e830ccd0f1fd"/><link data-rh="true" rel="alternate" type="application/json+oembed" href="https://www.nytimes.com/svc/oembed/json/?url=https%3A%2F%2Fwww.nytimes.com%2F2015%2F10%2F26%2Fworld%2Feurope%2Frussian-presence-near-undersea-cables-concerns-us.html" title="Russian Ships Near Data Cables Are Too Close for U.S. Comfort"/> <script data-rh="true" type="application/ld+json">{"@context":"http://schema.org","@type":"NewsArticle","description":"Russian submarines are aggressively operating near the undersea cables that carry almost all global Internet communications, raising concerns in the U.S.","image":{"@context":"http://schema.org","@type":"ImageObject","url":"https://static01.nyt.com/images/2015/10/26/world/JP-SUBS/JP-SUBS-superJumbo.jpg","height":1365,"width":2048,"contentUrl":"https://static01.nyt.com/images/2015/10/26/world/JP-SUBS/JP-SUBS-superJumbo.jpg","caption":"Adm. Mark Ferguson said the intensity of Russian submarine patrols had risen by almost 50 percent over the last year.","creditText":"Brendan Smialowski/Agence France-Presse — Getty Images"},"mainEntityOfPage":"https://www.nytimes.com/2015/10/26/world/europe/russian-presence-near-undersea-cables-concerns-us.html","url":"https://www.nytimes.com/2015/10/26/world/europe/russian-presence-near-undersea-cables-concerns-us.html","author":[{"@context":"http://schema.org","@type":"Person","url":"http://www.nytimes.com/by/david-e-sanger","name":"David E. Sanger"},{"@context":"http://schema.org","@type":"Person","url":"http://www.nytimes.com/by/eric-schmitt","name":"Eric Schmitt"}],"dateModified":"2015-10-25T17:02:34.000Z","datePublished":"2015-10-25T17:02:35.000Z","headline":"Russian Ships Near Data Cables Are Too Close for U.S. Comfort","alternativeHeadline":"Russian Ships Near Data Cables Are Too Close for U.S. Comfort","publisher":{"@id":"https://www.nytimes.com/#publisher","name":"The New York Times"},"hasPart":{"@type":"WebPageElement","isAccessibleForFree":false,"cssSelector":".meteredContent"},"copyrightHolder":{"@id":"https://www.nytimes.com/#publisher","name":"The New York Times"},"sourceOrganization":{"@id":"https://www.nytimes.com/#publisher","name":"The New York Times"},"copyrightYear":2024,"isAccessibleForFree":false,"isPartOf":{"@type":["CreativeWork","Product"],"name":"The New York Times","productID":"nytimes.com:basic"}}</script><script data-rh="true" type="application/ld+json">{"@context":"http://schema.org","@type":"NewsMediaOrganization","name":"The New York Times","logo":{"@context":"http://schema.org","@type":"ImageObject","url":"https://static01.nyt.com/images/icons/t_logo_291_black.png","height":291,"width":291,"contentUrl":"https://static01.nyt.com/images/icons/t_logo_291_black.png","creditText":"The New York Times"},"url":"https://www.nytimes.com/","@id":"https://www.nytimes.com/#publisher","diversityPolicy":"https://www.nytco.com/company/diversity-and-inclusion/","ethicsPolicy":"https://www.nytco.com/company/standards-ethics/","masthead":"https://www.nytimes.com/interactive/2023/01/28/admin/the-new-york-times-masthead.html","foundingDate":"1851-09-18","sameAs":"https://en.wikipedia.org/wiki/The_New_York_Times"}</script><script data-rh="true" type="application/ld+json">{"@context":"http://schema.org","@type":"BreadcrumbList","itemListElement":[{"@context":"http://schema.org","@type":"ListItem","name":"World","position":1,"item":"https://www.nytimes.com/section/world"},{"@context":"http://schema.org","@type":"ListItem","name":"Europe","position":2,"item":"https://www.nytimes.com/section/world/europe"}]}</script>+ <meta data-rh="true" property="article:published_time" content="2015-10-25T17:02:35.000Z"/><meta data-rh="true" property="article:modified_time" content="2015-10-25T17:02:34.000Z"/><meta data-rh="true" http-equiv="Content-Language" content="en"/><meta data-rh="true" name="articleid" content="100000003995797"/><meta data-rh="true" name="nyt_uri" content="nyt://article/a608af24-ea77-5f40-b5ee-e830ccd0f1fd"/><meta data-rh="true" name="pubp_event_id" content="pubp://event/7219283749a6486e8ec2777e0840ab41"/><meta data-rh="true" name="image" content="https://static01.nyt.com/images/2015/10/26/world/JP-SUBS/JP-SUBS-facebookJumbo.jpg?year=2015&h=550&w=1050&s=ceeb5555ae80dd2e2fe05553973d3948298bcba76c8d62a24907ae21e61140a7&k=ZQJBKqZ0VN"/><meta data-rh="true" name="byl" content="By David E. Sanger and Eric Schmitt"/><meta data-rh="true" name="news_keywords" content=",Russia,,Submarines and Submersibles,Wire and Cable,Computers and the Internet,Tech Industry,US,Homeland Security,Spying and Intelligence Agencies,US Military,Military,Mark Ferguson"/><meta data-rh="true" name="pdate" content="20151025"/><meta data-rh="true" property="article:section" content="World"/><meta data-rh="true" property="article:author" content="http://www.nytimes.com/by/david-e-sanger"/><meta data-rh="true" property="article:author" content="http://www.nytimes.com/by/eric-schmitt"/><meta data-rh="true" property="article:tag" content="Schmitt, Eric"/><meta data-rh="true" property="article:tag" content="Russia"/><meta data-rh="true" property="article:tag" content="Sanger, David E"/><meta data-rh="true" property="article:tag" content="Submarines and Submersibles"/><meta data-rh="true" property="article:tag" content="Wire and Cable"/><meta data-rh="true" property="article:tag" content="Computers and the Internet"/><meta data-rh="true" property="article:tag" content="United States"/><meta data-rh="true" property="article:tag" content="Homeland Security Department"/><meta data-rh="true" property="article:tag" content="Espionage and Intelligence Services"/><meta data-rh="true" property="article:tag" content="United States Defense and Military Forces"/><meta data-rh="true" property="article:tag" content="Defense and Military Forces"/><meta data-rh="true" property="article:tag" content="Ferguson, Mark E III"/><meta data-rh="true" property="article:opinion" content="false"/><meta data-rh="true" property="article:content_tier" content="metered"/><meta data-rh="true" name="CG" content="world"/><meta data-rh="true" name="SCG" content="europe"/><meta data-rh="true" name="CN" content=""/><meta data-rh="true" name="CT" content=""/><meta data-rh="true" name="PT" content="article"/><meta data-rh="true" name="PST" content="News"/><meta data-rh="true" name="url" content="https://www.nytimes.com/2015/10/26/world/europe/russian-presence-near-undersea-cables-concerns-us.html"/><meta data-rh="true" name="msapplication-starturl" content="https://www.nytimes.com"/><meta data-rh="true" property="al:android:url" content="nyt://article/a608af24-ea77-5f40-b5ee-e830ccd0f1fd"/><meta data-rh="true" property="al:android:package" content="com.nytimes.android"/><meta data-rh="true" property="al:android:app_name" content="NYTimes"/><meta data-rh="true" name="twitter:app:name:googleplay" content="NYTimes"/><meta data-rh="true" name="twitter:app:id:googleplay" content="com.nytimes.android"/><meta data-rh="true" name="twitter:app:url:googleplay" content="nyt://article/a608af24-ea77-5f40-b5ee-e830ccd0f1fd"/><meta data-rh="true" property="al:iphone:url" content="nytimes://www.nytimes.com/2015/10/26/world/europe/russian-presence-near-undersea-cables-concerns-us.html"/><meta data-rh="true" property="al:iphone:app_store_id" content="284862083"/><meta data-rh="true" property="al:iphone:app_name" content="NYTimes"/><meta data-rh="true" property="al:ipad:url" content="nytimes://www.nytimes.com/2015/10/26/world/europe/russian-presence-near-undersea-cables-concerns-us.html"/><meta data-rh="true" property="al:ipad:app_store_id" content="357066198"/><meta data-rh="true" property="al:ipad:app_name" content="NYTimes"/>++ </head>+ <body>+ TEST+ </body>+</html>+ "##, "https://www.example.com/whatever").unwrap();++ assert!(html.likely_has_paywall());+ }}diff --git a/core/src/webpage/schema_org/json_ld.rs b/core/src/webpage/schema_org/json_ld.rsindex a6670f21..aa02944e 100644--- a/core/src/webpage/schema_org/json_ld.rs+++ b/core/src/webpage/schema_org/json_ld.rs@@ -18,7 +18,7 @@ use kuchiki::NodeRef;use super::RawItem;-pub fn convert_all_ints_to_strings(json: &str) -> Result<String, serde_json::Error> {+pub fn convert_all_to_strings(json: &str) -> Result<String, serde_json::Error> {use serde_json::Value;fn convert_recursively(json: &mut Value) {@@ -26,6 +26,9 @@ pub fn convert_all_ints_to_strings(json: &str) -> Result<String, serde_json::ErrValue::Number(n) if n.is_u64() || n.is_i64() => {*json = Value::String(n.to_string());}+ Value::Bool(b) => {+ *json = Value::String(b.to_string());+ }Value::Array(a) => a.iter_mut().for_each(convert_recursively),Value::Object(o) => o.values_mut().for_each(convert_recursively),_ => (),@@ -50,9 +53,17 @@ pub(crate) fn parse(root: NodeRef) -> Vec<RawItem> {let text_contens = node.text_contents();let content = text_contens.trim();- if let Ok(schema) = convert_all_ints_to_strings(content) {- if let Ok(schema) = serde_json::from_str(&schema) {- res.push(schema);+ match convert_all_to_strings(content) {+ Ok(schema) => match serde_json::from_str(&schema) {+ Ok(schema) => {+ res.push(schema);+ }+ Err(e) => {+ tracing::debug!("Failed to parse schema.org JSON-LD: {}", e)+ }+ },+ Err(e) => {+ tracing::debug!("Failed to convert schema.org JSON-LD: {}", e)}}}@@ -171,4 +182,27 @@ mod tests {}]);}++ #[test]+ fn booleans() {+ let root = kuchiki::parse_html().one(+ r#"+ <html>+ <head>+ <script type="application/ld+json">+ {+ "someBoolean": false+ }+ </script>+ </head>+ <body>+ </body>+ </html>+ "#,+ );++ let res = parse(root);++ assert_eq!(res.len(), 1);+ }}diff --git a/frontend/src/lib/stores.ts b/frontend/src/lib/stores.tsindex 5e934273..35d5c31e 100644--- a/frontend/src/lib/stores.ts+++ b/frontend/src/lib/stores.ts@@ -70,6 +70,12 @@ export const searchQueryStore = writableLocalStorage<string | undefined>(SEARCH_const QUERY_ID_KEY = 'queryId';export const queryIdStore = writableLocalStorage<string | undefined>(QUERY_ID_KEY, void 0);+const MARK_PAGES_WITH_ADS_KEY = 'markPagesWithAds';+export const markPagesWithAdsStore = writableLocalStorage<boolean>(MARK_PAGES_WITH_ADS_KEY, false);++const MARK_PAGES_WITH_PAYWALL_KEY = 'markPagesWithPaywall';+export const markPagesWithPaywallStore = writableLocalStorage<boolean>(MARK_PAGES_WITH_PAYWALL_KEY, true);+const THEME_KEY = 'theme';export const themeStore = writableLocalStorage<string | void>(THEME_KEY, void 0);diff --git a/frontend/src/routes/search/Result.svelte b/frontend/src/routes/search/Result.svelteindex e8050ac7..3cfdb326 100644--- a/frontend/src/routes/search/Result.svelte+++ b/frontend/src/routes/search/Result.svelte@@ -1,9 +1,8 @@<script lang="ts">import AdjustVertical from '~icons/heroicons/adjustments-vertical';- import ShieldExclamation from '~icons/heroicons/shield-exclamation';import type { DisplayedWebpage } from '$lib/api';import { createEventDispatcher } from 'svelte';- import { clearSummary, summariesStore } from '$lib/stores';+ import { clearSummary, summariesStore, markPagesWithAdsStore, markPagesWithPaywallStore } from '$lib/stores';import Summary from './Summary.svelte';import { derived } from 'svelte/store';import { improvements } from '$lib/improvements';@@ -57,23 +56,23 @@<div class="line-clamp-3"><div class="inline"><span id="snippet-text" class="snippet-text">- {#if webpage.likelyHasAds && webpage.likelyHasPaywall}- <span title="page likely has ads and paywall">- <ShieldExclamation class="inline-block w-4 pb-0.5 h-[1em] text-primary" />+ {#if (webpage.likelyHasAds && $markPagesWithAdsStore ) && (webpage.likelyHasPaywall && $markPagesWithPaywallStore)}+ <span class="text-neutral text-center border p-0.5 rounded border-primary text-xs" title="page likely has ads and paywall">+ has ads + paywall</span>- {:else if webpage.likelyHasAds}- <span title="page likely has ads">- <ShieldExclamation class="inline-block w-4 pb-0.5 h-[1em] text-primary" />+ {:else if (webpage.likelyHasAds && $markPagesWithAdsStore)}+ <span class="text-neutral text-center border p-0.5 rounded border-primary text-xs" title="page likely has ads">+ has ads</span>- {:else if webpage.likelyHasPaywall}- <span title="page likely has paywall">- <ShieldExclamation class="inline-block w-4 pb-0.5 h-[1em] text-primary" />+ {:else if (webpage.likelyHasPaywall && $markPagesWithPaywallStore)}+ <span class="text-neutral text-center border p-0.5 rounded border-primary text-xs" title="page likely has paywall" >+ paywall</span>{/if}{#if webpage.snippet.date}<span class="text-neutral">- {webpage.snippet.date} -- </span>+ {webpage.snippet.date}+ </span> -{/if}<span><TextSnippet snippet={webpage.snippet.text} />diff --git a/frontend/src/routes/settings/+page.svelte b/frontend/src/routes/settings/+page.svelteindex f1b487a3..ca4b4458 100644--- a/frontend/src/routes/settings/+page.svelte+++ b/frontend/src/routes/settings/+page.svelte@@ -1,12 +1,9 @@<script lang="ts">import ThemeSelect from '$lib/components/ThemeSelect.svelte';- import PostSearchSelect from '$lib/components/PostSearchSelect.svelte';- import { safeSearchStore } from '$lib/stores';-- const safeSearchOptions = [- ['on', true],- ['off', false],- ] as const;+ import PostSearchSelect from './PostSearchSelect.svelte';+ import SafeSearchSelect from './SafeSearchSelect.svelte';+ import MarkPagesWithAdsSelect from './MarkPagesWithAdsSelect.svelte';+ import MarkPagesWithPaywallSelect from './MarkPagesWithPaywallSelect.svelte';const settings = [{@@ -24,6 +21,16 @@description: 'Send search queries using POST instead of GET',type: 'post',},+ {+ title: 'Mark results that contains ads',+ description: 'Add a marker to search results where it is likely that the page contains ads',+ type: 'mark-pages-with-ads',+ },+ {+ title: 'Mark results that contains paywall',+ description: 'Add a marker to search results where it is likely that the page contains a paywall',+ type: 'mark-pages-with-paywall',+ }] as const;</script>@@ -33,34 +40,22 @@<div class="flex flex-col space-y-5">{#each settings as setting}<div class="flex flex-col pr-1 sm:flex-row sm:justify-between">- <div class="flex flex-col">+ <div class="flex flex-col pr-5"><h2 class="text-lg">{setting.title}</h2>- <p>{setting.description}</p>+ <p class="text-sm">{setting.description}</p></div><div class="flex flex-col pr-5 sm:flex-row sm:justify-between sm:space-y-0"><div>{#if setting.type == 'safe-search'}- <div class="flex space-x-4 pr-2">- {#each safeSearchOptions as [name, state]}- <label- class="flex cursor-pointer flex-col items-center"- for="safe-search-{state}"- >- <input- type="radio"- name="safe-search"- id="safe-search-{state}"- value={state}- bind:group={$safeSearchStore}- />- <span class="capitalize">{name}</span>- </label>- {/each}- </div>+ <SafeSearchSelect />{:else if setting.type == 'theme'}<ThemeSelect />{:else if setting.type == 'post'}<PostSearchSelect />+ {:else if setting.type == 'mark-pages-with-ads'}+ <MarkPagesWithAdsSelect />+ {:else if setting.type == 'mark-pages-with-paywall'}+ <MarkPagesWithPaywallSelect />{/if}</div></div>diff --git a/frontend/src/routes/settings/MarkPagesWithAdsSelect.svelte b/frontend/src/routes/settings/MarkPagesWithAdsSelect.sveltenew file mode 100644index 00000000..b31c8fee--- /dev/null+++ b/frontend/src/routes/settings/MarkPagesWithAdsSelect.svelte@@ -0,0 +1,5 @@+<script lang="ts">+ import { markPagesWithAdsStore } from '$lib/stores';+ import RadioSelect from './RadioSelect.svelte';+</script>+<RadioSelect store={markPagesWithAdsStore} prefix="mark-ads" />diff --git a/frontend/src/routes/settings/MarkPagesWithPaywallSelect.svelte b/frontend/src/routes/settings/MarkPagesWithPaywallSelect.sveltenew file mode 100644index 00000000..3c8af1d0--- /dev/null+++ b/frontend/src/routes/settings/MarkPagesWithPaywallSelect.svelte@@ -0,0 +1,5 @@+<script lang="ts">+ import { markPagesWithPaywallStore } from '$lib/stores';+ import RadioSelect from './RadioSelect.svelte';+</script>+<RadioSelect store={markPagesWithPaywallStore} prefix="mark-paywalls" />\ No newline at end of filediff --git a/frontend/src/lib/components/PostSearchSelect.svelte b/frontend/src/routes/settings/PostSearchSelect.sveltesimilarity index 100%rename from frontend/src/lib/components/PostSearchSelect.svelterename to frontend/src/routes/settings/PostSearchSelect.sveltediff --git a/frontend/src/routes/settings/RadioSelect.svelte b/frontend/src/routes/settings/RadioSelect.sveltenew file mode 100644index 00000000..222a62c0--- /dev/null+++ b/frontend/src/routes/settings/RadioSelect.svelte@@ -0,0 +1,28 @@+<script lang="ts">+ import type { Writable } from "svelte/store";++ const options = [+ ['on', true],+ ['off', false],+ ] as const;++ export let store: Writable<boolean>;+ export let prefix: string;+</script>+<div class="flex space-x-4 pr-2">+ {#each options as [name, state]}+ <label+ class="flex cursor-pointer flex-col items-center"+ for="{prefix}-{state}"+ >+ <input+ type="radio"+ name="{prefix}"+ id="{prefix}-{state}"+ value={state}+ bind:group={$store}+ />+ <span class="capitalize">{name}</span>+ </label>+ {/each}+ </div>diff --git a/frontend/src/routes/settings/SafeSearchSelect.svelte b/frontend/src/routes/settings/SafeSearchSelect.sveltenew file mode 100644index 00000000..3b24ec78--- /dev/null+++ b/frontend/src/routes/settings/SafeSearchSelect.svelte@@ -0,0 +1,5 @@+<script lang="ts">+ import { safeSearchStore } from '$lib/stores';+ import RadioSelect from './RadioSelect.svelte';+</script>+<RadioSelect store={safeSearchStore} prefix="safe-search" />\ No newline at end of file