web search done right
convert RelFlags from a bitset to a vec of enums for public api
Mikkel Denker3763c51348c1
diff --git a/crates/core/src/api/docs.rs b/crates/core/src/api/docs.rsindex be38527e..952c08c1 100644--- a/crates/core/src/api/docs.rs+++ b/crates/core/src/api/docs.rs@@ -93,7 +93,8 @@ use utoipa_swagger_ui::SwaggerUi;explore::ExploreExportOpticParams,crate::webgraph::Node,- crate::webgraph::Edge,+ crate::webgraph::PrettyRelFlag,+ crate::webgraph::PrettyEdge,crate::search_prettifier::StructuredData,crate::search_prettifier::OneOrManyString,diff --git a/crates/core/src/api/webgraph.rs b/crates/core/src/api/webgraph.rsindex 282c2594..2d9d2431 100644--- a/crates/core/src/api/webgraph.rs+++ b/crates/core/src/api/webgraph.rs@@ -27,7 +27,7 @@ use crate::{FullBacklinksQuery, FullForwardlinksQuery, FullHostBacklinksQuery,FullHostForwardlinksQuery,},- Edge, EdgeLimit, Node,+ EdgeLimit, Node, PrettyEdge,},};@@ -94,8 +94,7 @@ pub mod host {path = "/beta/api/webgraph/host/knows",params(KnowsHostParams),responses(- (status = 200, description = "Whether the host is known", body = KnowsHost),- (status = 400, description = "Invalid host", body = String),+ (status = 200, description = "Whether the host is known", body = KnowsHost)))]pub async fn knows(@@ -126,7 +125,7 @@ pub mod host {path = "/beta/api/webgraph/host/ingoing",params(HostLinksParams),responses(- (status = 200, description = "Incoming links for a particular host", body = Vec<Edge>),+ (status = 200, description = "Incoming links for a particular host", body = Vec<PrettyEdge>),))]pub async fn ingoing_hosts(@@ -150,7 +149,7 @@ pub mod host {path = "/beta/api/webgraph/host/outgoing",params(HostLinksParams),responses(- (status = 200, description = "Outgoing links for a particular host", body = Vec<Edge>),+ (status = 200, description = "Outgoing links for a particular host", body = Vec<PrettyEdge>),))]pub async fn outgoing_hosts(@@ -188,7 +187,7 @@ pub mod page {path = "/beta/api/webgraph/page/ingoing",params(PageLinksParams),responses(- (status = 200, description = "Incoming links for a particular page", body = Vec<Edge>),+ (status = 200, description = "Incoming links for a particular page", body = Vec<PrettyEdge>),))]pub async fn ingoing_pages(@@ -211,7 +210,7 @@ pub mod page {path = "/beta/api/webgraph/page/outgoing",params(PageLinksParams),responses(- (status = 200, description = "Outgoing links for a particular page", body = Vec<Edge>),+ (status = 200, description = "Outgoing links for a particular page", body = Vec<PrettyEdge>),))]pub async fn outgoing_pages(@@ -235,20 +234,18 @@ async fn ingoing_links(state: Arc<State>,node: Node,level: WebgraphGranularity,-) -> anyhow::Result<Vec<Edge>> {+) -> anyhow::Result<Vec<PrettyEdge>> {match level {- WebgraphGranularity::Host => {- state- .webgraph- .search(FullHostBacklinksQuery::new(node).with_limit(EdgeLimit::Limit(1024)))- .await- }- WebgraphGranularity::Page => {- state- .webgraph- .search(FullBacklinksQuery::new(node).with_limit(EdgeLimit::Limit(1024)))- .await- }+ WebgraphGranularity::Host => state+ .webgraph+ .search(FullHostBacklinksQuery::new(node).with_limit(EdgeLimit::Limit(1024)))+ .await+ .map(|edges| edges.into_iter().map(PrettyEdge::from).collect()),+ WebgraphGranularity::Page => state+ .webgraph+ .search(FullBacklinksQuery::new(node).with_limit(EdgeLimit::Limit(1024)))+ .await+ .map(|edges| edges.into_iter().map(PrettyEdge::from).collect()),}}@@ -256,20 +253,18 @@ async fn outgoing_links(state: Arc<State>,node: Node,level: WebgraphGranularity,-) -> anyhow::Result<Vec<Edge>> {+) -> anyhow::Result<Vec<PrettyEdge>> {match level {- WebgraphGranularity::Host => {- state- .webgraph- .search(FullHostForwardlinksQuery::new(node).with_limit(EdgeLimit::Limit(1024)))- .await- }- WebgraphGranularity::Page => {- state- .webgraph- .search(FullForwardlinksQuery::new(node).with_limit(EdgeLimit::Limit(1024)))- .await- }+ WebgraphGranularity::Host => state+ .webgraph+ .search(FullHostForwardlinksQuery::new(node).with_limit(EdgeLimit::Limit(1024)))+ .await+ .map(|edges| edges.into_iter().map(PrettyEdge::from).collect()),+ WebgraphGranularity::Page => state+ .webgraph+ .search(FullForwardlinksQuery::new(node).with_limit(EdgeLimit::Limit(1024)))+ .await+ .map(|edges| edges.into_iter().map(PrettyEdge::from).collect()),}}diff --git a/crates/core/src/webgraph/document.rs b/crates/core/src/webgraph/document.rsindex f1f8c9d1..873b4cae 100644--- a/crates/core/src/webgraph/document.rs+++ b/crates/core/src/webgraph/document.rs@@ -45,12 +45,127 @@ pub struct SmallEdgeWithLabel {#[derive(Debug,Clone,+ bincode::Encode,+ bincode::Decode,serde::Serialize,serde::Deserialize,+ utoipa::ToSchema,+)]+#[serde(rename_all = "snake_case")]+pub enum PrettyRelFlag {+ Alternate,+ Author,+ Canonical,+ Help,+ Icon,+ License,+ Me,+ Next,+ NoFollow,+ Prev,+ PrivacyPolicy,+ Search,+ Stylesheet,+ Tag,+ TermsOfService,+ Sponsored,+ IsInFooter,+ IsInNavigation,+ LinkTag,+ ScriptTag,+ MetaTag,+ SameIcannDomain,+}++impl From<RelFlags> for Option<PrettyRelFlag> {+ fn from(flags: RelFlags) -> Self {+ if flags.contains(RelFlags::ALTERNATE) {+ Some(PrettyRelFlag::Alternate)+ } else if flags.contains(RelFlags::AUTHOR) {+ Some(PrettyRelFlag::Author)+ } else if flags.contains(RelFlags::CANONICAL) {+ Some(PrettyRelFlag::Canonical)+ } else if flags.contains(RelFlags::HELP) {+ Some(PrettyRelFlag::Help)+ } else if flags.contains(RelFlags::ICON) {+ Some(PrettyRelFlag::Icon)+ } else if flags.contains(RelFlags::LICENSE) {+ Some(PrettyRelFlag::License)+ } else if flags.contains(RelFlags::ME) {+ Some(PrettyRelFlag::Me)+ } else if flags.contains(RelFlags::NEXT) {+ Some(PrettyRelFlag::Next)+ } else if flags.contains(RelFlags::NOFOLLOW) {+ Some(PrettyRelFlag::NoFollow)+ } else if flags.contains(RelFlags::PREV) {+ Some(PrettyRelFlag::Prev)+ } else if flags.contains(RelFlags::PRIVACY_POLICY) {+ Some(PrettyRelFlag::PrivacyPolicy)+ } else if flags.contains(RelFlags::SEARCH) {+ Some(PrettyRelFlag::Search)+ } else if flags.contains(RelFlags::STYLESHEET) {+ Some(PrettyRelFlag::Stylesheet)+ } else if flags.contains(RelFlags::TAG) {+ Some(PrettyRelFlag::Tag)+ } else if flags.contains(RelFlags::TERMS_OF_SERVICE) {+ Some(PrettyRelFlag::TermsOfService)+ } else if flags.contains(RelFlags::SPONSORED) {+ Some(PrettyRelFlag::Sponsored)+ } else if flags.contains(RelFlags::IS_IN_FOOTER) {+ Some(PrettyRelFlag::IsInFooter)+ } else if flags.contains(RelFlags::IS_IN_NAVIGATION) {+ Some(PrettyRelFlag::IsInNavigation)+ } else if flags.contains(RelFlags::LINK_TAG) {+ Some(PrettyRelFlag::LinkTag)+ } else if flags.contains(RelFlags::SCRIPT_TAG) {+ Some(PrettyRelFlag::ScriptTag)+ } else if flags.contains(RelFlags::META_TAG) {+ Some(PrettyRelFlag::MetaTag)+ } else if flags.contains(RelFlags::SAME_ICANN_DOMAIN) {+ Some(PrettyRelFlag::SameIcannDomain)+ } else {+ None+ }+ }+}++impl From<RelFlags> for Vec<PrettyRelFlag> {+ fn from(flags: RelFlags) -> Self {+ flags+ .iter_names()+ .flat_map(|(_, flag)| <Option<PrettyRelFlag>>::from(flag))+ .collect()+ }+}++#[derive(+ Debug,+ Clone,bincode::Encode,bincode::Decode,+ serde::Serialize,+ serde::Deserialize,utoipa::ToSchema,)]+pub struct PrettyEdge {+ pub from: String,+ pub to: String,+ pub rel_flags: Vec<PrettyRelFlag>,+ pub label: String,+}++impl From<Edge> for PrettyEdge {+ fn from(edge: Edge) -> Self {+ Self {+ from: edge.from.as_str().to_string(),+ to: edge.to.as_str().to_string(),+ rel_flags: edge.rel_flags.into(),+ label: edge.label,+ }+ }+}++#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub struct Edge {pub from: Node,pub to: Node,diff --git a/frontend/src/lib/api/index.ts b/frontend/src/lib/api/index.tsindex b5a9103e..9ca94180 100644--- a/frontend/src/lib/api/index.ts+++ b/frontend/src/lib/api/index.ts@@ -128,7 +128,7 @@ export const api = {},options?: ApiOptions,) =>- requestJson<FullEdge[]>(+ requestJson<PrettyEdge[]>('POST',`/beta/api/webgraph/host/ingoing?${new URLSearchParams(query)}`,options,@@ -150,7 +150,7 @@ export const api = {},options?: ApiOptions,) =>- requestJson<FullEdge[]>(+ requestJson<PrettyEdge[]>('POST',`/beta/api/webgraph/host/outgoing?${new URLSearchParams(query)}`,options,@@ -163,7 +163,7 @@ export const api = {},options?: ApiOptions,) =>- requestJson<FullEdge[]>(+ requestJson<PrettyEdge[]>('POST',`/beta/api/webgraph/page/ingoing?${new URLSearchParams(query)}`,options,@@ -174,7 +174,7 @@ export const api = {},options?: ApiOptions,) =>- requestJson<FullEdge[]>(+ requestJson<PrettyEdge[]>('POST',`/beta/api/webgraph/page/outgoing?${new URLSearchParams(query)}`,options,@@ -296,11 +296,6 @@ export type ExploreExportOpticParams = {chosenHosts: string[];similarHosts: string[];};-export type FullEdge = {- from: Node;- label: string;- to: Node;-};export type HighlightedFragment = {kind: HighlightedKind;text: string;@@ -345,6 +340,59 @@ export type PartOfSpeechMeaning = {meanings: WordMeaning[];pos: PartOfSpeech;};+export type PrettyEdge = {+ from: string;+ label: string;+ rel_flags: PrettyRelFlag[];+ to: string;+};+export type PrettyRelFlag =+ | 'alternate'+ | 'author'+ | 'canonical'+ | 'help'+ | 'icon'+ | 'license'+ | 'me'+ | 'next'+ | 'no_follow'+ | 'prev'+ | 'privacy_policy'+ | 'search'+ | 'stylesheet'+ | 'tag'+ | 'terms_of_service'+ | 'sponsored'+ | 'is_in_footer'+ | 'is_in_navigation'+ | 'link_tag'+ | 'script_tag'+ | 'meta_tag'+ | 'same_icann_domain';+export const PRETTY_REL_FLAGS = [+ 'alternate',+ 'author',+ 'canonical',+ 'help',+ 'icon',+ 'license',+ 'me',+ 'next',+ 'no_follow',+ 'prev',+ 'privacy_policy',+ 'search',+ 'stylesheet',+ 'tag',+ 'terms_of_service',+ 'sponsored',+ 'is_in_footer',+ 'is_in_navigation',+ 'link_tag',+ 'script_tag',+ 'meta_tag',+ 'same_icann_domain',+] satisfies PrettyRelFlag[];export type Property = string | StructuredData;export type Region = 'All' | 'Denmark' | 'France' | 'Germany' | 'Spain' | 'US';export const REGIONS = ['All', 'Denmark', 'France', 'Germany', 'Spain', 'US'] satisfies Region[];