Skip to content

web search done right

convert RelFlags from a bitset to a vec of enums for public api

Mikkel Denker3763c51348c1
diff --git a/crates/core/src/api/docs.rs b/crates/core/src/api/docs.rs
index be38527e..952c08c1 100644
--- a/crates/core/src/api/docs.rs
+++ b/crates/core/src/api/docs.rs
@@ -93,7 +93,8 @@ use utoipa_swagger_ui::SwaggerUi;
explore::ExploreExportOpticParams,
crate::webgraph::Node,
- crate::webgraph::Edge,
+ crate::webgraph::PrettyRelFlag,
+ crate::webgraph::PrettyEdge,
crate::search_prettifier::StructuredData,
crate::search_prettifier::OneOrManyString,
diff --git a/crates/core/src/api/webgraph.rs b/crates/core/src/api/webgraph.rs
index 282c2594..2d9d2431 100644
--- a/crates/core/src/api/webgraph.rs
+++ b/crates/core/src/api/webgraph.rs
@@ -27,7 +27,7 @@ use crate::{
FullBacklinksQuery, FullForwardlinksQuery, FullHostBacklinksQuery,
FullHostForwardlinksQuery,
},
- Edge, EdgeLimit, Node,
+ EdgeLimit, Node, PrettyEdge,
},
};
@@ -94,8 +94,7 @@ pub mod host {
path = "/beta/api/webgraph/host/knows",
params(KnowsHostParams),
responses(
- (status = 200, description = "Whether the host is known", body = KnowsHost),
- (status = 400, description = "Invalid host", body = String),
+ (status = 200, description = "Whether the host is known", body = KnowsHost)
)
)]
pub async fn knows(
@@ -126,7 +125,7 @@ pub mod host {
path = "/beta/api/webgraph/host/ingoing",
params(HostLinksParams),
responses(
- (status = 200, description = "Incoming links for a particular host", body = Vec<Edge>),
+ (status = 200, description = "Incoming links for a particular host", body = Vec<PrettyEdge>),
)
)]
pub async fn ingoing_hosts(
@@ -150,7 +149,7 @@ pub mod host {
path = "/beta/api/webgraph/host/outgoing",
params(HostLinksParams),
responses(
- (status = 200, description = "Outgoing links for a particular host", body = Vec<Edge>),
+ (status = 200, description = "Outgoing links for a particular host", body = Vec<PrettyEdge>),
)
)]
pub async fn outgoing_hosts(
@@ -188,7 +187,7 @@ pub mod page {
path = "/beta/api/webgraph/page/ingoing",
params(PageLinksParams),
responses(
- (status = 200, description = "Incoming links for a particular page", body = Vec<Edge>),
+ (status = 200, description = "Incoming links for a particular page", body = Vec<PrettyEdge>),
)
)]
pub async fn ingoing_pages(
@@ -211,7 +210,7 @@ pub mod page {
path = "/beta/api/webgraph/page/outgoing",
params(PageLinksParams),
responses(
- (status = 200, description = "Outgoing links for a particular page", body = Vec<Edge>),
+ (status = 200, description = "Outgoing links for a particular page", body = Vec<PrettyEdge>),
)
)]
pub async fn outgoing_pages(
@@ -235,20 +234,18 @@ async fn ingoing_links(
state: Arc<State>,
node: Node,
level: WebgraphGranularity,
-) -> anyhow::Result<Vec<Edge>> {
+) -> anyhow::Result<Vec<PrettyEdge>> {
match level {
- WebgraphGranularity::Host => {
- state
- .webgraph
- .search(FullHostBacklinksQuery::new(node).with_limit(EdgeLimit::Limit(1024)))
- .await
- }
- WebgraphGranularity::Page => {
- state
- .webgraph
- .search(FullBacklinksQuery::new(node).with_limit(EdgeLimit::Limit(1024)))
- .await
- }
+ WebgraphGranularity::Host => state
+ .webgraph
+ .search(FullHostBacklinksQuery::new(node).with_limit(EdgeLimit::Limit(1024)))
+ .await
+ .map(|edges| edges.into_iter().map(PrettyEdge::from).collect()),
+ WebgraphGranularity::Page => state
+ .webgraph
+ .search(FullBacklinksQuery::new(node).with_limit(EdgeLimit::Limit(1024)))
+ .await
+ .map(|edges| edges.into_iter().map(PrettyEdge::from).collect()),
}
}
@@ -256,20 +253,18 @@ async fn outgoing_links(
state: Arc<State>,
node: Node,
level: WebgraphGranularity,
-) -> anyhow::Result<Vec<Edge>> {
+) -> anyhow::Result<Vec<PrettyEdge>> {
match level {
- WebgraphGranularity::Host => {
- state
- .webgraph
- .search(FullHostForwardlinksQuery::new(node).with_limit(EdgeLimit::Limit(1024)))
- .await
- }
- WebgraphGranularity::Page => {
- state
- .webgraph
- .search(FullForwardlinksQuery::new(node).with_limit(EdgeLimit::Limit(1024)))
- .await
- }
+ WebgraphGranularity::Host => state
+ .webgraph
+ .search(FullHostForwardlinksQuery::new(node).with_limit(EdgeLimit::Limit(1024)))
+ .await
+ .map(|edges| edges.into_iter().map(PrettyEdge::from).collect()),
+ WebgraphGranularity::Page => state
+ .webgraph
+ .search(FullForwardlinksQuery::new(node).with_limit(EdgeLimit::Limit(1024)))
+ .await
+ .map(|edges| edges.into_iter().map(PrettyEdge::from).collect()),
}
}
diff --git a/crates/core/src/webgraph/document.rs b/crates/core/src/webgraph/document.rs
index f1f8c9d1..873b4cae 100644
--- a/crates/core/src/webgraph/document.rs
+++ b/crates/core/src/webgraph/document.rs
@@ -45,12 +45,127 @@ pub struct SmallEdgeWithLabel {
#[derive(
Debug,
Clone,
+ bincode::Encode,
+ bincode::Decode,
serde::Serialize,
serde::Deserialize,
+ utoipa::ToSchema,
+)]
+#[serde(rename_all = "snake_case")]
+pub enum PrettyRelFlag {
+ Alternate,
+ Author,
+ Canonical,
+ Help,
+ Icon,
+ License,
+ Me,
+ Next,
+ NoFollow,
+ Prev,
+ PrivacyPolicy,
+ Search,
+ Stylesheet,
+ Tag,
+ TermsOfService,
+ Sponsored,
+ IsInFooter,
+ IsInNavigation,
+ LinkTag,
+ ScriptTag,
+ MetaTag,
+ SameIcannDomain,
+}
+
+impl From<RelFlags> for Option<PrettyRelFlag> {
+ fn from(flags: RelFlags) -> Self {
+ if flags.contains(RelFlags::ALTERNATE) {
+ Some(PrettyRelFlag::Alternate)
+ } else if flags.contains(RelFlags::AUTHOR) {
+ Some(PrettyRelFlag::Author)
+ } else if flags.contains(RelFlags::CANONICAL) {
+ Some(PrettyRelFlag::Canonical)
+ } else if flags.contains(RelFlags::HELP) {
+ Some(PrettyRelFlag::Help)
+ } else if flags.contains(RelFlags::ICON) {
+ Some(PrettyRelFlag::Icon)
+ } else if flags.contains(RelFlags::LICENSE) {
+ Some(PrettyRelFlag::License)
+ } else if flags.contains(RelFlags::ME) {
+ Some(PrettyRelFlag::Me)
+ } else if flags.contains(RelFlags::NEXT) {
+ Some(PrettyRelFlag::Next)
+ } else if flags.contains(RelFlags::NOFOLLOW) {
+ Some(PrettyRelFlag::NoFollow)
+ } else if flags.contains(RelFlags::PREV) {
+ Some(PrettyRelFlag::Prev)
+ } else if flags.contains(RelFlags::PRIVACY_POLICY) {
+ Some(PrettyRelFlag::PrivacyPolicy)
+ } else if flags.contains(RelFlags::SEARCH) {
+ Some(PrettyRelFlag::Search)
+ } else if flags.contains(RelFlags::STYLESHEET) {
+ Some(PrettyRelFlag::Stylesheet)
+ } else if flags.contains(RelFlags::TAG) {
+ Some(PrettyRelFlag::Tag)
+ } else if flags.contains(RelFlags::TERMS_OF_SERVICE) {
+ Some(PrettyRelFlag::TermsOfService)
+ } else if flags.contains(RelFlags::SPONSORED) {
+ Some(PrettyRelFlag::Sponsored)
+ } else if flags.contains(RelFlags::IS_IN_FOOTER) {
+ Some(PrettyRelFlag::IsInFooter)
+ } else if flags.contains(RelFlags::IS_IN_NAVIGATION) {
+ Some(PrettyRelFlag::IsInNavigation)
+ } else if flags.contains(RelFlags::LINK_TAG) {
+ Some(PrettyRelFlag::LinkTag)
+ } else if flags.contains(RelFlags::SCRIPT_TAG) {
+ Some(PrettyRelFlag::ScriptTag)
+ } else if flags.contains(RelFlags::META_TAG) {
+ Some(PrettyRelFlag::MetaTag)
+ } else if flags.contains(RelFlags::SAME_ICANN_DOMAIN) {
+ Some(PrettyRelFlag::SameIcannDomain)
+ } else {
+ None
+ }
+ }
+}
+
+impl From<RelFlags> for Vec<PrettyRelFlag> {
+ fn from(flags: RelFlags) -> Self {
+ flags
+ .iter_names()
+ .flat_map(|(_, flag)| <Option<PrettyRelFlag>>::from(flag))
+ .collect()
+ }
+}
+
+#[derive(
+ Debug,
+ Clone,
bincode::Encode,
bincode::Decode,
+ serde::Serialize,
+ serde::Deserialize,
utoipa::ToSchema,
)]
+pub struct PrettyEdge {
+ pub from: String,
+ pub to: String,
+ pub rel_flags: Vec<PrettyRelFlag>,
+ pub label: String,
+}
+
+impl From<Edge> for PrettyEdge {
+ fn from(edge: Edge) -> Self {
+ Self {
+ from: edge.from.as_str().to_string(),
+ to: edge.to.as_str().to_string(),
+ rel_flags: edge.rel_flags.into(),
+ label: edge.label,
+ }
+ }
+}
+
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]
pub struct Edge {
pub from: Node,
pub to: Node,
diff --git a/frontend/src/lib/api/index.ts b/frontend/src/lib/api/index.ts
index b5a9103e..9ca94180 100644
--- a/frontend/src/lib/api/index.ts
+++ b/frontend/src/lib/api/index.ts
@@ -128,7 +128,7 @@ export const api = {
},
options?: ApiOptions,
) =>
- requestJson<FullEdge[]>(
+ requestJson<PrettyEdge[]>(
'POST',
`/beta/api/webgraph/host/ingoing?${new URLSearchParams(query)}`,
options,
@@ -150,7 +150,7 @@ export const api = {
},
options?: ApiOptions,
) =>
- requestJson<FullEdge[]>(
+ requestJson<PrettyEdge[]>(
'POST',
`/beta/api/webgraph/host/outgoing?${new URLSearchParams(query)}`,
options,
@@ -163,7 +163,7 @@ export const api = {
},
options?: ApiOptions,
) =>
- requestJson<FullEdge[]>(
+ requestJson<PrettyEdge[]>(
'POST',
`/beta/api/webgraph/page/ingoing?${new URLSearchParams(query)}`,
options,
@@ -174,7 +174,7 @@ export const api = {
},
options?: ApiOptions,
) =>
- requestJson<FullEdge[]>(
+ requestJson<PrettyEdge[]>(
'POST',
`/beta/api/webgraph/page/outgoing?${new URLSearchParams(query)}`,
options,
@@ -296,11 +296,6 @@ export type ExploreExportOpticParams = {
chosenHosts: string[];
similarHosts: string[];
};
-export type FullEdge = {
- from: Node;
- label: string;
- to: Node;
-};
export type HighlightedFragment = {
kind: HighlightedKind;
text: string;
@@ -345,6 +340,59 @@ export type PartOfSpeechMeaning = {
meanings: WordMeaning[];
pos: PartOfSpeech;
};
+export type PrettyEdge = {
+ from: string;
+ label: string;
+ rel_flags: PrettyRelFlag[];
+ to: string;
+};
+export type PrettyRelFlag =
+ | 'alternate'
+ | 'author'
+ | 'canonical'
+ | 'help'
+ | 'icon'
+ | 'license'
+ | 'me'
+ | 'next'
+ | 'no_follow'
+ | 'prev'
+ | 'privacy_policy'
+ | 'search'
+ | 'stylesheet'
+ | 'tag'
+ | 'terms_of_service'
+ | 'sponsored'
+ | 'is_in_footer'
+ | 'is_in_navigation'
+ | 'link_tag'
+ | 'script_tag'
+ | 'meta_tag'
+ | 'same_icann_domain';
+export const PRETTY_REL_FLAGS = [
+ 'alternate',
+ 'author',
+ 'canonical',
+ 'help',
+ 'icon',
+ 'license',
+ 'me',
+ 'next',
+ 'no_follow',
+ 'prev',
+ 'privacy_policy',
+ 'search',
+ 'stylesheet',
+ 'tag',
+ 'terms_of_service',
+ 'sponsored',
+ 'is_in_footer',
+ 'is_in_navigation',
+ 'link_tag',
+ 'script_tag',
+ 'meta_tag',
+ 'same_icann_domain',
+] satisfies PrettyRelFlag[];
export type Property = string | StructuredData;
export type Region = 'All' | 'Denmark' | 'France' | 'Germany' | 'Spain' | 'US';
export const REGIONS = ['All', 'Denmark', 'France', 'Germany', 'Spain', 'US'] satisfies Region[];