World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
Web ingestion survives contact with real websites (#35)
WaylandYang159c89a37c78
diff --git a/crates/utopia-ingest/src/parsers.rs b/crates/utopia-ingest/src/parsers.rsindex 0084e61c..09e4726d 100644--- a/crates/utopia-ingest/src/parsers.rs+++ b/crates/utopia-ingest/src/parsers.rs@@ -86,7 +86,15 @@ pub fn spreadsheet(bytes: &[u8]) -> anyhow::Result<String> {Ok(out)}-/// HTML: DOM 遍历取文本,跳过 script/style/noscript/head。+/// HTML: 取正文文本。+///+/// 真实网页的 chrome 往往比正文还多——整站导航、语言列表、页脚法务、编辑工具。+/// 全量遍历会把它们一并喂给抽取器:既浪费每个分块一次 LLM 调用,又会把+/// "Main page""Privacy policy"之类抽成实体污染图谱(实测一篇维基条目 647KB+/// 产出 60 个分块,首块整块是侧栏菜单、末块整块是版权声明)。+///+/// 因此先认正文容器(main / role=main / article),只有都找不到才退回整篇;+/// 容器内仍可能嵌着导航与表单,交给 walk_html 的 SKIP 名单。pub fn html(bytes: &[u8]) -> String {let raw = plain_text(bytes);let doc = scraper::Html::parse_document(&raw);@@ -100,12 +108,24 @@ pub fn html(bytes: &[u8]) -> String {title.text().collect::<String>().trim()));}- walk_html(doc.root_element(), &mut out);+ let root = ["main", "[role=main]", "article"]+ .iter()+ .find_map(|sel| {+ scraper::Selector::parse(sel)+ .ok()+ .and_then(|s| doc.select(&s).next())+ })+ .unwrap_or_else(|| doc.root_element());+ walk_html(root, &mut out);out}fn walk_html(el: scraper::ElementRef, out: &mut String) {- const SKIP: &[&str] = &["script", "style", "noscript", "head", "svg", "template"];+ // 导航/页眉页脚/边栏/表单控件即便落在正文容器内也是 chrome,一律跳过+ const SKIP: &[&str] = &[+ "script", "style", "noscript", "head", "svg", "template", "nav", "header", "footer",+ "aside", "form", "button", "select", "iframe", "dialog",+ ];const BLOCK: &[&str] = &["p", "div", "li", "tr", "h1", "h2", "h3", "h4", "h5", "h6", "br", "section", "article",];diff --git a/crates/utopia-server/src/ingest_sources.rs b/crates/utopia-server/src/ingest_sources.rsindex f11bcc59..f1ea8bee 100644--- a/crates/utopia-server/src/ingest_sources.rs+++ b/crates/utopia-server/src/ingest_sources.rs@@ -11,6 +11,15 @@ use uuid::Uuid;/// 单次同步的新文档上限(防超长 feed/URL 列表拖垮任务)const MAX_NEW_PER_SYNC: usize = 200;+/// 抓取用的 User-Agent。reqwest 默认一个都不发,而维基百科明确拒绝匿名请求+/// (403),Cloudflare 前置的站点也普遍如此——URL 与 RSS 两类来源因此对一大批+/// 真实网站直接失效。自报家门也是爬虫礼节:站方能认出我们、能联系到我们。+const UA: &str = concat!(+ "Utopia/",+ env!("CARGO_PKG_VERSION"),+ " (+https://utopia.bi; self-hosted knowledge platform)"+);+/// 单次同步的产出统计(Moved/Unchanged 不计)。#[derive(Debug, Default, Clone, Copy)]pub struct SyncStats {@@ -281,6 +290,7 @@ async fn sync_urls(state: &AppState, source: &Source) -> anyhow::Result<SyncStatlet http = reqwest::Client::builder().timeout(std::time::Duration::from_secs(30))+ .user_agent(UA).build()?;let mut stats = SyncStats::default();let mut last_err: Option<String> = None;@@ -380,6 +390,7 @@ async fn sync_rss(state: &AppState, source: &Source) -> anyhow::Result<SyncStatslet http = reqwest::Client::builder().timeout(std::time::Duration::from_secs(30))+ .user_agent(UA).build()?;let resp = http.get(feed_url).send().await?;if !resp.status().is_success() {@@ -468,7 +479,9 @@ async fn sync_custom(state: &AppState, source: &Source) -> anyhow::Result<SyncSth.eq_ignore_ascii_case("localhost") || h == "127.0.0.1" || h == "::1" || h == "[::1]"}).unwrap_or(false);- let mut builder = reqwest::Client::builder().timeout(std::time::Duration::from_secs(30));+ let mut builder = reqwest::Client::builder()+ .timeout(std::time::Duration::from_secs(30))+ .user_agent(UA);if loopback {builder = builder.no_proxy();}