Skip to content

World's first open-source enterprise world model.

Web ingestion survives contact with real websites (#35)

WaylandYang159c89a37c78
diff --git a/crates/utopia-ingest/src/parsers.rs b/crates/utopia-ingest/src/parsers.rs
index 0084e61c..09e4726d 100644
--- a/crates/utopia-ingest/src/parsers.rs
+++ b/crates/utopia-ingest/src/parsers.rs
@@ -86,7 +86,15 @@ pub fn spreadsheet(bytes: &[u8]) -> anyhow::Result<String> {
Ok(out)
}
-/// HTML: DOM 遍历取文本,跳过 script/style/noscript/head。
+/// HTML: 取正文文本。
+///
+/// 真实网页的 chrome 往往比正文还多——整站导航、语言列表、页脚法务、编辑工具。
+/// 全量遍历会把它们一并喂给抽取器:既浪费每个分块一次 LLM 调用,又会把
+/// "Main page""Privacy policy"之类抽成实体污染图谱(实测一篇维基条目 647KB
+/// 产出 60 个分块,首块整块是侧栏菜单、末块整块是版权声明)。
+///
+/// 因此先认正文容器(main / role=main / article),只有都找不到才退回整篇;
+/// 容器内仍可能嵌着导航与表单,交给 walk_html 的 SKIP 名单。
pub fn html(bytes: &[u8]) -> String {
let raw = plain_text(bytes);
let doc = scraper::Html::parse_document(&raw);
@@ -100,12 +108,24 @@ pub fn html(bytes: &[u8]) -> String {
title.text().collect::<String>().trim()
));
}
- walk_html(doc.root_element(), &mut out);
+ let root = ["main", "[role=main]", "article"]
+ .iter()
+ .find_map(|sel| {
+ scraper::Selector::parse(sel)
+ .ok()
+ .and_then(|s| doc.select(&s).next())
+ })
+ .unwrap_or_else(|| doc.root_element());
+ walk_html(root, &mut out);
out
}
fn walk_html(el: scraper::ElementRef, out: &mut String) {
- const SKIP: &[&str] = &["script", "style", "noscript", "head", "svg", "template"];
+ // 导航/页眉页脚/边栏/表单控件即便落在正文容器内也是 chrome,一律跳过
+ const SKIP: &[&str] = &[
+ "script", "style", "noscript", "head", "svg", "template", "nav", "header", "footer",
+ "aside", "form", "button", "select", "iframe", "dialog",
+ ];
const BLOCK: &[&str] = &[
"p", "div", "li", "tr", "h1", "h2", "h3", "h4", "h5", "h6", "br", "section", "article",
];
diff --git a/crates/utopia-server/src/ingest_sources.rs b/crates/utopia-server/src/ingest_sources.rs
index f11bcc59..f1ea8bee 100644
--- a/crates/utopia-server/src/ingest_sources.rs
+++ b/crates/utopia-server/src/ingest_sources.rs
@@ -11,6 +11,15 @@ use uuid::Uuid;
/// 单次同步的新文档上限(防超长 feed/URL 列表拖垮任务)
const MAX_NEW_PER_SYNC: usize = 200;
+/// 抓取用的 User-Agent。reqwest 默认一个都不发,而维基百科明确拒绝匿名请求
+/// (403),Cloudflare 前置的站点也普遍如此——URL 与 RSS 两类来源因此对一大批
+/// 真实网站直接失效。自报家门也是爬虫礼节:站方能认出我们、能联系到我们。
+const UA: &str = concat!(
+ "Utopia/",
+ env!("CARGO_PKG_VERSION"),
+ " (+https://utopia.bi; self-hosted knowledge platform)"
+);
+
/// 单次同步的产出统计(Moved/Unchanged 不计)。
#[derive(Debug, Default, Clone, Copy)]
pub struct SyncStats {
@@ -281,6 +290,7 @@ async fn sync_urls(state: &AppState, source: &Source) -> anyhow::Result<SyncStat
let http = reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(30))
+ .user_agent(UA)
.build()?;
let mut stats = SyncStats::default();
let mut last_err: Option<String> = None;
@@ -380,6 +390,7 @@ async fn sync_rss(state: &AppState, source: &Source) -> anyhow::Result<SyncStats
let http = reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(30))
+ .user_agent(UA)
.build()?;
let resp = http.get(feed_url).send().await?;
if !resp.status().is_success() {
@@ -468,7 +479,9 @@ async fn sync_custom(state: &AppState, source: &Source) -> anyhow::Result<SyncSt
h.eq_ignore_ascii_case("localhost") || h == "127.0.0.1" || h == "::1" || h == "[::1]"
})
.unwrap_or(false);
- let mut builder = reqwest::Client::builder().timeout(std::time::Duration::from_secs(30));
+ let mut builder = reqwest::Client::builder()
+ .timeout(std::time::Duration::from_secs(30))
+ .user_agent(UA);
if loopback {
builder = builder.no_proxy();
}