Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(rss): refresh titles and source links when article text is unchanged (#3824)

诉说晚秋d84cdabac490
diff --git a/internal/datasource/connector/rss/connector.go b/internal/datasource/connector/rss/connector.go
index 98e7f2cb9..2f74fcc4d 100644
--- a/internal/datasource/connector/rss/connector.go
+++ b/internal/datasource/connector/rss/connector.go
@@ -124,7 +124,7 @@ func (c *Connector) FetchAll(
return items, err
}
-// FetchIncremental returns only items whose content fingerprint changed since
+// FetchIncremental returns only items whose item fingerprint changed since
// the prior cursor. Deletions are intentionally not emitted (feeds drop old
// items as a matter of course).
func (c *Connector) FetchIncremental(
@@ -171,7 +171,7 @@ func (c *Connector) FetchIncremental(
// walk is the shared implementation for FetchAll / FetchIncremental.
//
-// When incremental is true, items whose feed signal and content fingerprint are
+// When incremental is true, items whose feed signal and item fingerprint are
// both unchanged are omitted from the result (ingest skipped) without fetching
// article pages. The returned cursor always reflects the best-known item state
// so a later sync can detect changes.
@@ -250,7 +250,9 @@ func (c *Connector) walk(
if prevSignals != nil {
prevSig = prevSignals[itemID]
}
- if prevFP != "" && feedSig == prevSig {
+ // Body-only legacy cursors must be upgraded (below) before the
+ // no-fetch fast path can trust them.
+ if strings.HasPrefix(prevFP, itemFingerprintPrefix) && feedSig == prevSig {
newCursor.FeedItems[feedURL][itemID] = prevFP
newCursor.FeedSignals[feedURL][itemID] = feedSig
skipped++
@@ -259,6 +261,26 @@ func (c *Connector) walk(
}
resolved := c.resolveItem(ctx, cli, feed, item, feedURL, itemID, feedContent)
+ if incremental && prevItems != nil && isLegacyFingerprint(prevItems[itemID]) {
+ switch {
+ case legacyContentFingerprint(string(resolved.item.Content)) == prevItems[itemID]:
+ // The stored body is unchanged: upgrade the cursor in place
+ // instead of deleting and rebuilding the document.
+ newCursor.FeedItems[feedURL][itemID] = resolved.fingerprint
+ if !resolved.articleFailed {
+ newCursor.FeedSignals[feedURL][itemID] = feedSig
+ }
+ skipped++
+ continue
+ case resolved.articleFailed:
+ // A transient article failure must not replace the stored
+ // full text with the feed summary: keep the legacy cursor
+ // and retry on the next sync.
+ newCursor.FeedItems[feedURL][itemID] = prevItems[itemID]
+ skipped++
+ continue
+ }
+ }
newCursor.FeedItems[feedURL][itemID] = resolved.fingerprint
// Keep fallback deduplication, but retry failed full-text extraction on
// the next sync even when the feed entry itself has not changed.
@@ -337,7 +359,7 @@ func (c *Connector) resolveItem(
}
return resolvedFeedItem{
- fingerprint: contentFingerprint(content),
+ fingerprint: itemFingerprint(title, item.Link, content),
articleFailed: articleFailed,
item: types.FetchedItem{
ExternalID: itemExternalID(feedURL, itemID),
diff --git a/internal/datasource/connector/rss/item_identity_test.go b/internal/datasource/connector/rss/item_identity_test.go
new file mode 100644
index 000000000..484ef0828
--- /dev/null
+++ b/internal/datasource/connector/rss/item_identity_test.go
@@ -0,0 +1,220 @@
+package rss
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync/atomic"
+ "testing"
+
+ "github.com/Tencent/WeKnora/internal/types"
+)
+
+type identityFeed struct {
+ server *httptest.Server
+ xml atomic.Value
+ fetches atomic.Int32
+ articleStatus atomic.Int32
+ articleBody atomic.Value
+}
+
+func newIdentityFeed(t *testing.T) *identityFeed {
+ t.Helper()
+ f := &identityFeed{}
+ f.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path == "/feed.xml" {
+ w.Header().Set("Content-Type", "application/rss+xml")
+ _, _ = fmt.Fprint(w, f.xml.Load().(string))
+ return
+ }
+ f.fetches.Add(1)
+ if status := int(f.articleStatus.Load()); status != http.StatusOK {
+ w.WriteHeader(status)
+ return
+ }
+ w.Header().Set("Content-Type", "text/html; charset=utf-8")
+ _, _ = fmt.Fprintf(w, "<html><head><title>Stable article</title></head>"+
+ "<body><article>%s</article></body></html>", f.articleBody.Load().(string))
+ }))
+ t.Cleanup(f.server.Close)
+ f.articleStatus.Store(http.StatusOK)
+ f.articleBody.Store(longArticleBody)
+ f.publish("Original title", "/original", "Mon, 02 Jan 2006 15:04:05 GMT")
+ return f
+}
+
+func (f *identityFeed) publish(title, path, date string) {
+ f.xml.Store(fmt.Sprintf(`<rss version="2.0"><channel><title>Feed</title>
+<link>%s</link><description>Test</description>
+<item><guid isPermaLink="false">stable-guid</guid><title>%s</title><link>%s%s</link>
+<pubDate>%s</pubDate><description>Stable summary</description></item></channel></rss>`,
+ f.server.URL, title, f.server.URL, path, date))
+}
+
+func (f *identityFeed) sync(t *testing.T, cursor *types.SyncCursor) ([]types.FetchedItem, *types.SyncCursor) {
+ t.Helper()
+ config := makeConfig(f.server.URL+"/feed.xml", "")
+ items, next, err := NewConnector().FetchIncremental(context.Background(), config, cursor)
+ if err != nil {
+ t.Fatalf("FetchIncremental: %v", err)
+ }
+ return items, next
+}
+
+func TestIncrementalItemIdentityChanges(t *testing.T) {
+ for _, tt := range []struct{ name, title, path string }{
+ {"title only", "Corrected title", "/original"},
+ {"link only", "Original title", "/moved"},
+ } {
+ t.Run(tt.name, func(t *testing.T) {
+ f := newIdentityFeed(t)
+ original, cursor := f.sync(t, nil)
+ if len(original) != 1 {
+ t.Fatalf("expected 1 initial item, got %d", len(original))
+ }
+ f.publish(tt.title, tt.path, "Mon, 02 Jan 2006 15:04:05 GMT")
+ updated, next := f.sync(t, cursor)
+ if len(updated) != 1 {
+ t.Fatalf("expected 1 metadata update with unchanged body, got %d", len(updated))
+ }
+ if updated[0].ExternalID != original[0].ExternalID ||
+ string(updated[0].Content) != string(original[0].Content) {
+ t.Fatal("the stable GUID and body must be preserved")
+ }
+ if updated[0].Title != tt.title || updated[0].FileName != tt.title+".md" ||
+ updated[0].URL != f.server.URL+tt.path {
+ t.Fatalf("incorrect updated identity: %+v", updated[0])
+ }
+ if updated[0].Metadata["link"] != updated[0].URL {
+ t.Fatal("persisted source link must match the updated URL")
+ }
+ assertIdentityUnchanged(t, f, next)
+ })
+ }
+}
+
+func assertIdentityUnchanged(t *testing.T, f *identityFeed, cursor *types.SyncCursor) {
+ t.Helper()
+ before := f.fetches.Load()
+ items, _ := f.sync(t, cursor)
+ if len(items) != 0 || f.fetches.Load() != before {
+ t.Fatal("unchanged sync must not emit items or fetch the article")
+ }
+}
+
+func TestIncrementalItemTimestampOnly(t *testing.T) {
+ f := newIdentityFeed(t)
+ _, cursor := f.sync(t, nil)
+ f.publish("Original title", "/original", "Tue, 03 Jan 2006 15:04:05 GMT")
+ items, next := f.sync(t, cursor)
+ if len(items) != 0 {
+ t.Fatalf("expected no update for timestamp-only changes, got %d", len(items))
+ }
+ if f.fetches.Load() != 2 {
+ t.Fatal("changed feed signal must still resolve the article")
+ }
+ assertIdentityUnchanged(t, f, next)
+}
+
+// legacyIdentityCursor syncs once and rewrites the cursor the way a release
+// before item fingerprints stored it: a body-only "h:" hash.
+func legacyIdentityCursor(t *testing.T, f *identityFeed) (*types.SyncCursor, string) {
+ t.Helper()
+ items, cursor := f.sync(t, nil)
+ if len(items) != 1 {
+ t.Fatalf("expected 1 initial item, got %d", len(items))
+ }
+ sum := sha256.Sum256(items[0].Content)
+ legacy := fmt.Sprintf("h:%x", sum[:8])
+ cursor.ConnectorCursor["feed_items"] = map[string]map[string]string{
+ f.server.URL + "/feed.xml": {"stable-guid": legacy},
+ }
+ return cursor, legacy
+}
+
+func storedItemFingerprint(t *testing.T, f *identityFeed, cursor *types.SyncCursor) string {
+ t.Helper()
+ data, err := json.Marshal(cursor.ConnectorCursor)
+ if err != nil {
+ t.Fatalf("marshal cursor: %v", err)
+ }
+ var decoded rssCursor
+ if err := json.Unmarshal(data, &decoded); err != nil {
+ t.Fatalf("unmarshal cursor: %v", err)
+ }
+ return decoded.FeedItems[f.server.URL+"/feed.xml"]["stable-guid"]
+}
+
+// Upgrading must not delete and rebuild every stored item: an unchanged body
+// only has its cursor upgraded to an item fingerprint.
+func TestIncrementalItemLegacyCursorUnchangedBodyUpgradesInPlace(t *testing.T) {
+ f := newIdentityFeed(t)
+ cursor, _ := legacyIdentityCursor(t, f)
+ updated, next := f.sync(t, cursor)
+ if len(updated) != 0 {
+ t.Fatalf("unchanged legacy item must not be re-ingested, got %d", len(updated))
+ }
+ if fp := storedItemFingerprint(t, f, next); !strings.HasPrefix(fp, itemFingerprintPrefix) {
+ t.Fatalf("legacy cursor must be upgraded to an item fingerprint, got %q", fp)
+ }
+ assertIdentityUnchanged(t, f, next)
+}
+
+// A transient article failure during the upgrade must not replace the stored
+// full text with the feed summary. The legacy cursor is kept so the next sync
+// retries, and the upgrade completes once the article is reachable again.
+func TestIncrementalItemLegacyCursorArticleFailureKeepsCursor(t *testing.T) {
+ f := newIdentityFeed(t)
+ cursor, legacy := legacyIdentityCursor(t, f)
+ f.articleStatus.Store(http.StatusInternalServerError)
+ updated, next := f.sync(t, cursor)
+ if len(updated) != 0 {
+ t.Fatalf("article failure must not emit the feed summary, got %d", len(updated))
+ }
+ if fp := storedItemFingerprint(t, f, next); fp != legacy {
+ t.Fatalf("legacy cursor must be kept for retry, got %q want %q", fp, legacy)
+ }
+
+ f.articleStatus.Store(http.StatusOK)
+ before := f.fetches.Load()
+ updated, next = f.sync(t, next)
+ if len(updated) != 0 || f.fetches.Load() == before {
+ t.Fatalf("recovered article must be re-read and upgrade in place, emitted %d", len(updated))
+ }
+ if fp := storedItemFingerprint(t, f, next); !strings.HasPrefix(fp, itemFingerprintPrefix) {
+ t.Fatalf("legacy cursor must be upgraded after recovery, got %q", fp)
+ }
+}
+
+// A legacy item whose body really changed is still re-ingested.
+func TestIncrementalItemLegacyCursorChangedBodyEmits(t *testing.T) {
+ f := newIdentityFeed(t)
+ cursor, _ := legacyIdentityCursor(t, f)
+ f.articleBody.Store(longArticleBody + "<p>An appended paragraph that changes the stored body.</p>")
+ updated, next := f.sync(t, cursor)
+ if len(updated) != 1 || !strings.Contains(string(updated[0].Content), "appended paragraph") {
+ t.Fatalf("changed legacy item must be re-ingested with the new body, got %d items", len(updated))
+ }
+ if fp := storedItemFingerprint(t, f, next); !strings.HasPrefix(fp, itemFingerprintPrefix) {
+ t.Fatalf("re-ingested item must store an item fingerprint, got %q", fp)
+ }
+ assertIdentityUnchanged(t, f, next)
+}
+
+func TestIncrementalItemContentWithoutLink(t *testing.T) {
+ f := newIdentityFeed(t)
+ f.xml.Store(`<rss version="2.0"><channel><title>Feed</title><description>Test</description>
+<link>https://example.com</link>
+<item><guid>stable-guid</guid><title>Feed-only</title><description>Original body</description></item></channel></rss>`)
+ original, cursor := f.sync(t, nil)
+ f.xml.Store(strings.ReplaceAll(f.xml.Load().(string), "Original body", "Changed body"))
+ updated, next := f.sync(t, cursor)
+ if len(original) != 1 || len(updated) != 1 || string(updated[0].Content) != "Changed body" {
+ t.Fatal("body-only changes must still update feed-only entries")
+ }
+ assertIdentityUnchanged(t, f, next)
+}
diff --git a/internal/datasource/connector/rss/types.go b/internal/datasource/connector/rss/types.go
index 9fff63b5d..1c55aabdd 100644
--- a/internal/datasource/connector/rss/types.go
+++ b/internal/datasource/connector/rss/types.go
@@ -13,8 +13,8 @@
// Markdown. Feed-provided content (content:encoded / description) is used
// as a fallback.
// - Incremental: feed-level signals skip full-text article fetches when a feed
-// entry is unchanged; content fingerprints detect feed-body changes. Article-
-// only edits without feed updates may be missed until the feed entry changes.
+// entry is unchanged; item fingerprints detect title, link and body changes.
+// Article-only edits without feed updates may be missed until the feed entry changes.
// Deletions are NOT synced — feeds routinely drop old items.
//
// All outbound requests go through the SSRF-safe HTTP client so a malicious
@@ -144,7 +144,9 @@ func (c *Config) parseHeaders() map[string]string {
// rssCursor stores incremental sync state.
//
// FeedItems maps feedURL → itemID → fingerprint, where fingerprint is a
-// "h:<sha256-prefix>" hash of the final Markdown body that would be ingested.
+// "i:<sha256-prefix>" hash of the resolved title, link and Markdown body.
+// Legacy "h:" body-only fingerprints are upgraded in place when the body is
+// unchanged; only a changed body is re-ingested.
// FeedSignals maps feedURL → itemID → feed-only signal used to skip full-text
// article fetches when the feed entry itself has not changed.
type rssCursor struct {
@@ -153,12 +155,30 @@ type rssCursor struct {
FeedSignals map[string]map[string]string `json:"feed_signals,omitempty"`
}
-// contentFingerprint hashes ingested Markdown for incremental change detection.
-func contentFingerprint(markdown string) string {
+const itemFingerprintPrefix = "i:"
+
+// isLegacyFingerprint reports a body-only "h:" cursor entry written before
+// item fingerprints existed.
+func isLegacyFingerprint(fp string) bool {
+ return fp != "" && !strings.HasPrefix(fp, itemFingerprintPrefix)
+}
+
+// legacyContentFingerprint is the body-only fingerprint older cursors stored.
+// It is kept only to recognise unchanged bodies when upgrading those cursors.
+func legacyContentFingerprint(markdown string) string {
sum := sha256.Sum256([]byte(markdown))
return "h:" + hex.EncodeToString(sum[:])[:16]
}
+// itemFingerprint includes the document identity without timestamps, so date-only
+// feed changes still resolve articles without needlessly re-ingesting them.
+func itemFingerprint(title, link, markdown string) string {
+ // A string array cannot fail to marshal and preserves field boundaries.
+ data, _ := json.Marshal([3]string{title, link, markdown})
+ sum := sha256.Sum256(data)
+ return itemFingerprintPrefix + hex.EncodeToString(sum[:])[:16]
+}
+
// feedSignalFingerprint hashes feed-visible fields so incremental sync can skip
// article-page fetches when the entry has not changed in the feed.
func feedSignalFingerprint(item *gofeed.Item, feedContent string) string {