Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(rss): refresh titles and source links when article text is unchanged (#3824)
诉说晚秋d84cdabac490
diff --git a/internal/datasource/connector/rss/connector.go b/internal/datasource/connector/rss/connector.goindex 98e7f2cb9..2f74fcc4d 100644--- a/internal/datasource/connector/rss/connector.go+++ b/internal/datasource/connector/rss/connector.go@@ -124,7 +124,7 @@ func (c *Connector) FetchAll(return items, err}-// FetchIncremental returns only items whose content fingerprint changed since+// FetchIncremental returns only items whose item fingerprint changed since// the prior cursor. Deletions are intentionally not emitted (feeds drop old// items as a matter of course).func (c *Connector) FetchIncremental(@@ -171,7 +171,7 @@ func (c *Connector) FetchIncremental(// walk is the shared implementation for FetchAll / FetchIncremental.//-// When incremental is true, items whose feed signal and content fingerprint are+// When incremental is true, items whose feed signal and item fingerprint are// both unchanged are omitted from the result (ingest skipped) without fetching// article pages. The returned cursor always reflects the best-known item state// so a later sync can detect changes.@@ -250,7 +250,9 @@ func (c *Connector) walk(if prevSignals != nil {prevSig = prevSignals[itemID]}- if prevFP != "" && feedSig == prevSig {+ // Body-only legacy cursors must be upgraded (below) before the+ // no-fetch fast path can trust them.+ if strings.HasPrefix(prevFP, itemFingerprintPrefix) && feedSig == prevSig {newCursor.FeedItems[feedURL][itemID] = prevFPnewCursor.FeedSignals[feedURL][itemID] = feedSigskipped++@@ -259,6 +261,26 @@ func (c *Connector) walk(}resolved := c.resolveItem(ctx, cli, feed, item, feedURL, itemID, feedContent)+ if incremental && prevItems != nil && isLegacyFingerprint(prevItems[itemID]) {+ switch {+ case legacyContentFingerprint(string(resolved.item.Content)) == prevItems[itemID]:+ // The stored body is unchanged: upgrade the cursor in place+ // instead of deleting and rebuilding the document.+ newCursor.FeedItems[feedURL][itemID] = resolved.fingerprint+ if !resolved.articleFailed {+ newCursor.FeedSignals[feedURL][itemID] = feedSig+ }+ skipped+++ continue+ case resolved.articleFailed:+ // A transient article failure must not replace the stored+ // full text with the feed summary: keep the legacy cursor+ // and retry on the next sync.+ newCursor.FeedItems[feedURL][itemID] = prevItems[itemID]+ skipped+++ continue+ }+ }newCursor.FeedItems[feedURL][itemID] = resolved.fingerprint// Keep fallback deduplication, but retry failed full-text extraction on// the next sync even when the feed entry itself has not changed.@@ -337,7 +359,7 @@ func (c *Connector) resolveItem(}return resolvedFeedItem{- fingerprint: contentFingerprint(content),+ fingerprint: itemFingerprint(title, item.Link, content),articleFailed: articleFailed,item: types.FetchedItem{ExternalID: itemExternalID(feedURL, itemID),diff --git a/internal/datasource/connector/rss/item_identity_test.go b/internal/datasource/connector/rss/item_identity_test.gonew file mode 100644index 000000000..484ef0828--- /dev/null+++ b/internal/datasource/connector/rss/item_identity_test.go@@ -0,0 +1,220 @@+package rss++import (+ "context"+ "crypto/sha256"+ "encoding/json"+ "fmt"+ "net/http"+ "net/http/httptest"+ "strings"+ "sync/atomic"+ "testing"++ "github.com/Tencent/WeKnora/internal/types"+)++type identityFeed struct {+ server *httptest.Server+ xml atomic.Value+ fetches atomic.Int32+ articleStatus atomic.Int32+ articleBody atomic.Value+}++func newIdentityFeed(t *testing.T) *identityFeed {+ t.Helper()+ f := &identityFeed{}+ f.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Path == "/feed.xml" {+ w.Header().Set("Content-Type", "application/rss+xml")+ _, _ = fmt.Fprint(w, f.xml.Load().(string))+ return+ }+ f.fetches.Add(1)+ if status := int(f.articleStatus.Load()); status != http.StatusOK {+ w.WriteHeader(status)+ return+ }+ w.Header().Set("Content-Type", "text/html; charset=utf-8")+ _, _ = fmt.Fprintf(w, "<html><head><title>Stable article</title></head>"++ "<body><article>%s</article></body></html>", f.articleBody.Load().(string))+ }))+ t.Cleanup(f.server.Close)+ f.articleStatus.Store(http.StatusOK)+ f.articleBody.Store(longArticleBody)+ f.publish("Original title", "/original", "Mon, 02 Jan 2006 15:04:05 GMT")+ return f+}++func (f *identityFeed) publish(title, path, date string) {+ f.xml.Store(fmt.Sprintf(`<rss version="2.0"><channel><title>Feed</title>+<link>%s</link><description>Test</description>+<item><guid isPermaLink="false">stable-guid</guid><title>%s</title><link>%s%s</link>+<pubDate>%s</pubDate><description>Stable summary</description></item></channel></rss>`,+ f.server.URL, title, f.server.URL, path, date))+}++func (f *identityFeed) sync(t *testing.T, cursor *types.SyncCursor) ([]types.FetchedItem, *types.SyncCursor) {+ t.Helper()+ config := makeConfig(f.server.URL+"/feed.xml", "")+ items, next, err := NewConnector().FetchIncremental(context.Background(), config, cursor)+ if err != nil {+ t.Fatalf("FetchIncremental: %v", err)+ }+ return items, next+}++func TestIncrementalItemIdentityChanges(t *testing.T) {+ for _, tt := range []struct{ name, title, path string }{+ {"title only", "Corrected title", "/original"},+ {"link only", "Original title", "/moved"},+ } {+ t.Run(tt.name, func(t *testing.T) {+ f := newIdentityFeed(t)+ original, cursor := f.sync(t, nil)+ if len(original) != 1 {+ t.Fatalf("expected 1 initial item, got %d", len(original))+ }+ f.publish(tt.title, tt.path, "Mon, 02 Jan 2006 15:04:05 GMT")+ updated, next := f.sync(t, cursor)+ if len(updated) != 1 {+ t.Fatalf("expected 1 metadata update with unchanged body, got %d", len(updated))+ }+ if updated[0].ExternalID != original[0].ExternalID ||+ string(updated[0].Content) != string(original[0].Content) {+ t.Fatal("the stable GUID and body must be preserved")+ }+ if updated[0].Title != tt.title || updated[0].FileName != tt.title+".md" ||+ updated[0].URL != f.server.URL+tt.path {+ t.Fatalf("incorrect updated identity: %+v", updated[0])+ }+ if updated[0].Metadata["link"] != updated[0].URL {+ t.Fatal("persisted source link must match the updated URL")+ }+ assertIdentityUnchanged(t, f, next)+ })+ }+}++func assertIdentityUnchanged(t *testing.T, f *identityFeed, cursor *types.SyncCursor) {+ t.Helper()+ before := f.fetches.Load()+ items, _ := f.sync(t, cursor)+ if len(items) != 0 || f.fetches.Load() != before {+ t.Fatal("unchanged sync must not emit items or fetch the article")+ }+}++func TestIncrementalItemTimestampOnly(t *testing.T) {+ f := newIdentityFeed(t)+ _, cursor := f.sync(t, nil)+ f.publish("Original title", "/original", "Tue, 03 Jan 2006 15:04:05 GMT")+ items, next := f.sync(t, cursor)+ if len(items) != 0 {+ t.Fatalf("expected no update for timestamp-only changes, got %d", len(items))+ }+ if f.fetches.Load() != 2 {+ t.Fatal("changed feed signal must still resolve the article")+ }+ assertIdentityUnchanged(t, f, next)+}++// legacyIdentityCursor syncs once and rewrites the cursor the way a release+// before item fingerprints stored it: a body-only "h:" hash.+func legacyIdentityCursor(t *testing.T, f *identityFeed) (*types.SyncCursor, string) {+ t.Helper()+ items, cursor := f.sync(t, nil)+ if len(items) != 1 {+ t.Fatalf("expected 1 initial item, got %d", len(items))+ }+ sum := sha256.Sum256(items[0].Content)+ legacy := fmt.Sprintf("h:%x", sum[:8])+ cursor.ConnectorCursor["feed_items"] = map[string]map[string]string{+ f.server.URL + "/feed.xml": {"stable-guid": legacy},+ }+ return cursor, legacy+}++func storedItemFingerprint(t *testing.T, f *identityFeed, cursor *types.SyncCursor) string {+ t.Helper()+ data, err := json.Marshal(cursor.ConnectorCursor)+ if err != nil {+ t.Fatalf("marshal cursor: %v", err)+ }+ var decoded rssCursor+ if err := json.Unmarshal(data, &decoded); err != nil {+ t.Fatalf("unmarshal cursor: %v", err)+ }+ return decoded.FeedItems[f.server.URL+"/feed.xml"]["stable-guid"]+}++// Upgrading must not delete and rebuild every stored item: an unchanged body+// only has its cursor upgraded to an item fingerprint.+func TestIncrementalItemLegacyCursorUnchangedBodyUpgradesInPlace(t *testing.T) {+ f := newIdentityFeed(t)+ cursor, _ := legacyIdentityCursor(t, f)+ updated, next := f.sync(t, cursor)+ if len(updated) != 0 {+ t.Fatalf("unchanged legacy item must not be re-ingested, got %d", len(updated))+ }+ if fp := storedItemFingerprint(t, f, next); !strings.HasPrefix(fp, itemFingerprintPrefix) {+ t.Fatalf("legacy cursor must be upgraded to an item fingerprint, got %q", fp)+ }+ assertIdentityUnchanged(t, f, next)+}++// A transient article failure during the upgrade must not replace the stored+// full text with the feed summary. The legacy cursor is kept so the next sync+// retries, and the upgrade completes once the article is reachable again.+func TestIncrementalItemLegacyCursorArticleFailureKeepsCursor(t *testing.T) {+ f := newIdentityFeed(t)+ cursor, legacy := legacyIdentityCursor(t, f)+ f.articleStatus.Store(http.StatusInternalServerError)+ updated, next := f.sync(t, cursor)+ if len(updated) != 0 {+ t.Fatalf("article failure must not emit the feed summary, got %d", len(updated))+ }+ if fp := storedItemFingerprint(t, f, next); fp != legacy {+ t.Fatalf("legacy cursor must be kept for retry, got %q want %q", fp, legacy)+ }++ f.articleStatus.Store(http.StatusOK)+ before := f.fetches.Load()+ updated, next = f.sync(t, next)+ if len(updated) != 0 || f.fetches.Load() == before {+ t.Fatalf("recovered article must be re-read and upgrade in place, emitted %d", len(updated))+ }+ if fp := storedItemFingerprint(t, f, next); !strings.HasPrefix(fp, itemFingerprintPrefix) {+ t.Fatalf("legacy cursor must be upgraded after recovery, got %q", fp)+ }+}++// A legacy item whose body really changed is still re-ingested.+func TestIncrementalItemLegacyCursorChangedBodyEmits(t *testing.T) {+ f := newIdentityFeed(t)+ cursor, _ := legacyIdentityCursor(t, f)+ f.articleBody.Store(longArticleBody + "<p>An appended paragraph that changes the stored body.</p>")+ updated, next := f.sync(t, cursor)+ if len(updated) != 1 || !strings.Contains(string(updated[0].Content), "appended paragraph") {+ t.Fatalf("changed legacy item must be re-ingested with the new body, got %d items", len(updated))+ }+ if fp := storedItemFingerprint(t, f, next); !strings.HasPrefix(fp, itemFingerprintPrefix) {+ t.Fatalf("re-ingested item must store an item fingerprint, got %q", fp)+ }+ assertIdentityUnchanged(t, f, next)+}++func TestIncrementalItemContentWithoutLink(t *testing.T) {+ f := newIdentityFeed(t)+ f.xml.Store(`<rss version="2.0"><channel><title>Feed</title><description>Test</description>+<link>https://example.com</link>+<item><guid>stable-guid</guid><title>Feed-only</title><description>Original body</description></item></channel></rss>`)+ original, cursor := f.sync(t, nil)+ f.xml.Store(strings.ReplaceAll(f.xml.Load().(string), "Original body", "Changed body"))+ updated, next := f.sync(t, cursor)+ if len(original) != 1 || len(updated) != 1 || string(updated[0].Content) != "Changed body" {+ t.Fatal("body-only changes must still update feed-only entries")+ }+ assertIdentityUnchanged(t, f, next)+}diff --git a/internal/datasource/connector/rss/types.go b/internal/datasource/connector/rss/types.goindex 9fff63b5d..1c55aabdd 100644--- a/internal/datasource/connector/rss/types.go+++ b/internal/datasource/connector/rss/types.go@@ -13,8 +13,8 @@// Markdown. Feed-provided content (content:encoded / description) is used// as a fallback.// - Incremental: feed-level signals skip full-text article fetches when a feed-// entry is unchanged; content fingerprints detect feed-body changes. Article--// only edits without feed updates may be missed until the feed entry changes.+// entry is unchanged; item fingerprints detect title, link and body changes.+// Article-only edits without feed updates may be missed until the feed entry changes.// Deletions are NOT synced — feeds routinely drop old items.//// All outbound requests go through the SSRF-safe HTTP client so a malicious@@ -144,7 +144,9 @@ func (c *Config) parseHeaders() map[string]string {// rssCursor stores incremental sync state.//// FeedItems maps feedURL → itemID → fingerprint, where fingerprint is a-// "h:<sha256-prefix>" hash of the final Markdown body that would be ingested.+// "i:<sha256-prefix>" hash of the resolved title, link and Markdown body.+// Legacy "h:" body-only fingerprints are upgraded in place when the body is+// unchanged; only a changed body is re-ingested.// FeedSignals maps feedURL → itemID → feed-only signal used to skip full-text// article fetches when the feed entry itself has not changed.type rssCursor struct {@@ -153,12 +155,30 @@ type rssCursor struct {FeedSignals map[string]map[string]string `json:"feed_signals,omitempty"`}-// contentFingerprint hashes ingested Markdown for incremental change detection.-func contentFingerprint(markdown string) string {+const itemFingerprintPrefix = "i:"++// isLegacyFingerprint reports a body-only "h:" cursor entry written before+// item fingerprints existed.+func isLegacyFingerprint(fp string) bool {+ return fp != "" && !strings.HasPrefix(fp, itemFingerprintPrefix)+}++// legacyContentFingerprint is the body-only fingerprint older cursors stored.+// It is kept only to recognise unchanged bodies when upgrading those cursors.+func legacyContentFingerprint(markdown string) string {sum := sha256.Sum256([]byte(markdown))return "h:" + hex.EncodeToString(sum[:])[:16]}+// itemFingerprint includes the document identity without timestamps, so date-only+// feed changes still resolve articles without needlessly re-ingesting them.+func itemFingerprint(title, link, markdown string) string {+ // A string array cannot fail to marshal and preserves field boundaries.+ data, _ := json.Marshal([3]string{title, link, markdown})+ sum := sha256.Sum256(data)+ return itemFingerprintPrefix + hex.EncodeToString(sum[:])[:16]+}+// feedSignalFingerprint hashes feed-visible fields so incremental sync can skip// article-page fetches when the entry has not changed in the feed.func feedSignalFingerprint(item *gofeed.Item, feedContent string) string {