Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(rss): 修复全文回退后增量同步不重试

yeliheng-01015d484552140
diff --git a/internal/datasource/connector/rss/article_retry_test.go b/internal/datasource/connector/rss/article_retry_test.go
new file mode 100644
index 000000000..8cd5d7a4c
--- /dev/null
+++ b/internal/datasource/connector/rss/article_retry_test.go
@@ -0,0 +1,134 @@
+package rss
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "sync/atomic"
+ "testing"
+
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/stretchr/testify/require"
+)
+
+type articleRetryFeed struct {
+ server *httptest.Server
+ status atomic.Int32
+ articleReads atomic.Int32
+ healthyReads atomic.Int32
+ summary string
+ includeLink bool
+}
+
+func newArticleRetryFeed(t *testing.T) *articleRetryFeed {
+ t.Helper()
+ f := &articleRetryFeed{summary: "Temporary summary", includeLink: true}
+ f.status.Store(http.StatusServiceUnavailable)
+ f.server = httptest.NewServer(http.HandlerFunc(f.serveHTTP))
+ t.Cleanup(f.server.Close)
+ return f
+}
+
+func (f *articleRetryFeed) serveHTTP(w http.ResponseWriter, r *http.Request) {
+ switch r.URL.Path {
+ case "/feed.xml":
+ link := ""
+ if f.includeLink {
+ link = "<link>http://" + r.Host + "/article</link>"
+ }
+ w.Header().Set("Content-Type", "application/rss+xml")
+ _, _ = fmt.Fprintf(w, `<rss version="2.0"><channel><title>Retry feed</title>
+<item><guid>retry</guid><title>Retry article</title>%s<description><![CDATA[%s]]></description></item>
+<item><guid>healthy</guid><title>Healthy article</title><link>http://%s/healthy</link></item>
+</channel></rss>`, link, f.summary, r.Host)
+ case "/article":
+ f.articleReads.Add(1)
+ if status := int(f.status.Load()); status != http.StatusOK {
+ w.WriteHeader(status)
+ return
+ }
+ writeRetryArticle(w)
+ case "/healthy":
+ f.healthyReads.Add(1)
+ writeRetryArticle(w)
+ default:
+ http.NotFound(w, r)
+ }
+}
+
+func writeRetryArticle(w http.ResponseWriter) {
+ w.Header().Set("Content-Type", "text/html; charset=utf-8")
+ _, _ = fmt.Fprintf(w, "<html><head><title>Article</title></head><body><article>%s</article></body></html>",
+ longArticleBody)
+}
+
+func (f *articleRetryFeed) sync(t *testing.T, cursor *types.SyncCursor) ([]types.FetchedItem, *types.SyncCursor) {
+ t.Helper()
+ url := f.server.URL + "/feed.xml"
+ cfg := makeConfig(url, "")
+ cfg.ResourceIDs = []string{url}
+ items, next, err := NewConnector().FetchIncremental(context.Background(), cfg, cursor)
+ require.NoError(t, err)
+ require.NotNil(t, next)
+ return items, next
+}
+
+func TestIncrementalRetriesArticleAfterFallback(t *testing.T) {
+ for _, status := range []int{http.StatusNotFound, http.StatusServiceUnavailable} {
+ t.Run(http.StatusText(status), func(t *testing.T) {
+ f := newArticleRetryFeed(t)
+ f.status.Store(int32(status))
+ items, cursor := f.sync(t, nil)
+ require.Len(t, items, 2)
+ require.Equal(t, f.summary, string(items[0].Content))
+ items, cursor = f.sync(t, cursor)
+ require.Empty(t, items, "unchanged fallback must not be ingested again")
+ require.EqualValues(t, 2, f.articleReads.Load(), "failed article must be retried on the next sync")
+ require.EqualValues(t, 1, f.healthyReads.Load(), "successful sibling should stay cached")
+ f.status.Store(http.StatusOK)
+ items, cursor = f.sync(t, cursor)
+ require.Len(t, items, 1)
+ require.Contains(t, string(items[0].Content), "reasonably long article")
+ items, _ = f.sync(t, cursor)
+ require.Empty(t, items)
+ require.EqualValues(t, 3, f.articleReads.Load(), "recovered article should now stay cached")
+ require.EqualValues(t, 1, f.healthyReads.Load())
+ })
+ }
+}
+
+func TestIncrementalRetriesArticleWithEmptyFallback(t *testing.T) {
+ f := newArticleRetryFeed(t)
+ f.summary = ""
+ _, cursor := f.sync(t, nil)
+ f.status.Store(http.StatusOK)
+ items, _ := f.sync(t, cursor)
+ require.Len(t, items, 1)
+ require.NotEmpty(t, items[0].Content)
+ require.EqualValues(t, 2, f.articleReads.Load())
+}
+
+func TestIncrementalCachesRecoveryWithIdenticalContent(t *testing.T) {
+ f := newArticleRetryFeed(t)
+ f.summary = longArticleBody
+ _, cursor := f.sync(t, nil)
+ f.status.Store(http.StatusOK)
+ items, cursor := f.sync(t, cursor)
+ require.Empty(t, items, "identical recovered content does not need ingestion")
+ require.EqualValues(t, 2, f.articleReads.Load(), "the article must have been fetched successfully")
+ items, _ = f.sync(t, cursor)
+ require.Empty(t, items)
+ require.EqualValues(t, 2, f.articleReads.Load(), "success should cache the feed signal despite equal content")
+}
+
+func TestIncrementalCachesEntryWithoutArticleLink(t *testing.T) {
+ f := newArticleRetryFeed(t)
+ f.includeLink = false
+ items, cursor := f.sync(t, nil)
+ require.Len(t, items, 2)
+ items, _ = f.sync(t, cursor)
+ require.Empty(t, items)
+ require.Zero(t, f.articleReads.Load())
+ require.EqualValues(t, 1, f.healthyReads.Load())
+}
diff --git a/internal/datasource/connector/rss/connector.go b/internal/datasource/connector/rss/connector.go
index b84dc6ace..98e7f2cb9 100644
--- a/internal/datasource/connector/rss/connector.go
+++ b/internal/datasource/connector/rss/connector.go
@@ -260,7 +260,11 @@ func (c *Connector) walk(
resolved := c.resolveItem(ctx, cli, feed, item, feedURL, itemID, feedContent)
newCursor.FeedItems[feedURL][itemID] = resolved.fingerprint
- newCursor.FeedSignals[feedURL][itemID] = feedSig
+ // Keep fallback deduplication, but retry failed full-text extraction on
+ // the next sync even when the feed entry itself has not changed.
+ if !resolved.articleFailed {
+ newCursor.FeedSignals[feedURL][itemID] = feedSig
+ }
if incremental && prevItems != nil && prevItems[itemID] == resolved.fingerprint {
skipped++
@@ -285,8 +289,9 @@ func (c *Connector) walk(
}
type resolvedFeedItem struct {
- item types.FetchedItem
- fingerprint string
+ item types.FetchedItem
+ fingerprint string
+ articleFailed bool
}
// resolveItem assembles a FetchedItem for a single feed entry, resolving the
@@ -303,6 +308,7 @@ func (c *Connector) resolveItem(
// Prefer full article text; fall back to feed-provided content on failure.
contentHTML := feedContent
+ articleFailed := false
if strings.TrimSpace(item.Link) != "" {
if articleHTML, articleTitle, err := cli.extractArticle(ctx, item.Link); err == nil {
contentHTML = articleHTML
@@ -310,6 +316,7 @@ func (c *Connector) resolveItem(
title = articleTitle
}
} else {
+ articleFailed = true
logger.Warnf(ctx, "[RSS] full-text fetch failed for %s (using feed content): %v", item.Link, err)
}
}
@@ -330,7 +337,8 @@ func (c *Connector) resolveItem(
}
return resolvedFeedItem{
- fingerprint: contentFingerprint(content),
+ fingerprint: contentFingerprint(content),
+ articleFailed: articleFailed,
item: types.FetchedItem{
ExternalID: itemExternalID(feedURL, itemID),
Title: title,