Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(notion): 保留正文抓取失败时的同步游标 (#3693)

诉说晚秋b14bcbcd541f
diff --git a/internal/datasource/connector/notion/block_retry_test.go b/internal/datasource/connector/notion/block_retry_test.go
new file mode 100644
index 000000000..43a76df16
--- /dev/null
+++ b/internal/datasource/connector/notion/block_retry_test.go
@@ -0,0 +1,168 @@
+package notion
+
+import (
+ "context"
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/Tencent/WeKnora/internal/types"
+)
+
+const retryPageTime = "2026-09-24T10:00:00Z"
+
+type blockRetryFixture struct {
+ failed atomic.Bool
+ calls atomic.Int32
+ mode string
+}
+
+func (f *blockRetryFixture) serve(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ switch r.URL.Path {
+ case "/v1/search":
+ _, _ = w.Write([]byte(`{"results":[` + retryPage("page", "workspace") + `],"has_more":false}`))
+ case "/v1/pages/page":
+ _, _ = w.Write([]byte(retryPage("page", "workspace")))
+ case "/v1/pages/healthy":
+ _, _ = w.Write([]byte(retryPage("healthy", "workspace")))
+ case "/v1/pages/child":
+ _, _ = w.Write([]byte(retryPage("child", "page_id")))
+ case "/v1/blocks/page/children":
+ f.pageBlocks(w, r)
+ case "/v1/blocks/child/children":
+ f.content(w)
+ case "/v1/blocks/healthy/children":
+ _, _ = w.Write([]byte(`{"results":[{"id":"text","type":"paragraph","paragraph":{"rich_text":[` +
+ `{"type":"text","text":{"content":"healthy content"},"plain_text":"healthy content"}]}}]}`))
+ default:
+ http.NotFound(w, r)
+ }
+}
+
+func retryPage(id, parentType string) string {
+ return `{"id":"` + id + `","object":"page","last_edited_time":"` + retryPageTime +
+ `","parent":{"type":"` + parentType + `","page_id":"page"}}`
+}
+
+func (f *blockRetryFixture) pageBlocks(w http.ResponseWriter, r *http.Request) {
+ switch f.mode {
+ case "child":
+ _, _ = w.Write([]byte(`{"results":[{"id":"child","type":"child_page",` +
+ `"child_page":{"title":"Child"}}],"has_more":false}`))
+ case "later_page":
+ if r.URL.Query().Get("start_cursor") == "later" {
+ f.content(w)
+ return
+ }
+ _, _ = w.Write([]byte(`{"results":[],"has_more":true,"next_cursor":"later"}`))
+ case "empty":
+ _, _ = w.Write([]byte(`{"results":[],"has_more":false}`))
+ default:
+ f.content(w)
+ }
+}
+
+func (f *blockRetryFixture) content(w http.ResponseWriter) {
+ f.calls.Add(1)
+ if f.failed.Load() {
+ status := http.StatusUnauthorized
+ if f.mode == "server_error" {
+ status = http.StatusServiceUnavailable
+ }
+ w.WriteHeader(status)
+ _, _ = w.Write([]byte(`{"code":"unauthorized","message":"test block request rejected"}`))
+ return
+ }
+ _, _ = w.Write([]byte(`{"results":[{"id":"text","type":"paragraph","paragraph":{"rich_text":[` +
+ `{"type":"text","text":{"content":"recovered content"},"plain_text":"recovered content"}` +
+ `]}}],"has_more":false}`))
+}
+
+func newBlockRetryFixture(t *testing.T, mode string) (*blockRetryFixture, *types.DataSourceConfig) {
+ t.Helper()
+ allowNotionTestServer(t)
+ fixture := &blockRetryFixture{mode: mode}
+ fixture.failed.Store(true)
+ server := httptest.NewServer(http.HandlerFunc(fixture.serve))
+ t.Cleanup(server.Close)
+ return fixture, makeNotionConfig(&Config{APIKey: "test-token"}, server.URL, []string{"page"})
+}
+
+func TestNotionBlockFailurePreservesRetry(t *testing.T) {
+ for _, mode := range []string{"page", "server_error", "later_page", "child"} {
+ t.Run(mode, func(t *testing.T) {
+ fixture, config := newBlockRetryFixture(t, mode)
+ oldTime := time.Date(2026, 9, 23, 10, 0, 0, 0, time.UTC)
+ previous := buildCursor(map[string]time.Time{"page": oldTime, "removed": oldTime})
+ before, err := json.Marshal(previous)
+ require.NoError(t, err)
+ connector := NewConnector()
+ items, next, err := connector.FetchIncremental(context.Background(), config, previous)
+ require.ErrorContains(t, err, "get blocks")
+ require.Nil(t, items, "failed traversal must not emit content or deletion events")
+ require.Nil(t, next, "failed traversal must not acknowledge the new edit time")
+ after, marshalErr := json.Marshal(previous)
+ require.NoError(t, marshalErr)
+ require.Equal(t, before, after)
+ require.Positive(t, fixture.calls.Load())
+
+ fixture.failed.Store(false)
+ items, next, err = connector.FetchIncremental(context.Background(), config, previous)
+ require.NoError(t, err)
+ require.NotNil(t, next)
+ var foundContent, foundDeletion bool
+ for _, item := range items {
+ foundContent = foundContent || strings.Contains(string(item.Content), "recovered content")
+ foundDeletion = foundDeletion || (item.ExternalID == "removed" && item.IsDeleted)
+ }
+ require.True(t, foundContent, "recovered page must be fetched without another source edit")
+ require.True(t, foundDeletion, "successful traversal must still detect actual deletions")
+ calls := fixture.calls.Load()
+ items, _, err = connector.FetchIncremental(context.Background(), config, next)
+ require.NoError(t, err)
+ require.Empty(t, items)
+ require.Equal(t, calls, fixture.calls.Load(), "unchanged page must not be fetched again")
+ })
+ }
+}
+
+func TestNotionFirstSyncRejectsBlockFailure(t *testing.T) {
+ fixture, config := newBlockRetryFixture(t, "page")
+ connector := NewConnector()
+ items, next, err := connector.FetchIncremental(context.Background(), config, nil)
+ require.Error(t, err)
+ require.Nil(t, items)
+ require.Nil(t, next, "first sync must not substitute time.Now for a failed page")
+ fixture.failed.Store(false)
+ items, next, err = connector.FetchIncremental(context.Background(), config, nil)
+ require.NoError(t, err)
+ require.Len(t, items, 1)
+ require.NotNil(t, next)
+}
+
+func TestNotionFullSyncRejectsBlockFailure(t *testing.T) {
+ _, config := newBlockRetryFixture(t, "child")
+ config.ResourceIDs = []string{"healthy", "page"}
+ items, err := NewConnector().FetchAll(context.Background(), config, config.ResourceIDs)
+ require.Error(t, err)
+ require.Nil(t, items, "discard earlier successful items when a later page fails")
+}
+
+func TestNotionEmptyPageStillSucceeds(t *testing.T) {
+ _, config := newBlockRetryFixture(t, "empty")
+ previous := buildCursor(map[string]time.Time{"page": {}})
+ items, next, err := NewConnector().FetchIncremental(context.Background(), config, previous)
+ require.NoError(t, err)
+ require.Len(t, items, 1)
+ require.Equal(t, "page", items[0].ExternalID)
+ require.False(t, items[0].IsDeleted)
+ require.Equal(t, "# "+defaultUntitledName+"\n", string(items[0].Content))
+ require.NotNil(t, next)
+}
diff --git a/internal/datasource/connector/notion/connector.go b/internal/datasource/connector/notion/connector.go
index 9a224e0ed..77da8cf55 100644
--- a/internal/datasource/connector/notion/connector.go
+++ b/internal/datasource/connector/notion/connector.go
@@ -148,7 +148,10 @@ func (c *Connector) FetchAll(ctx context.Context, config *types.DataSourceConfig
for _, resourceID := range resourceIDs {
page, err := client.GetPage(ctx, resourceID)
if err == nil {
- pageItems, _ := c.fetchPage(ctx, client, page, visited)
+ pageItems, _, err := c.fetchPage(ctx, client, page, visited)
+ if err != nil {
+ return nil, err
+ }
allItems = append(allItems, pageItems...)
continue
}
@@ -268,7 +271,10 @@ func (c *Connector) FetchIncremental(ctx context.Context, config *types.DataSour
newEditTimes[rid] = rt
}
} else {
- items, truncated := c.fetchPage(ctx, client, pg, fetchVisited)
+ items, truncated, err := c.fetchPage(ctx, client, pg, fetchVisited)
+ if err != nil {
+ return nil, nil, err
+ }
queryTruncated = queryTruncated || truncated
changedItems = append(changedItems, items...)
}
@@ -334,19 +340,20 @@ func buildCursor(editTimes map[string]time.Time) *types.SyncCursor {
// If page is nil, it will be fetched from the API.
// The second result reports whether any query issued for this page (including
// child databases) was truncated at a vendor limit.
+// A page block fetch error aborts the round without acknowledging its edit time.
func (c *Connector) fetchPage(
ctx context.Context, client *notionClient, page *notionPage, visited map[string]bool,
-) ([]types.FetchedItem, bool) {
+) ([]types.FetchedItem, bool, error) {
if page == nil {
- return nil, false
+ return nil, false, nil
}
if visited[page.ID] {
- return nil, false
+ return nil, false, nil
}
visited[page.ID] = true
if page.InTrash {
- return nil, false
+ return nil, false, nil
}
// Database records store content in properties, not blocks — delegate to
@@ -362,15 +369,14 @@ func (c *Connector) fetchPage(
}
propNames := extractPropertySchema(*page)
if item := c.buildRecordItem(ctx, client, *page, propNames, dbTitle); item != nil {
- return []types.FetchedItem{*item}, false
+ return []types.FetchedItem{*item}, false, nil
}
- return nil, false
+ return nil, false, nil
}
blocks, err := client.GetBlockChildrenAll(ctx, page.ID)
if err != nil {
- logger.Warnf(ctx, "[Notion] failed to get blocks for page %s: %v", page.ID, err)
- return nil, false
+ return nil, false, fmt.Errorf("get blocks for notion page %s: %w", page.ID, err)
}
resolveFileUploads(ctx, client, blocks)
@@ -445,7 +451,10 @@ func (c *Connector) fetchPage(
logger.Warnf(ctx, "[Notion] failed to get child page %s: %v", block.ID, err)
continue
}
- childItems, childTruncated := c.fetchPage(ctx, client, childPage, visited)
+ childItems, childTruncated, err := c.fetchPage(ctx, client, childPage, visited)
+ if err != nil {
+ return nil, truncated || childTruncated, err
+ }
truncated = truncated || childTruncated
items = append(items, childItems...)
case "child_database":
@@ -455,7 +464,7 @@ func (c *Connector) fetchPage(
}
}
- return items, truncated
+ return items, truncated, nil
}
// fetchDatabase syncs each database record as an individual knowledge item (full sync).
diff --git a/internal/datasource/connector/notion/query_truncation_test.go b/internal/datasource/connector/notion/query_truncation_test.go
index d11691846..a92282f51 100644
--- a/internal/datasource/connector/notion/query_truncation_test.go
+++ b/internal/datasource/connector/notion/query_truncation_test.go
@@ -233,6 +233,7 @@ func TestFetchPagePropagatesTruncationFromChildDatabase(t *testing.T) {
Parent: notionParent{Type: parentTypeWorkspace},
}
- _, truncated := NewConnector().fetchPage(context.Background(), client, page, map[string]bool{})
+ _, truncated, err := NewConnector().fetchPage(context.Background(), client, page, map[string]bool{})
+ require.NoError(t, err)
require.True(t, truncated, "a capped child database query must reach the incremental round")
}