Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(notion): 附件下载失败中止本轮不再推进游标,附件不再永久缺失 (#4413)
linus-liu-webb2e81a9ac4e2
diff --git a/internal/datasource/connector/notion/attachment_retry_test.go b/internal/datasource/connector/notion/attachment_retry_test.gonew file mode 100644index 000000000..2144505ae--- /dev/null+++ b/internal/datasource/connector/notion/attachment_retry_test.go@@ -0,0 +1,199 @@+package notion++import (+ "context"+ "fmt"+ "net/http"+ "net/http/httptest"+ "sync/atomic"+ "testing"+ "time"++ "github.com/stretchr/testify/require"++ "github.com/Tencent/WeKnora/internal/types"+)++const attachmentPageEditTime = "2026-10-03T10:00:00Z"++const attachmentPageJSON = `{"id":"page","object":"page","url":"https://notion.so/page",` ++ `"last_edited_time":"` + attachmentPageEditTime + `","parent":{"type":"workspace"},` ++ `"properties":{"title":{"type":"title","title":[{"plain_text":"Attachment page"}]}}}`++// attachmentRetryFixture serves a single page whose whole content is one pdf+// block. The attachment URL keeps answering 500 until failing is cleared.+type attachmentRetryFixture struct {+ baseURL string+ attachmentURL string+ failing atomic.Bool+ blockFetches atomic.Int32+ downloads atomic.Int32+}++func (f *attachmentRetryFixture) serve(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ switch r.URL.Path {+ case "/v1/search":+ _, _ = fmt.Fprint(w, `{"results":[`+attachmentPageJSON+`],"has_more":false}`)+ case "/v1/pages/page":+ _, _ = fmt.Fprint(w, attachmentPageJSON)+ case "/v1/blocks/page/children":+ f.blockFetches.Add(1)+ _, _ = fmt.Fprintf(w, `{"results":[{"id":"pdf-block","type":"pdf","pdf":{"type":"file",`++ `"name":"reference.pdf","file":{"url":%q,"expiry_time":"2030-01-01T00:00:00Z"}}}],`++ `"has_more":false}`, f.attachmentURL)+ case "/att.pdf":+ f.downloads.Add(1)+ if f.failing.Load() {+ http.Error(w, "storage unavailable", http.StatusInternalServerError)+ return+ }+ _, _ = w.Write([]byte("%PDF-1.4 recovered attachment"))+ default:+ http.NotFound(w, r)+ }+}++func newAttachmentRetryFixture(t *testing.T) (*attachmentRetryFixture, *types.DataSourceConfig) {+ t.Helper()+ allowNotionTestServer(t)+ fixture := &attachmentRetryFixture{}+ fixture.failing.Store(true)+ server := httptest.NewServer(http.HandlerFunc(fixture.serve))+ t.Cleanup(server.Close)+ fixture.baseURL = server.URL+ fixture.attachmentURL = server.URL + "/att.pdf"+ return fixture, makeNotionConfig(&Config{APIKey: "test-token"}, server.URL, []string{"page"})+}++// fileUploadRetryFixture serves a page whose only file block is a file_upload+// that the source refuses to resolve.+type fileUploadRetryFixture struct {+ blockFetches atomic.Int32+}++func (f *fileUploadRetryFixture) serve(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ switch r.URL.Path {+ case "/v1/search":+ _, _ = fmt.Fprint(w, `{"results":[`+attachmentPageJSON+`],"has_more":false}`)+ case "/v1/pages/page":+ _, _ = fmt.Fprint(w, attachmentPageJSON)+ case "/v1/blocks/page/children":+ f.blockFetches.Add(1)+ _, _ = fmt.Fprint(w, `{"results":[{"id":"upload-block","type":"file","file":`++ `{"type":"file_upload","name":"uploaded.pdf","file_upload":{"id":"upload-1"}}}],`++ `"has_more":false}`)+ case "/v1/blocks/upload-block":+ http.Error(w, "block unavailable", http.StatusNotFound)+ default:+ http.NotFound(w, r)+ }+}++func newFileUploadRetryFixture(t *testing.T) (*fileUploadRetryFixture, *types.DataSourceConfig) {+ t.Helper()+ allowNotionTestServer(t)+ fixture := &fileUploadRetryFixture{}+ server := httptest.NewServer(http.HandlerFunc(fixture.serve))+ t.Cleanup(server.Close)+ return fixture, makeNotionConfig(&Config{APIKey: "test-token"}, server.URL, []string{"page"})+}++// A page that cannot store one of its attachments must not be reported as a+// complete fetch: the caller would otherwise persist the page edit time and the+// attachment would stay missing until a full re-sync.+func TestNotionAttachmentDownloadFailureFailsPageFetch(t *testing.T) {+ fixture, _ := newAttachmentRetryFixture(t)+ client := mustTestClient(t, "test-token", fixture.baseURL)+ page, err := client.GetPage(context.Background(), "page")+ require.NoError(t, err)+ items, truncated, err := NewConnector().fetchPage(+ context.Background(), client, page, map[string]bool{})+ if err == nil && !truncated {+ t.Fatalf("fetchPage reported a complete fetch (truncated=false, err=nil) after the attachment download failed")+ }+ require.Error(t, err, "a failed attachment download must abort the page fetch")+ require.Nil(t, items)+ require.Positive(t, fixture.downloads.Load())+}++// The sync loop persists the cursor of a round that reported success, so an+// attachment failure must leave the page edit time unacknowledged and the next+// round must fetch the unchanged page again.+func TestNotionAttachmentFailurePreservesRetry(t *testing.T) {+ fixture, config := newAttachmentRetryFixture(t)+ connector := NewConnector()+ previous := buildCursor(map[string]time.Time{"page": time.Date(2026, 10, 2, 10, 0, 0, 0, time.UTC)})++ items, next, err := connector.FetchIncremental(context.Background(), config, previous)+ require.Error(t, err, "a failed attachment download must fail the round")+ require.Nil(t, items, "a failed download must not emit a partial page")+ require.Nil(t, next, "a failed download must not acknowledge the page edit time")++ cursor := previous+ if next != nil {+ cursor = next+ }+ items, next, err = connector.FetchIncremental(context.Background(), config, cursor)+ require.Error(t, err, "the second round must retry the page whose attachment is still missing")+ require.Nil(t, items)+ require.Nil(t, next)+ require.GreaterOrEqual(t, fixture.blockFetches.Load(), int32(2),+ "the unchanged page must be fetched again until its attachment is stored")+}++// Once the source serves the attachment, the retried page must be ingested with+// its attachment and only then acknowledged.+func TestNotionAttachmentStoredAfterDownloadRecovers(t *testing.T) {+ fixture, config := newAttachmentRetryFixture(t)+ connector := NewConnector()+ previous := buildCursor(map[string]time.Time{"page": time.Date(2026, 10, 2, 10, 0, 0, 0, time.UTC)})++ _, next, err := connector.FetchIncremental(context.Background(), config, previous)+ require.Error(t, err)+ require.Nil(t, next)++ fixture.failing.Store(false)+ items, next, err := connector.FetchIncremental(context.Background(), config, previous)+ require.NoError(t, err)+ require.NotNil(t, next)++ var foundPage, foundAttachment bool+ for _, item := range items {+ if item.ExternalID == "page" && !item.IsDeleted {+ foundPage = true+ }+ if item.Metadata["object_type"] == objectTypeAttachment {+ foundAttachment = true+ require.Equal(t, "reference.pdf", item.FileName)+ require.Contains(t, string(item.Content), "recovered attachment")+ }+ }+ require.True(t, foundPage, "the page markdown must be emitted once the round succeeds")+ require.True(t, foundAttachment, "the recovered attachment must be emitted")++ items, _, err = connector.FetchIncremental(context.Background(), config, next)+ require.NoError(t, err)+ require.Empty(t, items, "an acknowledged page must not be fetched again")+ require.Equal(t, int32(2), fixture.blockFetches.Load())+}++// An unresolved file_upload leaves the file URL empty, which silently drops the+// attachment the same way a failed download does.+func TestNotionFileUploadResolutionFailurePreservesRetry(t *testing.T) {+ fixture, config := newFileUploadRetryFixture(t)+ connector := NewConnector()+ previous := buildCursor(map[string]time.Time{"page": time.Date(2026, 10, 2, 10, 0, 0, 0, time.UTC)})++ items, next, err := connector.FetchIncremental(context.Background(), config, previous)+ require.Error(t, err, "an unresolved file_upload must fail the round")+ require.Nil(t, items)+ require.Nil(t, next)++ items, next, err = connector.FetchIncremental(context.Background(), config, previous)+ require.Error(t, err, "the second round must retry the page whose file_upload is unresolved")+ require.Nil(t, items)+ require.Nil(t, next)+ require.GreaterOrEqual(t, fixture.blockFetches.Load(), int32(2))+}diff --git a/internal/datasource/connector/notion/connector.go b/internal/datasource/connector/notion/connector.goindex 77da8cf55..c7d7ed04a 100644--- a/internal/datasource/connector/notion/connector.go+++ b/internal/datasource/connector/notion/connector.go@@ -9,6 +9,7 @@ import ("strings""time"+ "github.com/Tencent/WeKnora/internal/datasource""github.com/Tencent/WeKnora/internal/logger""github.com/Tencent/WeKnora/internal/types")@@ -379,7 +380,9 @@ func (c *Connector) fetchPage(return nil, false, fmt.Errorf("get blocks for notion page %s: %w", page.ID, err)}- resolveFileUploads(ctx, client, blocks)+ if err := resolveFileUploads(ctx, client, blocks); err != nil {+ return nil, false, fmt.Errorf("resolve file uploads for notion page %s: %w", page.ID, err)+ }markdown, attachmentList := BlocksToMarkdown(blocks)// A successfully cleared page still exists. Keep its title as content so@@ -425,8 +428,11 @@ func (c *Connector) fetchPage(}data, err := client.DownloadFile(ctx, att.URL)if err != nil {- logger.Warnf(ctx, "[Notion] failed to download attachment %s: %v", att.FileName, err)- continue+ // The page cannot be stored completely: acknowledging its edit time+ // here would drop the attachment until a full re-sync. Abort the+ // round like a block fetch failure so the page is retried.+ return nil, false, fmt.Errorf("%w: download attachment %s for notion page %s: %w",+ datasource.ErrFetchFailed, att.FileName, page.ID, err)}items = append(items, types.FetchedItem{ExternalID: fmt.Sprintf("%s:%s", page.ID, att.FileName),@@ -632,7 +638,12 @@ func (c *Connector) buildRecordItem(ctx context.Context, client *notionClient, rif err != nil {logger.Warnf(ctx, "[Notion] failed to get blocks for record %s: %v", record.ID, err)} else if len(blocks) > 0 {- resolveFileUploads(ctx, client, blocks)+ // Database records are ingested through their properties: their block+ // files are not downloaded as attachments, so resolution here stays+ // best-effort (matching the GetBlockChildrenAll handling above).+ if err := resolveFileUploads(ctx, client, blocks); err != nil {+ logger.Warnf(ctx, "[Notion] failed to resolve file uploads for record %s: %v", record.ID, err)+ }markdown, _ := BlocksToMarkdown(blocks)if strings.TrimSpace(markdown) != "" {content.WriteString("\n" + markdown)@@ -720,7 +731,9 @@ func (c *Connector) buildDatabaseItem(ctx context.Context, client *notionClient,// Fetch blocks for the record, in case it has page contentblocks, err := client.GetBlockChildrenAll(ctx, record.ID)if err == nil && len(blocks) > 0 {- resolveFileUploads(ctx, client, blocks)+ if resolveErr := resolveFileUploads(ctx, client, blocks); resolveErr != nil {+ logger.Warnf(ctx, "[Notion] failed to resolve file uploads for record %s: %v", record.ID, resolveErr)+ }markdown, _ := BlocksToMarkdown(blocks)if strings.TrimSpace(markdown) != "" {extraContent.WriteString("\n## " + recordTitle + " 内容\n\n" + markdown + "\n")@@ -827,7 +840,9 @@ func (c *Connector) discoverAllResources(// resolveFileUploads walks the block tree and resolves file_upload type files// by re-fetching the block to get a temporary download URL.// This mutates blocks in place, replacing file_upload RawContent with resolved URLs.-func resolveFileUploads(ctx context.Context, client *notionClient, blocks []notionBlock) {+// A resolution failure is returned: an unresolved file_upload leaves the file URL+// empty, which silently drops the file the same way a failed download does.+func resolveFileUploads(ctx context.Context, client *notionClient, blocks []notionBlock) error {for i := range blocks {if isFileBlock(blocks[i].Type) && blocks[i].RawContent != nil {var file notionFile@@ -835,16 +850,19 @@ func resolveFileUploads(ctx context.Context, client *notionClient, blocks []notiif file.GetFileUploadID() != "" {resolved, err := client.ResolveBlock(ctx, blocks[i].ID)if err != nil {- logger.Warnf(ctx, "[Notion] failed to resolve file_upload in block %s: %v", blocks[i].ID, err)- continue+ return fmt.Errorf("%w: resolve file upload in block %s: %w",+ datasource.ErrFetchFailed, blocks[i].ID, err)}blocks[i].RawContent = resolved.RawContent}}if len(blocks[i].Children) > 0 {- resolveFileUploads(ctx, client, blocks[i].Children)+ if err := resolveFileUploads(ctx, client, blocks[i].Children); err != nil {+ return err+ }}}+ return nil}func isFileBlock(blockType string) bool {