RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(syncer): anchor sitemap PDF-pass checkpoints and declare regex dependency (#19383)
mkaaad14fa5597f1f1
diff --git a/internal/syncer/connector/sitemap.go b/internal/syncer/connector/sitemap.goindex 3109eec854..14cfa8572a 100644--- a/internal/syncer/connector/sitemap.go+++ b/internal/syncer/connector/sitemap.go@@ -522,9 +522,11 @@ func (s *sitemapSyncSession) Close() error {// applyResume advances past the last committed sitemap page when retrying a task.//-// Only sitemap-listed URLs can anchor a resume: PDFs discovered inside pages are-// not known before those pages are fetched, so a checkpoint taken during the-// PDF pass anchors on the last sitemap page instead and the PDF pass replays.+// Only sitemap-listed URLs can anchor a partial resume. A checkpoint taken+// during the PDF pass anchors on the PDF itself (see sitemapSyncCheckpoint);+// that anchor is not a sitemap-listed URL, so ErrSyncResumeInvalid is returned+// here and the runner restarts the whole window. Skipping past the parent page+// instead would silently drop any of its PDFs that were not committed yet.func (s *sitemapSyncSession) applyResume(checkpoint *SyncCheckpoint) error {if checkpoint == nil {return nil@@ -543,17 +545,18 @@ func (s *sitemapSyncSession) applyResume(checkpoint *SyncCheckpoint) error {}// sitemapSyncCheckpoint returns a resume point after a committed document.+//+// The anchor is the document's own SourceID, never the parent page. A+// checkpoint taken during the PDF pass therefore anchors on the PDF itself;+// that anchor is not a sitemap-listed URL, so applyResume reports+// ErrSyncResumeInvalid and the runner restarts the window instead of skipping+// the parent page and silently dropping the PDFs that had not been committed+// yet (those PDFs are only known again once the parent page is re-fetched).func sitemapSyncCheckpoint(doc SourceDocument) *SyncCheckpoint {- // Discovered PDFs carry parent_url; anchor on the parent page so the- // checkpoint always points at a sitemap-listed URL.- anchor := doc.SourceID- if parent, ok := doc.Metadata["parent_url"].(string); ok && parent != "" {- anchor = sitemapSourceID(parent)- }updatedAt := doc.UpdatedAtreturn &SyncCheckpoint{- Cursor: anchor,- SourceID: anchor,+ Cursor: doc.SourceID,+ SourceID: doc.SourceID,UpdatedAt: &updatedAt,}}diff --git a/internal/syncer/connector/sitemap_test.go b/internal/syncer/connector/sitemap_test.goindex fb9522a570..7bf78ad2c7 100644--- a/internal/syncer/connector/sitemap_test.go+++ b/internal/syncer/connector/sitemap_test.go@@ -219,7 +219,7 @@ func TestSitemapConnectorURLFilter(t *testing.T) {}// TestSitemapConnectorFollowPDFLinks verifies PDF discovery, domain restriction,-// deduplication, and the parent-anchored checkpoint of the PDF pass.+// deduplication, and the PDF-anchored checkpoint of the PDF pass.func TestSitemapConnectorFollowPDFLinks(t *testing.T) {restricted := newSitemapTestConnector(t, map[string]any{"follow_pdf_links": true, "url_filter": "/new$"}, defaultSitemapFixture())session, err := restricted.OpenSync(t.Context(), SyncRequest{FromBeginning: true})@@ -235,9 +235,12 @@ func TestSitemapConnectorFollowPDFLinks(t *testing.T) {if guide.Extension != ".pdf" || guide.Metadata["parent_url"] != "https://example.com/new" {t.Fatalf("discovered pdf = %+v", guide)}+ // The PDF-pass checkpoint anchors on the PDF itself, not on the parent+ // page, so a retry cannot skip the parent and drop uncommitted PDFs.last := batches[len(batches)-1]- if last.Checkpoint == nil || last.Checkpoint.SourceID != expectedSitemapSourceID("https://example.com/new") {- t.Fatalf("pdf pass checkpoint = %+v, want parent page anchor", last.Checkpoint)+ wantPDFAnchor := expectedSitemapSourceID("https://example.com/docs/manual.pdf?download=1#page=2")+ if last.Checkpoint == nil || last.Checkpoint.SourceID != wantPDFAnchor {+ t.Fatalf("pdf pass checkpoint = %+v, want pdf anchor %q", last.Checkpoint, wantPDFAnchor)}open := newSitemapTestConnector(t, map[string]any{"follow_pdf_links": "true", "restrict_pdf_to_domain": false, "url_filter": "/new$"}, defaultSitemapFixture())@@ -314,6 +317,43 @@ func TestSitemapConnectorResume(t *testing.T) {}}+// TestSitemapConnectorResumeAfterPDFPass verifies that a checkpoint taken during+// the PDF pass anchors on the PDF itself and cannot resume past it: the anchor is+// not a sitemap-listed URL, so OpenSync reports ErrSyncResumeInvalid and the+// runner restarts the window. Anchoring on the parent page instead would skip the+// parent and silently drop any of its PDFs that had not been committed yet.+func TestSitemapConnectorResumeAfterPDFPass(t *testing.T) {+ connector := newSitemapTestConnector(t, map[string]any{+ "batch_size": 1,+ "follow_pdf_links": true,+ "url_filter": "/new$",+ }, defaultSitemapFixture())+ session, err := connector.OpenSync(t.Context(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ batches, documents := drainSitemapSession(t, session)+ if len(documents) != 3 { // new page + guide.pdf + manual.pdf+ t.Fatalf("documents = %d, want 3", len(documents))+ }++ // batches: [new], [guide.pdf], [manual.pdf]. The first PDF-pass batch+ // checkpoint must anchor on the PDF itself.+ pdfBatch := batches[1]+ if pdfBatch.Checkpoint == nil {+ t.Fatalf("pdf pass batch has no checkpoint")+ }+ if got, want := pdfBatch.Checkpoint.SourceID, expectedSitemapSourceID("https://example.com/docs/guide.pdf"); got != want {+ t.Fatalf("pdf pass checkpoint = %q, want %q", got, want)+ }++ // Resuming from a PDF anchor must be rejected so the runner restarts the+ // window instead of skipping the parent page and losing manual.pdf.+ if _, err := connector.OpenSync(t.Context(), SyncRequest{FromBeginning: true, Resume: pdfBatch.Checkpoint}); !errors.Is(err, ErrSyncResumeInvalid) {+ t.Fatalf("resume from pdf anchor error = %v, want ErrSyncResumeInvalid", err)+ }+}+// TestSitemapConnectorValidate verifies configuration and reachability checks.func TestSitemapConnectorValidate(t *testing.T) {fixture := defaultSitemapFixture()diff --git a/pyproject.toml b/pyproject.tomlindex a43d3d1b11..bc1c97ba32 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -107,6 +107,7 @@ dependencies = ["ranx==0.3.20","rapidfuzz==3.14.5","readability-lxml>=0.8.4,<1.0.0",+ "regex>=2025.11.3","replicate==0.31.0","reportlab>=4.4.1","roman-numbers==1.0.2",@@ -205,7 +206,6 @@ test = ["python-docx>=1.1.2","python-pptx>=1.0.2","reportlab>=4.4.1",- "regex>=2025.11.3","requests>=2.32.2","requests-toolbelt>=1.0.0","pycryptodomex==3.20.0",diff --git a/uv.lock b/uv.lockindex fcf056dd4e..54e7ee9777 100644--- a/uv.lock+++ b/uv.lock@@ -8450,6 +8450,7 @@ requires-dist = [{ name = "ranx", specifier = "==0.3.20" },{ name = "rapidfuzz", specifier = "==3.14.5" },{ name = "readability-lxml", specifier = ">=0.8.4,<1.0.0" },+ { name = "regex", specifier = ">=2025.11.3" },{ name = "replicate", specifier = "==0.31.0" },{ name = "reportlab", specifier = ">=4.4.1" },{ name = "roman-numbers", specifier = "==1.0.2" },@@ -8497,7 +8498,6 @@ test = [{ name = "pytest-xdist", specifier = ">=3.8.0" },{ name = "python-docx", specifier = ">=1.1.2" },{ name = "python-pptx", specifier = ">=1.0.2" },- { name = "regex", specifier = ">=2025.11.3" },{ name = "reportlab", specifier = ">=4.4.1" },{ name = "requests", specifier = ">=2.32.2" },{ name = "requests-toolbelt", specifier = ">=1.0.0" },