Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(syncer): anchor sitemap PDF-pass checkpoints and declare regex dependency (#19383)

mkaaad14fa5597f1f1
diff --git a/internal/syncer/connector/sitemap.go b/internal/syncer/connector/sitemap.go
index 3109eec854..14cfa8572a 100644
--- a/internal/syncer/connector/sitemap.go
+++ b/internal/syncer/connector/sitemap.go
@@ -522,9 +522,11 @@ func (s *sitemapSyncSession) Close() error {
// applyResume advances past the last committed sitemap page when retrying a task.
//
-// Only sitemap-listed URLs can anchor a resume: PDFs discovered inside pages are
-// not known before those pages are fetched, so a checkpoint taken during the
-// PDF pass anchors on the last sitemap page instead and the PDF pass replays.
+// Only sitemap-listed URLs can anchor a partial resume. A checkpoint taken
+// during the PDF pass anchors on the PDF itself (see sitemapSyncCheckpoint);
+// that anchor is not a sitemap-listed URL, so ErrSyncResumeInvalid is returned
+// here and the runner restarts the whole window. Skipping past the parent page
+// instead would silently drop any of its PDFs that were not committed yet.
func (s *sitemapSyncSession) applyResume(checkpoint *SyncCheckpoint) error {
if checkpoint == nil {
return nil
@@ -543,17 +545,18 @@ func (s *sitemapSyncSession) applyResume(checkpoint *SyncCheckpoint) error {
}
// sitemapSyncCheckpoint returns a resume point after a committed document.
+//
+// The anchor is the document's own SourceID, never the parent page. A
+// checkpoint taken during the PDF pass therefore anchors on the PDF itself;
+// that anchor is not a sitemap-listed URL, so applyResume reports
+// ErrSyncResumeInvalid and the runner restarts the window instead of skipping
+// the parent page and silently dropping the PDFs that had not been committed
+// yet (those PDFs are only known again once the parent page is re-fetched).
func sitemapSyncCheckpoint(doc SourceDocument) *SyncCheckpoint {
- // Discovered PDFs carry parent_url; anchor on the parent page so the
- // checkpoint always points at a sitemap-listed URL.
- anchor := doc.SourceID
- if parent, ok := doc.Metadata["parent_url"].(string); ok && parent != "" {
- anchor = sitemapSourceID(parent)
- }
updatedAt := doc.UpdatedAt
return &SyncCheckpoint{
- Cursor: anchor,
- SourceID: anchor,
+ Cursor: doc.SourceID,
+ SourceID: doc.SourceID,
UpdatedAt: &updatedAt,
}
}
diff --git a/internal/syncer/connector/sitemap_test.go b/internal/syncer/connector/sitemap_test.go
index fb9522a570..7bf78ad2c7 100644
--- a/internal/syncer/connector/sitemap_test.go
+++ b/internal/syncer/connector/sitemap_test.go
@@ -219,7 +219,7 @@ func TestSitemapConnectorURLFilter(t *testing.T) {
}
// TestSitemapConnectorFollowPDFLinks verifies PDF discovery, domain restriction,
-// deduplication, and the parent-anchored checkpoint of the PDF pass.
+// deduplication, and the PDF-anchored checkpoint of the PDF pass.
func TestSitemapConnectorFollowPDFLinks(t *testing.T) {
restricted := newSitemapTestConnector(t, map[string]any{"follow_pdf_links": true, "url_filter": "/new$"}, defaultSitemapFixture())
session, err := restricted.OpenSync(t.Context(), SyncRequest{FromBeginning: true})
@@ -235,9 +235,12 @@ func TestSitemapConnectorFollowPDFLinks(t *testing.T) {
if guide.Extension != ".pdf" || guide.Metadata["parent_url"] != "https://example.com/new" {
t.Fatalf("discovered pdf = %+v", guide)
}
+ // The PDF-pass checkpoint anchors on the PDF itself, not on the parent
+ // page, so a retry cannot skip the parent and drop uncommitted PDFs.
last := batches[len(batches)-1]
- if last.Checkpoint == nil || last.Checkpoint.SourceID != expectedSitemapSourceID("https://example.com/new") {
- t.Fatalf("pdf pass checkpoint = %+v, want parent page anchor", last.Checkpoint)
+ wantPDFAnchor := expectedSitemapSourceID("https://example.com/docs/manual.pdf?download=1#page=2")
+ if last.Checkpoint == nil || last.Checkpoint.SourceID != wantPDFAnchor {
+ t.Fatalf("pdf pass checkpoint = %+v, want pdf anchor %q", last.Checkpoint, wantPDFAnchor)
}
open := newSitemapTestConnector(t, map[string]any{"follow_pdf_links": "true", "restrict_pdf_to_domain": false, "url_filter": "/new$"}, defaultSitemapFixture())
@@ -314,6 +317,43 @@ func TestSitemapConnectorResume(t *testing.T) {
}
}
+// TestSitemapConnectorResumeAfterPDFPass verifies that a checkpoint taken during
+// the PDF pass anchors on the PDF itself and cannot resume past it: the anchor is
+// not a sitemap-listed URL, so OpenSync reports ErrSyncResumeInvalid and the
+// runner restarts the window. Anchoring on the parent page instead would skip the
+// parent and silently drop any of its PDFs that had not been committed yet.
+func TestSitemapConnectorResumeAfterPDFPass(t *testing.T) {
+ connector := newSitemapTestConnector(t, map[string]any{
+ "batch_size": 1,
+ "follow_pdf_links": true,
+ "url_filter": "/new$",
+ }, defaultSitemapFixture())
+ session, err := connector.OpenSync(t.Context(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ batches, documents := drainSitemapSession(t, session)
+ if len(documents) != 3 { // new page + guide.pdf + manual.pdf
+ t.Fatalf("documents = %d, want 3", len(documents))
+ }
+
+ // batches: [new], [guide.pdf], [manual.pdf]. The first PDF-pass batch
+ // checkpoint must anchor on the PDF itself.
+ pdfBatch := batches[1]
+ if pdfBatch.Checkpoint == nil {
+ t.Fatalf("pdf pass batch has no checkpoint")
+ }
+ if got, want := pdfBatch.Checkpoint.SourceID, expectedSitemapSourceID("https://example.com/docs/guide.pdf"); got != want {
+ t.Fatalf("pdf pass checkpoint = %q, want %q", got, want)
+ }
+
+ // Resuming from a PDF anchor must be rejected so the runner restarts the
+ // window instead of skipping the parent page and losing manual.pdf.
+ if _, err := connector.OpenSync(t.Context(), SyncRequest{FromBeginning: true, Resume: pdfBatch.Checkpoint}); !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("resume from pdf anchor error = %v, want ErrSyncResumeInvalid", err)
+ }
+}
+
// TestSitemapConnectorValidate verifies configuration and reachability checks.
func TestSitemapConnectorValidate(t *testing.T) {
fixture := defaultSitemapFixture()
diff --git a/pyproject.toml b/pyproject.toml
index a43d3d1b11..bc1c97ba32 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -107,6 +107,7 @@ dependencies = [
"ranx==0.3.20",
"rapidfuzz==3.14.5",
"readability-lxml>=0.8.4,<1.0.0",
+ "regex>=2025.11.3",
"replicate==0.31.0",
"reportlab>=4.4.1",
"roman-numbers==1.0.2",
@@ -205,7 +206,6 @@ test = [
"python-docx>=1.1.2",
"python-pptx>=1.0.2",
"reportlab>=4.4.1",
- "regex>=2025.11.3",
"requests>=2.32.2",
"requests-toolbelt>=1.0.0",
"pycryptodomex==3.20.0",
diff --git a/uv.lock b/uv.lock
index fcf056dd4e..54e7ee9777 100644
--- a/uv.lock
+++ b/uv.lock
@@ -8450,6 +8450,7 @@ requires-dist = [
{ name = "ranx", specifier = "==0.3.20" },
{ name = "rapidfuzz", specifier = "==3.14.5" },
{ name = "readability-lxml", specifier = ">=0.8.4,<1.0.0" },
+ { name = "regex", specifier = ">=2025.11.3" },
{ name = "replicate", specifier = "==0.31.0" },
{ name = "reportlab", specifier = ">=4.4.1" },
{ name = "roman-numbers", specifier = "==1.0.2" },
@@ -8497,7 +8498,6 @@ test = [
{ name = "pytest-xdist", specifier = ">=3.8.0" },
{ name = "python-docx", specifier = ">=1.1.2" },
{ name = "python-pptx", specifier = ">=1.0.2" },
- { name = "regex", specifier = ">=2025.11.3" },
{ name = "reportlab", specifier = ">=4.4.1" },
{ name = "requests", specifier = ">=2.32.2" },
{ name = "requests-toolbelt", specifier = ">=1.0.0" },