Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(syncer): bound Jira/Confluence incremental windows with an exclusive next-minute upper bound so updates in the final minute are not lost (#21630)

ciela54d2b0c1798
diff --git a/internal/syncer/connector/confluence.go b/internal/syncer/connector/confluence.go
index 9d665b04b7..d60563dea8 100644
--- a/internal/syncer/connector/confluence.go
+++ b/internal/syncer/connector/confluence.go
@@ -515,7 +515,12 @@ func (c *ConfluenceConnector) pageCQL(start *time.Time, end time.Time, fromBegin
query += " and lastmodified >= '" + confluenceCQLTime(*start) + "'"
}
if !end.IsZero() {
- query += " and lastmodified <= '" + confluenceCQLTime(end) + "'"
+ // CQL timestamps carry minute granularity, so an inclusive `<=` on the
+ // truncated end drops everything updated in the rest of that minute and
+ // the next window's lower bound skips it for good. Bound with an
+ // exclusive comparison against the next minute tick; inConfluenceWindow
+ // drops the rows that lie past the window end.
+ query += " and lastmodified < '" + confluenceCQLTime(end.Add(time.Minute)) + "'"
}
return query + " order by lastmodified asc"
}
@@ -526,7 +531,7 @@ func (c *ConfluenceConnector) attachmentCQL(pageID string, start *time.Time, end
query += " and lastmodified >= '" + confluenceCQLTime(*start) + "'"
}
if !end.IsZero() {
- query += " and lastmodified <= '" + confluenceCQLTime(end) + "'"
+ query += " and lastmodified < '" + confluenceCQLTime(end.Add(time.Minute)) + "'"
}
return query + " order by lastmodified asc"
}
diff --git a/internal/syncer/connector/confluence_test.go b/internal/syncer/connector/confluence_test.go
index a394bbbb3b..3757080fe4 100644
--- a/internal/syncer/connector/confluence_test.go
+++ b/internal/syncer/connector/confluence_test.go
@@ -346,6 +346,31 @@ func TestConfluenceCQLQuoteEscapesBackslashes(t *testing.T) {
}
}
+// TestConfluenceCQLUpperBoundCoversWholeMinute pins the exclusive upper bound for
+// pages and attachments. CQL cannot express the seconds of the window end, so an
+// inclusive `<=` on the truncated value drops the rest of that minute and the
+// next window's lower bound skips those pages and attachments forever.
+func TestConfluenceCQLUpperBoundCoversWholeMinute(t *testing.T) {
+ connector := newTestConfluenceConnector(t, "https://example.atlassian.net/wiki", map[string]any{
+ "index_mode": "page",
+ "page_id": "123",
+ })
+ end := mustTime(t, "2026-01-04T00:00:30Z")
+
+ cases := map[string]string{
+ "page": connector.pageCQL(nil, end, false),
+ "attachment": connector.attachmentCQL("123", nil, end, false),
+ }
+ for name, cql := range cases {
+ if !strings.Contains(cql, `lastmodified < '2026-01-04 00:01'`) {
+ t.Fatalf("%s CQL = %q, want an exclusive bound at the next minute tick", name, cql)
+ }
+ if strings.Contains(cql, `lastmodified <=`) {
+ t.Fatalf("%s CQL = %q, want no inclusive upper bound", name, cql)
+ }
+ }
+}
+
func TestConfluenceSemanticIdentifierAlwaysUsesFullPath(t *testing.T) {
// Every identifier carries the full hierarchical path: space, ancestors, title.
got := confluenceSemanticIdentifier("Engineering", []string{"Root"}, "Page")
diff --git a/internal/syncer/connector/jira.go b/internal/syncer/connector/jira.go
index 71a23407db..669e3a2d34 100644
--- a/internal/syncer/connector/jira.go
+++ b/internal/syncer/connector/jira.go
@@ -238,7 +238,7 @@ func (c *JiraConnector) buildJQL(start *time.Time, end time.Time) string {
clauses = append(clauses, fmt.Sprintf("updated >= %q", c.formatJQLTime(adjusted)))
}
if !end.IsZero() {
- clauses = append(clauses, fmt.Sprintf("updated <= %q", c.formatJQLTime(end)))
+ clauses = append(clauses, fmt.Sprintf("updated < %q", c.formatJQLTime(end.Add(time.Minute))))
}
jql := strings.Join(clauses, " AND ")
if orderBy == "" {
@@ -638,6 +638,9 @@ func (s *jiraSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {
if s.windowStart != nil && !s.fromBeginning && !updatedAt.IsZero() && !updatedAt.After(*s.windowStart) {
continue
}
+ if !s.windowEnd.IsZero() && !updatedAt.IsZero() && updatedAt.After(s.windowEnd) {
+ continue
+ }
doc, ok := s.connector.issueDocument(issue)
if ok {
documents = append(documents, doc)
diff --git a/internal/syncer/connector/jira_test.go b/internal/syncer/connector/jira_test.go
index 3a2da7b4fa..a2596cf5b5 100644
--- a/internal/syncer/connector/jira_test.go
+++ b/internal/syncer/connector/jira_test.go
@@ -227,12 +227,92 @@ func TestJiraConnectorBuildJQLMovesUserOrderAfterFilters(t *testing.T) {
}
got := connector.buildJQL(&start, end)
- want := `(status = "Done") AND updated >= "2026-01-02 12:00" AND updated <= "2026-01-04 00:00" ORDER BY priority DESC, updated ASC`
+ want := `(status = "Done") AND updated >= "2026-01-02 12:00" AND updated < "2026-01-04 00:01" ORDER BY priority DESC, updated ASC`
if got != want {
t.Fatalf("buildJQL() = %q, want %q", got, want)
}
}
+// TestJiraConnectorBuildJQLUpperBoundCoversWholeMinute pins the exclusive upper
+// bound. JQL cannot express the seconds of the window end, so an inclusive `<=`
+// on the truncated value drops the rest of that minute and the next window's
+// lower bound skips those issues forever.
+func TestJiraConnectorBuildJQLUpperBoundCoversWholeMinute(t *testing.T) {
+ end := mustTime(t, "2026-01-04T00:00:30Z")
+ connector := &JiraConnector{
+ jqlQuery: `status = "Done"`,
+ timezoneOffset: 0,
+ }
+
+ got := connector.buildJQL(nil, end)
+ if !strings.Contains(got, `updated < "2026-01-04 00:01"`) {
+ t.Fatalf("buildJQL() = %q, want an exclusive bound at the next minute tick", got)
+ }
+ if strings.Contains(got, `updated <=`) {
+ t.Fatalf("buildJQL() = %q, want no inclusive upper bound", got)
+ }
+}
+
+// TestJiraSyncSessionDropsIssuesPastWindowEnd covers the client half: the JQL
+// upper bound spans the whole minute holding the window end, so issues updated
+// after that end come back from the server and must be dropped locally. The next
+// window re-queries them from its own lower bound.
+func TestJiraSyncSessionDropsIssuesPastWindowEnd(t *testing.T) {
+ withConnectorLoopbackTestHook(t)
+ var server *httptest.Server
+ mux := http.NewServeMux()
+ mux.HandleFunc("/rest/api/3/project/RAG", func(w http.ResponseWriter, r *http.Request) {
+ jiraWriteJSON(t, w, map[string]any{"key": "RAG"})
+ })
+ mux.HandleFunc("/rest/api/3/search/jql", func(w http.ResponseWriter, r *http.Request) {
+ jiraWriteJSON(t, w, map[string]any{"issues": []map[string]any{
+ {"id": "10000"},
+ {"id": "10001"},
+ }})
+ })
+ mux.HandleFunc("/rest/api/3/issue/bulkfetch", func(w http.ResponseWriter, r *http.Request) {
+ inWindow := jiraFixtureIssue(server.URL)
+ inWindow["id"] = "10000"
+ inWindow["key"] = "RAG-7"
+ inWindow["fields"].(map[string]any)["updated"] = "2026-01-03T10:00:00.000+0000"
+ inWindow["fields"].(map[string]any)["attachment"] = []map[string]any{}
+
+ pastWindow := jiraFixtureIssue(server.URL)
+ pastWindow["id"] = "10001"
+ pastWindow["key"] = "RAG-8"
+ pastWindow["fields"].(map[string]any)["updated"] = "2026-01-03T10:00:45.000+0000"
+ pastWindow["fields"].(map[string]any)["attachment"] = []map[string]any{}
+
+ jiraWriteJSON(t, w, map[string]any{"issues": []map[string]any{inWindow, pastWindow}})
+ })
+ server = httptest.NewServer(mux)
+ defer server.Close()
+
+ connector := mustJiraResumeConnector(t, server.URL)
+ start := mustTime(t, "2026-01-03T09:59:00Z")
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ WindowStart: &start,
+ WindowEnd: mustTime(t, "2026-01-03T10:00:30Z"),
+ })
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch failed: %v", err)
+ }
+ if len(batch.Documents) != 1 {
+ t.Fatalf("documents = %d, want 1 (only the issue updated at or before the window end)", len(batch.Documents))
+ }
+ if got := batch.Documents[0].SourceID; got != server.URL+"/browse/RAG-7" {
+ t.Fatalf("document source = %q, want the in-window issue", got)
+ }
+ if _, err := session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {
+ t.Fatalf("second NextBatch = %v, want io.EOF", err)
+ }
+}
+
func TestJiraConnectorBuildJQLKeepsDefaultOrder(t *testing.T) {
connector := &JiraConnector{
jqlQuery: `status = "Done"`,