RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(syncer): bound Jira/Confluence incremental windows with an exclusive next-minute upper bound so updates in the final minute are not lost (#21630)
ciela54d2b0c1798
diff --git a/internal/syncer/connector/confluence.go b/internal/syncer/connector/confluence.goindex 9d665b04b7..d60563dea8 100644--- a/internal/syncer/connector/confluence.go+++ b/internal/syncer/connector/confluence.go@@ -515,7 +515,12 @@ func (c *ConfluenceConnector) pageCQL(start *time.Time, end time.Time, fromBeginquery += " and lastmodified >= '" + confluenceCQLTime(*start) + "'"}if !end.IsZero() {- query += " and lastmodified <= '" + confluenceCQLTime(end) + "'"+ // CQL timestamps carry minute granularity, so an inclusive `<=` on the+ // truncated end drops everything updated in the rest of that minute and+ // the next window's lower bound skips it for good. Bound with an+ // exclusive comparison against the next minute tick; inConfluenceWindow+ // drops the rows that lie past the window end.+ query += " and lastmodified < '" + confluenceCQLTime(end.Add(time.Minute)) + "'"}return query + " order by lastmodified asc"}@@ -526,7 +531,7 @@ func (c *ConfluenceConnector) attachmentCQL(pageID string, start *time.Time, endquery += " and lastmodified >= '" + confluenceCQLTime(*start) + "'"}if !end.IsZero() {- query += " and lastmodified <= '" + confluenceCQLTime(end) + "'"+ query += " and lastmodified < '" + confluenceCQLTime(end.Add(time.Minute)) + "'"}return query + " order by lastmodified asc"}diff --git a/internal/syncer/connector/confluence_test.go b/internal/syncer/connector/confluence_test.goindex a394bbbb3b..3757080fe4 100644--- a/internal/syncer/connector/confluence_test.go+++ b/internal/syncer/connector/confluence_test.go@@ -346,6 +346,31 @@ func TestConfluenceCQLQuoteEscapesBackslashes(t *testing.T) {}}+// TestConfluenceCQLUpperBoundCoversWholeMinute pins the exclusive upper bound for+// pages and attachments. CQL cannot express the seconds of the window end, so an+// inclusive `<=` on the truncated value drops the rest of that minute and the+// next window's lower bound skips those pages and attachments forever.+func TestConfluenceCQLUpperBoundCoversWholeMinute(t *testing.T) {+ connector := newTestConfluenceConnector(t, "https://example.atlassian.net/wiki", map[string]any{+ "index_mode": "page",+ "page_id": "123",+ })+ end := mustTime(t, "2026-01-04T00:00:30Z")++ cases := map[string]string{+ "page": connector.pageCQL(nil, end, false),+ "attachment": connector.attachmentCQL("123", nil, end, false),+ }+ for name, cql := range cases {+ if !strings.Contains(cql, `lastmodified < '2026-01-04 00:01'`) {+ t.Fatalf("%s CQL = %q, want an exclusive bound at the next minute tick", name, cql)+ }+ if strings.Contains(cql, `lastmodified <=`) {+ t.Fatalf("%s CQL = %q, want no inclusive upper bound", name, cql)+ }+ }+}+func TestConfluenceSemanticIdentifierAlwaysUsesFullPath(t *testing.T) {// Every identifier carries the full hierarchical path: space, ancestors, title.got := confluenceSemanticIdentifier("Engineering", []string{"Root"}, "Page")diff --git a/internal/syncer/connector/jira.go b/internal/syncer/connector/jira.goindex 71a23407db..669e3a2d34 100644--- a/internal/syncer/connector/jira.go+++ b/internal/syncer/connector/jira.go@@ -238,7 +238,7 @@ func (c *JiraConnector) buildJQL(start *time.Time, end time.Time) string {clauses = append(clauses, fmt.Sprintf("updated >= %q", c.formatJQLTime(adjusted)))}if !end.IsZero() {- clauses = append(clauses, fmt.Sprintf("updated <= %q", c.formatJQLTime(end)))+ clauses = append(clauses, fmt.Sprintf("updated < %q", c.formatJQLTime(end.Add(time.Minute))))}jql := strings.Join(clauses, " AND ")if orderBy == "" {@@ -638,6 +638,9 @@ func (s *jiraSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {if s.windowStart != nil && !s.fromBeginning && !updatedAt.IsZero() && !updatedAt.After(*s.windowStart) {continue}+ if !s.windowEnd.IsZero() && !updatedAt.IsZero() && updatedAt.After(s.windowEnd) {+ continue+ }doc, ok := s.connector.issueDocument(issue)if ok {documents = append(documents, doc)diff --git a/internal/syncer/connector/jira_test.go b/internal/syncer/connector/jira_test.goindex 3a2da7b4fa..a2596cf5b5 100644--- a/internal/syncer/connector/jira_test.go+++ b/internal/syncer/connector/jira_test.go@@ -227,12 +227,92 @@ func TestJiraConnectorBuildJQLMovesUserOrderAfterFilters(t *testing.T) {}got := connector.buildJQL(&start, end)- want := `(status = "Done") AND updated >= "2026-01-02 12:00" AND updated <= "2026-01-04 00:00" ORDER BY priority DESC, updated ASC`+ want := `(status = "Done") AND updated >= "2026-01-02 12:00" AND updated < "2026-01-04 00:01" ORDER BY priority DESC, updated ASC`if got != want {t.Fatalf("buildJQL() = %q, want %q", got, want)}}+// TestJiraConnectorBuildJQLUpperBoundCoversWholeMinute pins the exclusive upper+// bound. JQL cannot express the seconds of the window end, so an inclusive `<=`+// on the truncated value drops the rest of that minute and the next window's+// lower bound skips those issues forever.+func TestJiraConnectorBuildJQLUpperBoundCoversWholeMinute(t *testing.T) {+ end := mustTime(t, "2026-01-04T00:00:30Z")+ connector := &JiraConnector{+ jqlQuery: `status = "Done"`,+ timezoneOffset: 0,+ }++ got := connector.buildJQL(nil, end)+ if !strings.Contains(got, `updated < "2026-01-04 00:01"`) {+ t.Fatalf("buildJQL() = %q, want an exclusive bound at the next minute tick", got)+ }+ if strings.Contains(got, `updated <=`) {+ t.Fatalf("buildJQL() = %q, want no inclusive upper bound", got)+ }+}++// TestJiraSyncSessionDropsIssuesPastWindowEnd covers the client half: the JQL+// upper bound spans the whole minute holding the window end, so issues updated+// after that end come back from the server and must be dropped locally. The next+// window re-queries them from its own lower bound.+func TestJiraSyncSessionDropsIssuesPastWindowEnd(t *testing.T) {+ withConnectorLoopbackTestHook(t)+ var server *httptest.Server+ mux := http.NewServeMux()+ mux.HandleFunc("/rest/api/3/project/RAG", func(w http.ResponseWriter, r *http.Request) {+ jiraWriteJSON(t, w, map[string]any{"key": "RAG"})+ })+ mux.HandleFunc("/rest/api/3/search/jql", func(w http.ResponseWriter, r *http.Request) {+ jiraWriteJSON(t, w, map[string]any{"issues": []map[string]any{+ {"id": "10000"},+ {"id": "10001"},+ }})+ })+ mux.HandleFunc("/rest/api/3/issue/bulkfetch", func(w http.ResponseWriter, r *http.Request) {+ inWindow := jiraFixtureIssue(server.URL)+ inWindow["id"] = "10000"+ inWindow["key"] = "RAG-7"+ inWindow["fields"].(map[string]any)["updated"] = "2026-01-03T10:00:00.000+0000"+ inWindow["fields"].(map[string]any)["attachment"] = []map[string]any{}++ pastWindow := jiraFixtureIssue(server.URL)+ pastWindow["id"] = "10001"+ pastWindow["key"] = "RAG-8"+ pastWindow["fields"].(map[string]any)["updated"] = "2026-01-03T10:00:45.000+0000"+ pastWindow["fields"].(map[string]any)["attachment"] = []map[string]any{}++ jiraWriteJSON(t, w, map[string]any{"issues": []map[string]any{inWindow, pastWindow}})+ })+ server = httptest.NewServer(mux)+ defer server.Close()++ connector := mustJiraResumeConnector(t, server.URL)+ start := mustTime(t, "2026-01-03T09:59:00Z")+ session, err := connector.OpenSync(context.Background(), SyncRequest{+ WindowStart: &start,+ WindowEnd: mustTime(t, "2026-01-03T10:00:30Z"),+ })+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }++ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch failed: %v", err)+ }+ if len(batch.Documents) != 1 {+ t.Fatalf("documents = %d, want 1 (only the issue updated at or before the window end)", len(batch.Documents))+ }+ if got := batch.Documents[0].SourceID; got != server.URL+"/browse/RAG-7" {+ t.Fatalf("document source = %q, want the in-window issue", got)+ }+ if _, err := session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("second NextBatch = %v, want io.EOF", err)+ }+}+func TestJiraConnectorBuildJQLKeepsDefaultOrder(t *testing.T) {connector := &JiraConnector{jqlQuery: `status = "Done"`,