RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(syncer): keep Confluence and Jira rows inside the watermark minute (#21577)
CCLUCKY3f8aab31b10a
diff --git a/internal/syncer/connector/confluence.go b/internal/syncer/connector/confluence.goindex d60563dea8..ccc81520b2 100644--- a/internal/syncer/connector/confluence.go+++ b/internal/syncer/connector/confluence.go@@ -511,8 +511,10 @@ func (c *ConfluenceConnector) basePageCQL() string {func (c *ConfluenceConnector) pageCQL(start *time.Time, end time.Time, fromBeginning bool) string {query := c.basePageCQL()- if !fromBeginning && start != nil && !start.IsZero() {- query += " and lastmodified >= '" + confluenceCQLTime(*start) + "'"+ if !fromBeginning {+ if lower := confluenceQueryLowerBound(start); lower != nil {+ query += " and lastmodified >= '" + confluenceCQLTime(*lower) + "'"+ }}if !end.IsZero() {// CQL timestamps carry minute granularity, so an inclusive `<=` on the@@ -527,8 +529,10 @@ func (c *ConfluenceConnector) pageCQL(start *time.Time, end time.Time, fromBeginfunc (c *ConfluenceConnector) attachmentCQL(pageID string, start *time.Time, end time.Time, fromBeginning bool) string {query := fmt.Sprintf("type=attachment and container='%s'", confluenceCQLQuote(pageID))- if !fromBeginning && start != nil && !start.IsZero() {- query += " and lastmodified >= '" + confluenceCQLTime(*start) + "'"+ if !fromBeginning {+ if lower := confluenceQueryLowerBound(start); lower != nil {+ query += " and lastmodified >= '" + confluenceCQLTime(*lower) + "'"+ }}if !end.IsZero() {query += " and lastmodified < '" + confluenceCQLTime(end.Add(time.Minute)) + "'"@@ -832,12 +836,26 @@ func inConfluenceWindow(updatedAt time.Time, start *time.Time, end time.Time) boif !end.IsZero() && updatedAt.After(end) {return false}- if start != nil && !updatedAt.After(*start) {+ if lower := confluenceQueryLowerBound(start); lower != nil && updatedAt.Before(*lower) {return false}return true}+// confluenceQueryLowerBound is the `lastmodified >=` bound both CQL builders+// emit. CQL only accepts minute precision, so a waterline inside a minute is+// queried from the start of that minute. Page filtering has to use the same+// bound: filtering at the unbuffered start drops rows the server already+// returned, and since the previous run's upper bound is the very same minute,+// a page updated between floor(start) and start is covered by neither run.+func confluenceQueryLowerBound(start *time.Time) *time.Time {+ if start == nil || start.IsZero() {+ return nil+ }+ floor := start.UTC().Truncate(time.Minute)+ return &floor+}+func parseConfluenceTime(value string) time.Time {value = strings.TrimSpace(value)if value == "" {diff --git a/internal/syncer/connector/confluence_test.go b/internal/syncer/connector/confluence_test.goindex 3757080fe4..336fdae430 100644--- a/internal/syncer/connector/confluence_test.go+++ b/internal/syncer/connector/confluence_test.go@@ -387,3 +387,23 @@ func TestConfluenceSemanticIdentifierAlwaysUsesFullPath(t *testing.T) {t.Fatalf("untitled identifier = %q, want Space / Untitled", unnamed)}}++func TestConfluenceWindowKeepsWhatTheCQLAlreadyReturned(t *testing.T) {+ // CQL accepts minute precision only, so pageCQL asks for everything from the+ // start of the waterline minute. The previous run's upper bound is the same+ // minute, so a page updated just before the waterline is covered by neither+ // run unless filtering accepts the same lower bound the query sent.+ start := mustTime(t, "2026-10-27T10:05:30Z")+ end := mustTime(t, "2026-10-27T10:06:00Z")+ if got := confluenceCQLTime(start); got != "2026-10-27 10:05" {+ t.Fatalf("confluenceCQLTime(%v) = %q, want minute precision", start, got)+ }+ inside := mustTime(t, "2026-10-27T10:05:10Z")+ if !inConfluenceWindow(inside, &start, end) {+ t.Fatalf("page updated at %v was dropped although the CQL lower bound covers it", inside)+ }+ before := mustTime(t, "2026-10-27T10:04:59Z")+ if inConfluenceWindow(before, &start, end) {+ t.Fatalf("page updated at %v belongs to the previous window", before)+ }+}diff --git a/internal/syncer/connector/jira.go b/internal/syncer/connector/jira.goindex 669e3a2d34..72202605d5 100644--- a/internal/syncer/connector/jira.go+++ b/internal/syncer/connector/jira.go@@ -227,15 +227,8 @@ func (c *JiraConnector) buildJQL(start *time.Time, end time.Time) string {sort.Strings(labels)clauses = append(clauses, "labels NOT IN ("+strings.Join(labels, ", ")+")")}- if start != nil {- adjusted := *start- if c.timeBuffer > 0 {- adjusted = adjusted.Add(-c.timeBuffer)- if adjusted.Before(time.Unix(0, 0)) {- adjusted = time.Unix(0, 0).UTC()- }- }- clauses = append(clauses, fmt.Sprintf("updated >= %q", c.formatJQLTime(adjusted)))+ if lower := c.jiraQueryLowerBound(start); lower != nil {+ clauses = append(clauses, fmt.Sprintf("updated >= %q", c.formatJQLTime(*lower)))}if !end.IsZero() {clauses = append(clauses, fmt.Sprintf("updated < %q", c.formatJQLTime(end.Add(time.Minute))))@@ -255,6 +248,27 @@ func (c *JiraConnector) formatJQLTime(t time.Time) string {return t.UTC().In(time.FixedZone("jira", offsetSeconds)).Format("2006-01-02 15:04")}+// jiraQueryLowerBound is the `updated >=` bound buildJQL sends, and the bound+// page filtering must apply. JQL accepts minute precision, so the value is+// floored to the minute. Filtering at the unbuffered start instead would make+// the time buffer widen only the server response and then throw the extra rows+// away; because the previous run's upper bound is floored the same way, an+// issue updated inside the waterline minute would be covered by neither run.+func (c *JiraConnector) jiraQueryLowerBound(start *time.Time) *time.Time {+ if start == nil || start.IsZero() {+ return nil+ }+ adjusted := start.UTC()+ if c.timeBuffer > 0 {+ adjusted = adjusted.Add(-c.timeBuffer)+ if adjusted.Before(time.Unix(0, 0)) {+ adjusted = time.Unix(0, 0).UTC()+ }+ }+ floor := adjusted.Truncate(time.Minute)+ return &floor+}+func splitJiraOrderBy(query string) (string, string) {cleaned := strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(query), ";"))orderStart := trailingJiraOrderByIndex(cleaned)@@ -635,8 +649,10 @@ func (s *jiraSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {continue}updatedAt := issue.Fields.Updated.Time()- if s.windowStart != nil && !s.fromBeginning && !updatedAt.IsZero() && !updatedAt.After(*s.windowStart) {- continue+ if !s.fromBeginning && !updatedAt.IsZero() {+ if lower := s.connector.jiraQueryLowerBound(s.windowStart); lower != nil && updatedAt.Before(*lower) {+ continue+ }}if !s.windowEnd.IsZero() && !updatedAt.IsZero() && updatedAt.After(s.windowEnd) {continuediff --git a/internal/syncer/connector/jira_test.go b/internal/syncer/connector/jira_test.goindex a2596cf5b5..7f22774029 100644--- a/internal/syncer/connector/jira_test.go+++ b/internal/syncer/connector/jira_test.go@@ -430,7 +430,16 @@ func TestJiraConnectorDownloadKeepsAuthorizationOnSameOriginRedirect(t *testing.}}+const jiraFixtureUpdatedAt = "2026-01-03T10:00:00.000+0000"+func jiraFixtureServer(t *testing.T) *httptest.Server {+ t.Helper()+ return jiraFixtureServerWithUpdated(t, jiraFixtureUpdatedAt, "")+}++// jiraFixtureServerWithUpdated serves the issue regardless of the JQL filter, so+// wantJQL lets a test pin the lower bound the connector actually requested.+func jiraFixtureServerWithUpdated(t *testing.T, updated, wantJQL string) *httptest.Server {t.Helper()withConnectorLoopbackTestHook(t)mux := http.NewServeMux()@@ -438,17 +447,22 @@ func jiraFixtureServer(t *testing.T) *httptest.Server {jiraWriteJSON(t, w, map[string]any{"key": "RAG"})})mux.HandleFunc("/rest/api/3/search/jql", func(w http.ResponseWriter, r *http.Request) {- if got := r.URL.Query().Get("jql"); !strings.Contains(got, `project = "RAG"`) {+ got := r.URL.Query().Get("jql")+ if !strings.Contains(got, `project = "RAG"`) {t.Errorf("jql = %q", got)return}+ if wantJQL != "" && !strings.Contains(got, wantJQL) {+ t.Errorf("jql = %q, want it to contain %q", got, wantJQL)+ return+ }jiraWriteJSON(t, w, map[string]any{"issues": []map[string]any{{"id": "10000"}},})})var server *httptest.Servermux.HandleFunc("/rest/api/3/issue/bulkfetch", func(w http.ResponseWriter, r *http.Request) {- jiraWriteJSON(t, w, map[string]any{"issues": []map[string]any{jiraFixtureIssue(server.URL)}})+ jiraWriteJSON(t, w, map[string]any{"issues": []map[string]any{jiraFixtureIssue(server.URL, updated)}})})mux.HandleFunc("/attachment/10001", func(w http.ResponseWriter, r *http.Request) {_, _ = w.Write([]byte("attachment body"))@@ -471,7 +485,7 @@ func jiraResumeFixtureServer(t *testing.T, keys []map[string]any) *httptest.Servmux.HandleFunc("/rest/api/3/issue/bulkfetch", func(w http.ResponseWriter, r *http.Request) {issues := make([]map[string]any, 0, len(keys))for _, meta := range keys {- issue := jiraFixtureIssue(server.URL)+ issue := jiraFixtureIssue(server.URL, jiraFixtureUpdatedAt)issue["id"] = meta["id"]issue["key"] = meta["key"]issues = append(issues, issue)@@ -482,9 +496,9 @@ func jiraResumeFixtureServer(t *testing.T, keys []map[string]any) *httptest.Servreturn server}-func mustJiraResumeConnector(t *testing.T, baseURL string) *JiraConnector {+func mustJiraResumeConnector(t *testing.T, baseURL string, overrides ...map[string]any) *JiraConnector {t.Helper()- connector, err := NewJiraConnector(map[string]any{+ config := map[string]any{"base_url": baseURL,"project_key": "RAG","batch_size": 10,@@ -493,21 +507,27 @@ func mustJiraResumeConnector(t *testing.T, baseURL string) *JiraConnector {"jira_api_token": "token","rest_api_version": "3",},- })+ }+ for _, override := range overrides {+ for key, value := range override {+ config[key] = value+ }+ }+ connector, err := NewJiraConnector(config)if err != nil {t.Fatalf("NewJiraConnector failed: %v", err)}return connector}-func jiraFixtureIssue(baseURL string) map[string]any {+func jiraFixtureIssue(baseURL, updated string) map[string]any {return map[string]any{"id": "10000","key": "RAG-7","fields": map[string]any{"summary": "Implement Jira sync","description": "Implement Jira sync in Go",- "updated": "2026-01-03T10:00:00.000+0000",+ "updated": updated,"created": "2026-01-02T10:00:00.000+0000","status": map[string]any{"name": "Open"},"priority": map[string]any{"name": "High"},@@ -550,3 +570,55 @@ func TestJiraTimeParsesOffset(t *testing.T) {t.Fatalf("time = %s, want %s", got, want)}}++func TestJiraSyncKeepsIssuesInsideTheWatermarkMinute(t *testing.T) {+ // The fixture issue was updated at 2026-01-03T10:00:00Z. A waterline at+ // 10:00:20Z asks JQL for `updated >= "2026-01-03 09:59"` (time buffer plus+ // minute precision); the page filter must accept that same span, or the+ // buffer only widens the response and the issue is dropped by both runs.+ server := jiraFixtureServer(t)+ defer server.Close()++ connector := mustJiraResumeConnector(t, server.URL)+ start := mustTime(t, "2026-01-03T10:00:20Z")+ end := mustTime(t, "2026-01-03T11:00:00Z")+ session, err := connector.OpenSync(context.Background(), SyncRequest{WindowStart: &start, WindowEnd: end})+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ defer func() { _ = session.Close() }()++ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch failed: %v", err)+ }+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != server.URL+"/browse/RAG-7" {+ t.Fatalf("documents = %#v, want the issue updated at 10:00:00Z", batch.Documents)+ }+}++func TestJiraSyncHonoursConfiguredTimeBuffer(t *testing.T) {+ // The issue was updated at 09:58:10Z. A configured 120s buffer — twice the+ // default — moves the 10:00:20Z watermark back to 09:58:20, floored to+ // 09:58, so both the requested JQL bound and the page filter must keep the+ // row. With the 60s default the bound would be 09:59 and the row dropped.+ server := jiraFixtureServerWithUpdated(t, "2026-01-03T09:58:10.000+0000", `updated >= "2026-01-03 09:58"`)+ defer server.Close()++ connector := mustJiraResumeConnector(t, server.URL, map[string]any{"time_buffer_seconds": 120})+ start := mustTime(t, "2026-01-03T10:00:20Z")+ end := mustTime(t, "2026-01-03T11:00:00Z")+ session, err := connector.OpenSync(context.Background(), SyncRequest{WindowStart: &start, WindowEnd: end})+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ defer func() { _ = session.Close() }()++ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch failed: %v", err)+ }+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != server.URL+"/browse/RAG-7" {+ t.Fatalf("documents = %#v, want the issue updated at 09:58:10Z", batch.Documents)+ }+}