Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(syncer): keep Confluence and Jira rows inside the watermark minute (#21577)

CCLUCKY3f8aab31b10a
diff --git a/internal/syncer/connector/confluence.go b/internal/syncer/connector/confluence.go
index d60563dea8..ccc81520b2 100644
--- a/internal/syncer/connector/confluence.go
+++ b/internal/syncer/connector/confluence.go
@@ -511,8 +511,10 @@ func (c *ConfluenceConnector) basePageCQL() string {
func (c *ConfluenceConnector) pageCQL(start *time.Time, end time.Time, fromBeginning bool) string {
query := c.basePageCQL()
- if !fromBeginning && start != nil && !start.IsZero() {
- query += " and lastmodified >= '" + confluenceCQLTime(*start) + "'"
+ if !fromBeginning {
+ if lower := confluenceQueryLowerBound(start); lower != nil {
+ query += " and lastmodified >= '" + confluenceCQLTime(*lower) + "'"
+ }
}
if !end.IsZero() {
// CQL timestamps carry minute granularity, so an inclusive `<=` on the
@@ -527,8 +529,10 @@ func (c *ConfluenceConnector) pageCQL(start *time.Time, end time.Time, fromBegin
func (c *ConfluenceConnector) attachmentCQL(pageID string, start *time.Time, end time.Time, fromBeginning bool) string {
query := fmt.Sprintf("type=attachment and container='%s'", confluenceCQLQuote(pageID))
- if !fromBeginning && start != nil && !start.IsZero() {
- query += " and lastmodified >= '" + confluenceCQLTime(*start) + "'"
+ if !fromBeginning {
+ if lower := confluenceQueryLowerBound(start); lower != nil {
+ query += " and lastmodified >= '" + confluenceCQLTime(*lower) + "'"
+ }
}
if !end.IsZero() {
query += " and lastmodified < '" + confluenceCQLTime(end.Add(time.Minute)) + "'"
@@ -832,12 +836,26 @@ func inConfluenceWindow(updatedAt time.Time, start *time.Time, end time.Time) bo
if !end.IsZero() && updatedAt.After(end) {
return false
}
- if start != nil && !updatedAt.After(*start) {
+ if lower := confluenceQueryLowerBound(start); lower != nil && updatedAt.Before(*lower) {
return false
}
return true
}
+// confluenceQueryLowerBound is the `lastmodified >=` bound both CQL builders
+// emit. CQL only accepts minute precision, so a waterline inside a minute is
+// queried from the start of that minute. Page filtering has to use the same
+// bound: filtering at the unbuffered start drops rows the server already
+// returned, and since the previous run's upper bound is the very same minute,
+// a page updated between floor(start) and start is covered by neither run.
+func confluenceQueryLowerBound(start *time.Time) *time.Time {
+ if start == nil || start.IsZero() {
+ return nil
+ }
+ floor := start.UTC().Truncate(time.Minute)
+ return &floor
+}
+
func parseConfluenceTime(value string) time.Time {
value = strings.TrimSpace(value)
if value == "" {
diff --git a/internal/syncer/connector/confluence_test.go b/internal/syncer/connector/confluence_test.go
index 3757080fe4..336fdae430 100644
--- a/internal/syncer/connector/confluence_test.go
+++ b/internal/syncer/connector/confluence_test.go
@@ -387,3 +387,23 @@ func TestConfluenceSemanticIdentifierAlwaysUsesFullPath(t *testing.T) {
t.Fatalf("untitled identifier = %q, want Space / Untitled", unnamed)
}
}
+
+func TestConfluenceWindowKeepsWhatTheCQLAlreadyReturned(t *testing.T) {
+ // CQL accepts minute precision only, so pageCQL asks for everything from the
+ // start of the waterline minute. The previous run's upper bound is the same
+ // minute, so a page updated just before the waterline is covered by neither
+ // run unless filtering accepts the same lower bound the query sent.
+ start := mustTime(t, "2026-10-27T10:05:30Z")
+ end := mustTime(t, "2026-10-27T10:06:00Z")
+ if got := confluenceCQLTime(start); got != "2026-10-27 10:05" {
+ t.Fatalf("confluenceCQLTime(%v) = %q, want minute precision", start, got)
+ }
+ inside := mustTime(t, "2026-10-27T10:05:10Z")
+ if !inConfluenceWindow(inside, &start, end) {
+ t.Fatalf("page updated at %v was dropped although the CQL lower bound covers it", inside)
+ }
+ before := mustTime(t, "2026-10-27T10:04:59Z")
+ if inConfluenceWindow(before, &start, end) {
+ t.Fatalf("page updated at %v belongs to the previous window", before)
+ }
+}
diff --git a/internal/syncer/connector/jira.go b/internal/syncer/connector/jira.go
index 669e3a2d34..72202605d5 100644
--- a/internal/syncer/connector/jira.go
+++ b/internal/syncer/connector/jira.go
@@ -227,15 +227,8 @@ func (c *JiraConnector) buildJQL(start *time.Time, end time.Time) string {
sort.Strings(labels)
clauses = append(clauses, "labels NOT IN ("+strings.Join(labels, ", ")+")")
}
- if start != nil {
- adjusted := *start
- if c.timeBuffer > 0 {
- adjusted = adjusted.Add(-c.timeBuffer)
- if adjusted.Before(time.Unix(0, 0)) {
- adjusted = time.Unix(0, 0).UTC()
- }
- }
- clauses = append(clauses, fmt.Sprintf("updated >= %q", c.formatJQLTime(adjusted)))
+ if lower := c.jiraQueryLowerBound(start); lower != nil {
+ clauses = append(clauses, fmt.Sprintf("updated >= %q", c.formatJQLTime(*lower)))
}
if !end.IsZero() {
clauses = append(clauses, fmt.Sprintf("updated < %q", c.formatJQLTime(end.Add(time.Minute))))
@@ -255,6 +248,27 @@ func (c *JiraConnector) formatJQLTime(t time.Time) string {
return t.UTC().In(time.FixedZone("jira", offsetSeconds)).Format("2006-01-02 15:04")
}
+// jiraQueryLowerBound is the `updated >=` bound buildJQL sends, and the bound
+// page filtering must apply. JQL accepts minute precision, so the value is
+// floored to the minute. Filtering at the unbuffered start instead would make
+// the time buffer widen only the server response and then throw the extra rows
+// away; because the previous run's upper bound is floored the same way, an
+// issue updated inside the waterline minute would be covered by neither run.
+func (c *JiraConnector) jiraQueryLowerBound(start *time.Time) *time.Time {
+ if start == nil || start.IsZero() {
+ return nil
+ }
+ adjusted := start.UTC()
+ if c.timeBuffer > 0 {
+ adjusted = adjusted.Add(-c.timeBuffer)
+ if adjusted.Before(time.Unix(0, 0)) {
+ adjusted = time.Unix(0, 0).UTC()
+ }
+ }
+ floor := adjusted.Truncate(time.Minute)
+ return &floor
+}
+
func splitJiraOrderBy(query string) (string, string) {
cleaned := strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(query), ";"))
orderStart := trailingJiraOrderByIndex(cleaned)
@@ -635,8 +649,10 @@ func (s *jiraSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {
continue
}
updatedAt := issue.Fields.Updated.Time()
- if s.windowStart != nil && !s.fromBeginning && !updatedAt.IsZero() && !updatedAt.After(*s.windowStart) {
- continue
+ if !s.fromBeginning && !updatedAt.IsZero() {
+ if lower := s.connector.jiraQueryLowerBound(s.windowStart); lower != nil && updatedAt.Before(*lower) {
+ continue
+ }
}
if !s.windowEnd.IsZero() && !updatedAt.IsZero() && updatedAt.After(s.windowEnd) {
continue
diff --git a/internal/syncer/connector/jira_test.go b/internal/syncer/connector/jira_test.go
index a2596cf5b5..7f22774029 100644
--- a/internal/syncer/connector/jira_test.go
+++ b/internal/syncer/connector/jira_test.go
@@ -430,7 +430,16 @@ func TestJiraConnectorDownloadKeepsAuthorizationOnSameOriginRedirect(t *testing.
}
}
+const jiraFixtureUpdatedAt = "2026-01-03T10:00:00.000+0000"
+
func jiraFixtureServer(t *testing.T) *httptest.Server {
+ t.Helper()
+ return jiraFixtureServerWithUpdated(t, jiraFixtureUpdatedAt, "")
+}
+
+// jiraFixtureServerWithUpdated serves the issue regardless of the JQL filter, so
+// wantJQL lets a test pin the lower bound the connector actually requested.
+func jiraFixtureServerWithUpdated(t *testing.T, updated, wantJQL string) *httptest.Server {
t.Helper()
withConnectorLoopbackTestHook(t)
mux := http.NewServeMux()
@@ -438,17 +447,22 @@ func jiraFixtureServer(t *testing.T) *httptest.Server {
jiraWriteJSON(t, w, map[string]any{"key": "RAG"})
})
mux.HandleFunc("/rest/api/3/search/jql", func(w http.ResponseWriter, r *http.Request) {
- if got := r.URL.Query().Get("jql"); !strings.Contains(got, `project = "RAG"`) {
+ got := r.URL.Query().Get("jql")
+ if !strings.Contains(got, `project = "RAG"`) {
t.Errorf("jql = %q", got)
return
}
+ if wantJQL != "" && !strings.Contains(got, wantJQL) {
+ t.Errorf("jql = %q, want it to contain %q", got, wantJQL)
+ return
+ }
jiraWriteJSON(t, w, map[string]any{
"issues": []map[string]any{{"id": "10000"}},
})
})
var server *httptest.Server
mux.HandleFunc("/rest/api/3/issue/bulkfetch", func(w http.ResponseWriter, r *http.Request) {
- jiraWriteJSON(t, w, map[string]any{"issues": []map[string]any{jiraFixtureIssue(server.URL)}})
+ jiraWriteJSON(t, w, map[string]any{"issues": []map[string]any{jiraFixtureIssue(server.URL, updated)}})
})
mux.HandleFunc("/attachment/10001", func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte("attachment body"))
@@ -471,7 +485,7 @@ func jiraResumeFixtureServer(t *testing.T, keys []map[string]any) *httptest.Serv
mux.HandleFunc("/rest/api/3/issue/bulkfetch", func(w http.ResponseWriter, r *http.Request) {
issues := make([]map[string]any, 0, len(keys))
for _, meta := range keys {
- issue := jiraFixtureIssue(server.URL)
+ issue := jiraFixtureIssue(server.URL, jiraFixtureUpdatedAt)
issue["id"] = meta["id"]
issue["key"] = meta["key"]
issues = append(issues, issue)
@@ -482,9 +496,9 @@ func jiraResumeFixtureServer(t *testing.T, keys []map[string]any) *httptest.Serv
return server
}
-func mustJiraResumeConnector(t *testing.T, baseURL string) *JiraConnector {
+func mustJiraResumeConnector(t *testing.T, baseURL string, overrides ...map[string]any) *JiraConnector {
t.Helper()
- connector, err := NewJiraConnector(map[string]any{
+ config := map[string]any{
"base_url": baseURL,
"project_key": "RAG",
"batch_size": 10,
@@ -493,21 +507,27 @@ func mustJiraResumeConnector(t *testing.T, baseURL string) *JiraConnector {
"jira_api_token": "token",
"rest_api_version": "3",
},
- })
+ }
+ for _, override := range overrides {
+ for key, value := range override {
+ config[key] = value
+ }
+ }
+ connector, err := NewJiraConnector(config)
if err != nil {
t.Fatalf("NewJiraConnector failed: %v", err)
}
return connector
}
-func jiraFixtureIssue(baseURL string) map[string]any {
+func jiraFixtureIssue(baseURL, updated string) map[string]any {
return map[string]any{
"id": "10000",
"key": "RAG-7",
"fields": map[string]any{
"summary": "Implement Jira sync",
"description": "Implement Jira sync in Go",
- "updated": "2026-01-03T10:00:00.000+0000",
+ "updated": updated,
"created": "2026-01-02T10:00:00.000+0000",
"status": map[string]any{"name": "Open"},
"priority": map[string]any{"name": "High"},
@@ -550,3 +570,55 @@ func TestJiraTimeParsesOffset(t *testing.T) {
t.Fatalf("time = %s, want %s", got, want)
}
}
+
+func TestJiraSyncKeepsIssuesInsideTheWatermarkMinute(t *testing.T) {
+ // The fixture issue was updated at 2026-01-03T10:00:00Z. A waterline at
+ // 10:00:20Z asks JQL for `updated >= "2026-01-03 09:59"` (time buffer plus
+ // minute precision); the page filter must accept that same span, or the
+ // buffer only widens the response and the issue is dropped by both runs.
+ server := jiraFixtureServer(t)
+ defer server.Close()
+
+ connector := mustJiraResumeConnector(t, server.URL)
+ start := mustTime(t, "2026-01-03T10:00:20Z")
+ end := mustTime(t, "2026-01-03T11:00:00Z")
+ session, err := connector.OpenSync(context.Background(), SyncRequest{WindowStart: &start, WindowEnd: end})
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ defer func() { _ = session.Close() }()
+
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch failed: %v", err)
+ }
+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != server.URL+"/browse/RAG-7" {
+ t.Fatalf("documents = %#v, want the issue updated at 10:00:00Z", batch.Documents)
+ }
+}
+
+func TestJiraSyncHonoursConfiguredTimeBuffer(t *testing.T) {
+ // The issue was updated at 09:58:10Z. A configured 120s buffer — twice the
+ // default — moves the 10:00:20Z watermark back to 09:58:20, floored to
+ // 09:58, so both the requested JQL bound and the page filter must keep the
+ // row. With the 60s default the bound would be 09:59 and the row dropped.
+ server := jiraFixtureServerWithUpdated(t, "2026-01-03T09:58:10.000+0000", `updated >= "2026-01-03 09:58"`)
+ defer server.Close()
+
+ connector := mustJiraResumeConnector(t, server.URL, map[string]any{"time_buffer_seconds": 120})
+ start := mustTime(t, "2026-01-03T10:00:20Z")
+ end := mustTime(t, "2026-01-03T11:00:00Z")
+ session, err := connector.OpenSync(context.Background(), SyncRequest{WindowStart: &start, WindowEnd: end})
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ defer func() { _ = session.Close() }()
+
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch failed: %v", err)
+ }
+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != server.URL+"/browse/RAG-7" {
+ t.Fatalf("documents = %#v, want the issue updated at 09:58:10Z", batch.Documents)
+ }
+}