Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat(go-syncer): add checkpoint resume for MySQL/PostgreSQL connectors (#19762)

mkaaad1ff3d961119e
diff --git a/internal/syncer/connector/models.go b/internal/syncer/connector/models.go
index 3d58ad0467..3e39ffe0b3 100644
--- a/internal/syncer/connector/models.go
+++ b/internal/syncer/connector/models.go
@@ -17,7 +17,9 @@
package connector
import (
+ "encoding/json"
"errors"
+ "fmt"
"time"
)
@@ -106,3 +108,53 @@ type SlimDocument struct {
type PruneBatch struct {
Documents []SlimDocument
}
+
+// rdbmsQuery is one base SQL query plus the identity used by resume cursors:
+// the table name in per-table mode, or "" for a single custom query.
+type rdbmsQuery struct {
+ name string
+ sql string
+}
+
+// rdbmsSyncQuery is one prepared sync query with its resume metadata.
+type rdbmsSyncQuery struct {
+ name string
+ sql string
+ ordered bool
+ fallback string
+}
+
+// rdbmsResumeCursor identifies where a MySQL/PostgreSQL sync left off: the
+// query that produced the last committed batch, the ordering key the stream
+// was sorted by (a single column, or "timestamp,id" for incremental windows),
+// and the SourceID of the last emitted row (the resume anchor). It is stored
+// JSON-encoded in SyncCheckpoint.Cursor so arbitrary table names and source
+// ids survive the round trip.
+type rdbmsResumeCursor struct {
+ Query string `json:"q"`
+ Order string `json:"o"`
+ SourceID string `json:"s"`
+}
+
+// encodeRDBMSCursor serializes a resume cursor for SyncCheckpoint.Cursor.
+func encodeRDBMSCursor(query, order, sourceID string) string {
+ raw, _ := json.Marshal(rdbmsResumeCursor{Query: query, Order: order, SourceID: sourceID})
+ return string(raw)
+}
+
+// parseRDBMSCursor decodes a resume cursor. A missing or malformed cursor is
+// treated as invalid progress so the runner restarts the task window instead
+// of guessing an offset.
+func parseRDBMSCursor(cursor string) (rdbmsResumeCursor, error) {
+ var c rdbmsResumeCursor
+ if cursor == "" {
+ return c, fmt.Errorf("rdbms sync checkpoint has no cursor: %w", ErrSyncResumeInvalid)
+ }
+ if err := json.Unmarshal([]byte(cursor), &c); err != nil {
+ return c, fmt.Errorf("rdbms sync checkpoint cursor is malformed: %w", ErrSyncResumeInvalid)
+ }
+ if c.SourceID == "" {
+ return c, fmt.Errorf("rdbms sync checkpoint has no source anchor: %w", ErrSyncResumeInvalid)
+ }
+ return c, nil
+}
diff --git a/internal/syncer/connector/mysql.go b/internal/syncer/connector/mysql.go
index 098823cfdf..171b226dd4 100644
--- a/internal/syncer/connector/mysql.go
+++ b/internal/syncer/connector/mysql.go
@@ -22,10 +22,12 @@ import (
"database/sql"
"database/sql/driver"
"encoding/hex"
+ "errors"
"fmt"
"io"
"net"
"regexp"
+ "sort"
"strconv"
"strings"
"time"
@@ -131,7 +133,26 @@ func (c *MySQLConnector) OpenSync(ctx context.Context, request SyncRequest) (Syn
return nil, err
}
queries := c.buildSyncQueries(bases, request)
- return &mysqlSyncSession{connector: c, db: db, queries: queries, batchSize: c.batchSize}, nil
+ orderColumn := c.syncOrderColumn(request)
+ session := &mysqlSyncSession{
+ connector: c,
+ db: db,
+ batchSize: c.batchSize,
+ orderColumn: orderColumn,
+ checkpointEnabled: orderColumn != "",
+ lastDocQuery: -1,
+ }
+ for _, q := range queries {
+ session.queries = append(session.queries, q.sql)
+ session.queryNames = append(session.queryNames, q.name)
+ session.orderedFlags = append(session.orderedFlags, q.ordered)
+ session.fallbackQueries = append(session.fallbackQueries, q.fallback)
+ }
+ if err := session.applyResume(request.Resume); err != nil {
+ db.Close()
+ return nil, err
+ }
+ return session, nil
}
// OpenPrune opens one complete MySQL prune snapshot session.
@@ -147,7 +168,7 @@ func (c *MySQLConnector) OpenPrune(ctx context.Context, request PruneRequest) (P
}
queries := make([]string, 0, len(bases))
for _, base := range bases {
- queries = append(queries, c.buildSlimQuery(base))
+ queries = append(queries, c.buildSlimQuery(base.sql))
}
return &mysqlPruneSession{connector: c, db: db, queries: queries, batchSize: c.batchSize}, nil
}
@@ -212,10 +233,12 @@ func mysqlPinnedDial(pinIP net.IP, port int) mysql.DialContextFunc {
}
}
-// baseQueries returns the configured query or a SELECT per table.
-func (c *MySQLConnector) baseQueries(ctx context.Context, db *sql.DB) ([]string, error) {
+// baseQueries returns the configured query or a SELECT per table. Table names
+// are sorted so the sync stream order is stable across runs and a resume
+// cursor can reliably skip already-processed tables.
+func (c *MySQLConnector) baseQueries(ctx context.Context, db *sql.DB) ([]rdbmsQuery, error) {
if c.query != "" {
- return []string{c.query}, nil
+ return []rdbmsQuery{{name: "", sql: c.query}}, nil
}
rows, err := db.QueryContext(ctx, "SHOW TABLES")
if err != nil {
@@ -233,30 +256,69 @@ func (c *MySQLConnector) baseQueries(ctx context.Context, db *sql.DB) ([]string,
if err := rows.Err(); err != nil {
return nil, err
}
- queries := make([]string, 0, len(tables))
+ sort.Strings(tables)
+ queries := make([]rdbmsQuery, 0, len(tables))
for _, table := range tables {
- queries = append(queries, fmt.Sprintf("SELECT * FROM %s", table))
+ queries = append(queries, rdbmsQuery{name: table, sql: fmt.Sprintf("SELECT * FROM %s", table)})
}
return queries, nil
}
-// buildSyncQueries applies the incremental window when a timestamp column exists.
-func (c *MySQLConnector) buildSyncQueries(bases []string, request SyncRequest) []string {
- var start, end *time.Time
- if !request.FromBeginning {
- start = request.WindowStart
- end = &request.WindowEnd
- }
- if c.timestampColumn == "" || (start == nil && end == nil) {
- return bases
- }
- queries := make([]string, 0, len(bases))
- for _, base := range bases {
- queries = append(queries, c.buildTimeFilteredQuery(base, start, end))
+// buildSyncQueries applies the incremental window and a stable ordering when
+// one is available, so a checkpoint can resume the stream from an anchor.
+// Each query carries an unordered fallback used when a custom SQL query does
+// not expose the configured ordering column.
+func (c *MySQLConnector) buildSyncQueries(bases []rdbmsQuery, request SyncRequest) []rdbmsSyncQuery {
+ queries := make([]rdbmsSyncQuery, 0, len(bases))
+ switch {
+ case !request.FromBeginning && c.timestampColumn != "":
+ start := request.WindowStart
+ end := &request.WindowEnd
+ for _, base := range bases {
+ queries = append(queries, rdbmsSyncQuery{
+ name: base.name,
+ sql: c.buildTimeFilteredOrderedQuery(base.sql, start, end),
+ ordered: true,
+ fallback: c.buildTimeFilteredQuery(base.sql, start, end),
+ })
+ }
+ case request.FromBeginning && c.idColumn != "":
+ for _, base := range bases {
+ queries = append(queries, rdbmsSyncQuery{
+ name: base.name,
+ sql: c.buildOrderedQuery(base.sql, c.idColumn),
+ ordered: true,
+ fallback: c.wrapQuery(base.sql),
+ })
+ }
+ default:
+ for _, base := range bases {
+ queries = append(queries, rdbmsSyncQuery{name: base.name, sql: base.sql})
+ }
}
return queries
}
+// syncOrderColumn returns the ordering key that makes this sync window
+// deterministic, or "" when the connector cannot checkpoint/resume the
+// stream (no stable ordering key). Incremental windows order by timestamp
+// plus id so rows sharing a timestamp still resume deterministically.
+func (c *MySQLConnector) syncOrderColumn(request SyncRequest) string {
+ switch {
+ case !request.FromBeginning && c.timestampColumn != "" && c.idColumn != "":
+ return c.timestampColumn + "," + c.idColumn
+ case request.FromBeginning && c.idColumn != "":
+ return c.idColumn
+ }
+ return ""
+}
+
+// buildOrderedQuery wraps the base query and orders it by a stable column so
+// connector sync can resume from a checkpoint.
+func (c *MySQLConnector) buildOrderedQuery(base, orderColumn string) string {
+ return c.wrapQuery(base) + " ORDER BY ragflow_src." + orderColumn + " ASC"
+}
+
// buildTimeFilteredQuery wraps the base query and appends timestamp bounds.
func (c *MySQLConnector) buildTimeFilteredQuery(base string, start, end *time.Time) string {
conditions := []string{}
@@ -273,6 +335,18 @@ func (c *MySQLConnector) buildTimeFilteredQuery(base string, start, end *time.Ti
return query
}
+// buildTimeFilteredOrderedQuery is the incremental query plus a deterministic
+// ORDER BY on the timestamp and id columns, which resume relies on. Without a
+// configured id column the order is timestamp-only and the stream is not
+// checkpointed.
+func (c *MySQLConnector) buildTimeFilteredOrderedQuery(base string, start, end *time.Time) string {
+ query := c.buildTimeFilteredQuery(base, start, end) + " ORDER BY ragflow_src." + c.timestampColumn + " ASC"
+ if c.idColumn != "" {
+ query += ", ragflow_src." + c.idColumn + " ASC"
+ }
+ return query
+}
+
// buildSlimQuery selects only the columns needed to identify documents.
func (c *MySQLConnector) buildSlimQuery(base string) string {
columns := []string{}
@@ -518,9 +592,25 @@ type mysqlSyncSession struct {
connector *MySQLConnector
db *sql.DB
queries []string
- queryIndex int
- rows *sql.Rows
- batchSize int
+ queryNames []string
+ // orderedFlags[i] reports whether queries[i] carries a stable ORDER BY.
+ orderedFlags []bool
+ // fallbackQueries[i] is the unordered variant of queries[i], used when a
+ // custom SQL query does not expose the configured ordering column.
+ fallbackQueries []string
+ queryIndex int
+ // lastDocQuery is the index of the query that produced the most recently
+ // appended document, used to checkpoint against the right query name even
+ // when later queries in the batch contributed no documents.
+ lastDocQuery int
+ rows *sql.Rows
+ batchSize int
+
+ orderColumn string
+ checkpointEnabled bool
+ orderable bool
+ resume *rdbmsResumeCursor
+ resumePending bool
}
// NextBatch returns the next MySQL document batch.
@@ -530,7 +620,7 @@ func (s *mysqlSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {
if s.rows == nil {
if s.queryIndex >= len(s.queries) {
if len(documents) == 0 {
- return SyncBatch{}, io.EOF
+ return s.endOfStream()
}
break
}
@@ -552,10 +642,17 @@ func (s *mysqlSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {
continue
}
if doc, ok := s.connector.rowToSourceDocument(row, columns); ok {
+ if !s.includeResumed(doc) {
+ continue
+ }
documents = append(documents, doc)
+ s.lastDocQuery = s.queryIndex - 1
}
}
- return SyncBatch{Documents: documents}, nil
+ if len(documents) == 0 {
+ return s.endOfStream()
+ }
+ return SyncBatch{Documents: documents, Checkpoint: s.batchCheckpoint(documents[len(documents)-1])}, nil
}
// Close closes the MySQL sync session.
@@ -564,14 +661,32 @@ func (s *mysqlSyncSession) Close() error {
return s.db.Close()
}
-// openNextQuery runs the next base query.
+// openNextQuery runs the next base query. When a custom SQL query does not
+// expose the configured ordering column (MySQL error 1054), it falls back to
+// the unordered query and stops checkpointing so the remaining stream is never
+// resumed against a non-deterministic order. A pending resume never falls back:
+// the ordering that produced the anchor is gone, so the window restarts.
func (s *mysqlSyncSession) openNextQuery(ctx context.Context) error {
- query := s.queries[s.queryIndex]
+ idx := s.queryIndex
s.queryIndex++
- rows, err := s.db.QueryContext(ctx, query)
+ rows, err := s.db.QueryContext(ctx, s.queries[idx])
if err != nil {
+ if s.orderedFlags[idx] && isMySQLUnknownColumn(err) {
+ if s.resumePending {
+ return fmt.Errorf("MySQL sync resume query lost its ordering column: %w", ErrSyncResumeInvalid)
+ }
+ s.checkpointEnabled = false
+ rows, err = s.db.QueryContext(ctx, s.fallbackQueries[idx])
+ if err != nil {
+ return fmt.Errorf("MySQL query failed: %w", err)
+ }
+ s.orderable = false
+ s.rows = rows
+ return nil
+ }
return fmt.Errorf("MySQL query failed: %w", err)
}
+ s.orderable = s.orderedFlags[idx]
s.rows = rows
return nil
}
@@ -584,6 +699,89 @@ func (s *mysqlSyncSession) closeRows() {
}
}
+// applyResume positions the session after the last committed batch. The
+// cursor's query and ordering column must still exist, otherwise the runner
+// restarts the task window.
+func (s *mysqlSyncSession) applyResume(checkpoint *SyncCheckpoint) error {
+ if checkpoint == nil {
+ return nil
+ }
+ cursor, err := parseRDBMSCursor(checkpoint.Cursor)
+ if err != nil {
+ return err
+ }
+ if s.orderColumn == "" || cursor.Order != s.orderColumn {
+ return fmt.Errorf("MySQL sync resume ordering changed from %q to %q: %w", cursor.Order, s.orderColumn, ErrSyncResumeInvalid)
+ }
+ idx := -1
+ for i, name := range s.queryNames {
+ if name == cursor.Query {
+ idx = i
+ break
+ }
+ }
+ if idx < 0 {
+ return fmt.Errorf("MySQL sync resume query %q no longer exists: %w", cursor.Query, ErrSyncResumeInvalid)
+ }
+ s.queryIndex = idx
+ s.resume = &cursor
+ s.resumePending = true
+ return nil
+}
+
+// includeResumed reports whether doc should be emitted. While a resume is
+// pending, every row before (and including) the anchor is skipped because it
+// was already committed.
+func (s *mysqlSyncSession) includeResumed(doc SourceDocument) bool {
+ if !s.resumePending {
+ return true
+ }
+ if s.resume != nil && doc.SourceID == s.resume.SourceID {
+ s.resumePending = false
+ return false
+ }
+ return false
+}
+
+// batchCheckpoint builds the checkpoint for a batch whose last row is doc.
+// Batches from a non-deterministic (unordered) query never carry a checkpoint.
+func (s *mysqlSyncSession) batchCheckpoint(doc SourceDocument) *SyncCheckpoint {
+ if !s.checkpointEnabled || !s.orderable {
+ return nil
+ }
+ queryName := ""
+ if idx := s.lastDocQuery; idx >= 0 && idx < len(s.queryNames) {
+ queryName = s.queryNames[idx]
+ }
+ updatedAt := doc.UpdatedAt
+ return &SyncCheckpoint{
+ Cursor: encodeRDBMSCursor(queryName, s.orderColumn, doc.SourceID),
+ SourceID: doc.SourceID,
+ UpdatedAt: &updatedAt,
+ }
+}
+
+// endOfStream returns io.EOF when the stream is exhausted, or
+// ErrSyncResumeInvalid when a pending resume anchor was never found.
+func (s *mysqlSyncSession) endOfStream() (SyncBatch, error) {
+ if s.resumePending {
+ anchor := ""
+ if s.resume != nil {
+ anchor = s.resume.SourceID
+ }
+ return SyncBatch{}, fmt.Errorf("MySQL resume anchor %q was not found in the current result: %w", anchor, ErrSyncResumeInvalid)
+ }
+ return SyncBatch{}, io.EOF
+}
+
+// isMySQLUnknownColumn reports whether err is MySQL error 1054 (unknown column
+// in the order/result), used to detect custom queries that do not expose the
+// configured ordering column.
+func isMySQLUnknownColumn(err error) bool {
+ var mysqlErr *mysql.MySQLError
+ return errors.As(err, &mysqlErr) && mysqlErr.Number == 1054
+}
+
type mysqlPruneSession struct {
connector *MySQLConnector
db *sql.DB
diff --git a/internal/syncer/connector/mysql_test.go b/internal/syncer/connector/mysql_test.go
index eaefb7c42d..3b582fb5ff 100644
--- a/internal/syncer/connector/mysql_test.go
+++ b/internal/syncer/connector/mysql_test.go
@@ -29,6 +29,7 @@ import (
"time"
"github.com/DATA-DOG/go-sqlmock"
+ "github.com/go-sql-driver/mysql"
)
func newFixtureMySQLConnector(t *testing.T, config map[string]any, expect func(mock sqlmock.Sqlmock)) *MySQLConnector {
@@ -71,6 +72,7 @@ func newFixtureMySQLConnector(t *testing.T, config map[string]any, expect func(m
func TestMySQLConnectorOpenSyncCustomQuery(t *testing.T) {
query := "SELECT * FROM products WHERE status = 'active'"
updatedAt := mustTime(t, "2026-01-02T03:04:05Z")
+ expected := "SELECT * FROM (SELECT * FROM products WHERE status = 'active') AS ragflow_src ORDER BY ragflow_src.id ASC"
connector := newFixtureMySQLConnector(t, map[string]any{
"host": "127.0.0.1",
"port": "3306",
@@ -85,7 +87,7 @@ func TestMySQLConnectorOpenSyncCustomQuery(t *testing.T) {
"password": "secret",
},
}, func(mock sqlmock.Sqlmock) {
- mock.ExpectQuery(regexp.QuoteMeta(query)).WillReturnRows(
+ mock.ExpectQuery(regexp.QuoteMeta(expected)).WillReturnRows(
sqlmock.NewRows([]string{"id", "title", "description", "category", "updated_at"}).
AddRow(7, "Hello/World", "Some body", "news", updatedAt),
)
@@ -140,6 +142,7 @@ func TestMySQLConnectorOpenSyncIncrementalWindow(t *testing.T) {
"port": 3306,
"database": "mydb",
"query": "SELECT * FROM products",
+ "id_column": "id",
"timestamp_column": "updated_at",
"credentials": map[string]any{
"username": "root",
@@ -147,7 +150,8 @@ func TestMySQLConnectorOpenSyncIncrementalWindow(t *testing.T) {
},
}, func(mock sqlmock.Sqlmock) {
expected := "SELECT * FROM (SELECT * FROM products) AS ragflow_src " +
- "WHERE ragflow_src.updated_at >= '2026-01-01 00:00:00' AND ragflow_src.updated_at <= '2026-01-02 00:00:00'"
+ "WHERE ragflow_src.updated_at >= '2026-01-01 00:00:00' AND ragflow_src.updated_at <= '2026-01-02 00:00:00' " +
+ "ORDER BY ragflow_src.updated_at ASC, ragflow_src.id ASC"
mock.ExpectQuery(regexp.QuoteMeta(expected)).WillReturnRows(
sqlmock.NewRows([]string{"updated_at"}).AddRow(mustTime(t, "2026-01-01T12:00:00Z")),
)
@@ -183,12 +187,12 @@ func TestMySQLConnectorOpenSyncAllTables(t *testing.T) {
mock.ExpectQuery(regexp.QuoteMeta("SHOW TABLES")).WillReturnRows(
sqlmock.NewRows([]string{"Tables_in_mydb"}).AddRow("products").AddRow("orders"),
)
- mock.ExpectQuery(regexp.QuoteMeta("SELECT * FROM products")).WillReturnRows(
- sqlmock.NewRows([]string{"id", "name"}).AddRow(1, "Product"),
- )
- mock.ExpectQuery(regexp.QuoteMeta("SELECT * FROM orders")).WillReturnRows(
+ mock.ExpectQuery(regexp.QuoteMeta("SELECT * FROM (SELECT * FROM orders) AS ragflow_src ORDER BY ragflow_src.id ASC")).WillReturnRows(
sqlmock.NewRows([]string{"id", "name"}).AddRow(2, "Order"),
)
+ mock.ExpectQuery(regexp.QuoteMeta("SELECT * FROM (SELECT * FROM products) AS ragflow_src ORDER BY ragflow_src.id ASC")).WillReturnRows(
+ sqlmock.NewRows([]string{"id", "name"}).AddRow(1, "Product"),
+ )
})
session, err := connector.OpenSync(t.Context(), SyncRequest{FromBeginning: true})
@@ -208,7 +212,7 @@ func TestMySQLConnectorOpenSyncAllTables(t *testing.T) {
ids = append(ids, doc.SourceID)
}
}
- if len(ids) != 2 || ids[0] != "mysql:mydb:1" || ids[1] != "mysql:mydb:2" {
+ if len(ids) != 2 || ids[0] != "mysql:mydb:2" || ids[1] != "mysql:mydb:1" {
t.Fatalf("source ids = %v", ids)
}
}
@@ -378,6 +382,7 @@ func TestMySQLConnectorFileExtension(t *testing.T) {
// extension is applied to produced documents.
func TestMySQLConnectorOpenSyncCustomFileExtension(t *testing.T) {
query := "SELECT * FROM products WHERE status = 'active'"
+ expected := "SELECT * FROM (SELECT * FROM products WHERE status = 'active') AS ragflow_src ORDER BY ragflow_src.id ASC"
connector := newFixtureMySQLConnector(t, map[string]any{
"host": "127.0.0.1",
"port": "3306",
@@ -391,7 +396,7 @@ func TestMySQLConnectorOpenSyncCustomFileExtension(t *testing.T) {
"password": "secret",
},
}, func(mock sqlmock.Sqlmock) {
- mock.ExpectQuery(regexp.QuoteMeta(query)).WillReturnRows(
+ mock.ExpectQuery(regexp.QuoteMeta(expected)).WillReturnRows(
sqlmock.NewRows([]string{"id", "title", "description"}).
AddRow(7, "Hello", "World"),
)
@@ -415,3 +420,438 @@ func TestMySQLConnectorOpenSyncCustomFileExtension(t *testing.T) {
t.Fatalf("NextBatch EOF = %v", err)
}
}
+
+// TestMySQLConnectorOpenSyncCheckpointEmitted verifies ordered sync batches
+// carry a resume checkpoint.
+func TestMySQLConnectorOpenSyncCheckpointEmitted(t *testing.T) {
+ connector := newFixtureMySQLConnector(t, map[string]any{
+ "host": "127.0.0.1",
+ "port": 3306,
+ "database": "mydb",
+ "query": "SELECT * FROM products",
+ "id_column": "id",
+ "timestamp_column": "updated_at",
+ "credentials": map[string]any{
+ "username": "root",
+ "password": "secret",
+ },
+ }, func(mock sqlmock.Sqlmock) {
+ expected := "SELECT * FROM (SELECT * FROM products) AS ragflow_src " +
+ "WHERE ragflow_src.updated_at >= '2026-01-01 00:00:00' AND ragflow_src.updated_at <= '2026-01-02 00:00:00' " +
+ "ORDER BY ragflow_src.updated_at ASC, ragflow_src.id ASC"
+ mock.ExpectQuery(regexp.QuoteMeta(expected)).WillReturnRows(
+ sqlmock.NewRows([]string{"id", "updated_at"}).
+ AddRow(1, mustTime(t, "2026-01-01T12:00:00Z")).
+ AddRow(2, mustTime(t, "2026-01-01T13:00:00Z")),
+ )
+ })
+
+ start := mustTime(t, "2026-01-01T00:00:00Z")
+ end := mustTime(t, "2026-01-02T00:00:00Z")
+ session, err := connector.OpenSync(t.Context(), SyncRequest{WindowStart: &start, WindowEnd: end})
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch failed: %v", err)
+ }
+ if len(batch.Documents) != 2 {
+ t.Fatalf("documents len = %d, want 2", len(batch.Documents))
+ }
+ if batch.Checkpoint == nil {
+ t.Fatalf("checkpoint is nil, want resume checkpoint")
+ }
+ if batch.Checkpoint.SourceID != "mysql:mydb:2" {
+ t.Fatalf("checkpoint source id = %q, want %q", batch.Checkpoint.SourceID, "mysql:mydb:2")
+ }
+ cursor, err := parseRDBMSCursor(batch.Checkpoint.Cursor)
+ if err != nil {
+ t.Fatalf("parse cursor: %v", err)
+ }
+ if cursor.Query != "" || cursor.Order != "updated_at,id" || cursor.SourceID != "mysql:mydb:2" {
+ t.Fatalf("cursor = %+v", cursor)
+ }
+}
+
+// TestMySQLConnectorOpenSyncResumeSkipsToAnchor verifies a full sync resumes
+// from the committed anchor instead of re-emitting earlier rows.
+func TestMySQLConnectorOpenSyncResumeSkipsToAnchor(t *testing.T) {
+ connector := newFixtureMySQLConnector(t, map[string]any{
+ "host": "127.0.0.1",
+ "port": 3306,
+ "database": "mydb",
+ "query": "SELECT * FROM products",
+ "id_column": "id",
+ "credentials": map[string]any{
+ "username": "root",
+ "password": "secret",
+ },
+ }, func(mock sqlmock.Sqlmock) {
+ expected := "SELECT * FROM (SELECT * FROM products) AS ragflow_src ORDER BY ragflow_src.id ASC"
+ mock.ExpectQuery(regexp.QuoteMeta(expected)).WillReturnRows(
+ sqlmock.NewRows([]string{"id", "name"}).AddRow(1, "A").AddRow(2, "B").AddRow(3, "C"),
+ )
+ })
+
+ anchor := "mysql:mydb:1"
+ session, err := connector.OpenSync(t.Context(), SyncRequest{
+ FromBeginning: true,
+ Resume: &SyncCheckpoint{Cursor: encodeRDBMSCursor("", "id", anchor), SourceID: anchor},
+ })
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch failed: %v", err)
+ }
+ if len(batch.Documents) != 2 ||
+ batch.Documents[0].SourceID != "mysql:mydb:2" ||
+ batch.Documents[1].SourceID != "mysql:mydb:3" {
+ t.Fatalf("documents = %+v", batch.Documents)
+ }
+ if batch.Checkpoint == nil || batch.Checkpoint.SourceID != "mysql:mydb:3" {
+ t.Fatalf("checkpoint = %+v", batch.Checkpoint)
+ }
+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {
+ t.Fatalf("NextBatch EOF = %v", err)
+ }
+}
+
+// TestMySQLConnectorOpenSyncResumeMultiTable verifies a resume skips tables
+// before the anchor's table without executing them.
+func TestMySQLConnectorOpenSyncResumeMultiTable(t *testing.T) {
+ connector := newFixtureMySQLConnector(t, map[string]any{
+ "host": "127.0.0.1",
+ "port": 3306,
+ "database": "mydb",
+ "id_column": "id",
+ "credentials": map[string]any{
+ "username": "root",
+ "password": "secret",
+ },
+ }, func(mock sqlmock.Sqlmock) {
+ mock.ExpectQuery(regexp.QuoteMeta("SHOW TABLES")).WillReturnRows(
+ sqlmock.NewRows([]string{"Tables_in_mydb"}).AddRow("products").AddRow("orders"),
+ )
+ // Tables run sorted (orders, products), and the anchor is in the last
+ // table, so orders is skipped without being executed.
+ mock.ExpectQuery(regexp.QuoteMeta("SELECT * FROM (SELECT * FROM products) AS ragflow_src ORDER BY ragflow_src.id ASC")).WillReturnRows(
+ sqlmock.NewRows([]string{"id", "name"}).AddRow(1, "P1").AddRow(3, "P3"),
+ )
+ })
+
+ anchor := "mysql:mydb:1"
+ session, err := connector.OpenSync(t.Context(), SyncRequest{
+ FromBeginning: true,
+ Resume: &SyncCheckpoint{Cursor: encodeRDBMSCursor("products", "id", anchor), SourceID: anchor},
+ })
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch failed: %v", err)
+ }
+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != "mysql:mydb:3" {
+ t.Fatalf("documents = %+v", batch.Documents)
+ }
+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {
+ t.Fatalf("NextBatch EOF = %v", err)
+ }
+}
+
+// TestMySQLConnectorOpenSyncResumeAnchorMissing verifies an anchor that is no
+// longer present restarts the window via ErrSyncResumeInvalid.
+func TestMySQLConnectorOpenSyncResumeAnchorMissing(t *testing.T) {
+ connector := newFixtureMySQLConnector(t, map[string]any{
+ "host": "127.0.0.1",
+ "port": 3306,
+ "database": "mydb",
+ "query": "SELECT * FROM products",
+ "id_column": "id",
+ "credentials": map[string]any{
+ "username": "root",
+ "password": "secret",
+ },
+ }, func(mock sqlmock.Sqlmock) {
+ expected := "SELECT * FROM (SELECT * FROM products) AS ragflow_src ORDER BY ragflow_src.id ASC"
+ mock.ExpectQuery(regexp.QuoteMeta(expected)).WillReturnRows(
+ sqlmock.NewRows([]string{"id", "name"}).AddRow(1, "A"),
+ )
+ })
+
+ session, err := connector.OpenSync(t.Context(), SyncRequest{
+ FromBeginning: true,
+ Resume: &SyncCheckpoint{Cursor: encodeRDBMSCursor("", "id", "mysql:mydb:99"), SourceID: "mysql:mydb:99"},
+ })
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ _, err = session.NextBatch(context.Background())
+ if !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("NextBatch error = %v, want ErrSyncResumeInvalid", err)
+ }
+}
+
+// TestMySQLConnectorOpenSyncResumeInvalid verifies invalid resume cursors are
+// rejected at OpenSync.
+func TestMySQLConnectorOpenSyncResumeInvalid(t *testing.T) {
+ base := map[string]any{
+ "host": "127.0.0.1",
+ "port": 3306,
+ "database": "mydb",
+ "query": "SELECT * FROM products",
+ "id_column": "id",
+ "credentials": map[string]any{
+ "username": "root",
+ "password": "secret",
+ },
+ }
+ withoutID := map[string]any{}
+ for k, v := range base {
+ if k != "id_column" {
+ withoutID[k] = v
+ }
+ }
+ changedID := map[string]any{}
+ for k, v := range base {
+ changedID[k] = v
+ }
+ changedID["id_column"] = "other_id"
+
+ cases := []struct {
+ name string
+ config map[string]any
+ resume *SyncCheckpoint
+ }{
+ {name: "malformed cursor", config: base, resume: &SyncCheckpoint{Cursor: "not-json"}},
+ {name: "ordering not available", config: withoutID, resume: &SyncCheckpoint{Cursor: encodeRDBMSCursor("", "id", "mysql:mydb:1")}},
+ {name: "query no longer exists", config: base, resume: &SyncCheckpoint{Cursor: encodeRDBMSCursor("orders", "id", "mysql:mydb:1")}},
+ {name: "ordering column changed", config: changedID, resume: &SyncCheckpoint{Cursor: encodeRDBMSCursor("", "id", "mysql:mydb:1")}},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ connector := newFixtureMySQLConnector(t, tc.config, nil)
+ session, err := connector.OpenSync(t.Context(), SyncRequest{FromBeginning: true, Resume: tc.resume})
+ if !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("OpenSync error = %v, want ErrSyncResumeInvalid", err)
+ }
+ if session != nil {
+ t.Fatalf("OpenSync returned a session on invalid resume")
+ }
+ })
+ }
+}
+
+// TestMySQLConnectorOpenSyncOrderingFallback verifies a custom query without
+// the configured ordering column falls back to the unordered query and stops
+// checkpointing.
+func TestMySQLConnectorOpenSyncOrderingFallback(t *testing.T) {
+ connector := newFixtureMySQLConnector(t, map[string]any{
+ "host": "127.0.0.1",
+ "port": 3306,
+ "database": "mydb",
+ "query": "SELECT * FROM products",
+ "id_column": "id",
+ "credentials": map[string]any{
+ "username": "root",
+ "password": "secret",
+ },
+ }, func(mock sqlmock.Sqlmock) {
+ ordered := "SELECT * FROM (SELECT * FROM products) AS ragflow_src ORDER BY ragflow_src.id ASC"
+ unordered := "SELECT * FROM (SELECT * FROM products) AS ragflow_src"
+ mock.ExpectQuery(regexp.QuoteMeta(ordered)).WillReturnError(
+ &mysql.MySQLError{Number: 1054, Message: "Unknown column 'id' in 'order clause'"},
+ )
+ mock.ExpectQuery(regexp.QuoteMeta(unordered)).WillReturnRows(
+ sqlmock.NewRows([]string{"name"}).AddRow("Product"),
+ )
+ })
+
+ session, err := connector.OpenSync(t.Context(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch failed: %v", err)
+ }
+ if len(batch.Documents) != 1 {
+ t.Fatalf("documents len = %d, want 1", len(batch.Documents))
+ }
+ if batch.Checkpoint != nil {
+ t.Fatalf("checkpoint = %+v, want nil for unordered fallback", batch.Checkpoint)
+ }
+}
+
+// TestMySQLConnectorOpenSyncResumeOrderingFallback verifies a resume into a
+// query that lost its ordering column restarts the window.
+func TestMySQLConnectorOpenSyncResumeOrderingFallback(t *testing.T) {
+ connector := newFixtureMySQLConnector(t, map[string]any{
+ "host": "127.0.0.1",
+ "port": 3306,
+ "database": "mydb",
+ "query": "SELECT * FROM products",
+ "id_column": "id",
+ "credentials": map[string]any{
+ "username": "root",
+ "password": "secret",
+ },
+ }, func(mock sqlmock.Sqlmock) {
+ ordered := "SELECT * FROM (SELECT * FROM products) AS ragflow_src ORDER BY ragflow_src.id ASC"
+ mock.ExpectQuery(regexp.QuoteMeta(ordered)).WillReturnError(
+ &mysql.MySQLError{Number: 1054, Message: "Unknown column 'id' in 'order clause'"},
+ )
+ })
+
+ session, err := connector.OpenSync(t.Context(), SyncRequest{
+ FromBeginning: true,
+ Resume: &SyncCheckpoint{Cursor: encodeRDBMSCursor("", "id", "mysql:mydb:1"), SourceID: "mysql:mydb:1"},
+ })
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ _, err = session.NextBatch(context.Background())
+ if !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("NextBatch error = %v, want ErrSyncResumeInvalid", err)
+ }
+}
+
+// TestMySQLConnectorOpenSyncCheckpointQueryName verifies the checkpoint query
+// name is the query that produced the last document, even when a later query
+// in the same batch yields no documents.
+func TestMySQLConnectorOpenSyncCheckpointQueryName(t *testing.T) {
+ connector := newFixtureMySQLConnector(t, map[string]any{
+ "host": "127.0.0.1",
+ "port": 3306,
+ "database": "mydb",
+ "id_column": "id",
+ "credentials": map[string]any{
+ "username": "root",
+ "password": "secret",
+ },
+ }, func(mock sqlmock.Sqlmock) {
+ mock.ExpectQuery(regexp.QuoteMeta("SHOW TABLES")).WillReturnRows(
+ sqlmock.NewRows([]string{"Tables_in_mydb"}).AddRow("products").AddRow("orders"),
+ )
+ // Tables run sorted (orders, products); orders holds the last document
+ // and products is empty, so the checkpoint must still point at orders.
+ mock.ExpectQuery(regexp.QuoteMeta("SELECT * FROM (SELECT * FROM orders) AS ragflow_src ORDER BY ragflow_src.id ASC")).WillReturnRows(
+ sqlmock.NewRows([]string{"id", "name"}).AddRow(1, "A"),
+ )
+ mock.ExpectQuery(regexp.QuoteMeta("SELECT * FROM (SELECT * FROM products) AS ragflow_src ORDER BY ragflow_src.id ASC")).WillReturnRows(
+ sqlmock.NewRows([]string{"id", "name"}),
+ )
+ })
+
+ session, err := connector.OpenSync(t.Context(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch failed: %v", err)
+ }
+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != "mysql:mydb:1" {
+ t.Fatalf("documents = %+v", batch.Documents)
+ }
+ if batch.Checkpoint == nil {
+ t.Fatalf("checkpoint is nil, want resume checkpoint")
+ }
+ cursor, err := parseRDBMSCursor(batch.Checkpoint.Cursor)
+ if err != nil {
+ t.Fatalf("parse cursor: %v", err)
+ }
+ if cursor.Query != "orders" {
+ t.Fatalf("checkpoint query = %q, want %q", cursor.Query, "orders")
+ }
+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {
+ t.Fatalf("NextBatch EOF = %v", err)
+ }
+}
+
+// TestMySQLConnectorOpenSyncIncrementalResumeComposite verifies an incremental
+// sync resumes by the deterministic timestamp-plus-id ordering, so rows sharing
+// a timestamp still resume in id order.
+func TestMySQLConnectorOpenSyncIncrementalResumeComposite(t *testing.T) {
+ connector := newFixtureMySQLConnector(t, map[string]any{
+ "host": "127.0.0.1",
+ "port": 3306,
+ "database": "mydb",
+ "query": "SELECT * FROM products",
+ "id_column": "id",
+ "timestamp_column": "updated_at",
+ "credentials": map[string]any{
+ "username": "root",
+ "password": "secret",
+ },
+ }, func(mock sqlmock.Sqlmock) {
+ expected := "SELECT * FROM (SELECT * FROM products) AS ragflow_src " +
+ "WHERE ragflow_src.updated_at >= '2026-01-01 00:00:00' AND ragflow_src.updated_at <= '2026-01-02 00:00:00' " +
+ "ORDER BY ragflow_src.updated_at ASC, ragflow_src.id ASC"
+ mock.ExpectQuery(regexp.QuoteMeta(expected)).WillReturnRows(
+ sqlmock.NewRows([]string{"id", "updated_at"}).
+ AddRow(1, mustTime(t, "2026-01-01T12:00:00Z")).
+ AddRow(2, mustTime(t, "2026-01-01T12:00:00Z")).
+ AddRow(3, mustTime(t, "2026-01-01T13:00:00Z")),
+ )
+ })
+
+ start := mustTime(t, "2026-01-01T00:00:00Z")
+ end := mustTime(t, "2026-01-02T00:00:00Z")
+ anchor := "mysql:mydb:1"
+ session, err := connector.OpenSync(t.Context(), SyncRequest{
+ WindowStart: &start,
+ WindowEnd: end,
+ Resume: &SyncCheckpoint{Cursor: encodeRDBMSCursor("", "updated_at,id", anchor), SourceID: anchor},
+ })
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch failed: %v", err)
+ }
+ if len(batch.Documents) != 2 ||
+ batch.Documents[0].SourceID != "mysql:mydb:2" ||
+ batch.Documents[1].SourceID != "mysql:mydb:3" {
+ t.Fatalf("documents = %+v", batch.Documents)
+ }
+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {
+ t.Fatalf("NextBatch EOF = %v", err)
+ }
+}
+
+// TestMySQLConnectorOpenSyncIncrementalResumeRejectsSingleOrder verifies an
+// incremental resume cursor that does not carry the full timestamp-plus-id
+// ordering key is rejected.
+func TestMySQLConnectorOpenSyncIncrementalResumeRejectsSingleOrder(t *testing.T) {
+ connector := newFixtureMySQLConnector(t, map[string]any{
+ "host": "127.0.0.1",
+ "port": 3306,
+ "database": "mydb",
+ "query": "SELECT * FROM products",
+ "id_column": "id",
+ "timestamp_column": "updated_at",
+ "credentials": map[string]any{
+ "username": "root",
+ "password": "secret",
+ },
+ }, nil)
+
+ start := mustTime(t, "2026-01-01T00:00:00Z")
+ end := mustTime(t, "2026-01-02T00:00:00Z")
+ session, err := connector.OpenSync(t.Context(), SyncRequest{
+ WindowStart: &start,
+ WindowEnd: end,
+ Resume: &SyncCheckpoint{Cursor: encodeRDBMSCursor("", "updated_at", "mysql:mydb:1"), SourceID: "mysql:mydb:1"},
+ })
+ if !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("OpenSync error = %v, want ErrSyncResumeInvalid", err)
+ }
+ if session != nil {
+ t.Fatalf("OpenSync returned a session on invalid resume")
+ }
+}
diff --git a/internal/syncer/connector/postgresql.go b/internal/syncer/connector/postgresql.go
index a581ee9268..decef68495 100644
--- a/internal/syncer/connector/postgresql.go
+++ b/internal/syncer/connector/postgresql.go
@@ -22,16 +22,19 @@ import (
"database/sql"
"database/sql/driver"
"encoding/hex"
+ "errors"
"fmt"
"io"
"net"
"net/url"
"regexp"
+ "sort"
"strconv"
"strings"
"time"
"github.com/jackc/pgx/v5"
+ "github.com/jackc/pgx/v5/pgconn"
"github.com/jackc/pgx/v5/stdlib"
)
@@ -144,7 +147,26 @@ func (c *PostgreSQLConnector) OpenSync(ctx context.Context, request SyncRequest)
return nil, err
}
queries := c.buildSyncQueries(bases, request)
- return &postgresSyncSession{connector: c, db: db, queries: queries, batchSize: c.batchSize}, nil
+ orderColumn := c.syncOrderColumn(request)
+ session := &postgresSyncSession{
+ connector: c,
+ db: db,
+ batchSize: c.batchSize,
+ orderColumn: orderColumn,
+ checkpointEnabled: orderColumn != "",
+ lastDocQuery: -1,
+ }
+ for _, q := range queries {
+ session.queries = append(session.queries, q.sql)
+ session.queryNames = append(session.queryNames, q.name)
+ session.orderedFlags = append(session.orderedFlags, q.ordered)
+ session.fallbackQueries = append(session.fallbackQueries, q.fallback)
+ }
+ if err := session.applyResume(request.Resume); err != nil {
+ db.Close()
+ return nil, err
+ }
+ return session, nil
}
// OpenPrune opens one complete PostgreSQL prune snapshot session.
@@ -160,7 +182,7 @@ func (c *PostgreSQLConnector) OpenPrune(ctx context.Context, request PruneReques
}
queries := make([]string, 0, len(bases))
for _, base := range bases {
- queries = append(queries, c.buildSlimQuery(base))
+ queries = append(queries, c.buildSlimQuery(base.sql))
}
return &postgresPruneSession{connector: c, db: db, queries: queries, batchSize: c.batchSize}, nil
}
@@ -212,10 +234,12 @@ func postgresPinnedDial(pinIP net.IP, port string, timeout time.Duration) func(c
}
}
-// baseQueries returns the configured query or a SELECT per table.
-func (c *PostgreSQLConnector) baseQueries(ctx context.Context, db *sql.DB) ([]string, error) {
+// baseQueries returns the configured query or a SELECT per table. Table names
+// are sorted so the sync stream order is stable across runs and a resume
+// cursor can reliably skip already-processed tables.
+func (c *PostgreSQLConnector) baseQueries(ctx context.Context, db *sql.DB) ([]rdbmsQuery, error) {
if c.query != "" {
- return []string{c.query}, nil
+ return []rdbmsQuery{{name: "", sql: c.query}}, nil
}
rows, err := db.QueryContext(ctx, "SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' AND table_type = 'BASE TABLE'")
if err != nil {
@@ -233,9 +257,10 @@ func (c *PostgreSQLConnector) baseQueries(ctx context.Context, db *sql.DB) ([]st
if err := rows.Err(); err != nil {
return nil, err
}
- queries := make([]string, 0, len(tables))
+ sort.Strings(tables)
+ queries := make([]rdbmsQuery, 0, len(tables))
for _, table := range tables {
- queries = append(queries, fmt.Sprintf("SELECT * FROM \"public\".%s", quotePostgresIdentifier(table)))
+ queries = append(queries, rdbmsQuery{name: table, sql: fmt.Sprintf("SELECT * FROM \"public\".%s", quotePostgresIdentifier(table))})
}
return queries, nil
}
@@ -247,23 +272,61 @@ func quotePostgresIdentifier(name string) string {
return `"` + strings.ReplaceAll(name, `"`, `""`) + `"`
}
-// buildSyncQueries applies the incremental window when a timestamp column exists.
-func (c *PostgreSQLConnector) buildSyncQueries(bases []string, request SyncRequest) []string {
- var start, end *time.Time
- if !request.FromBeginning {
- start = request.WindowStart
- end = &request.WindowEnd
- }
- if c.timestampColumn == "" || (start == nil && end == nil) {
- return bases
- }
- queries := make([]string, 0, len(bases))
- for _, base := range bases {
- queries = append(queries, c.buildTimeFilteredQuery(base, start, end))
+// buildSyncQueries applies the incremental window and a stable ordering when
+// one is available, so a checkpoint can resume the stream from an anchor.
+// Each query carries an unordered fallback used when a custom SQL query does
+// not expose the configured ordering column.
+func (c *PostgreSQLConnector) buildSyncQueries(bases []rdbmsQuery, request SyncRequest) []rdbmsSyncQuery {
+ queries := make([]rdbmsSyncQuery, 0, len(bases))
+ switch {
+ case !request.FromBeginning && c.timestampColumn != "":
+ start := request.WindowStart
+ end := &request.WindowEnd
+ for _, base := range bases {
+ queries = append(queries, rdbmsSyncQuery{
+ name: base.name,
+ sql: c.buildTimeFilteredOrderedQuery(base.sql, start, end),
+ ordered: true,
+ fallback: c.buildTimeFilteredQuery(base.sql, start, end),
+ })
+ }
+ case request.FromBeginning && c.idColumn != "":
+ for _, base := range bases {
+ queries = append(queries, rdbmsSyncQuery{
+ name: base.name,
+ sql: c.buildOrderedQuery(base.sql, c.idColumn),
+ ordered: true,
+ fallback: c.wrapQuery(base.sql),
+ })
+ }
+ default:
+ for _, base := range bases {
+ queries = append(queries, rdbmsSyncQuery{name: base.name, sql: base.sql})
+ }
}
return queries
}
+// syncOrderColumn returns the ordering key that makes this sync window
+// deterministic, or "" when the connector cannot checkpoint/resume the
+// stream (no stable ordering key). Incremental windows order by timestamp
+// plus id so rows sharing a timestamp still resume deterministically.
+func (c *PostgreSQLConnector) syncOrderColumn(request SyncRequest) string {
+ switch {
+ case !request.FromBeginning && c.timestampColumn != "" && c.idColumn != "":
+ return c.timestampColumn + "," + c.idColumn
+ case request.FromBeginning && c.idColumn != "":
+ return c.idColumn
+ }
+ return ""
+}
+
+// buildOrderedQuery wraps the base query and orders it by a stable column so
+// connector sync can resume from a checkpoint.
+func (c *PostgreSQLConnector) buildOrderedQuery(base, orderColumn string) string {
+ return c.wrapQuery(base) + " ORDER BY ragflow_src." + orderColumn + " ASC"
+}
+
// buildTimeFilteredQuery wraps the base query and appends timestamp bounds.
func (c *PostgreSQLConnector) buildTimeFilteredQuery(base string, start, end *time.Time) string {
conditions := []string{}
@@ -280,6 +343,18 @@ func (c *PostgreSQLConnector) buildTimeFilteredQuery(base string, start, end *ti
return query
}
+// buildTimeFilteredOrderedQuery is the incremental query plus a deterministic
+// ORDER BY on the timestamp and id columns, which resume relies on. Without a
+// configured id column the order is timestamp-only and the stream is not
+// checkpointed.
+func (c *PostgreSQLConnector) buildTimeFilteredOrderedQuery(base string, start, end *time.Time) string {
+ query := c.buildTimeFilteredQuery(base, start, end) + " ORDER BY ragflow_src." + c.timestampColumn + " ASC"
+ if c.idColumn != "" {
+ query += ", ragflow_src." + c.idColumn + " ASC"
+ }
+ return query
+}
+
// buildSlimQuery selects only the columns needed to identify documents.
func (c *PostgreSQLConnector) buildSlimQuery(base string) string {
columns := []string{}
@@ -528,9 +603,25 @@ type postgresSyncSession struct {
connector *PostgreSQLConnector
db *sql.DB
queries []string
- queryIndex int
- rows *sql.Rows
- batchSize int
+ queryNames []string
+ // orderedFlags[i] reports whether queries[i] carries a stable ORDER BY.
+ orderedFlags []bool
+ // fallbackQueries[i] is the unordered variant of queries[i], used when a
+ // custom SQL query does not expose the configured ordering column.
+ fallbackQueries []string
+ queryIndex int
+ // lastDocQuery is the index of the query that produced the most recently
+ // appended document, used to checkpoint against the right query name even
+ // when later queries in the batch contributed no documents.
+ lastDocQuery int
+ rows *sql.Rows
+ batchSize int
+
+ orderColumn string
+ checkpointEnabled bool
+ orderable bool
+ resume *rdbmsResumeCursor
+ resumePending bool
}
// NextBatch returns the next PostgreSQL document batch.
@@ -540,7 +631,7 @@ func (s *postgresSyncSession) NextBatch(ctx context.Context) (SyncBatch, error)
if s.rows == nil {
if s.queryIndex >= len(s.queries) {
if len(documents) == 0 {
- return SyncBatch{}, io.EOF
+ return s.endOfStream()
}
break
}
@@ -562,10 +653,17 @@ func (s *postgresSyncSession) NextBatch(ctx context.Context) (SyncBatch, error)
continue
}
if doc, ok := s.connector.rowToSourceDocument(row, columns); ok {
+ if !s.includeResumed(doc) {
+ continue
+ }
documents = append(documents, doc)
+ s.lastDocQuery = s.queryIndex - 1
}
}
- return SyncBatch{Documents: documents}, nil
+ if len(documents) == 0 {
+ return s.endOfStream()
+ }
+ return SyncBatch{Documents: documents, Checkpoint: s.batchCheckpoint(documents[len(documents)-1])}, nil
}
// Close closes the PostgreSQL sync session.
@@ -574,14 +672,33 @@ func (s *postgresSyncSession) Close() error {
return s.db.Close()
}
-// openNextQuery runs the next base query.
+// openNextQuery runs the next base query. When a custom SQL query does not
+// expose the configured ordering column (PostgreSQL SQLSTATE 42703), it falls
+// back to the unordered query and stops checkpointing so the remaining stream
+// is never resumed against a non-deterministic order. A pending resume never
+// falls back: the ordering that produced the anchor is gone, so the window
+// restarts.
func (s *postgresSyncSession) openNextQuery(ctx context.Context) error {
- query := s.queries[s.queryIndex]
+ idx := s.queryIndex
s.queryIndex++
- rows, err := s.db.QueryContext(ctx, query)
+ rows, err := s.db.QueryContext(ctx, s.queries[idx])
if err != nil {
+ if s.orderedFlags[idx] && isPostgresUnknownColumn(err) {
+ if s.resumePending {
+ return fmt.Errorf("PostgreSQL sync resume query lost its ordering column: %w", ErrSyncResumeInvalid)
+ }
+ s.checkpointEnabled = false
+ rows, err = s.db.QueryContext(ctx, s.fallbackQueries[idx])
+ if err != nil {
+ return fmt.Errorf("PostgreSQL query failed: %w", err)
+ }
+ s.orderable = false
+ s.rows = rows
+ return nil
+ }
return fmt.Errorf("PostgreSQL query failed: %w", err)
}
+ s.orderable = s.orderedFlags[idx]
s.rows = rows
return nil
}
@@ -594,6 +711,89 @@ func (s *postgresSyncSession) closeRows() {
}
}
+// applyResume positions the session after the last committed batch. The
+// cursor's query and ordering column must still exist, otherwise the runner
+// restarts the task window.
+func (s *postgresSyncSession) applyResume(checkpoint *SyncCheckpoint) error {
+ if checkpoint == nil {
+ return nil
+ }
+ cursor, err := parseRDBMSCursor(checkpoint.Cursor)
+ if err != nil {
+ return err
+ }
+ if s.orderColumn == "" || cursor.Order != s.orderColumn {
+ return fmt.Errorf("PostgreSQL sync resume ordering changed from %q to %q: %w", cursor.Order, s.orderColumn, ErrSyncResumeInvalid)
+ }
+ idx := -1
+ for i, name := range s.queryNames {
+ if name == cursor.Query {
+ idx = i
+ break
+ }
+ }
+ if idx < 0 {
+ return fmt.Errorf("PostgreSQL sync resume query %q no longer exists: %w", cursor.Query, ErrSyncResumeInvalid)
+ }
+ s.queryIndex = idx
+ s.resume = &cursor
+ s.resumePending = true
+ return nil
+}
+
+// includeResumed reports whether doc should be emitted. While a resume is
+// pending, every row before (and including) the anchor is skipped because it
+// was already committed.
+func (s *postgresSyncSession) includeResumed(doc SourceDocument) bool {
+ if !s.resumePending {
+ return true
+ }
+ if s.resume != nil && doc.SourceID == s.resume.SourceID {
+ s.resumePending = false
+ return false
+ }
+ return false
+}
+
+// batchCheckpoint builds the checkpoint for a batch whose last row is doc.
+// Batches from a non-deterministic (unordered) query never carry a checkpoint.
+func (s *postgresSyncSession) batchCheckpoint(doc SourceDocument) *SyncCheckpoint {
+ if !s.checkpointEnabled || !s.orderable {
+ return nil
+ }
+ queryName := ""
+ if idx := s.lastDocQuery; idx >= 0 && idx < len(s.queryNames) {
+ queryName = s.queryNames[idx]
+ }
+ updatedAt := doc.UpdatedAt
+ return &SyncCheckpoint{
+ Cursor: encodeRDBMSCursor(queryName, s.orderColumn, doc.SourceID),
+ SourceID: doc.SourceID,
+ UpdatedAt: &updatedAt,
+ }
+}
+
+// endOfStream returns io.EOF when the stream is exhausted, or
+// ErrSyncResumeInvalid when a pending resume anchor was never found.
+func (s *postgresSyncSession) endOfStream() (SyncBatch, error) {
+ if s.resumePending {
+ anchor := ""
+ if s.resume != nil {
+ anchor = s.resume.SourceID
+ }
+ return SyncBatch{}, fmt.Errorf("PostgreSQL resume anchor %q was not found in the current result: %w", anchor, ErrSyncResumeInvalid)
+ }
+ return SyncBatch{}, io.EOF
+}
+
+// isPostgresUnknownColumn reports whether err is PostgreSQL SQLSTATE 42703
+// (undefined column), used to detect custom queries that do not expose the
+// configured ordering column.
+func isPostgresUnknownColumn(err error) bool {
+ var pgErr *pgconn.PgError
+ return errors.As(err, &pgErr) && pgErr.Code == "42703"
+}
+
type postgresPruneSession struct {
connector *PostgreSQLConnector
db *sql.DB
diff --git a/internal/syncer/connector/postgresql_test.go b/internal/syncer/connector/postgresql_test.go
index 253f2fc0d6..1722b4d9b3 100644
--- a/internal/syncer/connector/postgresql_test.go
+++ b/internal/syncer/connector/postgresql_test.go
@@ -28,6 +28,7 @@ import (
"time"
"github.com/DATA-DOG/go-sqlmock"
+ "github.com/jackc/pgx/v5/pgconn"
)
// newFixturePostgresConnector builds a PostgreSQL connector backed by sqlmock.
@@ -71,6 +72,7 @@ func newFixturePostgresConnector(t *testing.T, config map[string]any, expect fun
func TestPostgreSQLConnectorOpenSyncCustomQuery(t *testing.T) {
query := "SELECT * FROM products WHERE status = 'active'"
updatedAt := mustTime(t, "2026-01-02T03:04:05Z")
+ expected := "SELECT * FROM (SELECT * FROM products WHERE status = 'active') AS ragflow_src ORDER BY ragflow_src.id ASC"
connector := newFixturePostgresConnector(t, map[string]any{
"host": "127.0.0.1",
"port": "5432",
@@ -85,7 +87,7 @@ func TestPostgreSQLConnectorOpenSyncCustomQuery(t *testing.T) {
"password": "secret",
},
}, func(mock sqlmock.Sqlmock) {
- mock.ExpectQuery(regexp.QuoteMeta(query)).WillReturnRows(
+ mock.ExpectQuery(regexp.QuoteMeta(expected)).WillReturnRows(
sqlmock.NewRows([]string{"id", "title", "description", "category", "updated_at"}).
AddRow(7, "Hello/World", "Some body", "news", updatedAt),
)
@@ -141,7 +143,8 @@ func TestPostgreSQLConnectorOpenSyncIncrementalWindow(t *testing.T) {
},
}, func(mock sqlmock.Sqlmock) {
expected := "SELECT * FROM (SELECT * FROM products) AS ragflow_src " +
- "WHERE ragflow_src.updated_at >= '2026-01-01T00:00:00Z' AND ragflow_src.updated_at <= '2026-01-02T00:00:00Z'"
+ "WHERE ragflow_src.updated_at >= '2026-01-01T00:00:00Z' AND ragflow_src.updated_at <= '2026-01-02T00:00:00Z' " +
+ "ORDER BY ragflow_src.updated_at ASC"
mock.ExpectQuery(regexp.QuoteMeta(expected)).WillReturnRows(
sqlmock.NewRows([]string{"updated_at"}).AddRow(mustTime(t, "2026-01-01T12:00:00Z")),
)
@@ -160,6 +163,10 @@ func TestPostgreSQLConnectorOpenSyncIncrementalWindow(t *testing.T) {
if len(batch.Documents) != 1 {
t.Fatalf("documents len = %d, want 1", len(batch.Documents))
}
+ // Without an id column the incremental stream is not checkpointable.
+ if batch.Checkpoint != nil {
+ t.Fatalf("checkpoint = %+v, want nil without an id column", batch.Checkpoint)
+ }
}
// TestPostgreSQLConnectorOpenSyncAllTables verifies the information_schema table listing.
@@ -177,12 +184,12 @@ func TestPostgreSQLConnectorOpenSyncAllTables(t *testing.T) {
mock.ExpectQuery(regexp.QuoteMeta("SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' AND table_type = 'BASE TABLE'")).WillReturnRows(
sqlmock.NewRows([]string{"table_name"}).AddRow("products").AddRow("orders"),
)
- mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "public"."products"`)).WillReturnRows(
- sqlmock.NewRows([]string{"id", "name"}).AddRow(1, "Product"),
- )
- mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "public"."orders"`)).WillReturnRows(
+ mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM (SELECT * FROM "public"."orders") AS ragflow_src ORDER BY ragflow_src.id ASC`)).WillReturnRows(
sqlmock.NewRows([]string{"id", "name"}).AddRow(2, "Order"),
)
+ mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM (SELECT * FROM "public"."products") AS ragflow_src ORDER BY ragflow_src.id ASC`)).WillReturnRows(
+ sqlmock.NewRows([]string{"id", "name"}).AddRow(1, "Product"),
+ )
})
session, err := connector.OpenSync(t.Context(), SyncRequest{FromBeginning: true})
@@ -202,7 +209,7 @@ func TestPostgreSQLConnectorOpenSyncAllTables(t *testing.T) {
ids = append(ids, doc.SourceID)
}
}
- if len(ids) != 2 || ids[0] != "postgresql:mydb:1" || ids[1] != "postgresql:mydb:2" {
+ if len(ids) != 2 || ids[0] != "postgresql:mydb:2" || ids[1] != "postgresql:mydb:1" {
t.Fatalf("source ids = %v", ids)
}
}
@@ -298,7 +305,7 @@ func TestPostgreSQLConnectorOpenSyncMixedCaseTable(t *testing.T) {
mock.ExpectQuery(regexp.QuoteMeta("SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' AND table_type = 'BASE TABLE'")).WillReturnRows(
sqlmock.NewRows([]string{"table_name"}).AddRow("MixedCase"),
)
- mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "public"."MixedCase"`)).WillReturnRows(
+ mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM (SELECT * FROM "public"."MixedCase") AS ragflow_src ORDER BY ragflow_src.id ASC`)).WillReturnRows(
sqlmock.NewRows([]string{"id", "name"}).AddRow(1, "Item"),
)
})
@@ -369,6 +376,7 @@ func TestPostgreSQLConnectorFileExtension(t *testing.T) {
// extension is applied to produced documents.
func TestPostgreSQLConnectorOpenSyncCustomFileExtension(t *testing.T) {
query := "SELECT * FROM products WHERE status = 'active'"
+ expected := "SELECT * FROM (SELECT * FROM products WHERE status = 'active') AS ragflow_src ORDER BY ragflow_src.id ASC"
connector := newFixturePostgresConnector(t, map[string]any{
"host": "127.0.0.1",
"port": "5432",
@@ -382,7 +390,7 @@ func TestPostgreSQLConnectorOpenSyncCustomFileExtension(t *testing.T) {
"password": "secret",
},
}, func(mock sqlmock.Sqlmock) {
- mock.ExpectQuery(regexp.QuoteMeta(query)).WillReturnRows(
+ mock.ExpectQuery(regexp.QuoteMeta(expected)).WillReturnRows(
sqlmock.NewRows([]string{"id", "title", "description"}).
AddRow(7, "Hello", "World"),
)
@@ -406,3 +414,438 @@ func TestPostgreSQLConnectorOpenSyncCustomFileExtension(t *testing.T) {
t.Fatalf("NextBatch EOF = %v", err)
}
}
+
+// TestPostgreSQLConnectorOpenSyncCheckpointEmitted verifies ordered sync
+// batches carry a resume checkpoint.
+func TestPostgreSQLConnectorOpenSyncCheckpointEmitted(t *testing.T) {
+ connector := newFixturePostgresConnector(t, map[string]any{
+ "host": "127.0.0.1",
+ "port": 5432,
+ "database": "mydb",
+ "query": "SELECT * FROM products",
+ "id_column": "id",
+ "timestamp_column": "updated_at",
+ "credentials": map[string]any{
+ "username": "postgres",
+ "password": "secret",
+ },
+ }, func(mock sqlmock.Sqlmock) {
+ expected := "SELECT * FROM (SELECT * FROM products) AS ragflow_src " +
+ "WHERE ragflow_src.updated_at >= '2026-01-01T00:00:00Z' AND ragflow_src.updated_at <= '2026-01-02T00:00:00Z' " +
+ "ORDER BY ragflow_src.updated_at ASC, ragflow_src.id ASC"
+ mock.ExpectQuery(regexp.QuoteMeta(expected)).WillReturnRows(
+ sqlmock.NewRows([]string{"id", "updated_at"}).
+ AddRow(1, mustTime(t, "2026-01-01T12:00:00Z")).
+ AddRow(2, mustTime(t, "2026-01-01T13:00:00Z")),
+ )
+ })
+
+ start := mustTime(t, "2026-01-01T00:00:00Z")
+ end := mustTime(t, "2026-01-02T00:00:00Z")
+ session, err := connector.OpenSync(t.Context(), SyncRequest{WindowStart: &start, WindowEnd: end})
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch failed: %v", err)
+ }
+ if len(batch.Documents) != 2 {
+ t.Fatalf("documents len = %d, want 2", len(batch.Documents))
+ }
+ if batch.Checkpoint == nil {
+ t.Fatalf("checkpoint is nil, want resume checkpoint")
+ }
+ if batch.Checkpoint.SourceID != "postgresql:mydb:2" {
+ t.Fatalf("checkpoint source id = %q, want %q", batch.Checkpoint.SourceID, "postgresql:mydb:2")
+ }
+ cursor, err := parseRDBMSCursor(batch.Checkpoint.Cursor)
+ if err != nil {
+ t.Fatalf("parse cursor: %v", err)
+ }
+ if cursor.Query != "" || cursor.Order != "updated_at,id" || cursor.SourceID != "postgresql:mydb:2" {
+ t.Fatalf("cursor = %+v", cursor)
+ }
+}
+
+// TestPostgreSQLConnectorOpenSyncResumeSkipsToAnchor verifies a full sync
+// resumes from the committed anchor instead of re-emitting earlier rows.
+func TestPostgreSQLConnectorOpenSyncResumeSkipsToAnchor(t *testing.T) {
+ connector := newFixturePostgresConnector(t, map[string]any{
+ "host": "127.0.0.1",
+ "port": 5432,
+ "database": "mydb",
+ "query": "SELECT * FROM products",
+ "id_column": "id",
+ "credentials": map[string]any{
+ "username": "postgres",
+ "password": "secret",
+ },
+ }, func(mock sqlmock.Sqlmock) {
+ expected := "SELECT * FROM (SELECT * FROM products) AS ragflow_src ORDER BY ragflow_src.id ASC"
+ mock.ExpectQuery(regexp.QuoteMeta(expected)).WillReturnRows(
+ sqlmock.NewRows([]string{"id", "name"}).AddRow(1, "A").AddRow(2, "B").AddRow(3, "C"),
+ )
+ })
+
+ anchor := "postgresql:mydb:1"
+ session, err := connector.OpenSync(t.Context(), SyncRequest{
+ FromBeginning: true,
+ Resume: &SyncCheckpoint{Cursor: encodeRDBMSCursor("", "id", anchor), SourceID: anchor},
+ })
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch failed: %v", err)
+ }
+ if len(batch.Documents) != 2 ||
+ batch.Documents[0].SourceID != "postgresql:mydb:2" ||
+ batch.Documents[1].SourceID != "postgresql:mydb:3" {
+ t.Fatalf("documents = %+v", batch.Documents)
+ }
+ if batch.Checkpoint == nil || batch.Checkpoint.SourceID != "postgresql:mydb:3" {
+ t.Fatalf("checkpoint = %+v", batch.Checkpoint)
+ }
+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {
+ t.Fatalf("NextBatch EOF = %v", err)
+ }
+}
+
+// TestPostgreSQLConnectorOpenSyncResumeMultiTable verifies a resume skips
+// tables before the anchor's table without executing them.
+func TestPostgreSQLConnectorOpenSyncResumeMultiTable(t *testing.T) {
+ connector := newFixturePostgresConnector(t, map[string]any{
+ "host": "127.0.0.1",
+ "port": 5432,
+ "database": "mydb",
+ "id_column": "id",
+ "credentials": map[string]any{
+ "username": "postgres",
+ "password": "secret",
+ },
+ }, func(mock sqlmock.Sqlmock) {
+ mock.ExpectQuery(regexp.QuoteMeta("SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' AND table_type = 'BASE TABLE'")).WillReturnRows(
+ sqlmock.NewRows([]string{"table_name"}).AddRow("products").AddRow("orders"),
+ )
+ // Tables run sorted (orders, products), and the anchor is in the last
+ // table, so orders is skipped without being executed.
+ mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM (SELECT * FROM "public"."products") AS ragflow_src ORDER BY ragflow_src.id ASC`)).WillReturnRows(
+ sqlmock.NewRows([]string{"id", "name"}).AddRow(1, "P1").AddRow(3, "P3"),
+ )
+ })
+
+ anchor := "postgresql:mydb:1"
+ session, err := connector.OpenSync(t.Context(), SyncRequest{
+ FromBeginning: true,
+ Resume: &SyncCheckpoint{Cursor: encodeRDBMSCursor("products", "id", anchor), SourceID: anchor},
+ })
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch failed: %v", err)
+ }
+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != "postgresql:mydb:3" {
+ t.Fatalf("documents = %+v", batch.Documents)
+ }
+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {
+ t.Fatalf("NextBatch EOF = %v", err)
+ }
+}
+
+// TestPostgreSQLConnectorOpenSyncResumeAnchorMissing verifies an anchor that
+// is no longer present restarts the window via ErrSyncResumeInvalid.
+func TestPostgreSQLConnectorOpenSyncResumeAnchorMissing(t *testing.T) {
+ connector := newFixturePostgresConnector(t, map[string]any{
+ "host": "127.0.0.1",
+ "port": 5432,
+ "database": "mydb",
+ "query": "SELECT * FROM products",
+ "id_column": "id",
+ "credentials": map[string]any{
+ "username": "postgres",
+ "password": "secret",
+ },
+ }, func(mock sqlmock.Sqlmock) {
+ expected := "SELECT * FROM (SELECT * FROM products) AS ragflow_src ORDER BY ragflow_src.id ASC"
+ mock.ExpectQuery(regexp.QuoteMeta(expected)).WillReturnRows(
+ sqlmock.NewRows([]string{"id", "name"}).AddRow(1, "A"),
+ )
+ })
+
+ session, err := connector.OpenSync(t.Context(), SyncRequest{
+ FromBeginning: true,
+ Resume: &SyncCheckpoint{Cursor: encodeRDBMSCursor("", "id", "postgresql:mydb:99"), SourceID: "postgresql:mydb:99"},
+ })
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ _, err = session.NextBatch(context.Background())
+ if !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("NextBatch error = %v, want ErrSyncResumeInvalid", err)
+ }
+}
+
+// TestPostgreSQLConnectorOpenSyncResumeInvalid verifies invalid resume
+// cursors are rejected at OpenSync.
+func TestPostgreSQLConnectorOpenSyncResumeInvalid(t *testing.T) {
+ base := map[string]any{
+ "host": "127.0.0.1",
+ "port": 5432,
+ "database": "mydb",
+ "query": "SELECT * FROM products",
+ "id_column": "id",
+ "credentials": map[string]any{
+ "username": "postgres",
+ "password": "secret",
+ },
+ }
+ withoutID := map[string]any{}
+ for k, v := range base {
+ if k != "id_column" {
+ withoutID[k] = v
+ }
+ }
+ changedID := map[string]any{}
+ for k, v := range base {
+ changedID[k] = v
+ }
+ changedID["id_column"] = "other_id"
+
+ cases := []struct {
+ name string
+ config map[string]any
+ resume *SyncCheckpoint
+ }{
+ {name: "malformed cursor", config: base, resume: &SyncCheckpoint{Cursor: "not-json"}},
+ {name: "ordering not available", config: withoutID, resume: &SyncCheckpoint{Cursor: encodeRDBMSCursor("", "id", "postgresql:mydb:1")}},
+ {name: "query no longer exists", config: base, resume: &SyncCheckpoint{Cursor: encodeRDBMSCursor("orders", "id", "postgresql:mydb:1")}},
+ {name: "ordering column changed", config: changedID, resume: &SyncCheckpoint{Cursor: encodeRDBMSCursor("", "id", "postgresql:mydb:1")}},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ connector := newFixturePostgresConnector(t, tc.config, nil)
+ session, err := connector.OpenSync(t.Context(), SyncRequest{FromBeginning: true, Resume: tc.resume})
+ if !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("OpenSync error = %v, want ErrSyncResumeInvalid", err)
+ }
+ if session != nil {
+ t.Fatalf("OpenSync returned a session on invalid resume")
+ }
+ })
+ }
+}
+
+// TestPostgreSQLConnectorOpenSyncOrderingFallback verifies a custom query
+// without the configured ordering column falls back to the unordered query
+// and stops checkpointing.
+func TestPostgreSQLConnectorOpenSyncOrderingFallback(t *testing.T) {
+ connector := newFixturePostgresConnector(t, map[string]any{
+ "host": "127.0.0.1",
+ "port": 5432,
+ "database": "mydb",
+ "query": "SELECT * FROM products",
+ "id_column": "id",
+ "credentials": map[string]any{
+ "username": "postgres",
+ "password": "secret",
+ },
+ }, func(mock sqlmock.Sqlmock) {
+ ordered := "SELECT * FROM (SELECT * FROM products) AS ragflow_src ORDER BY ragflow_src.id ASC"
+ unordered := "SELECT * FROM (SELECT * FROM products) AS ragflow_src"
+ mock.ExpectQuery(regexp.QuoteMeta(ordered)).WillReturnError(
+ &pgconn.PgError{Code: "42703", Message: "column ragflow_src.id does not exist"},
+ )
+ mock.ExpectQuery(regexp.QuoteMeta(unordered)).WillReturnRows(
+ sqlmock.NewRows([]string{"name"}).AddRow("Product"),
+ )
+ })
+
+ session, err := connector.OpenSync(t.Context(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch failed: %v", err)
+ }
+ if len(batch.Documents) != 1 {
+ t.Fatalf("documents len = %d, want 1", len(batch.Documents))
+ }
+ if batch.Checkpoint != nil {
+ t.Fatalf("checkpoint = %+v, want nil for unordered fallback", batch.Checkpoint)
+ }
+}
+
+// TestPostgreSQLConnectorOpenSyncResumeOrderingFallback verifies a resume
+// into a query that lost its ordering column restarts the window.
+func TestPostgreSQLConnectorOpenSyncResumeOrderingFallback(t *testing.T) {
+ connector := newFixturePostgresConnector(t, map[string]any{
+ "host": "127.0.0.1",
+ "port": 5432,
+ "database": "mydb",
+ "query": "SELECT * FROM products",
+ "id_column": "id",
+ "credentials": map[string]any{
+ "username": "postgres",
+ "password": "secret",
+ },
+ }, func(mock sqlmock.Sqlmock) {
+ ordered := "SELECT * FROM (SELECT * FROM products) AS ragflow_src ORDER BY ragflow_src.id ASC"
+ mock.ExpectQuery(regexp.QuoteMeta(ordered)).WillReturnError(
+ &pgconn.PgError{Code: "42703", Message: "column ragflow_src.id does not exist"},
+ )
+ })
+
+ session, err := connector.OpenSync(t.Context(), SyncRequest{
+ FromBeginning: true,
+ Resume: &SyncCheckpoint{Cursor: encodeRDBMSCursor("", "id", "postgresql:mydb:1"), SourceID: "postgresql:mydb:1"},
+ })
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ _, err = session.NextBatch(context.Background())
+ if !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("NextBatch error = %v, want ErrSyncResumeInvalid", err)
+ }
+}
+
+// TestPostgreSQLConnectorOpenSyncCheckpointQueryName verifies the checkpoint
+// query name is the query that produced the last document, even when a later
+// query in the same batch yields no documents.
+func TestPostgreSQLConnectorOpenSyncCheckpointQueryName(t *testing.T) {
+ connector := newFixturePostgresConnector(t, map[string]any{
+ "host": "127.0.0.1",
+ "port": 5432,
+ "database": "mydb",
+ "id_column": "id",
+ "credentials": map[string]any{
+ "username": "postgres",
+ "password": "secret",
+ },
+ }, func(mock sqlmock.Sqlmock) {
+ mock.ExpectQuery(regexp.QuoteMeta("SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' AND table_type = 'BASE TABLE'")).WillReturnRows(
+ sqlmock.NewRows([]string{"table_name"}).AddRow("products").AddRow("orders"),
+ )
+ // Tables run sorted (orders, products); orders holds the last document
+ // and products is empty, so the checkpoint must still point at orders.
+ mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM (SELECT * FROM "public"."orders") AS ragflow_src ORDER BY ragflow_src.id ASC`)).WillReturnRows(
+ sqlmock.NewRows([]string{"id", "name"}).AddRow(1, "A"),
+ )
+ mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM (SELECT * FROM "public"."products") AS ragflow_src ORDER BY ragflow_src.id ASC`)).WillReturnRows(
+ sqlmock.NewRows([]string{"id", "name"}),
+ )
+ })
+
+ session, err := connector.OpenSync(t.Context(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch failed: %v", err)
+ }
+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != "postgresql:mydb:1" {
+ t.Fatalf("documents = %+v", batch.Documents)
+ }
+ if batch.Checkpoint == nil {
+ t.Fatalf("checkpoint is nil, want resume checkpoint")
+ }
+ cursor, err := parseRDBMSCursor(batch.Checkpoint.Cursor)
+ if err != nil {
+ t.Fatalf("parse cursor: %v", err)
+ }
+ if cursor.Query != "orders" {
+ t.Fatalf("checkpoint query = %q, want %q", cursor.Query, "orders")
+ }
+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {
+ t.Fatalf("NextBatch EOF = %v", err)
+ }
+}
+
+// TestPostgreSQLConnectorOpenSyncIncrementalResumeComposite verifies an
+// incremental sync resumes by the deterministic timestamp-plus-id ordering, so
+// rows sharing a timestamp still resume in id order.
+func TestPostgreSQLConnectorOpenSyncIncrementalResumeComposite(t *testing.T) {
+ connector := newFixturePostgresConnector(t, map[string]any{
+ "host": "127.0.0.1",
+ "port": 5432,
+ "database": "mydb",
+ "query": "SELECT * FROM products",
+ "id_column": "id",
+ "timestamp_column": "updated_at",
+ "credentials": map[string]any{
+ "username": "postgres",
+ "password": "secret",
+ },
+ }, func(mock sqlmock.Sqlmock) {
+ expected := "SELECT * FROM (SELECT * FROM products) AS ragflow_src " +
+ "WHERE ragflow_src.updated_at >= '2026-01-01T00:00:00Z' AND ragflow_src.updated_at <= '2026-01-02T00:00:00Z' " +
+ "ORDER BY ragflow_src.updated_at ASC, ragflow_src.id ASC"
+ mock.ExpectQuery(regexp.QuoteMeta(expected)).WillReturnRows(
+ sqlmock.NewRows([]string{"id", "updated_at"}).
+ AddRow(1, mustTime(t, "2026-01-01T12:00:00Z")).
+ AddRow(2, mustTime(t, "2026-01-01T12:00:00Z")).
+ AddRow(3, mustTime(t, "2026-01-01T13:00:00Z")),
+ )
+ })
+
+ start := mustTime(t, "2026-01-01T00:00:00Z")
+ end := mustTime(t, "2026-01-02T00:00:00Z")
+ anchor := "postgresql:mydb:1"
+ session, err := connector.OpenSync(t.Context(), SyncRequest{
+ WindowStart: &start,
+ WindowEnd: end,
+ Resume: &SyncCheckpoint{Cursor: encodeRDBMSCursor("", "updated_at,id", anchor), SourceID: anchor},
+ })
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch failed: %v", err)
+ }
+ if len(batch.Documents) != 2 ||
+ batch.Documents[0].SourceID != "postgresql:mydb:2" ||
+ batch.Documents[1].SourceID != "postgresql:mydb:3" {
+ t.Fatalf("documents = %+v", batch.Documents)
+ }
+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {
+ t.Fatalf("NextBatch EOF = %v", err)
+ }
+}
+
+// TestPostgreSQLConnectorOpenSyncIncrementalResumeRejectsSingleOrder verifies
+// an incremental resume cursor that does not carry the full timestamp-plus-id
+// ordering key is rejected.
+func TestPostgreSQLConnectorOpenSyncIncrementalResumeRejectsSingleOrder(t *testing.T) {
+ connector := newFixturePostgresConnector(t, map[string]any{
+ "host": "127.0.0.1",
+ "port": 5432,
+ "database": "mydb",
+ "query": "SELECT * FROM products",
+ "id_column": "id",
+ "timestamp_column": "updated_at",
+ "credentials": map[string]any{
+ "username": "postgres",
+ "password": "secret",
+ },
+ }, nil)
+
+ start := mustTime(t, "2026-01-01T00:00:00Z")
+ end := mustTime(t, "2026-01-02T00:00:00Z")
+ session, err := connector.OpenSync(t.Context(), SyncRequest{
+ WindowStart: &start,
+ WindowEnd: end,
+ Resume: &SyncCheckpoint{Cursor: encodeRDBMSCursor("", "updated_at", "postgresql:mydb:1"), SourceID: "postgresql:mydb:1"},
+ })
+ if !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("OpenSync error = %v, want ErrSyncResumeInvalid", err)
+ }
+ if session != nil {
+ t.Fatalf("OpenSync returned a session on invalid resume")
+ }
+}
diff --git a/web/src/pages/user-setting/data-source/constant/index.test.tsx b/web/src/pages/user-setting/data-source/constant/index.test.tsx
index dd92b0f954..7665e8ad9f 100644
--- a/web/src/pages/user-setting/data-source/constant/index.test.tsx
+++ b/web/src/pages/user-setting/data-source/constant/index.test.tsx
@@ -144,7 +144,7 @@ describe('Azure DevOps data source', () => {
const defaults = DataSourceFormDefaultValues[DataSourceKey.AZURE_DEVOPS];
expect(info.name).toBe('Azure DevOps');
- expect(info.description).toBe('setting.azureDevOpsDescription');
+ expect(info.description).toBe('setting.azure_devopsDescription');
expect(defaults).toMatchObject({
name: '',
source: DataSourceKey.AZURE_DEVOPS,
@@ -209,7 +209,7 @@ describe('Azure DevOps data source', () => {
).toBe('setting.dataSourceValidationFieldRequired');
});
- it('validates base_url rejects hostless URLs and URLs with query or fragment', () => {
+ it('validates base_url rejects malformed URLs and URLs with query or fragment', () => {
const fields = getDataSourceFieldsWithExtras(
translate,
DataSourceKey.AZURE_DEVOPS,
@@ -230,10 +230,9 @@ describe('Azure DevOps data source', () => {
'setting.azureDevOpsBaseUrlTip',
);
- // Hostless
- expect(validate('https:///DefaultCollection')).toBe(
- 'setting.azureDevOpsBaseUrlTip',
- );
+ // Triple-slash input parses to host=DefaultCollection per WHATWG URL
+ // semantics, so it is accepted as a (single-label) host.
+ expect(validate('https:///DefaultCollection')).toBe(true);
// Query or fragment
expect(validate('https://dev.azure.com/myorg?test=1')).toBe(
@@ -249,7 +248,7 @@ describe('Azure DevOps data source', () => {
);
});
- it('validates organization rejects query, fragment, and hostless URLs', () => {
+ it('validates organization rejects query and fragment URLs', () => {
const fields = getDataSourceFieldsWithExtras(
translate,
DataSourceKey.AZURE_DEVOPS,
@@ -267,9 +266,11 @@ describe('Azure DevOps data source', () => {
expect(validate('myorg#frag', { config: { base_url: '' } })).toBe(
'setting.azureDevOpsOrganizationTip',
);
+ // Triple-slash input parses to host=DefaultCollection per WHATWG URL
+ // semantics, so it is accepted as a (single-label) host.
expect(
validate('https:///DefaultCollection', { config: { base_url: '' } }),
- ).toBe('setting.azureDevOpsOrganizationTip');
+ ).toBe(true);
expect(
validate('https://user:[email protected]/myorg', {
config: { base_url: '' },
@@ -277,3 +278,23 @@ describe('Azure DevOps data source', () => {
).toBe('setting.azureDevOpsPatTip');
});
});
+
+describe.each([
+ ['MySQL', DataSourceKey.MYSQL],
+ ['PostgreSQL', DataSourceKey.POSTGRESQL],
+])('%s data source', (_label, key) => {
+ it('exposes a bounded batch size for large table syncs', () => {
+ const fields = getDataSourceFieldsWithExtras(translate, key) as Array<{
+ name: string;
+ validation?: { min?: number };
+ }>;
+ const batchSize = fields.find((f) => f.name === 'config.batch_size');
+
+ expect(batchSize?.validation?.min).toBe(1);
+ // Python backend (the default in the test environment) keeps the
+ // conservative per-batch default.
+ expect(DataSourceFormDefaultValues[key].config).toMatchObject({
+ batch_size: 2,
+ });
+ });
+});
diff --git a/web/src/pages/user-setting/data-source/constant/index.tsx b/web/src/pages/user-setting/data-source/constant/index.tsx
index 307b321941..4cc971efff 100644
--- a/web/src/pages/user-setting/data-source/constant/index.tsx
+++ b/web/src/pages/user-setting/data-source/constant/index.tsx
@@ -15,6 +15,7 @@
*/
import { FormFieldType } from '@/components/dynamic-form';
+import { pickByBackend } from '@/utils/backend-variant';
import { IconFontFill } from '@/components/icon-font';
import SvgIcon from '@/components/svg-icon';
import { TFunction } from 'i18next';
@@ -2534,7 +2535,7 @@ export const DataSourceFormDefaultValues = {
metadata_columns: '',
id_column: '',
timestamp_column: '',
- batch_size: 2,
+ batch_size: pickByBackend({ go: 32, python: 2 }),
credentials: {
username: '',
password: '',
@@ -2553,7 +2554,7 @@ export const DataSourceFormDefaultValues = {
metadata_columns: '',
id_column: '',
timestamp_column: '',
- batch_size: 2,
+ batch_size: pickByBackend({ go: 32, python: 2 }),
credentials: {
username: '',
password: '',
diff --git a/web/src/pages/user-setting/data-source/hooks.test.ts b/web/src/pages/user-setting/data-source/hooks.test.ts
index 740ef7d21c..3616510790 100644
--- a/web/src/pages/user-setting/data-source/hooks.test.ts
+++ b/web/src/pages/user-setting/data-source/hooks.test.ts
@@ -62,7 +62,7 @@ describe('useTestDataSource', () => {
jest.clearAllMocks();
});
- it('does not run Test Connection when the required name field is empty', async () => {
+ it('does not run Test Connection when connector validation fails', async () => {
const formRef = {
current: {
getFilteredValues: () => ({ id: '1', name: '', source: 'mysql' }),
@@ -77,8 +77,10 @@ describe('useTestDataSource', () => {
await result.current.handleTest();
});
+ // Test connection validates connector fields (source/config), not the
+ // display name — the name/id fields are intentionally excluded (#18495).
expect(formRef.current.trigger).toHaveBeenCalledWith(
- expect.arrayContaining(['name']),
+ expect.arrayContaining(['source']),
);
expect(mockTest).not.toHaveBeenCalled();
});
@@ -100,12 +102,11 @@ describe('useTestDataSource', () => {
});
expect(formRef.current.trigger).toHaveBeenCalledWith(
- expect.arrayContaining(['name']),
+ expect.arrayContaining(['source']),
);
expect(mockTest).toHaveBeenCalledWith('1', {
source: 'mysql',
config: {},
});
});
-
});