RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(go-syncer): add checkpoint resume for MySQL/PostgreSQL connectors (#19762)
mkaaad1ff3d961119e
diff --git a/internal/syncer/connector/models.go b/internal/syncer/connector/models.goindex 3d58ad0467..3e39ffe0b3 100644--- a/internal/syncer/connector/models.go+++ b/internal/syncer/connector/models.go@@ -17,7 +17,9 @@package connectorimport (+ "encoding/json""errors"+ "fmt""time")@@ -106,3 +108,53 @@ type SlimDocument struct {type PruneBatch struct {Documents []SlimDocument}++// rdbmsQuery is one base SQL query plus the identity used by resume cursors:+// the table name in per-table mode, or "" for a single custom query.+type rdbmsQuery struct {+ name string+ sql string+}++// rdbmsSyncQuery is one prepared sync query with its resume metadata.+type rdbmsSyncQuery struct {+ name string+ sql string+ ordered bool+ fallback string+}++// rdbmsResumeCursor identifies where a MySQL/PostgreSQL sync left off: the+// query that produced the last committed batch, the ordering key the stream+// was sorted by (a single column, or "timestamp,id" for incremental windows),+// and the SourceID of the last emitted row (the resume anchor). It is stored+// JSON-encoded in SyncCheckpoint.Cursor so arbitrary table names and source+// ids survive the round trip.+type rdbmsResumeCursor struct {+ Query string `json:"q"`+ Order string `json:"o"`+ SourceID string `json:"s"`+}++// encodeRDBMSCursor serializes a resume cursor for SyncCheckpoint.Cursor.+func encodeRDBMSCursor(query, order, sourceID string) string {+ raw, _ := json.Marshal(rdbmsResumeCursor{Query: query, Order: order, SourceID: sourceID})+ return string(raw)+}++// parseRDBMSCursor decodes a resume cursor. A missing or malformed cursor is+// treated as invalid progress so the runner restarts the task window instead+// of guessing an offset.+func parseRDBMSCursor(cursor string) (rdbmsResumeCursor, error) {+ var c rdbmsResumeCursor+ if cursor == "" {+ return c, fmt.Errorf("rdbms sync checkpoint has no cursor: %w", ErrSyncResumeInvalid)+ }+ if err := json.Unmarshal([]byte(cursor), &c); err != nil {+ return c, fmt.Errorf("rdbms sync checkpoint cursor is malformed: %w", ErrSyncResumeInvalid)+ }+ if c.SourceID == "" {+ return c, fmt.Errorf("rdbms sync checkpoint has no source anchor: %w", ErrSyncResumeInvalid)+ }+ return c, nil+}diff --git a/internal/syncer/connector/mysql.go b/internal/syncer/connector/mysql.goindex 098823cfdf..171b226dd4 100644--- a/internal/syncer/connector/mysql.go+++ b/internal/syncer/connector/mysql.go@@ -22,10 +22,12 @@ import ("database/sql""database/sql/driver""encoding/hex"+ "errors""fmt""io""net""regexp"+ "sort""strconv""strings""time"@@ -131,7 +133,26 @@ func (c *MySQLConnector) OpenSync(ctx context.Context, request SyncRequest) (Synreturn nil, err}queries := c.buildSyncQueries(bases, request)- return &mysqlSyncSession{connector: c, db: db, queries: queries, batchSize: c.batchSize}, nil+ orderColumn := c.syncOrderColumn(request)+ session := &mysqlSyncSession{+ connector: c,+ db: db,+ batchSize: c.batchSize,+ orderColumn: orderColumn,+ checkpointEnabled: orderColumn != "",+ lastDocQuery: -1,+ }+ for _, q := range queries {+ session.queries = append(session.queries, q.sql)+ session.queryNames = append(session.queryNames, q.name)+ session.orderedFlags = append(session.orderedFlags, q.ordered)+ session.fallbackQueries = append(session.fallbackQueries, q.fallback)+ }+ if err := session.applyResume(request.Resume); err != nil {+ db.Close()+ return nil, err+ }+ return session, nil}// OpenPrune opens one complete MySQL prune snapshot session.@@ -147,7 +168,7 @@ func (c *MySQLConnector) OpenPrune(ctx context.Context, request PruneRequest) (P}queries := make([]string, 0, len(bases))for _, base := range bases {- queries = append(queries, c.buildSlimQuery(base))+ queries = append(queries, c.buildSlimQuery(base.sql))}return &mysqlPruneSession{connector: c, db: db, queries: queries, batchSize: c.batchSize}, nil}@@ -212,10 +233,12 @@ func mysqlPinnedDial(pinIP net.IP, port int) mysql.DialContextFunc {}}-// baseQueries returns the configured query or a SELECT per table.-func (c *MySQLConnector) baseQueries(ctx context.Context, db *sql.DB) ([]string, error) {+// baseQueries returns the configured query or a SELECT per table. Table names+// are sorted so the sync stream order is stable across runs and a resume+// cursor can reliably skip already-processed tables.+func (c *MySQLConnector) baseQueries(ctx context.Context, db *sql.DB) ([]rdbmsQuery, error) {if c.query != "" {- return []string{c.query}, nil+ return []rdbmsQuery{{name: "", sql: c.query}}, nil}rows, err := db.QueryContext(ctx, "SHOW TABLES")if err != nil {@@ -233,30 +256,69 @@ func (c *MySQLConnector) baseQueries(ctx context.Context, db *sql.DB) ([]string,if err := rows.Err(); err != nil {return nil, err}- queries := make([]string, 0, len(tables))+ sort.Strings(tables)+ queries := make([]rdbmsQuery, 0, len(tables))for _, table := range tables {- queries = append(queries, fmt.Sprintf("SELECT * FROM %s", table))+ queries = append(queries, rdbmsQuery{name: table, sql: fmt.Sprintf("SELECT * FROM %s", table)})}return queries, nil}-// buildSyncQueries applies the incremental window when a timestamp column exists.-func (c *MySQLConnector) buildSyncQueries(bases []string, request SyncRequest) []string {- var start, end *time.Time- if !request.FromBeginning {- start = request.WindowStart- end = &request.WindowEnd- }- if c.timestampColumn == "" || (start == nil && end == nil) {- return bases- }- queries := make([]string, 0, len(bases))- for _, base := range bases {- queries = append(queries, c.buildTimeFilteredQuery(base, start, end))+// buildSyncQueries applies the incremental window and a stable ordering when+// one is available, so a checkpoint can resume the stream from an anchor.+// Each query carries an unordered fallback used when a custom SQL query does+// not expose the configured ordering column.+func (c *MySQLConnector) buildSyncQueries(bases []rdbmsQuery, request SyncRequest) []rdbmsSyncQuery {+ queries := make([]rdbmsSyncQuery, 0, len(bases))+ switch {+ case !request.FromBeginning && c.timestampColumn != "":+ start := request.WindowStart+ end := &request.WindowEnd+ for _, base := range bases {+ queries = append(queries, rdbmsSyncQuery{+ name: base.name,+ sql: c.buildTimeFilteredOrderedQuery(base.sql, start, end),+ ordered: true,+ fallback: c.buildTimeFilteredQuery(base.sql, start, end),+ })+ }+ case request.FromBeginning && c.idColumn != "":+ for _, base := range bases {+ queries = append(queries, rdbmsSyncQuery{+ name: base.name,+ sql: c.buildOrderedQuery(base.sql, c.idColumn),+ ordered: true,+ fallback: c.wrapQuery(base.sql),+ })+ }+ default:+ for _, base := range bases {+ queries = append(queries, rdbmsSyncQuery{name: base.name, sql: base.sql})+ }}return queries}+// syncOrderColumn returns the ordering key that makes this sync window+// deterministic, or "" when the connector cannot checkpoint/resume the+// stream (no stable ordering key). Incremental windows order by timestamp+// plus id so rows sharing a timestamp still resume deterministically.+func (c *MySQLConnector) syncOrderColumn(request SyncRequest) string {+ switch {+ case !request.FromBeginning && c.timestampColumn != "" && c.idColumn != "":+ return c.timestampColumn + "," + c.idColumn+ case request.FromBeginning && c.idColumn != "":+ return c.idColumn+ }+ return ""+}++// buildOrderedQuery wraps the base query and orders it by a stable column so+// connector sync can resume from a checkpoint.+func (c *MySQLConnector) buildOrderedQuery(base, orderColumn string) string {+ return c.wrapQuery(base) + " ORDER BY ragflow_src." + orderColumn + " ASC"+}+// buildTimeFilteredQuery wraps the base query and appends timestamp bounds.func (c *MySQLConnector) buildTimeFilteredQuery(base string, start, end *time.Time) string {conditions := []string{}@@ -273,6 +335,18 @@ func (c *MySQLConnector) buildTimeFilteredQuery(base string, start, end *time.Tireturn query}+// buildTimeFilteredOrderedQuery is the incremental query plus a deterministic+// ORDER BY on the timestamp and id columns, which resume relies on. Without a+// configured id column the order is timestamp-only and the stream is not+// checkpointed.+func (c *MySQLConnector) buildTimeFilteredOrderedQuery(base string, start, end *time.Time) string {+ query := c.buildTimeFilteredQuery(base, start, end) + " ORDER BY ragflow_src." + c.timestampColumn + " ASC"+ if c.idColumn != "" {+ query += ", ragflow_src." + c.idColumn + " ASC"+ }+ return query+}+// buildSlimQuery selects only the columns needed to identify documents.func (c *MySQLConnector) buildSlimQuery(base string) string {columns := []string{}@@ -518,9 +592,25 @@ type mysqlSyncSession struct {connector *MySQLConnectordb *sql.DBqueries []string- queryIndex int- rows *sql.Rows- batchSize int+ queryNames []string+ // orderedFlags[i] reports whether queries[i] carries a stable ORDER BY.+ orderedFlags []bool+ // fallbackQueries[i] is the unordered variant of queries[i], used when a+ // custom SQL query does not expose the configured ordering column.+ fallbackQueries []string+ queryIndex int+ // lastDocQuery is the index of the query that produced the most recently+ // appended document, used to checkpoint against the right query name even+ // when later queries in the batch contributed no documents.+ lastDocQuery int+ rows *sql.Rows+ batchSize int++ orderColumn string+ checkpointEnabled bool+ orderable bool+ resume *rdbmsResumeCursor+ resumePending bool}// NextBatch returns the next MySQL document batch.@@ -530,7 +620,7 @@ func (s *mysqlSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {if s.rows == nil {if s.queryIndex >= len(s.queries) {if len(documents) == 0 {- return SyncBatch{}, io.EOF+ return s.endOfStream()}break}@@ -552,10 +642,17 @@ func (s *mysqlSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {continue}if doc, ok := s.connector.rowToSourceDocument(row, columns); ok {+ if !s.includeResumed(doc) {+ continue+ }documents = append(documents, doc)+ s.lastDocQuery = s.queryIndex - 1}}- return SyncBatch{Documents: documents}, nil+ if len(documents) == 0 {+ return s.endOfStream()+ }+ return SyncBatch{Documents: documents, Checkpoint: s.batchCheckpoint(documents[len(documents)-1])}, nil}// Close closes the MySQL sync session.@@ -564,14 +661,32 @@ func (s *mysqlSyncSession) Close() error {return s.db.Close()}-// openNextQuery runs the next base query.+// openNextQuery runs the next base query. When a custom SQL query does not+// expose the configured ordering column (MySQL error 1054), it falls back to+// the unordered query and stops checkpointing so the remaining stream is never+// resumed against a non-deterministic order. A pending resume never falls back:+// the ordering that produced the anchor is gone, so the window restarts.func (s *mysqlSyncSession) openNextQuery(ctx context.Context) error {- query := s.queries[s.queryIndex]+ idx := s.queryIndexs.queryIndex++- rows, err := s.db.QueryContext(ctx, query)+ rows, err := s.db.QueryContext(ctx, s.queries[idx])if err != nil {+ if s.orderedFlags[idx] && isMySQLUnknownColumn(err) {+ if s.resumePending {+ return fmt.Errorf("MySQL sync resume query lost its ordering column: %w", ErrSyncResumeInvalid)+ }+ s.checkpointEnabled = false+ rows, err = s.db.QueryContext(ctx, s.fallbackQueries[idx])+ if err != nil {+ return fmt.Errorf("MySQL query failed: %w", err)+ }+ s.orderable = false+ s.rows = rows+ return nil+ }return fmt.Errorf("MySQL query failed: %w", err)}+ s.orderable = s.orderedFlags[idx]s.rows = rowsreturn nil}@@ -584,6 +699,89 @@ func (s *mysqlSyncSession) closeRows() {}}+// applyResume positions the session after the last committed batch. The+// cursor's query and ordering column must still exist, otherwise the runner+// restarts the task window.+func (s *mysqlSyncSession) applyResume(checkpoint *SyncCheckpoint) error {+ if checkpoint == nil {+ return nil+ }+ cursor, err := parseRDBMSCursor(checkpoint.Cursor)+ if err != nil {+ return err+ }+ if s.orderColumn == "" || cursor.Order != s.orderColumn {+ return fmt.Errorf("MySQL sync resume ordering changed from %q to %q: %w", cursor.Order, s.orderColumn, ErrSyncResumeInvalid)+ }+ idx := -1+ for i, name := range s.queryNames {+ if name == cursor.Query {+ idx = i+ break+ }+ }+ if idx < 0 {+ return fmt.Errorf("MySQL sync resume query %q no longer exists: %w", cursor.Query, ErrSyncResumeInvalid)+ }+ s.queryIndex = idx+ s.resume = &cursor+ s.resumePending = true+ return nil+}++// includeResumed reports whether doc should be emitted. While a resume is+// pending, every row before (and including) the anchor is skipped because it+// was already committed.+func (s *mysqlSyncSession) includeResumed(doc SourceDocument) bool {+ if !s.resumePending {+ return true+ }+ if s.resume != nil && doc.SourceID == s.resume.SourceID {+ s.resumePending = false+ return false+ }+ return false+}++// batchCheckpoint builds the checkpoint for a batch whose last row is doc.+// Batches from a non-deterministic (unordered) query never carry a checkpoint.+func (s *mysqlSyncSession) batchCheckpoint(doc SourceDocument) *SyncCheckpoint {+ if !s.checkpointEnabled || !s.orderable {+ return nil+ }+ queryName := ""+ if idx := s.lastDocQuery; idx >= 0 && idx < len(s.queryNames) {+ queryName = s.queryNames[idx]+ }+ updatedAt := doc.UpdatedAt+ return &SyncCheckpoint{+ Cursor: encodeRDBMSCursor(queryName, s.orderColumn, doc.SourceID),+ SourceID: doc.SourceID,+ UpdatedAt: &updatedAt,+ }+}++// endOfStream returns io.EOF when the stream is exhausted, or+// ErrSyncResumeInvalid when a pending resume anchor was never found.+func (s *mysqlSyncSession) endOfStream() (SyncBatch, error) {+ if s.resumePending {+ anchor := ""+ if s.resume != nil {+ anchor = s.resume.SourceID+ }+ return SyncBatch{}, fmt.Errorf("MySQL resume anchor %q was not found in the current result: %w", anchor, ErrSyncResumeInvalid)+ }+ return SyncBatch{}, io.EOF+}++// isMySQLUnknownColumn reports whether err is MySQL error 1054 (unknown column+// in the order/result), used to detect custom queries that do not expose the+// configured ordering column.+func isMySQLUnknownColumn(err error) bool {+ var mysqlErr *mysql.MySQLError+ return errors.As(err, &mysqlErr) && mysqlErr.Number == 1054+}+type mysqlPruneSession struct {connector *MySQLConnectordb *sql.DBdiff --git a/internal/syncer/connector/mysql_test.go b/internal/syncer/connector/mysql_test.goindex eaefb7c42d..3b582fb5ff 100644--- a/internal/syncer/connector/mysql_test.go+++ b/internal/syncer/connector/mysql_test.go@@ -29,6 +29,7 @@ import ("time""github.com/DATA-DOG/go-sqlmock"+ "github.com/go-sql-driver/mysql")func newFixtureMySQLConnector(t *testing.T, config map[string]any, expect func(mock sqlmock.Sqlmock)) *MySQLConnector {@@ -71,6 +72,7 @@ func newFixtureMySQLConnector(t *testing.T, config map[string]any, expect func(mfunc TestMySQLConnectorOpenSyncCustomQuery(t *testing.T) {query := "SELECT * FROM products WHERE status = 'active'"updatedAt := mustTime(t, "2026-01-02T03:04:05Z")+ expected := "SELECT * FROM (SELECT * FROM products WHERE status = 'active') AS ragflow_src ORDER BY ragflow_src.id ASC"connector := newFixtureMySQLConnector(t, map[string]any{"host": "127.0.0.1","port": "3306",@@ -85,7 +87,7 @@ func TestMySQLConnectorOpenSyncCustomQuery(t *testing.T) {"password": "secret",},}, func(mock sqlmock.Sqlmock) {- mock.ExpectQuery(regexp.QuoteMeta(query)).WillReturnRows(+ mock.ExpectQuery(regexp.QuoteMeta(expected)).WillReturnRows(sqlmock.NewRows([]string{"id", "title", "description", "category", "updated_at"}).AddRow(7, "Hello/World", "Some body", "news", updatedAt),)@@ -140,6 +142,7 @@ func TestMySQLConnectorOpenSyncIncrementalWindow(t *testing.T) {"port": 3306,"database": "mydb","query": "SELECT * FROM products",+ "id_column": "id","timestamp_column": "updated_at","credentials": map[string]any{"username": "root",@@ -147,7 +150,8 @@ func TestMySQLConnectorOpenSyncIncrementalWindow(t *testing.T) {},}, func(mock sqlmock.Sqlmock) {expected := "SELECT * FROM (SELECT * FROM products) AS ragflow_src " +- "WHERE ragflow_src.updated_at >= '2026-01-01 00:00:00' AND ragflow_src.updated_at <= '2026-01-02 00:00:00'"+ "WHERE ragflow_src.updated_at >= '2026-01-01 00:00:00' AND ragflow_src.updated_at <= '2026-01-02 00:00:00' " ++ "ORDER BY ragflow_src.updated_at ASC, ragflow_src.id ASC"mock.ExpectQuery(regexp.QuoteMeta(expected)).WillReturnRows(sqlmock.NewRows([]string{"updated_at"}).AddRow(mustTime(t, "2026-01-01T12:00:00Z")),)@@ -183,12 +187,12 @@ func TestMySQLConnectorOpenSyncAllTables(t *testing.T) {mock.ExpectQuery(regexp.QuoteMeta("SHOW TABLES")).WillReturnRows(sqlmock.NewRows([]string{"Tables_in_mydb"}).AddRow("products").AddRow("orders"),)- mock.ExpectQuery(regexp.QuoteMeta("SELECT * FROM products")).WillReturnRows(- sqlmock.NewRows([]string{"id", "name"}).AddRow(1, "Product"),- )- mock.ExpectQuery(regexp.QuoteMeta("SELECT * FROM orders")).WillReturnRows(+ mock.ExpectQuery(regexp.QuoteMeta("SELECT * FROM (SELECT * FROM orders) AS ragflow_src ORDER BY ragflow_src.id ASC")).WillReturnRows(sqlmock.NewRows([]string{"id", "name"}).AddRow(2, "Order"),)+ mock.ExpectQuery(regexp.QuoteMeta("SELECT * FROM (SELECT * FROM products) AS ragflow_src ORDER BY ragflow_src.id ASC")).WillReturnRows(+ sqlmock.NewRows([]string{"id", "name"}).AddRow(1, "Product"),+ )})session, err := connector.OpenSync(t.Context(), SyncRequest{FromBeginning: true})@@ -208,7 +212,7 @@ func TestMySQLConnectorOpenSyncAllTables(t *testing.T) {ids = append(ids, doc.SourceID)}}- if len(ids) != 2 || ids[0] != "mysql:mydb:1" || ids[1] != "mysql:mydb:2" {+ if len(ids) != 2 || ids[0] != "mysql:mydb:2" || ids[1] != "mysql:mydb:1" {t.Fatalf("source ids = %v", ids)}}@@ -378,6 +382,7 @@ func TestMySQLConnectorFileExtension(t *testing.T) {// extension is applied to produced documents.func TestMySQLConnectorOpenSyncCustomFileExtension(t *testing.T) {query := "SELECT * FROM products WHERE status = 'active'"+ expected := "SELECT * FROM (SELECT * FROM products WHERE status = 'active') AS ragflow_src ORDER BY ragflow_src.id ASC"connector := newFixtureMySQLConnector(t, map[string]any{"host": "127.0.0.1","port": "3306",@@ -391,7 +396,7 @@ func TestMySQLConnectorOpenSyncCustomFileExtension(t *testing.T) {"password": "secret",},}, func(mock sqlmock.Sqlmock) {- mock.ExpectQuery(regexp.QuoteMeta(query)).WillReturnRows(+ mock.ExpectQuery(regexp.QuoteMeta(expected)).WillReturnRows(sqlmock.NewRows([]string{"id", "title", "description"}).AddRow(7, "Hello", "World"),)@@ -415,3 +420,438 @@ func TestMySQLConnectorOpenSyncCustomFileExtension(t *testing.T) {t.Fatalf("NextBatch EOF = %v", err)}}++// TestMySQLConnectorOpenSyncCheckpointEmitted verifies ordered sync batches+// carry a resume checkpoint.+func TestMySQLConnectorOpenSyncCheckpointEmitted(t *testing.T) {+ connector := newFixtureMySQLConnector(t, map[string]any{+ "host": "127.0.0.1",+ "port": 3306,+ "database": "mydb",+ "query": "SELECT * FROM products",+ "id_column": "id",+ "timestamp_column": "updated_at",+ "credentials": map[string]any{+ "username": "root",+ "password": "secret",+ },+ }, func(mock sqlmock.Sqlmock) {+ expected := "SELECT * FROM (SELECT * FROM products) AS ragflow_src " ++ "WHERE ragflow_src.updated_at >= '2026-01-01 00:00:00' AND ragflow_src.updated_at <= '2026-01-02 00:00:00' " ++ "ORDER BY ragflow_src.updated_at ASC, ragflow_src.id ASC"+ mock.ExpectQuery(regexp.QuoteMeta(expected)).WillReturnRows(+ sqlmock.NewRows([]string{"id", "updated_at"}).+ AddRow(1, mustTime(t, "2026-01-01T12:00:00Z")).+ AddRow(2, mustTime(t, "2026-01-01T13:00:00Z")),+ )+ })++ start := mustTime(t, "2026-01-01T00:00:00Z")+ end := mustTime(t, "2026-01-02T00:00:00Z")+ session, err := connector.OpenSync(t.Context(), SyncRequest{WindowStart: &start, WindowEnd: end})+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch failed: %v", err)+ }+ if len(batch.Documents) != 2 {+ t.Fatalf("documents len = %d, want 2", len(batch.Documents))+ }+ if batch.Checkpoint == nil {+ t.Fatalf("checkpoint is nil, want resume checkpoint")+ }+ if batch.Checkpoint.SourceID != "mysql:mydb:2" {+ t.Fatalf("checkpoint source id = %q, want %q", batch.Checkpoint.SourceID, "mysql:mydb:2")+ }+ cursor, err := parseRDBMSCursor(batch.Checkpoint.Cursor)+ if err != nil {+ t.Fatalf("parse cursor: %v", err)+ }+ if cursor.Query != "" || cursor.Order != "updated_at,id" || cursor.SourceID != "mysql:mydb:2" {+ t.Fatalf("cursor = %+v", cursor)+ }+}++// TestMySQLConnectorOpenSyncResumeSkipsToAnchor verifies a full sync resumes+// from the committed anchor instead of re-emitting earlier rows.+func TestMySQLConnectorOpenSyncResumeSkipsToAnchor(t *testing.T) {+ connector := newFixtureMySQLConnector(t, map[string]any{+ "host": "127.0.0.1",+ "port": 3306,+ "database": "mydb",+ "query": "SELECT * FROM products",+ "id_column": "id",+ "credentials": map[string]any{+ "username": "root",+ "password": "secret",+ },+ }, func(mock sqlmock.Sqlmock) {+ expected := "SELECT * FROM (SELECT * FROM products) AS ragflow_src ORDER BY ragflow_src.id ASC"+ mock.ExpectQuery(regexp.QuoteMeta(expected)).WillReturnRows(+ sqlmock.NewRows([]string{"id", "name"}).AddRow(1, "A").AddRow(2, "B").AddRow(3, "C"),+ )+ })++ anchor := "mysql:mydb:1"+ session, err := connector.OpenSync(t.Context(), SyncRequest{+ FromBeginning: true,+ Resume: &SyncCheckpoint{Cursor: encodeRDBMSCursor("", "id", anchor), SourceID: anchor},+ })+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch failed: %v", err)+ }+ if len(batch.Documents) != 2 ||+ batch.Documents[0].SourceID != "mysql:mydb:2" ||+ batch.Documents[1].SourceID != "mysql:mydb:3" {+ t.Fatalf("documents = %+v", batch.Documents)+ }+ if batch.Checkpoint == nil || batch.Checkpoint.SourceID != "mysql:mydb:3" {+ t.Fatalf("checkpoint = %+v", batch.Checkpoint)+ }+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("NextBatch EOF = %v", err)+ }+}++// TestMySQLConnectorOpenSyncResumeMultiTable verifies a resume skips tables+// before the anchor's table without executing them.+func TestMySQLConnectorOpenSyncResumeMultiTable(t *testing.T) {+ connector := newFixtureMySQLConnector(t, map[string]any{+ "host": "127.0.0.1",+ "port": 3306,+ "database": "mydb",+ "id_column": "id",+ "credentials": map[string]any{+ "username": "root",+ "password": "secret",+ },+ }, func(mock sqlmock.Sqlmock) {+ mock.ExpectQuery(regexp.QuoteMeta("SHOW TABLES")).WillReturnRows(+ sqlmock.NewRows([]string{"Tables_in_mydb"}).AddRow("products").AddRow("orders"),+ )+ // Tables run sorted (orders, products), and the anchor is in the last+ // table, so orders is skipped without being executed.+ mock.ExpectQuery(regexp.QuoteMeta("SELECT * FROM (SELECT * FROM products) AS ragflow_src ORDER BY ragflow_src.id ASC")).WillReturnRows(+ sqlmock.NewRows([]string{"id", "name"}).AddRow(1, "P1").AddRow(3, "P3"),+ )+ })++ anchor := "mysql:mydb:1"+ session, err := connector.OpenSync(t.Context(), SyncRequest{+ FromBeginning: true,+ Resume: &SyncCheckpoint{Cursor: encodeRDBMSCursor("products", "id", anchor), SourceID: anchor},+ })+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch failed: %v", err)+ }+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != "mysql:mydb:3" {+ t.Fatalf("documents = %+v", batch.Documents)+ }+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("NextBatch EOF = %v", err)+ }+}++// TestMySQLConnectorOpenSyncResumeAnchorMissing verifies an anchor that is no+// longer present restarts the window via ErrSyncResumeInvalid.+func TestMySQLConnectorOpenSyncResumeAnchorMissing(t *testing.T) {+ connector := newFixtureMySQLConnector(t, map[string]any{+ "host": "127.0.0.1",+ "port": 3306,+ "database": "mydb",+ "query": "SELECT * FROM products",+ "id_column": "id",+ "credentials": map[string]any{+ "username": "root",+ "password": "secret",+ },+ }, func(mock sqlmock.Sqlmock) {+ expected := "SELECT * FROM (SELECT * FROM products) AS ragflow_src ORDER BY ragflow_src.id ASC"+ mock.ExpectQuery(regexp.QuoteMeta(expected)).WillReturnRows(+ sqlmock.NewRows([]string{"id", "name"}).AddRow(1, "A"),+ )+ })++ session, err := connector.OpenSync(t.Context(), SyncRequest{+ FromBeginning: true,+ Resume: &SyncCheckpoint{Cursor: encodeRDBMSCursor("", "id", "mysql:mydb:99"), SourceID: "mysql:mydb:99"},+ })+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ _, err = session.NextBatch(context.Background())+ if !errors.Is(err, ErrSyncResumeInvalid) {+ t.Fatalf("NextBatch error = %v, want ErrSyncResumeInvalid", err)+ }+}++// TestMySQLConnectorOpenSyncResumeInvalid verifies invalid resume cursors are+// rejected at OpenSync.+func TestMySQLConnectorOpenSyncResumeInvalid(t *testing.T) {+ base := map[string]any{+ "host": "127.0.0.1",+ "port": 3306,+ "database": "mydb",+ "query": "SELECT * FROM products",+ "id_column": "id",+ "credentials": map[string]any{+ "username": "root",+ "password": "secret",+ },+ }+ withoutID := map[string]any{}+ for k, v := range base {+ if k != "id_column" {+ withoutID[k] = v+ }+ }+ changedID := map[string]any{}+ for k, v := range base {+ changedID[k] = v+ }+ changedID["id_column"] = "other_id"++ cases := []struct {+ name string+ config map[string]any+ resume *SyncCheckpoint+ }{+ {name: "malformed cursor", config: base, resume: &SyncCheckpoint{Cursor: "not-json"}},+ {name: "ordering not available", config: withoutID, resume: &SyncCheckpoint{Cursor: encodeRDBMSCursor("", "id", "mysql:mydb:1")}},+ {name: "query no longer exists", config: base, resume: &SyncCheckpoint{Cursor: encodeRDBMSCursor("orders", "id", "mysql:mydb:1")}},+ {name: "ordering column changed", config: changedID, resume: &SyncCheckpoint{Cursor: encodeRDBMSCursor("", "id", "mysql:mydb:1")}},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ connector := newFixtureMySQLConnector(t, tc.config, nil)+ session, err := connector.OpenSync(t.Context(), SyncRequest{FromBeginning: true, Resume: tc.resume})+ if !errors.Is(err, ErrSyncResumeInvalid) {+ t.Fatalf("OpenSync error = %v, want ErrSyncResumeInvalid", err)+ }+ if session != nil {+ t.Fatalf("OpenSync returned a session on invalid resume")+ }+ })+ }+}++// TestMySQLConnectorOpenSyncOrderingFallback verifies a custom query without+// the configured ordering column falls back to the unordered query and stops+// checkpointing.+func TestMySQLConnectorOpenSyncOrderingFallback(t *testing.T) {+ connector := newFixtureMySQLConnector(t, map[string]any{+ "host": "127.0.0.1",+ "port": 3306,+ "database": "mydb",+ "query": "SELECT * FROM products",+ "id_column": "id",+ "credentials": map[string]any{+ "username": "root",+ "password": "secret",+ },+ }, func(mock sqlmock.Sqlmock) {+ ordered := "SELECT * FROM (SELECT * FROM products) AS ragflow_src ORDER BY ragflow_src.id ASC"+ unordered := "SELECT * FROM (SELECT * FROM products) AS ragflow_src"+ mock.ExpectQuery(regexp.QuoteMeta(ordered)).WillReturnError(+ &mysql.MySQLError{Number: 1054, Message: "Unknown column 'id' in 'order clause'"},+ )+ mock.ExpectQuery(regexp.QuoteMeta(unordered)).WillReturnRows(+ sqlmock.NewRows([]string{"name"}).AddRow("Product"),+ )+ })++ session, err := connector.OpenSync(t.Context(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch failed: %v", err)+ }+ if len(batch.Documents) != 1 {+ t.Fatalf("documents len = %d, want 1", len(batch.Documents))+ }+ if batch.Checkpoint != nil {+ t.Fatalf("checkpoint = %+v, want nil for unordered fallback", batch.Checkpoint)+ }+}++// TestMySQLConnectorOpenSyncResumeOrderingFallback verifies a resume into a+// query that lost its ordering column restarts the window.+func TestMySQLConnectorOpenSyncResumeOrderingFallback(t *testing.T) {+ connector := newFixtureMySQLConnector(t, map[string]any{+ "host": "127.0.0.1",+ "port": 3306,+ "database": "mydb",+ "query": "SELECT * FROM products",+ "id_column": "id",+ "credentials": map[string]any{+ "username": "root",+ "password": "secret",+ },+ }, func(mock sqlmock.Sqlmock) {+ ordered := "SELECT * FROM (SELECT * FROM products) AS ragflow_src ORDER BY ragflow_src.id ASC"+ mock.ExpectQuery(regexp.QuoteMeta(ordered)).WillReturnError(+ &mysql.MySQLError{Number: 1054, Message: "Unknown column 'id' in 'order clause'"},+ )+ })++ session, err := connector.OpenSync(t.Context(), SyncRequest{+ FromBeginning: true,+ Resume: &SyncCheckpoint{Cursor: encodeRDBMSCursor("", "id", "mysql:mydb:1"), SourceID: "mysql:mydb:1"},+ })+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ _, err = session.NextBatch(context.Background())+ if !errors.Is(err, ErrSyncResumeInvalid) {+ t.Fatalf("NextBatch error = %v, want ErrSyncResumeInvalid", err)+ }+}++// TestMySQLConnectorOpenSyncCheckpointQueryName verifies the checkpoint query+// name is the query that produced the last document, even when a later query+// in the same batch yields no documents.+func TestMySQLConnectorOpenSyncCheckpointQueryName(t *testing.T) {+ connector := newFixtureMySQLConnector(t, map[string]any{+ "host": "127.0.0.1",+ "port": 3306,+ "database": "mydb",+ "id_column": "id",+ "credentials": map[string]any{+ "username": "root",+ "password": "secret",+ },+ }, func(mock sqlmock.Sqlmock) {+ mock.ExpectQuery(regexp.QuoteMeta("SHOW TABLES")).WillReturnRows(+ sqlmock.NewRows([]string{"Tables_in_mydb"}).AddRow("products").AddRow("orders"),+ )+ // Tables run sorted (orders, products); orders holds the last document+ // and products is empty, so the checkpoint must still point at orders.+ mock.ExpectQuery(regexp.QuoteMeta("SELECT * FROM (SELECT * FROM orders) AS ragflow_src ORDER BY ragflow_src.id ASC")).WillReturnRows(+ sqlmock.NewRows([]string{"id", "name"}).AddRow(1, "A"),+ )+ mock.ExpectQuery(regexp.QuoteMeta("SELECT * FROM (SELECT * FROM products) AS ragflow_src ORDER BY ragflow_src.id ASC")).WillReturnRows(+ sqlmock.NewRows([]string{"id", "name"}),+ )+ })++ session, err := connector.OpenSync(t.Context(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch failed: %v", err)+ }+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != "mysql:mydb:1" {+ t.Fatalf("documents = %+v", batch.Documents)+ }+ if batch.Checkpoint == nil {+ t.Fatalf("checkpoint is nil, want resume checkpoint")+ }+ cursor, err := parseRDBMSCursor(batch.Checkpoint.Cursor)+ if err != nil {+ t.Fatalf("parse cursor: %v", err)+ }+ if cursor.Query != "orders" {+ t.Fatalf("checkpoint query = %q, want %q", cursor.Query, "orders")+ }+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("NextBatch EOF = %v", err)+ }+}++// TestMySQLConnectorOpenSyncIncrementalResumeComposite verifies an incremental+// sync resumes by the deterministic timestamp-plus-id ordering, so rows sharing+// a timestamp still resume in id order.+func TestMySQLConnectorOpenSyncIncrementalResumeComposite(t *testing.T) {+ connector := newFixtureMySQLConnector(t, map[string]any{+ "host": "127.0.0.1",+ "port": 3306,+ "database": "mydb",+ "query": "SELECT * FROM products",+ "id_column": "id",+ "timestamp_column": "updated_at",+ "credentials": map[string]any{+ "username": "root",+ "password": "secret",+ },+ }, func(mock sqlmock.Sqlmock) {+ expected := "SELECT * FROM (SELECT * FROM products) AS ragflow_src " ++ "WHERE ragflow_src.updated_at >= '2026-01-01 00:00:00' AND ragflow_src.updated_at <= '2026-01-02 00:00:00' " ++ "ORDER BY ragflow_src.updated_at ASC, ragflow_src.id ASC"+ mock.ExpectQuery(regexp.QuoteMeta(expected)).WillReturnRows(+ sqlmock.NewRows([]string{"id", "updated_at"}).+ AddRow(1, mustTime(t, "2026-01-01T12:00:00Z")).+ AddRow(2, mustTime(t, "2026-01-01T12:00:00Z")).+ AddRow(3, mustTime(t, "2026-01-01T13:00:00Z")),+ )+ })++ start := mustTime(t, "2026-01-01T00:00:00Z")+ end := mustTime(t, "2026-01-02T00:00:00Z")+ anchor := "mysql:mydb:1"+ session, err := connector.OpenSync(t.Context(), SyncRequest{+ WindowStart: &start,+ WindowEnd: end,+ Resume: &SyncCheckpoint{Cursor: encodeRDBMSCursor("", "updated_at,id", anchor), SourceID: anchor},+ })+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch failed: %v", err)+ }+ if len(batch.Documents) != 2 ||+ batch.Documents[0].SourceID != "mysql:mydb:2" ||+ batch.Documents[1].SourceID != "mysql:mydb:3" {+ t.Fatalf("documents = %+v", batch.Documents)+ }+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("NextBatch EOF = %v", err)+ }+}++// TestMySQLConnectorOpenSyncIncrementalResumeRejectsSingleOrder verifies an+// incremental resume cursor that does not carry the full timestamp-plus-id+// ordering key is rejected.+func TestMySQLConnectorOpenSyncIncrementalResumeRejectsSingleOrder(t *testing.T) {+ connector := newFixtureMySQLConnector(t, map[string]any{+ "host": "127.0.0.1",+ "port": 3306,+ "database": "mydb",+ "query": "SELECT * FROM products",+ "id_column": "id",+ "timestamp_column": "updated_at",+ "credentials": map[string]any{+ "username": "root",+ "password": "secret",+ },+ }, nil)++ start := mustTime(t, "2026-01-01T00:00:00Z")+ end := mustTime(t, "2026-01-02T00:00:00Z")+ session, err := connector.OpenSync(t.Context(), SyncRequest{+ WindowStart: &start,+ WindowEnd: end,+ Resume: &SyncCheckpoint{Cursor: encodeRDBMSCursor("", "updated_at", "mysql:mydb:1"), SourceID: "mysql:mydb:1"},+ })+ if !errors.Is(err, ErrSyncResumeInvalid) {+ t.Fatalf("OpenSync error = %v, want ErrSyncResumeInvalid", err)+ }+ if session != nil {+ t.Fatalf("OpenSync returned a session on invalid resume")+ }+}diff --git a/internal/syncer/connector/postgresql.go b/internal/syncer/connector/postgresql.goindex a581ee9268..decef68495 100644--- a/internal/syncer/connector/postgresql.go+++ b/internal/syncer/connector/postgresql.go@@ -22,16 +22,19 @@ import ("database/sql""database/sql/driver""encoding/hex"+ "errors""fmt""io""net""net/url""regexp"+ "sort""strconv""strings""time""github.com/jackc/pgx/v5"+ "github.com/jackc/pgx/v5/pgconn""github.com/jackc/pgx/v5/stdlib")@@ -144,7 +147,26 @@ func (c *PostgreSQLConnector) OpenSync(ctx context.Context, request SyncRequest)return nil, err}queries := c.buildSyncQueries(bases, request)- return &postgresSyncSession{connector: c, db: db, queries: queries, batchSize: c.batchSize}, nil+ orderColumn := c.syncOrderColumn(request)+ session := &postgresSyncSession{+ connector: c,+ db: db,+ batchSize: c.batchSize,+ orderColumn: orderColumn,+ checkpointEnabled: orderColumn != "",+ lastDocQuery: -1,+ }+ for _, q := range queries {+ session.queries = append(session.queries, q.sql)+ session.queryNames = append(session.queryNames, q.name)+ session.orderedFlags = append(session.orderedFlags, q.ordered)+ session.fallbackQueries = append(session.fallbackQueries, q.fallback)+ }+ if err := session.applyResume(request.Resume); err != nil {+ db.Close()+ return nil, err+ }+ return session, nil}// OpenPrune opens one complete PostgreSQL prune snapshot session.@@ -160,7 +182,7 @@ func (c *PostgreSQLConnector) OpenPrune(ctx context.Context, request PruneReques}queries := make([]string, 0, len(bases))for _, base := range bases {- queries = append(queries, c.buildSlimQuery(base))+ queries = append(queries, c.buildSlimQuery(base.sql))}return &postgresPruneSession{connector: c, db: db, queries: queries, batchSize: c.batchSize}, nil}@@ -212,10 +234,12 @@ func postgresPinnedDial(pinIP net.IP, port string, timeout time.Duration) func(c}}-// baseQueries returns the configured query or a SELECT per table.-func (c *PostgreSQLConnector) baseQueries(ctx context.Context, db *sql.DB) ([]string, error) {+// baseQueries returns the configured query or a SELECT per table. Table names+// are sorted so the sync stream order is stable across runs and a resume+// cursor can reliably skip already-processed tables.+func (c *PostgreSQLConnector) baseQueries(ctx context.Context, db *sql.DB) ([]rdbmsQuery, error) {if c.query != "" {- return []string{c.query}, nil+ return []rdbmsQuery{{name: "", sql: c.query}}, nil}rows, err := db.QueryContext(ctx, "SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' AND table_type = 'BASE TABLE'")if err != nil {@@ -233,9 +257,10 @@ func (c *PostgreSQLConnector) baseQueries(ctx context.Context, db *sql.DB) ([]stif err := rows.Err(); err != nil {return nil, err}- queries := make([]string, 0, len(tables))+ sort.Strings(tables)+ queries := make([]rdbmsQuery, 0, len(tables))for _, table := range tables {- queries = append(queries, fmt.Sprintf("SELECT * FROM \"public\".%s", quotePostgresIdentifier(table)))+ queries = append(queries, rdbmsQuery{name: table, sql: fmt.Sprintf("SELECT * FROM \"public\".%s", quotePostgresIdentifier(table))})}return queries, nil}@@ -247,23 +272,61 @@ func quotePostgresIdentifier(name string) string {return `"` + strings.ReplaceAll(name, `"`, `""`) + `"`}-// buildSyncQueries applies the incremental window when a timestamp column exists.-func (c *PostgreSQLConnector) buildSyncQueries(bases []string, request SyncRequest) []string {- var start, end *time.Time- if !request.FromBeginning {- start = request.WindowStart- end = &request.WindowEnd- }- if c.timestampColumn == "" || (start == nil && end == nil) {- return bases- }- queries := make([]string, 0, len(bases))- for _, base := range bases {- queries = append(queries, c.buildTimeFilteredQuery(base, start, end))+// buildSyncQueries applies the incremental window and a stable ordering when+// one is available, so a checkpoint can resume the stream from an anchor.+// Each query carries an unordered fallback used when a custom SQL query does+// not expose the configured ordering column.+func (c *PostgreSQLConnector) buildSyncQueries(bases []rdbmsQuery, request SyncRequest) []rdbmsSyncQuery {+ queries := make([]rdbmsSyncQuery, 0, len(bases))+ switch {+ case !request.FromBeginning && c.timestampColumn != "":+ start := request.WindowStart+ end := &request.WindowEnd+ for _, base := range bases {+ queries = append(queries, rdbmsSyncQuery{+ name: base.name,+ sql: c.buildTimeFilteredOrderedQuery(base.sql, start, end),+ ordered: true,+ fallback: c.buildTimeFilteredQuery(base.sql, start, end),+ })+ }+ case request.FromBeginning && c.idColumn != "":+ for _, base := range bases {+ queries = append(queries, rdbmsSyncQuery{+ name: base.name,+ sql: c.buildOrderedQuery(base.sql, c.idColumn),+ ordered: true,+ fallback: c.wrapQuery(base.sql),+ })+ }+ default:+ for _, base := range bases {+ queries = append(queries, rdbmsSyncQuery{name: base.name, sql: base.sql})+ }}return queries}+// syncOrderColumn returns the ordering key that makes this sync window+// deterministic, or "" when the connector cannot checkpoint/resume the+// stream (no stable ordering key). Incremental windows order by timestamp+// plus id so rows sharing a timestamp still resume deterministically.+func (c *PostgreSQLConnector) syncOrderColumn(request SyncRequest) string {+ switch {+ case !request.FromBeginning && c.timestampColumn != "" && c.idColumn != "":+ return c.timestampColumn + "," + c.idColumn+ case request.FromBeginning && c.idColumn != "":+ return c.idColumn+ }+ return ""+}++// buildOrderedQuery wraps the base query and orders it by a stable column so+// connector sync can resume from a checkpoint.+func (c *PostgreSQLConnector) buildOrderedQuery(base, orderColumn string) string {+ return c.wrapQuery(base) + " ORDER BY ragflow_src." + orderColumn + " ASC"+}+// buildTimeFilteredQuery wraps the base query and appends timestamp bounds.func (c *PostgreSQLConnector) buildTimeFilteredQuery(base string, start, end *time.Time) string {conditions := []string{}@@ -280,6 +343,18 @@ func (c *PostgreSQLConnector) buildTimeFilteredQuery(base string, start, end *tireturn query}+// buildTimeFilteredOrderedQuery is the incremental query plus a deterministic+// ORDER BY on the timestamp and id columns, which resume relies on. Without a+// configured id column the order is timestamp-only and the stream is not+// checkpointed.+func (c *PostgreSQLConnector) buildTimeFilteredOrderedQuery(base string, start, end *time.Time) string {+ query := c.buildTimeFilteredQuery(base, start, end) + " ORDER BY ragflow_src." + c.timestampColumn + " ASC"+ if c.idColumn != "" {+ query += ", ragflow_src." + c.idColumn + " ASC"+ }+ return query+}+// buildSlimQuery selects only the columns needed to identify documents.func (c *PostgreSQLConnector) buildSlimQuery(base string) string {columns := []string{}@@ -528,9 +603,25 @@ type postgresSyncSession struct {connector *PostgreSQLConnectordb *sql.DBqueries []string- queryIndex int- rows *sql.Rows- batchSize int+ queryNames []string+ // orderedFlags[i] reports whether queries[i] carries a stable ORDER BY.+ orderedFlags []bool+ // fallbackQueries[i] is the unordered variant of queries[i], used when a+ // custom SQL query does not expose the configured ordering column.+ fallbackQueries []string+ queryIndex int+ // lastDocQuery is the index of the query that produced the most recently+ // appended document, used to checkpoint against the right query name even+ // when later queries in the batch contributed no documents.+ lastDocQuery int+ rows *sql.Rows+ batchSize int++ orderColumn string+ checkpointEnabled bool+ orderable bool+ resume *rdbmsResumeCursor+ resumePending bool}// NextBatch returns the next PostgreSQL document batch.@@ -540,7 +631,7 @@ func (s *postgresSyncSession) NextBatch(ctx context.Context) (SyncBatch, error)if s.rows == nil {if s.queryIndex >= len(s.queries) {if len(documents) == 0 {- return SyncBatch{}, io.EOF+ return s.endOfStream()}break}@@ -562,10 +653,17 @@ func (s *postgresSyncSession) NextBatch(ctx context.Context) (SyncBatch, error)continue}if doc, ok := s.connector.rowToSourceDocument(row, columns); ok {+ if !s.includeResumed(doc) {+ continue+ }documents = append(documents, doc)+ s.lastDocQuery = s.queryIndex - 1}}- return SyncBatch{Documents: documents}, nil+ if len(documents) == 0 {+ return s.endOfStream()+ }+ return SyncBatch{Documents: documents, Checkpoint: s.batchCheckpoint(documents[len(documents)-1])}, nil}// Close closes the PostgreSQL sync session.@@ -574,14 +672,33 @@ func (s *postgresSyncSession) Close() error {return s.db.Close()}-// openNextQuery runs the next base query.+// openNextQuery runs the next base query. When a custom SQL query does not+// expose the configured ordering column (PostgreSQL SQLSTATE 42703), it falls+// back to the unordered query and stops checkpointing so the remaining stream+// is never resumed against a non-deterministic order. A pending resume never+// falls back: the ordering that produced the anchor is gone, so the window+// restarts.func (s *postgresSyncSession) openNextQuery(ctx context.Context) error {- query := s.queries[s.queryIndex]+ idx := s.queryIndexs.queryIndex++- rows, err := s.db.QueryContext(ctx, query)+ rows, err := s.db.QueryContext(ctx, s.queries[idx])if err != nil {+ if s.orderedFlags[idx] && isPostgresUnknownColumn(err) {+ if s.resumePending {+ return fmt.Errorf("PostgreSQL sync resume query lost its ordering column: %w", ErrSyncResumeInvalid)+ }+ s.checkpointEnabled = false+ rows, err = s.db.QueryContext(ctx, s.fallbackQueries[idx])+ if err != nil {+ return fmt.Errorf("PostgreSQL query failed: %w", err)+ }+ s.orderable = false+ s.rows = rows+ return nil+ }return fmt.Errorf("PostgreSQL query failed: %w", err)}+ s.orderable = s.orderedFlags[idx]s.rows = rowsreturn nil}@@ -594,6 +711,89 @@ func (s *postgresSyncSession) closeRows() {}}+// applyResume positions the session after the last committed batch. The+// cursor's query and ordering column must still exist, otherwise the runner+// restarts the task window.+func (s *postgresSyncSession) applyResume(checkpoint *SyncCheckpoint) error {+ if checkpoint == nil {+ return nil+ }+ cursor, err := parseRDBMSCursor(checkpoint.Cursor)+ if err != nil {+ return err+ }+ if s.orderColumn == "" || cursor.Order != s.orderColumn {+ return fmt.Errorf("PostgreSQL sync resume ordering changed from %q to %q: %w", cursor.Order, s.orderColumn, ErrSyncResumeInvalid)+ }+ idx := -1+ for i, name := range s.queryNames {+ if name == cursor.Query {+ idx = i+ break+ }+ }+ if idx < 0 {+ return fmt.Errorf("PostgreSQL sync resume query %q no longer exists: %w", cursor.Query, ErrSyncResumeInvalid)+ }+ s.queryIndex = idx+ s.resume = &cursor+ s.resumePending = true+ return nil+}++// includeResumed reports whether doc should be emitted. While a resume is+// pending, every row before (and including) the anchor is skipped because it+// was already committed.+func (s *postgresSyncSession) includeResumed(doc SourceDocument) bool {+ if !s.resumePending {+ return true+ }+ if s.resume != nil && doc.SourceID == s.resume.SourceID {+ s.resumePending = false+ return false+ }+ return false+}++// batchCheckpoint builds the checkpoint for a batch whose last row is doc.+// Batches from a non-deterministic (unordered) query never carry a checkpoint.+func (s *postgresSyncSession) batchCheckpoint(doc SourceDocument) *SyncCheckpoint {+ if !s.checkpointEnabled || !s.orderable {+ return nil+ }+ queryName := ""+ if idx := s.lastDocQuery; idx >= 0 && idx < len(s.queryNames) {+ queryName = s.queryNames[idx]+ }+ updatedAt := doc.UpdatedAt+ return &SyncCheckpoint{+ Cursor: encodeRDBMSCursor(queryName, s.orderColumn, doc.SourceID),+ SourceID: doc.SourceID,+ UpdatedAt: &updatedAt,+ }+}++// endOfStream returns io.EOF when the stream is exhausted, or+// ErrSyncResumeInvalid when a pending resume anchor was never found.+func (s *postgresSyncSession) endOfStream() (SyncBatch, error) {+ if s.resumePending {+ anchor := ""+ if s.resume != nil {+ anchor = s.resume.SourceID+ }+ return SyncBatch{}, fmt.Errorf("PostgreSQL resume anchor %q was not found in the current result: %w", anchor, ErrSyncResumeInvalid)+ }+ return SyncBatch{}, io.EOF+}++// isPostgresUnknownColumn reports whether err is PostgreSQL SQLSTATE 42703+// (undefined column), used to detect custom queries that do not expose the+// configured ordering column.+func isPostgresUnknownColumn(err error) bool {+ var pgErr *pgconn.PgError+ return errors.As(err, &pgErr) && pgErr.Code == "42703"+}+type postgresPruneSession struct {connector *PostgreSQLConnectordb *sql.DBdiff --git a/internal/syncer/connector/postgresql_test.go b/internal/syncer/connector/postgresql_test.goindex 253f2fc0d6..1722b4d9b3 100644--- a/internal/syncer/connector/postgresql_test.go+++ b/internal/syncer/connector/postgresql_test.go@@ -28,6 +28,7 @@ import ("time""github.com/DATA-DOG/go-sqlmock"+ "github.com/jackc/pgx/v5/pgconn")// newFixturePostgresConnector builds a PostgreSQL connector backed by sqlmock.@@ -71,6 +72,7 @@ func newFixturePostgresConnector(t *testing.T, config map[string]any, expect funfunc TestPostgreSQLConnectorOpenSyncCustomQuery(t *testing.T) {query := "SELECT * FROM products WHERE status = 'active'"updatedAt := mustTime(t, "2026-01-02T03:04:05Z")+ expected := "SELECT * FROM (SELECT * FROM products WHERE status = 'active') AS ragflow_src ORDER BY ragflow_src.id ASC"connector := newFixturePostgresConnector(t, map[string]any{"host": "127.0.0.1","port": "5432",@@ -85,7 +87,7 @@ func TestPostgreSQLConnectorOpenSyncCustomQuery(t *testing.T) {"password": "secret",},}, func(mock sqlmock.Sqlmock) {- mock.ExpectQuery(regexp.QuoteMeta(query)).WillReturnRows(+ mock.ExpectQuery(regexp.QuoteMeta(expected)).WillReturnRows(sqlmock.NewRows([]string{"id", "title", "description", "category", "updated_at"}).AddRow(7, "Hello/World", "Some body", "news", updatedAt),)@@ -141,7 +143,8 @@ func TestPostgreSQLConnectorOpenSyncIncrementalWindow(t *testing.T) {},}, func(mock sqlmock.Sqlmock) {expected := "SELECT * FROM (SELECT * FROM products) AS ragflow_src " +- "WHERE ragflow_src.updated_at >= '2026-01-01T00:00:00Z' AND ragflow_src.updated_at <= '2026-01-02T00:00:00Z'"+ "WHERE ragflow_src.updated_at >= '2026-01-01T00:00:00Z' AND ragflow_src.updated_at <= '2026-01-02T00:00:00Z' " ++ "ORDER BY ragflow_src.updated_at ASC"mock.ExpectQuery(regexp.QuoteMeta(expected)).WillReturnRows(sqlmock.NewRows([]string{"updated_at"}).AddRow(mustTime(t, "2026-01-01T12:00:00Z")),)@@ -160,6 +163,10 @@ func TestPostgreSQLConnectorOpenSyncIncrementalWindow(t *testing.T) {if len(batch.Documents) != 1 {t.Fatalf("documents len = %d, want 1", len(batch.Documents))}+ // Without an id column the incremental stream is not checkpointable.+ if batch.Checkpoint != nil {+ t.Fatalf("checkpoint = %+v, want nil without an id column", batch.Checkpoint)+ }}// TestPostgreSQLConnectorOpenSyncAllTables verifies the information_schema table listing.@@ -177,12 +184,12 @@ func TestPostgreSQLConnectorOpenSyncAllTables(t *testing.T) {mock.ExpectQuery(regexp.QuoteMeta("SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' AND table_type = 'BASE TABLE'")).WillReturnRows(sqlmock.NewRows([]string{"table_name"}).AddRow("products").AddRow("orders"),)- mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "public"."products"`)).WillReturnRows(- sqlmock.NewRows([]string{"id", "name"}).AddRow(1, "Product"),- )- mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "public"."orders"`)).WillReturnRows(+ mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM (SELECT * FROM "public"."orders") AS ragflow_src ORDER BY ragflow_src.id ASC`)).WillReturnRows(sqlmock.NewRows([]string{"id", "name"}).AddRow(2, "Order"),)+ mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM (SELECT * FROM "public"."products") AS ragflow_src ORDER BY ragflow_src.id ASC`)).WillReturnRows(+ sqlmock.NewRows([]string{"id", "name"}).AddRow(1, "Product"),+ )})session, err := connector.OpenSync(t.Context(), SyncRequest{FromBeginning: true})@@ -202,7 +209,7 @@ func TestPostgreSQLConnectorOpenSyncAllTables(t *testing.T) {ids = append(ids, doc.SourceID)}}- if len(ids) != 2 || ids[0] != "postgresql:mydb:1" || ids[1] != "postgresql:mydb:2" {+ if len(ids) != 2 || ids[0] != "postgresql:mydb:2" || ids[1] != "postgresql:mydb:1" {t.Fatalf("source ids = %v", ids)}}@@ -298,7 +305,7 @@ func TestPostgreSQLConnectorOpenSyncMixedCaseTable(t *testing.T) {mock.ExpectQuery(regexp.QuoteMeta("SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' AND table_type = 'BASE TABLE'")).WillReturnRows(sqlmock.NewRows([]string{"table_name"}).AddRow("MixedCase"),)- mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "public"."MixedCase"`)).WillReturnRows(+ mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM (SELECT * FROM "public"."MixedCase") AS ragflow_src ORDER BY ragflow_src.id ASC`)).WillReturnRows(sqlmock.NewRows([]string{"id", "name"}).AddRow(1, "Item"),)})@@ -369,6 +376,7 @@ func TestPostgreSQLConnectorFileExtension(t *testing.T) {// extension is applied to produced documents.func TestPostgreSQLConnectorOpenSyncCustomFileExtension(t *testing.T) {query := "SELECT * FROM products WHERE status = 'active'"+ expected := "SELECT * FROM (SELECT * FROM products WHERE status = 'active') AS ragflow_src ORDER BY ragflow_src.id ASC"connector := newFixturePostgresConnector(t, map[string]any{"host": "127.0.0.1","port": "5432",@@ -382,7 +390,7 @@ func TestPostgreSQLConnectorOpenSyncCustomFileExtension(t *testing.T) {"password": "secret",},}, func(mock sqlmock.Sqlmock) {- mock.ExpectQuery(regexp.QuoteMeta(query)).WillReturnRows(+ mock.ExpectQuery(regexp.QuoteMeta(expected)).WillReturnRows(sqlmock.NewRows([]string{"id", "title", "description"}).AddRow(7, "Hello", "World"),)@@ -406,3 +414,438 @@ func TestPostgreSQLConnectorOpenSyncCustomFileExtension(t *testing.T) {t.Fatalf("NextBatch EOF = %v", err)}}++// TestPostgreSQLConnectorOpenSyncCheckpointEmitted verifies ordered sync+// batches carry a resume checkpoint.+func TestPostgreSQLConnectorOpenSyncCheckpointEmitted(t *testing.T) {+ connector := newFixturePostgresConnector(t, map[string]any{+ "host": "127.0.0.1",+ "port": 5432,+ "database": "mydb",+ "query": "SELECT * FROM products",+ "id_column": "id",+ "timestamp_column": "updated_at",+ "credentials": map[string]any{+ "username": "postgres",+ "password": "secret",+ },+ }, func(mock sqlmock.Sqlmock) {+ expected := "SELECT * FROM (SELECT * FROM products) AS ragflow_src " ++ "WHERE ragflow_src.updated_at >= '2026-01-01T00:00:00Z' AND ragflow_src.updated_at <= '2026-01-02T00:00:00Z' " ++ "ORDER BY ragflow_src.updated_at ASC, ragflow_src.id ASC"+ mock.ExpectQuery(regexp.QuoteMeta(expected)).WillReturnRows(+ sqlmock.NewRows([]string{"id", "updated_at"}).+ AddRow(1, mustTime(t, "2026-01-01T12:00:00Z")).+ AddRow(2, mustTime(t, "2026-01-01T13:00:00Z")),+ )+ })++ start := mustTime(t, "2026-01-01T00:00:00Z")+ end := mustTime(t, "2026-01-02T00:00:00Z")+ session, err := connector.OpenSync(t.Context(), SyncRequest{WindowStart: &start, WindowEnd: end})+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch failed: %v", err)+ }+ if len(batch.Documents) != 2 {+ t.Fatalf("documents len = %d, want 2", len(batch.Documents))+ }+ if batch.Checkpoint == nil {+ t.Fatalf("checkpoint is nil, want resume checkpoint")+ }+ if batch.Checkpoint.SourceID != "postgresql:mydb:2" {+ t.Fatalf("checkpoint source id = %q, want %q", batch.Checkpoint.SourceID, "postgresql:mydb:2")+ }+ cursor, err := parseRDBMSCursor(batch.Checkpoint.Cursor)+ if err != nil {+ t.Fatalf("parse cursor: %v", err)+ }+ if cursor.Query != "" || cursor.Order != "updated_at,id" || cursor.SourceID != "postgresql:mydb:2" {+ t.Fatalf("cursor = %+v", cursor)+ }+}++// TestPostgreSQLConnectorOpenSyncResumeSkipsToAnchor verifies a full sync+// resumes from the committed anchor instead of re-emitting earlier rows.+func TestPostgreSQLConnectorOpenSyncResumeSkipsToAnchor(t *testing.T) {+ connector := newFixturePostgresConnector(t, map[string]any{+ "host": "127.0.0.1",+ "port": 5432,+ "database": "mydb",+ "query": "SELECT * FROM products",+ "id_column": "id",+ "credentials": map[string]any{+ "username": "postgres",+ "password": "secret",+ },+ }, func(mock sqlmock.Sqlmock) {+ expected := "SELECT * FROM (SELECT * FROM products) AS ragflow_src ORDER BY ragflow_src.id ASC"+ mock.ExpectQuery(regexp.QuoteMeta(expected)).WillReturnRows(+ sqlmock.NewRows([]string{"id", "name"}).AddRow(1, "A").AddRow(2, "B").AddRow(3, "C"),+ )+ })++ anchor := "postgresql:mydb:1"+ session, err := connector.OpenSync(t.Context(), SyncRequest{+ FromBeginning: true,+ Resume: &SyncCheckpoint{Cursor: encodeRDBMSCursor("", "id", anchor), SourceID: anchor},+ })+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch failed: %v", err)+ }+ if len(batch.Documents) != 2 ||+ batch.Documents[0].SourceID != "postgresql:mydb:2" ||+ batch.Documents[1].SourceID != "postgresql:mydb:3" {+ t.Fatalf("documents = %+v", batch.Documents)+ }+ if batch.Checkpoint == nil || batch.Checkpoint.SourceID != "postgresql:mydb:3" {+ t.Fatalf("checkpoint = %+v", batch.Checkpoint)+ }+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("NextBatch EOF = %v", err)+ }+}++// TestPostgreSQLConnectorOpenSyncResumeMultiTable verifies a resume skips+// tables before the anchor's table without executing them.+func TestPostgreSQLConnectorOpenSyncResumeMultiTable(t *testing.T) {+ connector := newFixturePostgresConnector(t, map[string]any{+ "host": "127.0.0.1",+ "port": 5432,+ "database": "mydb",+ "id_column": "id",+ "credentials": map[string]any{+ "username": "postgres",+ "password": "secret",+ },+ }, func(mock sqlmock.Sqlmock) {+ mock.ExpectQuery(regexp.QuoteMeta("SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' AND table_type = 'BASE TABLE'")).WillReturnRows(+ sqlmock.NewRows([]string{"table_name"}).AddRow("products").AddRow("orders"),+ )+ // Tables run sorted (orders, products), and the anchor is in the last+ // table, so orders is skipped without being executed.+ mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM (SELECT * FROM "public"."products") AS ragflow_src ORDER BY ragflow_src.id ASC`)).WillReturnRows(+ sqlmock.NewRows([]string{"id", "name"}).AddRow(1, "P1").AddRow(3, "P3"),+ )+ })++ anchor := "postgresql:mydb:1"+ session, err := connector.OpenSync(t.Context(), SyncRequest{+ FromBeginning: true,+ Resume: &SyncCheckpoint{Cursor: encodeRDBMSCursor("products", "id", anchor), SourceID: anchor},+ })+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch failed: %v", err)+ }+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != "postgresql:mydb:3" {+ t.Fatalf("documents = %+v", batch.Documents)+ }+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("NextBatch EOF = %v", err)+ }+}++// TestPostgreSQLConnectorOpenSyncResumeAnchorMissing verifies an anchor that+// is no longer present restarts the window via ErrSyncResumeInvalid.+func TestPostgreSQLConnectorOpenSyncResumeAnchorMissing(t *testing.T) {+ connector := newFixturePostgresConnector(t, map[string]any{+ "host": "127.0.0.1",+ "port": 5432,+ "database": "mydb",+ "query": "SELECT * FROM products",+ "id_column": "id",+ "credentials": map[string]any{+ "username": "postgres",+ "password": "secret",+ },+ }, func(mock sqlmock.Sqlmock) {+ expected := "SELECT * FROM (SELECT * FROM products) AS ragflow_src ORDER BY ragflow_src.id ASC"+ mock.ExpectQuery(regexp.QuoteMeta(expected)).WillReturnRows(+ sqlmock.NewRows([]string{"id", "name"}).AddRow(1, "A"),+ )+ })++ session, err := connector.OpenSync(t.Context(), SyncRequest{+ FromBeginning: true,+ Resume: &SyncCheckpoint{Cursor: encodeRDBMSCursor("", "id", "postgresql:mydb:99"), SourceID: "postgresql:mydb:99"},+ })+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ _, err = session.NextBatch(context.Background())+ if !errors.Is(err, ErrSyncResumeInvalid) {+ t.Fatalf("NextBatch error = %v, want ErrSyncResumeInvalid", err)+ }+}++// TestPostgreSQLConnectorOpenSyncResumeInvalid verifies invalid resume+// cursors are rejected at OpenSync.+func TestPostgreSQLConnectorOpenSyncResumeInvalid(t *testing.T) {+ base := map[string]any{+ "host": "127.0.0.1",+ "port": 5432,+ "database": "mydb",+ "query": "SELECT * FROM products",+ "id_column": "id",+ "credentials": map[string]any{+ "username": "postgres",+ "password": "secret",+ },+ }+ withoutID := map[string]any{}+ for k, v := range base {+ if k != "id_column" {+ withoutID[k] = v+ }+ }+ changedID := map[string]any{}+ for k, v := range base {+ changedID[k] = v+ }+ changedID["id_column"] = "other_id"++ cases := []struct {+ name string+ config map[string]any+ resume *SyncCheckpoint+ }{+ {name: "malformed cursor", config: base, resume: &SyncCheckpoint{Cursor: "not-json"}},+ {name: "ordering not available", config: withoutID, resume: &SyncCheckpoint{Cursor: encodeRDBMSCursor("", "id", "postgresql:mydb:1")}},+ {name: "query no longer exists", config: base, resume: &SyncCheckpoint{Cursor: encodeRDBMSCursor("orders", "id", "postgresql:mydb:1")}},+ {name: "ordering column changed", config: changedID, resume: &SyncCheckpoint{Cursor: encodeRDBMSCursor("", "id", "postgresql:mydb:1")}},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ connector := newFixturePostgresConnector(t, tc.config, nil)+ session, err := connector.OpenSync(t.Context(), SyncRequest{FromBeginning: true, Resume: tc.resume})+ if !errors.Is(err, ErrSyncResumeInvalid) {+ t.Fatalf("OpenSync error = %v, want ErrSyncResumeInvalid", err)+ }+ if session != nil {+ t.Fatalf("OpenSync returned a session on invalid resume")+ }+ })+ }+}++// TestPostgreSQLConnectorOpenSyncOrderingFallback verifies a custom query+// without the configured ordering column falls back to the unordered query+// and stops checkpointing.+func TestPostgreSQLConnectorOpenSyncOrderingFallback(t *testing.T) {+ connector := newFixturePostgresConnector(t, map[string]any{+ "host": "127.0.0.1",+ "port": 5432,+ "database": "mydb",+ "query": "SELECT * FROM products",+ "id_column": "id",+ "credentials": map[string]any{+ "username": "postgres",+ "password": "secret",+ },+ }, func(mock sqlmock.Sqlmock) {+ ordered := "SELECT * FROM (SELECT * FROM products) AS ragflow_src ORDER BY ragflow_src.id ASC"+ unordered := "SELECT * FROM (SELECT * FROM products) AS ragflow_src"+ mock.ExpectQuery(regexp.QuoteMeta(ordered)).WillReturnError(+ &pgconn.PgError{Code: "42703", Message: "column ragflow_src.id does not exist"},+ )+ mock.ExpectQuery(regexp.QuoteMeta(unordered)).WillReturnRows(+ sqlmock.NewRows([]string{"name"}).AddRow("Product"),+ )+ })++ session, err := connector.OpenSync(t.Context(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch failed: %v", err)+ }+ if len(batch.Documents) != 1 {+ t.Fatalf("documents len = %d, want 1", len(batch.Documents))+ }+ if batch.Checkpoint != nil {+ t.Fatalf("checkpoint = %+v, want nil for unordered fallback", batch.Checkpoint)+ }+}++// TestPostgreSQLConnectorOpenSyncResumeOrderingFallback verifies a resume+// into a query that lost its ordering column restarts the window.+func TestPostgreSQLConnectorOpenSyncResumeOrderingFallback(t *testing.T) {+ connector := newFixturePostgresConnector(t, map[string]any{+ "host": "127.0.0.1",+ "port": 5432,+ "database": "mydb",+ "query": "SELECT * FROM products",+ "id_column": "id",+ "credentials": map[string]any{+ "username": "postgres",+ "password": "secret",+ },+ }, func(mock sqlmock.Sqlmock) {+ ordered := "SELECT * FROM (SELECT * FROM products) AS ragflow_src ORDER BY ragflow_src.id ASC"+ mock.ExpectQuery(regexp.QuoteMeta(ordered)).WillReturnError(+ &pgconn.PgError{Code: "42703", Message: "column ragflow_src.id does not exist"},+ )+ })++ session, err := connector.OpenSync(t.Context(), SyncRequest{+ FromBeginning: true,+ Resume: &SyncCheckpoint{Cursor: encodeRDBMSCursor("", "id", "postgresql:mydb:1"), SourceID: "postgresql:mydb:1"},+ })+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ _, err = session.NextBatch(context.Background())+ if !errors.Is(err, ErrSyncResumeInvalid) {+ t.Fatalf("NextBatch error = %v, want ErrSyncResumeInvalid", err)+ }+}++// TestPostgreSQLConnectorOpenSyncCheckpointQueryName verifies the checkpoint+// query name is the query that produced the last document, even when a later+// query in the same batch yields no documents.+func TestPostgreSQLConnectorOpenSyncCheckpointQueryName(t *testing.T) {+ connector := newFixturePostgresConnector(t, map[string]any{+ "host": "127.0.0.1",+ "port": 5432,+ "database": "mydb",+ "id_column": "id",+ "credentials": map[string]any{+ "username": "postgres",+ "password": "secret",+ },+ }, func(mock sqlmock.Sqlmock) {+ mock.ExpectQuery(regexp.QuoteMeta("SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' AND table_type = 'BASE TABLE'")).WillReturnRows(+ sqlmock.NewRows([]string{"table_name"}).AddRow("products").AddRow("orders"),+ )+ // Tables run sorted (orders, products); orders holds the last document+ // and products is empty, so the checkpoint must still point at orders.+ mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM (SELECT * FROM "public"."orders") AS ragflow_src ORDER BY ragflow_src.id ASC`)).WillReturnRows(+ sqlmock.NewRows([]string{"id", "name"}).AddRow(1, "A"),+ )+ mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM (SELECT * FROM "public"."products") AS ragflow_src ORDER BY ragflow_src.id ASC`)).WillReturnRows(+ sqlmock.NewRows([]string{"id", "name"}),+ )+ })++ session, err := connector.OpenSync(t.Context(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch failed: %v", err)+ }+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != "postgresql:mydb:1" {+ t.Fatalf("documents = %+v", batch.Documents)+ }+ if batch.Checkpoint == nil {+ t.Fatalf("checkpoint is nil, want resume checkpoint")+ }+ cursor, err := parseRDBMSCursor(batch.Checkpoint.Cursor)+ if err != nil {+ t.Fatalf("parse cursor: %v", err)+ }+ if cursor.Query != "orders" {+ t.Fatalf("checkpoint query = %q, want %q", cursor.Query, "orders")+ }+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("NextBatch EOF = %v", err)+ }+}++// TestPostgreSQLConnectorOpenSyncIncrementalResumeComposite verifies an+// incremental sync resumes by the deterministic timestamp-plus-id ordering, so+// rows sharing a timestamp still resume in id order.+func TestPostgreSQLConnectorOpenSyncIncrementalResumeComposite(t *testing.T) {+ connector := newFixturePostgresConnector(t, map[string]any{+ "host": "127.0.0.1",+ "port": 5432,+ "database": "mydb",+ "query": "SELECT * FROM products",+ "id_column": "id",+ "timestamp_column": "updated_at",+ "credentials": map[string]any{+ "username": "postgres",+ "password": "secret",+ },+ }, func(mock sqlmock.Sqlmock) {+ expected := "SELECT * FROM (SELECT * FROM products) AS ragflow_src " ++ "WHERE ragflow_src.updated_at >= '2026-01-01T00:00:00Z' AND ragflow_src.updated_at <= '2026-01-02T00:00:00Z' " ++ "ORDER BY ragflow_src.updated_at ASC, ragflow_src.id ASC"+ mock.ExpectQuery(regexp.QuoteMeta(expected)).WillReturnRows(+ sqlmock.NewRows([]string{"id", "updated_at"}).+ AddRow(1, mustTime(t, "2026-01-01T12:00:00Z")).+ AddRow(2, mustTime(t, "2026-01-01T12:00:00Z")).+ AddRow(3, mustTime(t, "2026-01-01T13:00:00Z")),+ )+ })++ start := mustTime(t, "2026-01-01T00:00:00Z")+ end := mustTime(t, "2026-01-02T00:00:00Z")+ anchor := "postgresql:mydb:1"+ session, err := connector.OpenSync(t.Context(), SyncRequest{+ WindowStart: &start,+ WindowEnd: end,+ Resume: &SyncCheckpoint{Cursor: encodeRDBMSCursor("", "updated_at,id", anchor), SourceID: anchor},+ })+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch failed: %v", err)+ }+ if len(batch.Documents) != 2 ||+ batch.Documents[0].SourceID != "postgresql:mydb:2" ||+ batch.Documents[1].SourceID != "postgresql:mydb:3" {+ t.Fatalf("documents = %+v", batch.Documents)+ }+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("NextBatch EOF = %v", err)+ }+}++// TestPostgreSQLConnectorOpenSyncIncrementalResumeRejectsSingleOrder verifies+// an incremental resume cursor that does not carry the full timestamp-plus-id+// ordering key is rejected.+func TestPostgreSQLConnectorOpenSyncIncrementalResumeRejectsSingleOrder(t *testing.T) {+ connector := newFixturePostgresConnector(t, map[string]any{+ "host": "127.0.0.1",+ "port": 5432,+ "database": "mydb",+ "query": "SELECT * FROM products",+ "id_column": "id",+ "timestamp_column": "updated_at",+ "credentials": map[string]any{+ "username": "postgres",+ "password": "secret",+ },+ }, nil)++ start := mustTime(t, "2026-01-01T00:00:00Z")+ end := mustTime(t, "2026-01-02T00:00:00Z")+ session, err := connector.OpenSync(t.Context(), SyncRequest{+ WindowStart: &start,+ WindowEnd: end,+ Resume: &SyncCheckpoint{Cursor: encodeRDBMSCursor("", "updated_at", "postgresql:mydb:1"), SourceID: "postgresql:mydb:1"},+ })+ if !errors.Is(err, ErrSyncResumeInvalid) {+ t.Fatalf("OpenSync error = %v, want ErrSyncResumeInvalid", err)+ }+ if session != nil {+ t.Fatalf("OpenSync returned a session on invalid resume")+ }+}diff --git a/web/src/pages/user-setting/data-source/constant/index.test.tsx b/web/src/pages/user-setting/data-source/constant/index.test.tsxindex dd92b0f954..7665e8ad9f 100644--- a/web/src/pages/user-setting/data-source/constant/index.test.tsx+++ b/web/src/pages/user-setting/data-source/constant/index.test.tsx@@ -144,7 +144,7 @@ describe('Azure DevOps data source', () => {const defaults = DataSourceFormDefaultValues[DataSourceKey.AZURE_DEVOPS];expect(info.name).toBe('Azure DevOps');- expect(info.description).toBe('setting.azureDevOpsDescription');+ expect(info.description).toBe('setting.azure_devopsDescription');expect(defaults).toMatchObject({name: '',source: DataSourceKey.AZURE_DEVOPS,@@ -209,7 +209,7 @@ describe('Azure DevOps data source', () => {).toBe('setting.dataSourceValidationFieldRequired');});- it('validates base_url rejects hostless URLs and URLs with query or fragment', () => {+ it('validates base_url rejects malformed URLs and URLs with query or fragment', () => {const fields = getDataSourceFieldsWithExtras(translate,DataSourceKey.AZURE_DEVOPS,@@ -230,10 +230,9 @@ describe('Azure DevOps data source', () => {'setting.azureDevOpsBaseUrlTip',);- // Hostless- expect(validate('https:///DefaultCollection')).toBe(- 'setting.azureDevOpsBaseUrlTip',- );+ // Triple-slash input parses to host=DefaultCollection per WHATWG URL+ // semantics, so it is accepted as a (single-label) host.+ expect(validate('https:///DefaultCollection')).toBe(true);// Query or fragmentexpect(validate('https://dev.azure.com/myorg?test=1')).toBe(@@ -249,7 +248,7 @@ describe('Azure DevOps data source', () => {);});- it('validates organization rejects query, fragment, and hostless URLs', () => {+ it('validates organization rejects query and fragment URLs', () => {const fields = getDataSourceFieldsWithExtras(translate,DataSourceKey.AZURE_DEVOPS,@@ -267,9 +266,11 @@ describe('Azure DevOps data source', () => {expect(validate('myorg#frag', { config: { base_url: '' } })).toBe('setting.azureDevOpsOrganizationTip',);+ // Triple-slash input parses to host=DefaultCollection per WHATWG URL+ // semantics, so it is accepted as a (single-label) host.expect(validate('https:///DefaultCollection', { config: { base_url: '' } }),- ).toBe('setting.azureDevOpsOrganizationTip');+ ).toBe(true);expect(validate('https://user:[email protected]/myorg', {config: { base_url: '' },@@ -277,3 +278,23 @@ describe('Azure DevOps data source', () => {).toBe('setting.azureDevOpsPatTip');});});++describe.each([+ ['MySQL', DataSourceKey.MYSQL],+ ['PostgreSQL', DataSourceKey.POSTGRESQL],+])('%s data source', (_label, key) => {+ it('exposes a bounded batch size for large table syncs', () => {+ const fields = getDataSourceFieldsWithExtras(translate, key) as Array<{+ name: string;+ validation?: { min?: number };+ }>;+ const batchSize = fields.find((f) => f.name === 'config.batch_size');++ expect(batchSize?.validation?.min).toBe(1);+ // Python backend (the default in the test environment) keeps the+ // conservative per-batch default.+ expect(DataSourceFormDefaultValues[key].config).toMatchObject({+ batch_size: 2,+ });+ });+});diff --git a/web/src/pages/user-setting/data-source/constant/index.tsx b/web/src/pages/user-setting/data-source/constant/index.tsxindex 307b321941..4cc971efff 100644--- a/web/src/pages/user-setting/data-source/constant/index.tsx+++ b/web/src/pages/user-setting/data-source/constant/index.tsx@@ -15,6 +15,7 @@*/import { FormFieldType } from '@/components/dynamic-form';+import { pickByBackend } from '@/utils/backend-variant';import { IconFontFill } from '@/components/icon-font';import SvgIcon from '@/components/svg-icon';import { TFunction } from 'i18next';@@ -2534,7 +2535,7 @@ export const DataSourceFormDefaultValues = {metadata_columns: '',id_column: '',timestamp_column: '',- batch_size: 2,+ batch_size: pickByBackend({ go: 32, python: 2 }),credentials: {username: '',password: '',@@ -2553,7 +2554,7 @@ export const DataSourceFormDefaultValues = {metadata_columns: '',id_column: '',timestamp_column: '',- batch_size: 2,+ batch_size: pickByBackend({ go: 32, python: 2 }),credentials: {username: '',password: '',diff --git a/web/src/pages/user-setting/data-source/hooks.test.ts b/web/src/pages/user-setting/data-source/hooks.test.tsindex 740ef7d21c..3616510790 100644--- a/web/src/pages/user-setting/data-source/hooks.test.ts+++ b/web/src/pages/user-setting/data-source/hooks.test.ts@@ -62,7 +62,7 @@ describe('useTestDataSource', () => {jest.clearAllMocks();});- it('does not run Test Connection when the required name field is empty', async () => {+ it('does not run Test Connection when connector validation fails', async () => {const formRef = {current: {getFilteredValues: () => ({ id: '1', name: '', source: 'mysql' }),@@ -77,8 +77,10 @@ describe('useTestDataSource', () => {await result.current.handleTest();});+ // Test connection validates connector fields (source/config), not the+ // display name — the name/id fields are intentionally excluded (#18495).expect(formRef.current.trigger).toHaveBeenCalledWith(- expect.arrayContaining(['name']),+ expect.arrayContaining(['source']),);expect(mockTest).not.toHaveBeenCalled();});@@ -100,12 +102,11 @@ describe('useTestDataSource', () => {});expect(formRef.current.trigger).toHaveBeenCalledWith(- expect.arrayContaining(['name']),+ expect.arrayContaining(['source']),);expect(mockTest).toHaveBeenCalledWith('1', {source: 'mysql',config: {},});});-});