Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat(go-syncer): add BigQuery connector (#19106)

mkaaadc63c2374801c
diff --git a/go.mod b/go.mod
index d07769e6e6..3a9f622f18 100644
--- a/go.mod
+++ b/go.mod
@@ -3,6 +3,7 @@ module ragflow
go 1.26.4
require (
+ cloud.google.com/go/bigquery v1.82.0
cloud.google.com/go/storage v1.63.0
github.com/AkmalOt/gomsg v0.0.0-20260407083308-985c3a1a76b7
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0
@@ -69,12 +70,12 @@ require (
go.opentelemetry.io/otel/sdk v1.44.0
go.opentelemetry.io/otel/trace v1.44.0
go.uber.org/zap v1.27.1
- golang.org/x/crypto v0.53.0
+ golang.org/x/crypto v0.54.0
golang.org/x/image v0.45.0
- golang.org/x/net v0.56.0
+ golang.org/x/net v0.57.0
golang.org/x/oauth2 v0.36.0
golang.org/x/sync v0.22.0
- golang.org/x/term v0.44.0
+ golang.org/x/term v0.45.0
golang.org/x/text v0.41.0
gonum.org/v1/gonum v0.17.0
google.golang.org/api v0.287.1
@@ -88,13 +89,20 @@ require (
require (
github.com/PuerkitoBio/goquery v1.9.2 // indirect
github.com/andybalholm/cascadia v1.3.2 // indirect
+ github.com/apache/arrow/go/v15 v15.0.2 // indirect
github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 // indirect
+ github.com/google/flatbuffers v23.5.26+incompatible // indirect
+ github.com/pierrec/lz4/v4 v4.1.18 // indirect
+ golang.org/x/mod v0.38.0 // indirect
+ golang.org/x/telemetry v0.0.0-20260708182218-49f421fb7959 // indirect
+ golang.org/x/tools v0.48.0 // indirect
+ golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
)
require (
buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20260415201107-50325440f8f2.1 // indirect
cel.dev/expr v0.25.1 // indirect
- cloud.google.com/go v0.123.0 // indirect
+ cloud.google.com/go v0.123.0
cloud.google.com/go/auth v0.21.0 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
cloud.google.com/go/compute/metadata v0.9.0 // indirect
@@ -232,13 +240,13 @@ require (
go.uber.org/multierr v1.11.0 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/arch v0.27.0 // indirect
- golang.org/x/exp v0.0.0-20231226003508-02704c960a9b // indirect
+ golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
golang.org/x/sys v0.47.0 // indirect
golang.org/x/time v0.15.0 // indirect
google.golang.org/genproto v0.0.0-20260519071638-aa98bba5eb94 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260630182238-925bb5da69e7 // indirect
- google.golang.org/grpc v1.82.0 // indirect
+ google.golang.org/grpc v1.82.1 // indirect
google.golang.org/protobuf v1.36.11 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
modernc.org/libc v1.22.5 // indirect
diff --git a/go.sum b/go.sum
index 4c7646356f..e3a0399e75 100644
--- a/go.sum
+++ b/go.sum
@@ -9,14 +9,18 @@ cloud.google.com/go/auth v0.21.0 h1:g/QwYfYb2Ai6HH8oomAOyBaIHLbscZ4+T/F/f5JZHkE=
cloud.google.com/go/auth v0.21.0/go.mod h1:M9o2Oz+YI2jAfxewJgb1vyI3vceHF+eohmxyzmrl+9s=
cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=
cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
+cloud.google.com/go/bigquery v1.82.0 h1:NcnkqYN8jjtn6B1o7W/aUNl7DjJiNqgO0xnDNGxW+2s=
+cloud.google.com/go/bigquery v1.82.0/go.mod h1:cc0XscySNQNuHBxuZSg5yyxFsg/ZHAfViAG49gJbWew=
cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs=
cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10=
+cloud.google.com/go/datacatalog v1.32.0 h1:fyYn8ODkGil5y3zTIqgIhOfzTu1ACaU2o+C750CO6Ac=
+cloud.google.com/go/datacatalog v1.32.0/go.mod h1:DE272tynQUwheJeQAyVfV+nO8yrdkuDyOgH2LtOrkWM=
cloud.google.com/go/iam v1.11.0 h1:KieQ9Pb+LLPak1O3Rv3GgCxhnmkYf7Xyh0P5HfF1jFM=
cloud.google.com/go/iam v1.11.0/go.mod h1:KP+nKGugNJW4LcLx1uEZcq1ok5sQHFaQehQNl4QDgV4=
cloud.google.com/go/logging v1.18.0 h1:KhzZq+1cSkPH9YUaKLLhLtQxIHitVayBmk0sGfoM9+k=
cloud.google.com/go/logging v1.18.0/go.mod h1:ZGKnpBaURITh+g/uom2VhbiFoFWvejcrHPDhxFtU/gI=
-cloud.google.com/go/longrunning v1.0.0 h1:lwzWEYD8+NkYV7dhexOz6kmlvajZA70+bW/xMhRVVdY=
-cloud.google.com/go/longrunning v1.0.0/go.mod h1:8nqFBPOO1U/XkhWl0I19AMZEphrHi73VNABIpKYaTwM=
+cloud.google.com/go/longrunning v1.2.0 h1:WjYH3YHBGCxGJP9M4dWGHBfXr/cFIjMkNgWcJj7/iMM=
+cloud.google.com/go/longrunning v1.2.0/go.mod h1:5KMQALFGOCtFoi2xSOA1u3H7WKlhmckgiyFw7+LGQp0=
cloud.google.com/go/monitoring v1.29.0 h1:AHhDsFaSax1/4k+qlIDX/SDGe6hggnfXJ9dkgD9qBPY=
cloud.google.com/go/monitoring v1.29.0/go.mod h1:72NOVjJXHY/HBfoLT0+qlCZBT059+9VXLeAnL2PeeVM=
cloud.google.com/go/storage v1.63.0 h1:hvXF2xfg9I32bjujggxgkEZn/Ej6sJ9pieFgeueBLrQ=
@@ -111,6 +115,8 @@ github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsVi
github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
+github.com/apache/arrow/go/v15 v15.0.2 h1:60IliRbiyTWCWjERBCkO1W4Qun9svcYoZrSLcyOsMLE=
+github.com/apache/arrow/go/v15 v15.0.2/go.mod h1:DGXsR3ajT524njufqf95822i+KTh+yea1jass9YXgjA=
github.com/apache/thrift v0.23.0 h1:wKR6YnefQSEnxpEfmgTPuJibNG4bF0p2TK34tHLWi3s=
github.com/apache/thrift v0.23.0/go.mod h1:zPt6WxgvTOM6hF92y8C+MkEM5LMxZuk4JcQOiU4Esvs=
github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ=
@@ -299,6 +305,8 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/gomarkdown/markdown v0.0.0-20260614204949-e08cff860f76 h1:Ltt9ldIaSYEsjA7sPY2c8r9dOmnKM1vlzhh3dxlhBHM=
github.com/gomarkdown/markdown v0.0.0-20260614204949-e08cff860f76/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA=
+github.com/google/flatbuffers v23.5.26+incompatible h1:M9dgRyhJemaM4Sw8+66GHBu8ioaQmyPLg1b8VwK5WJg=
+github.com/google/flatbuffers v23.5.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -443,6 +451,8 @@ github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
github.com/phpdave11/gofpdi v1.0.14-0.20211212211723-1f10f9844311 h1:zyWXQ6vu27ETMpYsEMAsisQ+GqJ4e1TPvSNfdOPF0no=
github.com/phpdave11/gofpdi v1.0.14-0.20211212211723-1f10f9844311/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI=
+github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ=
+github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA=
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
@@ -572,8 +582,6 @@ github.com/xuri/excelize/v2 v2.11.0 h1:HxaEFl6sRN2+8J5a8HaKq+0M4FsjBGMnWWtjOCPSG
github.com/xuri/excelize/v2 v2.11.0/go.mod h1:jxFLbzaIwGQ5ufFNvYfUOHqXhfPaNmP14KWfmNz2Uak=
github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9 h1:+C0TIdyyYmzadGaL/HBLbf3WdLgC29pgyhTjAT/0nuE=
github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ=
-github.com/yalue/onnxruntime_go v1.23.0 h1:Hin0mFphwGOeT7xEQrAIi/p2O6ngmSy4uz0yXkC9yCw=
-github.com/yalue/onnxruntime_go v1.23.0/go.mod h1:b4X26A8pekNb1ACJ58wAXgNKeUCGEAQ9dmACut9Sm/4=
github.com/yargevad/filepathx v1.0.0 h1:SYcT+N3tYGi+NvazubCNlvgIPbzAk7i7y2dwg3I5FYc=
github.com/yargevad/filepathx v1.0.0/go.mod h1:BprfX/gpYNJHJfc35GjRRpVcwWXS89gGulUIU5tK3tA=
github.com/yfedoseev/office_oxide/go v0.1.8 h1:Q6aQrrs0pN/B6fBoWN6PN95QSQ7EKGMb5HExh04mJJc=
@@ -648,11 +656,11 @@ golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOM
golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
-golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto=
-golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio=
+golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw=
+golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20231226003508-02704c960a9b h1:kLiC65FbiHWFAOu+lxwNPujcsl8VYyTYYEZnsOO1WK4=
-golang.org/x/exp v0.0.0-20231226003508-02704c960a9b/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI=
+golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=
+golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
golang.org/x/image v0.45.0 h1:FMb1nTbH5H9vF55SriQHgFw5GnNL9Jg6L25BwXKzhB0=
golang.org/x/image v0.45.0/go.mod h1:n62x/7RqlwXDvGsSU4u6IUTUf6KghUZ9Bt7cG/T9Fx4=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -665,6 +673,8 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk=
+golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -692,8 +702,8 @@ golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
-golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=
-golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec=
+golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
+golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
@@ -739,6 +749,8 @@ golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
+golang.org/x/telemetry v0.0.0-20260708182218-49f421fb7959 h1:RJhm5l6Fo4rmEIcndxDllNhhf/fAx8qIm4t6A7vpm2A=
+golang.org/x/telemetry v0.0.0-20260708182218-49f421fb7959/go.mod h1:LV7u5Oco+Z/g6XI7PqN+EUUUGGkEcmB1uj2ceI0fOVg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
@@ -752,8 +764,8 @@ golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0=
-golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc=
-golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y=
+golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
+golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -783,10 +795,14 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
+golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE=
+golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY=
+golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
google.golang.org/api v0.287.1 h1:LiyJx32VU3cwQfLchn/513qKhc25hq0pEANYJoWNnnI=
@@ -807,8 +823,8 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc v1.82.0 h1:vguDnZUPjE26w09A63VoxZPnvPjB5Riyc0mkXPFmAIU=
-google.golang.org/grpc v1.82.0/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA=
+google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE=
+google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
diff --git a/internal/syncer/connector/bigquery.go b/internal/syncer/connector/bigquery.go
new file mode 100644
index 0000000000..f8e8aa0e84
--- /dev/null
+++ b/internal/syncer/connector/bigquery.go
@@ -0,0 +1,1104 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package connector
+
+import (
+ "context"
+ "crypto/md5"
+ "encoding/base64"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "math/big"
+ "regexp"
+ "sort"
+ "strconv"
+ "strings"
+ "time"
+
+ gcpbigquery "cloud.google.com/go/bigquery"
+ "cloud.google.com/go/civil"
+ "google.golang.org/api/iterator"
+ "google.golang.org/api/option"
+)
+
+const (
+ defaultBigQueryBatchSize = 2
+ defaultBigQueryPageSize = 1000
+ defaultBigQueryMaxBytes = int64(1024 * 1024 * 1024)
+ bigQueryCursorTypeKey = "__ragflow_bq_cursor_type__"
+ bigQueryIdentifierPattern = `^[A-Za-z_][A-Za-z0-9_]*$`
+ bigQueryProjectIDPattern = `^[A-Za-z][A-Za-z0-9_-]*$`
+)
+
+var bigQueryIdentifierRE = regexp.MustCompile(bigQueryIdentifierPattern)
+var bigQueryProjectIDRE = regexp.MustCompile(bigQueryProjectIDPattern)
+
+// BigQueryConnector imports rows from Google BigQuery tables or custom queries.
+//
+// It mirrors the Python BigQueryConnector: selected content columns form the
+// document body, metadata columns are copied into document metadata, an
+// optional id column gives stable document ids, and an optional timestamp
+// column drives cursor-based incremental sync and pruning.
+type BigQueryConnector struct {
+ projectID string
+ datasetID string
+ tableID string
+ location string
+ query string
+ contentColumns []string
+ metadataColumns []string
+ idColumn string
+ timestampColumn string
+ batchSize int
+ pageSize int
+ maximumBytesBilled int64
+ jobTimeout time.Duration
+ useQueryCache bool
+ credentialsJSON []byte
+ newClient func(context.Context) (bigQueryClient, error)
+}
+
+type bigQueryClient interface {
+ Query(query string) bigQueryQuery
+ Close() error
+}
+
+type bigQueryQuery interface {
+ Run(ctx context.Context) (bigQueryJob, error)
+ Read(ctx context.Context) (bigQueryRowIterator, error)
+ SetDryRun(bool)
+ SetDisableQueryCache(bool)
+ SetMaxBytesBilled(int64)
+ SetJobTimeout(time.Duration)
+ SetParameters([]gcpbigquery.QueryParameter)
+}
+
+type bigQueryJob interface {
+ Status(ctx context.Context) (bigQueryJobStatus, error)
+ Wait(ctx context.Context) (bigQueryJobStatus, error)
+ Read(ctx context.Context) (bigQueryRowIterator, error)
+}
+
+type bigQueryJobStatus interface {
+ Done() bool
+ Err() error
+ Statistics() bigQueryJobStatistics
+}
+
+type bigQueryJobStatistics interface {
+ QuerySchema() []bigQueryField
+ TotalBytesProcessed() int64
+}
+
+type bigQueryRowIterator interface {
+ Next(dst any) error
+ Schema() []bigQueryField
+ SetPageSize(int)
+}
+
+type bigQueryField struct {
+ Name string
+ Type string
+}
+
+type gcpBigQueryClient struct {
+ client *gcpbigquery.Client
+}
+
+func (c *gcpBigQueryClient) Query(query string) bigQueryQuery {
+ return &gcpBigQueryQuery{q: c.client.Query(query)}
+}
+
+func (c *gcpBigQueryClient) Close() error { return c.client.Close() }
+
+type gcpBigQueryQuery struct {
+ q *gcpbigquery.Query
+}
+
+func (q *gcpBigQueryQuery) Run(ctx context.Context) (bigQueryJob, error) {
+ job, err := q.q.Run(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return &gcpBigQueryJob{j: job}, nil
+}
+
+func (q *gcpBigQueryQuery) Read(ctx context.Context) (bigQueryRowIterator, error) {
+ it, err := q.q.Read(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return &gcpBigQueryRowIterator{it: it}, nil
+}
+
+func (q *gcpBigQueryQuery) SetDryRun(value bool) { q.q.DryRun = value }
+func (q *gcpBigQueryQuery) SetDisableQueryCache(value bool) { q.q.DisableQueryCache = value }
+func (q *gcpBigQueryQuery) SetMaxBytesBilled(value int64) { q.q.MaxBytesBilled = value }
+func (q *gcpBigQueryQuery) SetJobTimeout(value time.Duration) { q.q.JobTimeout = value }
+func (q *gcpBigQueryQuery) SetParameters(parameters []gcpbigquery.QueryParameter) {
+ q.q.Parameters = append([]gcpbigquery.QueryParameter(nil), parameters...)
+}
+
+type gcpBigQueryJob struct{ j *gcpbigquery.Job }
+
+func (j *gcpBigQueryJob) Status(ctx context.Context) (bigQueryJobStatus, error) {
+ status, err := j.j.Status(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return &gcpBigQueryJobStatus{status: status}, nil
+}
+
+func (j *gcpBigQueryJob) Wait(ctx context.Context) (bigQueryJobStatus, error) {
+ status, err := j.j.Wait(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return &gcpBigQueryJobStatus{status: status}, nil
+}
+
+func (j *gcpBigQueryJob) Read(ctx context.Context) (bigQueryRowIterator, error) {
+ it, err := j.j.Read(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return &gcpBigQueryRowIterator{it: it}, nil
+}
+
+type gcpBigQueryJobStatus struct{ status *gcpbigquery.JobStatus }
+
+func (s *gcpBigQueryJobStatus) Done() bool { return s.status.Done() }
+func (s *gcpBigQueryJobStatus) Err() error { return s.status.Err() }
+func (s *gcpBigQueryJobStatus) Statistics() bigQueryJobStatistics {
+ return &gcpBigQueryJobStatistics{stats: s.status.Statistics}
+}
+
+type gcpBigQueryJobStatistics struct{ stats *gcpbigquery.JobStatistics }
+
+func (s *gcpBigQueryJobStatistics) QuerySchema() []bigQueryField {
+ if s.stats == nil || s.stats.Details == nil {
+ return nil
+ }
+ queryStats, ok := s.stats.Details.(*gcpbigquery.QueryStatistics)
+ if !ok {
+ return nil
+ }
+ return toBigQueryFields(queryStats.Schema)
+}
+
+func (s *gcpBigQueryJobStatistics) TotalBytesProcessed() int64 {
+ if s.stats == nil {
+ return 0
+ }
+ return s.stats.TotalBytesProcessed
+}
+
+type gcpBigQueryRowIterator struct{ it *gcpbigquery.RowIterator }
+
+func (i *gcpBigQueryRowIterator) Next(dst any) error {
+ err := i.it.Next(dst)
+ if errors.Is(err, iterator.Done) {
+ return io.EOF
+ }
+ return err
+}
+
+func (i *gcpBigQueryRowIterator) Schema() []bigQueryField { return toBigQueryFields(i.it.Schema) }
+
+func (i *gcpBigQueryRowIterator) SetPageSize(size int) {
+ if size <= 0 || i.it == nil || i.it.PageInfo() == nil {
+ return
+ }
+ i.it.PageInfo().MaxSize = size
+}
+
+// NewBigQueryConnector creates a BigQuery connector from Python-compatible config.
+func NewBigQueryConnector(config map[string]any) (*BigQueryConnector, error) {
+ credentials := configAnyMap(config["credentials"])
+ rawCredential := credentials["service_account_json"]
+ var credentialBytes []byte
+ switch value := rawCredential.(type) {
+ case string:
+ credentialBytes = []byte(strings.TrimSpace(value))
+ case map[string]any:
+ data, err := json.Marshal(value)
+ if err != nil {
+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("BigQuery: service_account_json is not valid JSON: %v", err)}
+ }
+ credentialBytes = data
+ default:
+ if rawCredential != nil {
+ return nil, &ConnectorValidationError{Message: "BigQuery: service_account_json must be a JSON string or object"}
+ }
+ }
+
+ connector := &BigQueryConnector{
+ projectID: strings.TrimSpace(stringConfig(config["project_id"])),
+ datasetID: strings.TrimSpace(stringConfig(config["dataset_id"])),
+ tableID: strings.TrimSpace(stringConfig(config["table_id"])),
+ location: strings.TrimSpace(stringConfig(config["location"])),
+ query: sanitizeBigQueryQuery(stringConfig(config["query"])),
+ idColumn: strings.TrimSpace(stringConfig(config["id_column"])),
+ timestampColumn: strings.TrimSpace(stringConfig(config["timestamp_column"])),
+ batchSize: configInt(config["batch_size"], defaultBigQueryBatchSize),
+ pageSize: configInt(config["page_size"], defaultBigQueryPageSize),
+ maximumBytesBilled: configInt64(config["maximum_bytes_billed"], defaultBigQueryMaxBytes),
+ useQueryCache: configBoolDefault(config["use_query_cache"], true),
+ credentialsJSON: credentialBytes,
+ }
+ if connector.maximumBytesBilled <= 0 {
+ connector.maximumBytesBilled = defaultBigQueryMaxBytes
+ }
+ if value := config["job_timeout_ms"]; value != nil {
+ connector.jobTimeout = time.Duration(configInt(value, 0)) * time.Millisecond
+ }
+ connector.contentColumns = connector.splitColumns(config["content_columns"])
+ connector.metadataColumns = connector.splitColumns(config["metadata_columns"])
+ if err := connector.validateIdentifiers(); err != nil {
+ return nil, err
+ }
+ return connector, nil
+}
+
+// Validate validates BigQuery settings and credentials.
+func (c *BigQueryConnector) Validate(ctx context.Context) error {
+ if err := c.validateStatic(); err != nil {
+ return err
+ }
+ client, err := c.openClient(ctx)
+ if err != nil {
+ return err
+ }
+ defer client.Close()
+
+ connectivity := c.configuredQuery(client, "SELECT 1")
+ if err := waitBigQueryJob(ctx, connectivity); err != nil {
+ return &ConnectorValidationError{Message: fmt.Sprintf("BigQuery validation failed: %v", err)}
+ }
+
+ dryRun := c.configuredQuery(client, c.wrapQuery(c.buildBaseQuery(), "*"))
+ dryRun.SetDryRun(true)
+ dryRun.SetDisableQueryCache(true)
+ job, err := dryRun.Run(ctx)
+ if err != nil {
+ return &ConnectorValidationError{Message: fmt.Sprintf("BigQuery validation failed: %v", err)}
+ }
+ status, err := job.Wait(ctx)
+ if err != nil {
+ return &ConnectorValidationError{Message: fmt.Sprintf("BigQuery validation failed: %v", err)}
+ }
+ if !status.Done() {
+ return &ConnectorValidationError{Message: "BigQuery validation dry-run did not complete"}
+ }
+ if err := status.Err(); err != nil {
+ return &ConnectorValidationError{Message: fmt.Sprintf("BigQuery validation failed: %v", err)}
+ }
+ schema := status.Statistics().QuerySchema()
+ if len(schema) == 0 {
+ return &ConnectorValidationError{Message: "BigQuery validation dry-run returned no schema"}
+ }
+ return c.validateSchema(schema)
+}
+
+// ValidateConnectorSetting validates an unsaved BigQuery config.
+func (c *BigQueryConnector) ValidateConnectorSetting(ctx context.Context, request map[string]any) error {
+ ctx, cancel := context.WithTimeout(ctx, connectorSettingValidationTimeout)
+ defer cancel()
+ candidate, err := NewBigQueryConnector(request)
+ if err != nil {
+ return err
+ }
+ return candidate.Validate(ctx)
+}
+
+// OpenSync opens one BigQuery sync session.
+func (c *BigQueryConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {
+ if err := c.validateStatic(); err != nil {
+ return nil, err
+ }
+ client, err := c.openClient(ctx)
+ if err != nil {
+ return nil, err
+ }
+ queryText, parameters := c.buildSyncQuery(request)
+ query := c.configuredQuery(client, queryText)
+ query.SetParameters(parameters)
+ iterator, err := query.Read(ctx)
+ if err != nil {
+ client.Close()
+ return nil, err
+ }
+ iterator.SetPageSize(c.effectivePageSize())
+ session := &bigQuerySyncSession{
+ connector: c,
+ client: client,
+ rows: iterator,
+ batchSize: c.effectiveBatchSize(),
+ }
+ if err := session.applyResume(request.Resume); err != nil {
+ client.Close()
+ return nil, err
+ }
+ return session, nil
+}
+
+// OpenPrune opens one complete BigQuery prune snapshot session.
+func (c *BigQueryConnector) OpenPrune(ctx context.Context, request PruneRequest) (PruneSession, error) {
+ if err := c.validateStatic(); err != nil {
+ return nil, err
+ }
+ client, err := c.openClient(ctx)
+ if err != nil {
+ return nil, err
+ }
+ query := c.configuredQuery(client, c.buildSlimQuery(c.buildBaseQuery()))
+ iterator, err := query.Read(ctx)
+ if err != nil {
+ client.Close()
+ return nil, err
+ }
+ iterator.SetPageSize(c.effectivePageSize())
+ return &bigQueryPruneSession{
+ connector: c,
+ client: client,
+ rows: iterator,
+ batchSize: c.effectiveBatchSize(),
+ }, nil
+}
+
+func (c *BigQueryConnector) openClient(ctx context.Context) (bigQueryClient, error) {
+ if c.newClient != nil {
+ return c.newClient(ctx)
+ }
+ if len(c.credentialsJSON) == 0 {
+ return nil, &ConnectorMissingCredentialError{Message: "BigQuery: missing service_account_json"}
+ }
+ if !json.Valid(c.credentialsJSON) {
+ return nil, &ConnectorValidationError{Message: "BigQuery: service_account_json is not valid JSON"}
+ }
+ options := []option.ClientOption{option.WithCredentialsJSON(c.credentialsJSON)}
+ client, err := gcpbigquery.NewClient(ctx, c.projectID, options...)
+ if err != nil {
+ return nil, fmt.Errorf("BigQuery: failed to create client: %w", err)
+ }
+ if c.location != "" {
+ client.Location = c.location
+ }
+ return &gcpBigQueryClient{client: client}, nil
+}
+
+func (c *BigQueryConnector) configuredQuery(client bigQueryClient, query string) bigQueryQuery {
+ q := client.Query(query)
+ q.SetMaxBytesBilled(c.maximumBytesBilled)
+ q.SetJobTimeout(c.jobTimeout)
+ q.SetDisableQueryCache(!c.useQueryCache)
+ return q
+}
+
+func (c *BigQueryConnector) validateStatic() error {
+ if c == nil {
+ return &ConnectorValidationError{Message: "BigQuery connector is nil"}
+ }
+ if c.projectID == "" {
+ return &ConnectorValidationError{Message: "BigQuery project_id is required."}
+ }
+ if len(c.contentColumns) == 0 {
+ return &ConnectorValidationError{Message: "At least one content column must be specified."}
+ }
+ if c.query == "" && (c.datasetID == "" || c.tableID == "") {
+ return &ConnectorValidationError{Message: "BigQuery requires either a custom query or both dataset_id and table_id."}
+ }
+ if c.batchSize <= 0 {
+ return &ConnectorValidationError{Message: "BigQuery batch_size must be a positive integer"}
+ }
+ if c.pageSize <= 0 {
+ return &ConnectorValidationError{Message: "BigQuery page_size must be a positive integer"}
+ }
+ if len(c.credentialsJSON) == 0 {
+ return &ConnectorMissingCredentialError{Message: "BigQuery: missing service_account_json"}
+ }
+ return nil
+}
+
+func (c *BigQueryConnector) validateIdentifiers() error {
+ if c.projectID != "" && !bigQueryProjectIDRE.MatchString(c.projectID) {
+ return &ConnectorValidationError{Message: fmt.Sprintf("Invalid BigQuery identifier for %q", "project_id")}
+ }
+ for _, column := range c.contentColumns {
+ if err := validateBigQueryIdentifier(column, fmt.Sprintf("content_columns[%q]", column)); err != nil {
+ return err
+ }
+ }
+ for _, column := range c.metadataColumns {
+ if err := validateBigQueryIdentifier(column, fmt.Sprintf("metadata_columns[%q]", column)); err != nil {
+ return err
+ }
+ }
+ for name, column := range map[string]string{"dataset_id": c.datasetID, "table_id": c.tableID} {
+ if column != "" {
+ if err := validateBigQueryIdentifier(column, name); err != nil {
+ return err
+ }
+ }
+ }
+ if c.idColumn != "" {
+ if err := validateBigQueryIdentifier(c.idColumn, "id_column"); err != nil {
+ return err
+ }
+ }
+ if c.timestampColumn != "" {
+ if err := validateBigQueryIdentifier(c.timestampColumn, "timestamp_column"); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func validateBigQueryIdentifier(value, name string) error {
+ if value == "" {
+ return nil
+ }
+ if !bigQueryIdentifierRE.MatchString(value) {
+ return &ConnectorValidationError{Message: fmt.Sprintf("Invalid BigQuery identifier for %q", name)}
+ }
+ return nil
+}
+
+func (c *BigQueryConnector) effectiveBatchSize() int {
+ if c.batchSize > 0 {
+ return c.batchSize
+ }
+ return defaultBigQueryBatchSize
+}
+
+func (c *BigQueryConnector) effectivePageSize() int {
+ if c.pageSize > 0 {
+ return c.pageSize
+ }
+ return defaultBigQueryPageSize
+}
+
+func (c *BigQueryConnector) buildBaseQuery() string {
+ if c.query != "" {
+ return c.query
+ }
+ return fmt.Sprintf("SELECT * FROM `%s.%s.%s`", c.projectID, c.datasetID, c.tableID)
+}
+
+func (c *BigQueryConnector) usesTableMode() bool { return c.query == "" }
+
+func (c *BigQueryConnector) wrapQuery(base, selectClause string) string {
+ return c.wrapFilteredQuery(base, selectClause, "")
+}
+
+func (c *BigQueryConnector) wrapFilteredQuery(base, selectClause, where string) string {
+ query := fmt.Sprintf("SELECT %s FROM (%s) AS ragflow_src", selectClause, base)
+ if where != "" {
+ query += " WHERE " + where
+ }
+ return query + c.stableOrderClause()
+}
+
+func (c *BigQueryConnector) stableOrderClause() string {
+ columns := make([]string, 0, 2)
+ if c.idColumn != "" {
+ columns = append(columns, c.idColumn)
+ }
+ if c.timestampColumn != "" && c.timestampColumn != c.idColumn {
+ columns = append(columns, c.timestampColumn)
+ }
+ if len(columns) == 0 {
+ return ""
+ }
+ order := make([]string, 0, len(columns))
+ for _, column := range columns {
+ order = append(order, fmt.Sprintf("ragflow_src.%s ASC", column))
+ }
+ return " ORDER BY " + strings.Join(order, ", ")
+}
+
+func (c *BigQueryConnector) validateSchema(schema []bigQueryField) error {
+ names := map[string]struct{}{}
+ for _, field := range schema {
+ names[field.Name] = struct{}{}
+ }
+ required := map[string]struct{}{}
+ for _, column := range c.contentColumns {
+ required[column] = struct{}{}
+ }
+ optional := map[string]struct{}{}
+ for _, column := range c.metadataColumns {
+ optional[column] = struct{}{}
+ }
+ if c.idColumn != "" {
+ optional[c.idColumn] = struct{}{}
+ }
+ if c.timestampColumn != "" {
+ optional[c.timestampColumn] = struct{}{}
+ }
+ var missing []string
+ for column := range required {
+ if _, ok := names[column]; !ok {
+ missing = append(missing, column)
+ }
+ }
+ for column := range optional {
+ if _, ok := names[column]; !ok {
+ missing = append(missing, column)
+ }
+ }
+ sort.Strings(missing)
+ if len(missing) > 0 {
+ return &ConnectorValidationError{Message: fmt.Sprintf("BigQuery configured columns not found in schema: %s", strings.Join(missing, ", "))}
+ }
+ if c.timestampColumn != "" {
+ if _, ok := names[c.timestampColumn]; !ok {
+ return &ConnectorValidationError{Message: fmt.Sprintf("BigQuery timestamp column '%s' was not found in the schema.", c.timestampColumn)}
+ }
+ }
+ return nil
+}
+
+// buildSyncQuery returns the wrapped query and cursor parameters.
+func (c *BigQueryConnector) buildSyncQuery(request SyncRequest) (string, []gcpbigquery.QueryParameter) {
+ base := c.buildBaseQuery()
+ if c.timestampColumn == "" || request.FromBeginning {
+ return c.wrapQuery(base, "*"), nil
+ }
+ conditions := []string{}
+ parameters := []gcpbigquery.QueryParameter{}
+ if request.WindowStart != nil {
+ conditions = append(conditions, fmt.Sprintf("ragflow_src.%s >= @start_cursor", c.timestampColumn))
+ parameters = append(parameters, gcpbigquery.QueryParameter{Name: "start_cursor", Value: request.WindowStart.UTC()})
+ }
+ if !request.WindowEnd.IsZero() {
+ conditions = append(conditions, fmt.Sprintf("ragflow_src.%s <= @end_cursor", c.timestampColumn))
+ parameters = append(parameters, gcpbigquery.QueryParameter{Name: "end_cursor", Value: request.WindowEnd.UTC()})
+ }
+ if len(conditions) == 0 {
+ return c.wrapQuery(base, "*"), nil
+ }
+ return c.wrapFilteredQuery(base, "*", strings.Join(conditions, " AND ")), parameters
+}
+
+func (c *BigQueryConnector) buildSlimQuery(base string) string {
+ columns := c.contentColumns
+ if c.idColumn != "" {
+ columns = []string{c.idColumn}
+ }
+ if len(columns) == 0 {
+ return c.wrapQuery(base, "*")
+ }
+ selects := make([]string, 0, len(columns))
+ for _, column := range columns {
+ selects = append(selects, fmt.Sprintf("ragflow_src.%s", column))
+ }
+ return c.wrapQuery(base, strings.Join(selects, ", "))
+}
+
+func (c *BigQueryConnector) splitColumns(value any) []string {
+ switch typed := value.(type) {
+ case string:
+ parts := strings.Split(typed, ",")
+ columns := make([]string, 0, len(parts))
+ for _, part := range parts {
+ if column := strings.TrimSpace(part); column != "" {
+ columns = append(columns, column)
+ }
+ }
+ return columns
+ case []any:
+ columns := make([]string, 0, len(typed))
+ for _, item := range typed {
+ if column := strings.TrimSpace(stringConfig(item)); column != "" {
+ columns = append(columns, column)
+ }
+ }
+ return columns
+ }
+ return nil
+}
+
+func (c *BigQueryConnector) contentColumnsForRow(row map[string]any, schema []bigQueryField) []string {
+ if len(c.contentColumns) > 0 {
+ return c.contentColumns
+ }
+ excluded := map[string]bool{}
+ if c.idColumn != "" {
+ excluded[c.idColumn] = true
+ }
+ if c.timestampColumn != "" {
+ excluded[c.timestampColumn] = true
+ }
+ columns := make([]string, 0, len(schema))
+ for _, field := range schema {
+ if _, ok := row[field.Name]; ok && !excluded[field.Name] {
+ columns = append(columns, field.Name)
+ }
+ }
+ return columns
+}
+
+func (c *BigQueryConnector) buildContent(row map[string]any, columns []string) string {
+ parts := []string{}
+ for _, column := range columns {
+ value, ok := row[column]
+ if !ok || value == nil {
+ continue
+ }
+ rendered := renderBigQueryContentValue(value)
+ if rendered == "" {
+ continue
+ }
+ parts = append(parts, fmt.Sprintf("【%s】:\n%s", column, rendered))
+ }
+ return strings.Join(parts, "\n\n")
+}
+
+func (c *BigQueryConnector) buildDocumentID(row map[string]any) string {
+ prefix := c.idPrefix()
+ if c.idColumn != "" {
+ if value, ok := row[c.idColumn]; ok && value != nil {
+ return prefix + ":" + bigQueryValueString(value)
+ }
+ }
+ content := c.buildContent(row, c.contentColumns)
+ sum := md5.Sum([]byte(content))
+ return prefix + ":" + hex.EncodeToString(sum[:])
+}
+
+func (c *BigQueryConnector) idPrefix() string {
+ if c.usesTableMode() {
+ return fmt.Sprintf("bigquery:%s:%s.%s", c.projectID, c.datasetID, c.tableID)
+ }
+ return fmt.Sprintf("bigquery:%s:query", c.projectID)
+}
+
+func (c *BigQueryConnector) rowToSourceDocument(row map[string]any, schema []bigQueryField) (SourceDocument, bool) {
+ contentColumns := c.contentColumnsForRow(row, schema)
+ content := c.buildContent(row, contentColumns)
+
+ metadata := map[string]any{}
+ for _, column := range c.metadataColumns {
+ value, ok := row[column]
+ if !ok || value == nil {
+ continue
+ }
+ metadata[column] = renderBigQueryMetadataValue(value)
+ }
+
+ updatedAt := time.Now().UTC()
+ if c.timestampColumn != "" {
+ if value, ok := row[c.timestampColumn]; ok && value != nil {
+ updatedAt = parseBigQueryUpdatedAt(value)
+ }
+ }
+
+ semanticID := "bigquery_record"
+ if len(contentColumns) > 0 {
+ if value, ok := row[contentColumns[0]]; ok && value != nil {
+ semanticID = strings.TrimSpace(strings.ReplaceAll(strings.ReplaceAll(bigQueryValueString(value), "\n", " "), "\r", " "))
+ if semanticID == "" {
+ semanticID = "bigquery_record"
+ } else if len(semanticID) > 100 {
+ semanticID = semanticID[:100]
+ }
+ }
+ }
+
+ sourceID := c.buildDocumentID(row)
+ blob := []byte(content)
+ return SourceDocument{
+ SourceID: sourceID,
+ SemanticIdentifier: semanticID,
+ Extension: ".txt",
+ Blob: blob,
+ UpdatedAt: updatedAt,
+ SizeBytes: int64(len(blob)),
+ Metadata: metadata,
+ Fingerprint: stableFingerprint(map[string]any{
+ "id": sourceID,
+ "content": content,
+ "metadata": metadata,
+ }),
+ }, true
+}
+
+func renderBigQueryContentValue(value any) string {
+ switch typed := value.(type) {
+ case []byte:
+ return ""
+ case time.Time:
+ return typed.Format(time.RFC3339)
+ case civil.Date:
+ return typed.String()
+ case civil.Time:
+ return typed.String()
+ case civil.DateTime:
+ return typed.String()
+ case *big.Rat:
+ return typed.RatString()
+ case map[string]any:
+ data, err := json.Marshal(typed)
+ if err != nil {
+ return fmt.Sprint(value)
+ }
+ return string(data)
+ case []any:
+ data, err := json.Marshal(typed)
+ if err != nil {
+ return fmt.Sprint(value)
+ }
+ return string(data)
+ default:
+ return fmt.Sprint(value)
+ }
+}
+
+func renderBigQueryMetadataValue(value any) string {
+ switch typed := value.(type) {
+ case []byte:
+ return base64.StdEncoding.EncodeToString(typed)
+ case time.Time:
+ return typed.Format(time.RFC3339)
+ case civil.Date:
+ return typed.String()
+ case civil.Time:
+ return typed.String()
+ case civil.DateTime:
+ return typed.String()
+ case *big.Rat:
+ return typed.RatString()
+ case map[string]any:
+ data, err := json.Marshal(typed)
+ if err != nil {
+ return fmt.Sprint(value)
+ }
+ return string(data)
+ case []any:
+ data, err := json.Marshal(typed)
+ if err != nil {
+ return fmt.Sprint(value)
+ }
+ return string(data)
+ default:
+ return fmt.Sprint(value)
+ }
+}
+
+func parseBigQueryUpdatedAt(value any) time.Time {
+ switch typed := value.(type) {
+ case time.Time:
+ return typed.UTC()
+ case civil.DateTime:
+ return typed.In(time.UTC)
+ case civil.Date:
+ return time.Date(typed.Year, typed.Month, typed.Day, 0, 0, 0, 0, time.UTC)
+ }
+ return time.Now().UTC()
+}
+
+func bigQueryValueString(value any) string {
+ switch typed := value.(type) {
+ case []byte:
+ return string(typed)
+ case time.Time:
+ return typed.Format(time.RFC3339)
+ case civil.Date:
+ return typed.String()
+ case civil.Time:
+ return typed.String()
+ case civil.DateTime:
+ return typed.String()
+ case *big.Rat:
+ return typed.RatString()
+ case map[string]any:
+ data, err := json.Marshal(typed)
+ if err != nil {
+ return fmt.Sprint(value)
+ }
+ return string(data)
+ case []any:
+ data, err := json.Marshal(typed)
+ if err != nil {
+ return fmt.Sprint(value)
+ }
+ return string(data)
+ default:
+ return fmt.Sprint(value)
+ }
+}
+
+func waitBigQueryJob(ctx context.Context, query bigQueryQuery) error {
+ job, err := query.Run(ctx)
+ if err != nil {
+ return err
+ }
+ status, err := job.Wait(ctx)
+ if err != nil {
+ return err
+ }
+ if !status.Done() {
+ return fmt.Errorf("BigQuery query did not complete")
+ }
+ if err := status.Err(); err != nil {
+ return err
+ }
+ return nil
+}
+
+func toBigQueryFields(schema gcpbigquery.Schema) []bigQueryField {
+ if len(schema) == 0 {
+ return nil
+ }
+ fields := make([]bigQueryField, 0, len(schema))
+ for _, field := range schema {
+ if field == nil {
+ continue
+ }
+ fields = append(fields, bigQueryField{Name: field.Name, Type: string(field.Type)})
+ }
+ return fields
+}
+
+func sanitizeBigQueryQuery(raw string) string {
+ query := strings.TrimSpace(raw)
+ if query == "" {
+ return ""
+ }
+ if strings.HasPrefix(query, "```") {
+ query = strings.TrimPrefix(query, "```")
+ if strings.HasSuffix(query, "```") {
+ query = strings.TrimSuffix(query, "```")
+ }
+ if head, tail, ok := strings.Cut(query, "\n"); ok {
+ head = strings.TrimSpace(head)
+ if head == "sql" || head == "google-sql" || head == "bigquery" {
+ head = ""
+ }
+ query = strings.TrimSpace(head) + "\n" + strings.TrimSpace(tail)
+ }
+ }
+ return strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(query), ";"))
+}
+
+// serializeBigQueryCursor mirrors Python's cursor serialization used for
+// connector config persistence. Only temporal values need a marker wrapper.
+func serializeBigQueryCursor(value any) any {
+ switch typed := value.(type) {
+ case time.Time:
+ return map[string]any{bigQueryCursorTypeKey: "datetime", "value": typed.Format(time.RFC3339)}
+ case civil.Date:
+ return map[string]any{bigQueryCursorTypeKey: "date", "value": typed.String()}
+ case civil.Time:
+ return map[string]any{bigQueryCursorTypeKey: "time", "value": typed.String()}
+ case civil.DateTime:
+ return map[string]any{bigQueryCursorTypeKey: "datetime", "value": typed.String()}
+ case *big.Rat:
+ return map[string]any{bigQueryCursorTypeKey: "decimal", "value": typed.RatString()}
+ default:
+ return value
+ }
+}
+
+func deserializeBigQueryCursor(value any) any {
+ typed, ok := value.(map[string]any)
+ if !ok {
+ return value
+ }
+ raw, ok := typed[bigQueryCursorTypeKey].(string)
+ if !ok {
+ return value
+ }
+ switch raw {
+ case "datetime":
+ if parsed, err := time.Parse(time.RFC3339, fmt.Sprint(typed["value"])); err == nil {
+ return parsed
+ }
+ if parsed, err := time.Parse("2006-01-02 15:04:05", fmt.Sprint(typed["value"])); err == nil {
+ return parsed
+ }
+ case "date":
+ if parsed, err := time.Parse("2006-01-02", fmt.Sprint(typed["value"])); err == nil {
+ return parsed
+ }
+ case "time":
+ if parsed, err := civil.ParseTime(fmt.Sprint(typed["value"])); err == nil {
+ return parsed
+ }
+ case "decimal":
+ if parsed, ok := new(big.Rat).SetString(fmt.Sprint(typed["value"])); ok {
+ return parsed
+ }
+ }
+ return value
+}
+
+func configInt64(value any, fallback int64) int64 {
+ if value == nil {
+ return fallback
+ }
+ switch typed := value.(type) {
+ case int:
+ return int64(typed)
+ case int64:
+ return typed
+ case float64:
+ return int64(typed)
+ case json.Number:
+ if parsed, err := typed.Int64(); err == nil {
+ return parsed
+ }
+ case string:
+ parsed, err := strconv.ParseInt(strings.TrimSpace(typed), 10, 64)
+ if err == nil {
+ return parsed
+ }
+ }
+ return fallback
+}
+
+type bigQuerySyncSession struct {
+ connector *BigQueryConnector
+ client bigQueryClient
+ rows bigQueryRowIterator
+ batchSize int
+ resume *SyncCheckpoint
+ closed bool
+}
+
+func (s *bigQuerySyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {
+ if s.closed {
+ return SyncBatch{}, io.EOF
+ }
+ documents := make([]SourceDocument, 0, s.batchSize)
+ for len(documents) < s.batchSize {
+ var row map[string]any
+ err := s.rows.Next(&row)
+ if errors.Is(err, io.EOF) {
+ if len(documents) == 0 {
+ if s.resume != nil {
+ return SyncBatch{}, fmt.Errorf("bigquery resume anchor %q was not found in the current result: %w", s.resume.SourceID, ErrSyncResumeInvalid)
+ }
+ return SyncBatch{}, io.EOF
+ }
+ break
+ }
+ if err != nil {
+ return SyncBatch{}, err
+ }
+ doc, ok := s.connector.rowToSourceDocument(row, s.rows.Schema())
+ if !ok {
+ continue
+ }
+ if !s.includeResumed(doc) {
+ if s.resume != nil && doc.SourceID == s.resume.SourceID {
+ // The anchor was found; continue emitting rows after it.
+ }
+ continue
+ }
+ documents = append(documents, doc)
+ }
+ if len(documents) == 0 {
+ return SyncBatch{}, io.EOF
+ }
+ return SyncBatch{Documents: documents, Checkpoint: bigQuerySyncCheckpoint(documents[len(documents)-1])}, nil
+}
+
+func (s *bigQuerySyncSession) Close() error {
+ if s.closed {
+ return nil
+ }
+ s.closed = true
+ return s.client.Close()
+}
+
+func (s *bigQuerySyncSession) includeResumed(doc SourceDocument) bool {
+ if s.resume == nil || s.resume.SourceID == "" {
+ return true
+ }
+ if doc.SourceID == s.resume.SourceID {
+ s.resume = nil
+ return false
+ }
+ return false
+}
+
+func (s *bigQuerySyncSession) applyResume(checkpoint *SyncCheckpoint) error {
+ if checkpoint == nil {
+ return nil
+ }
+ sourceID := firstNonEmpty(checkpoint.SourceID, checkpoint.Cursor)
+ if sourceID == "" {
+ return fmt.Errorf("bigquery sync checkpoint has no source anchor: %w", ErrSyncResumeInvalid)
+ }
+ if s.connector.stableOrderClause() == "" {
+ return fmt.Errorf("bigquery sync resume requires a stable ordering: %w", ErrSyncResumeInvalid)
+ }
+ s.resume = &SyncCheckpoint{SourceID: sourceID}
+ return nil
+}
+
+func bigQuerySyncCheckpoint(document SourceDocument) *SyncCheckpoint {
+ updatedAt := document.UpdatedAt
+ return &SyncCheckpoint{
+ Cursor: document.SourceID,
+ SourceID: document.SourceID,
+ UpdatedAt: &updatedAt,
+ }
+}
+
+type bigQueryPruneSession struct {
+ connector *BigQueryConnector
+ client bigQueryClient
+ rows bigQueryRowIterator
+ batchSize int
+ closed bool
+}
+
+func (s *bigQueryPruneSession) NextBatch(ctx context.Context) (PruneBatch, error) {
+ if s.closed {
+ return PruneBatch{}, io.EOF
+ }
+ documents := make([]SlimDocument, 0, s.batchSize)
+ for len(documents) < s.batchSize {
+ var row map[string]any
+ err := s.rows.Next(&row)
+ if errors.Is(err, io.EOF) {
+ break
+ }
+ if err != nil {
+ return PruneBatch{}, err
+ }
+ documents = append(documents, SlimDocument{SourceID: s.connector.buildDocumentID(row)})
+ }
+ if len(documents) == 0 {
+ return PruneBatch{}, io.EOF
+ }
+ return PruneBatch{Documents: documents}, nil
+}
+
+func (s *bigQueryPruneSession) Close() error {
+ if s.closed {
+ return nil
+ }
+ s.closed = true
+ return s.client.Close()
+}
diff --git a/internal/syncer/connector/bigquery_test.go b/internal/syncer/connector/bigquery_test.go
new file mode 100644
index 0000000000..83ef77888f
--- /dev/null
+++ b/internal/syncer/connector/bigquery_test.go
@@ -0,0 +1,536 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package connector
+
+import (
+ "context"
+ "crypto/md5"
+ "encoding/hex"
+ "errors"
+ "io"
+ "strings"
+ "testing"
+ "time"
+
+ gcpbigquery "cloud.google.com/go/bigquery"
+ "cloud.google.com/go/civil"
+)
+
+type fakeBigQueryClient struct {
+ queries []*fakeBigQueryQuery
+ rows []map[string]any
+ schema []bigQueryField
+ closed bool
+}
+
+func (f *fakeBigQueryClient) Query(query string) bigQueryQuery {
+ q := &fakeBigQueryQuery{
+ text: query,
+ readIterator: &fakeBigQueryRowIterator{
+ rows: f.rows,
+ schema: f.schema,
+ },
+ }
+ if strings.Contains(query, "FROM (") {
+ q.runJob = &fakeBigQueryJob{status: &fakeBigQueryJobStatus{done: true, schema: f.schema}}
+ }
+ f.queries = append(f.queries, q)
+ return q
+}
+
+func (f *fakeBigQueryClient) Close() error {
+ f.closed = true
+ return nil
+}
+
+type fakeBigQueryQuery struct {
+ text string
+ dryRun bool
+ disableCache bool
+ maxBytes int64
+ jobTimeout time.Duration
+ parameters []gcpbigquery.QueryParameter
+ runJob *fakeBigQueryJob
+ readIterator *fakeBigQueryRowIterator
+ runErr, readErr error
+}
+
+func (f *fakeBigQueryQuery) Run(context.Context) (bigQueryJob, error) {
+ if f.runErr != nil {
+ return nil, f.runErr
+ }
+ if f.runJob == nil {
+ return &fakeBigQueryJob{status: &fakeBigQueryJobStatus{done: true}}, nil
+ }
+ return f.runJob, nil
+}
+
+func (f *fakeBigQueryQuery) Read(context.Context) (bigQueryRowIterator, error) {
+ if f.readErr != nil {
+ return nil, f.readErr
+ }
+ if f.readIterator == nil {
+ return &fakeBigQueryRowIterator{}, nil
+ }
+ return f.readIterator, nil
+}
+
+func (f *fakeBigQueryQuery) SetDryRun(v bool) { f.dryRun = v }
+func (f *fakeBigQueryQuery) SetDisableQueryCache(v bool) { f.disableCache = v }
+func (f *fakeBigQueryQuery) SetMaxBytesBilled(v int64) { f.maxBytes = v }
+func (f *fakeBigQueryQuery) SetJobTimeout(v time.Duration) { f.jobTimeout = v }
+func (f *fakeBigQueryQuery) SetParameters(v []gcpbigquery.QueryParameter) {
+ f.parameters = append([]gcpbigquery.QueryParameter(nil), v...)
+}
+
+type fakeBigQueryJob struct {
+ status *fakeBigQueryJobStatus
+}
+
+func (f *fakeBigQueryJob) Status(context.Context) (bigQueryJobStatus, error) {
+ if f.status == nil {
+ return &fakeBigQueryJobStatus{done: true}, nil
+ }
+ return f.status, nil
+}
+
+func (f *fakeBigQueryJob) Wait(context.Context) (bigQueryJobStatus, error) {
+ return f.Status(context.Background())
+}
+
+func (f *fakeBigQueryJob) Read(context.Context) (bigQueryRowIterator, error) {
+ return &fakeBigQueryRowIterator{}, nil
+}
+
+type fakeBigQueryJobStatus struct {
+ done bool
+ err error
+ schema []bigQueryField
+ bytes int64
+}
+
+func (f *fakeBigQueryJobStatus) Done() bool { return f.done }
+func (f *fakeBigQueryJobStatus) Err() error { return f.err }
+func (f *fakeBigQueryJobStatus) Statistics() bigQueryJobStatistics {
+ return &fakeBigQueryJobStatistics{schema: f.schema, bytes: f.bytes}
+}
+
+type fakeBigQueryJobStatistics struct {
+ schema []bigQueryField
+ bytes int64
+}
+
+func (f *fakeBigQueryJobStatistics) QuerySchema() []bigQueryField { return f.schema }
+func (f *fakeBigQueryJobStatistics) TotalBytesProcessed() int64 { return f.bytes }
+
+type fakeBigQueryRowIterator struct {
+ rows []map[string]any
+ schema []bigQueryField
+ index int
+ page int
+}
+
+func (f *fakeBigQueryRowIterator) Next(dst any) error {
+ if f.index >= len(f.rows) {
+ return io.EOF
+ }
+ row := f.rows[f.index]
+ f.index++
+ target, ok := dst.(*map[string]any)
+ if !ok {
+ return errors.New("fake iterator expects *map[string]any")
+ }
+ *target = row
+ return nil
+}
+
+func (f *fakeBigQueryRowIterator) Schema() []bigQueryField { return f.schema }
+
+func (f *fakeBigQueryRowIterator) SetPageSize(size int) { f.page = size }
+
+func newFakeBigQueryConnector(t *testing.T, config map[string]any, fake *fakeBigQueryClient) *BigQueryConnector {
+ t.Helper()
+ if config == nil {
+ config = map[string]any{
+ "project_id": "my-proj",
+ "dataset_id": "ds",
+ "table_id": "tbl",
+ "content_columns": "name,description",
+ "id_column": "id",
+ "credentials": map[string]any{
+ "service_account_json": `{"type":"service_account","project_id":"my-proj"}`,
+ },
+ }
+ }
+ connector, err := NewBigQueryConnector(config)
+ if err != nil {
+ t.Fatalf("NewBigQueryConnector failed: %v", err)
+ }
+ connector.newClient = func(context.Context) (bigQueryClient, error) {
+ return fake, nil
+ }
+ return connector
+}
+
+func bigQuerySchema() []bigQueryField {
+ return []bigQueryField{
+ {Name: "id", Type: "INT64"},
+ {Name: "name", Type: "STRING"},
+ {Name: "description", Type: "STRING"},
+ {Name: "category", Type: "STRING"},
+ {Name: "updated_at", Type: "TIMESTAMP"},
+ }
+}
+
+func TestBigQueryConnectorValidate(t *testing.T) {
+ fake := &fakeBigQueryClient{schema: bigQuerySchema()}
+ connector := newFakeBigQueryConnector(t, nil, fake)
+ if err := connector.Validate(context.Background()); err != nil {
+ t.Fatalf("Validate failed: %v", err)
+ }
+ if len(fake.queries) != 2 {
+ t.Fatalf("query count = %d, want 2", len(fake.queries))
+ }
+ if fake.queries[0].text != "SELECT 1" {
+ t.Fatalf("connectivity query = %q", fake.queries[0].text)
+ }
+ if !fake.queries[1].dryRun || !fake.queries[1].disableCache {
+ t.Fatalf("dry-run flags = dry:%v cache:%v", fake.queries[1].dryRun, fake.queries[1].disableCache)
+ }
+}
+
+func TestBigQueryConnectorValidateMissingColumn(t *testing.T) {
+ fake := &fakeBigQueryClient{schema: []bigQueryField{
+ {Name: "id", Type: "INT64"},
+ {Name: "name", Type: "STRING"},
+ }}
+ connector := newFakeBigQueryConnector(t, nil, fake)
+ err := connector.Validate(context.Background())
+ if err == nil || !strings.Contains(err.Error(), "description") {
+ t.Fatalf("Validate error = %v, want missing description", err)
+ }
+}
+
+func TestBigQueryConnectorOpenSyncFull(t *testing.T) {
+ updatedAt := mustTime(t, "2026-01-02T03:04:05Z")
+ fake := &fakeBigQueryClient{schema: bigQuerySchema(), rows: []map[string]any{
+ {"id": int64(7), "name": "Hello/World", "description": "Some body", "category": "news", "updated_at": updatedAt},
+ }}
+ connector := newFakeBigQueryConnector(t, map[string]any{
+ "project_id": "my-proj",
+ "dataset_id": "ds",
+ "table_id": "tbl",
+ "content_columns": "name,description",
+ "metadata_columns": "id,category,updated_at",
+ "id_column": "id",
+ "timestamp_column": "updated_at",
+ "credentials": map[string]any{
+ "service_account_json": `{"type":"service_account","project_id":"my-proj"}`,
+ },
+ }, fake)
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch failed: %v", err)
+ }
+ if len(batch.Documents) != 1 {
+ t.Fatalf("documents len = %d", len(batch.Documents))
+ }
+ doc := batch.Documents[0]
+ if doc.SourceID != "bigquery:my-proj:ds.tbl:7" {
+ t.Fatalf("source id = %q", doc.SourceID)
+ }
+ if doc.SemanticIdentifier != "Hello/World" {
+ t.Fatalf("semantic id = %q", doc.SemanticIdentifier)
+ }
+ if doc.Extension != ".txt" {
+ t.Fatalf("extension = %q", doc.Extension)
+ }
+ blob := string(doc.Blob)
+ if !strings.Contains(blob, "【name】:\nHello/World") || !strings.Contains(blob, "【description】:\nSome body") {
+ t.Fatalf("blob = %q", blob)
+ }
+ if fake.queries[0].readIterator.page != defaultBigQueryPageSize {
+ t.Fatalf("page size = %d, want %d", fake.queries[0].readIterator.page, defaultBigQueryPageSize)
+ }
+ if !strings.Contains(fake.queries[0].text, "ORDER BY ragflow_src.id ASC, ragflow_src.updated_at ASC") {
+ t.Fatalf("sync query = %q, want stable ordering", fake.queries[0].text)
+ }
+ if !doc.UpdatedAt.Equal(updatedAt) {
+ t.Fatalf("updated at = %s", doc.UpdatedAt)
+ }
+ if doc.Metadata["category"] != "news" || doc.Metadata["id"] != "7" {
+ t.Fatalf("metadata = %v", doc.Metadata)
+ }
+ if doc.Metadata["updated_at"] != updatedAt.Format(time.RFC3339) {
+ t.Fatalf("metadata updated_at = %v", doc.Metadata["updated_at"])
+ }
+ if doc.Fingerprint == "" {
+ t.Fatalf("fingerprint is empty")
+ }
+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {
+ t.Fatalf("NextBatch EOF = %v", err)
+ }
+}
+
+func TestBigQueryConnectorIncrementalParameters(t *testing.T) {
+ fake := &fakeBigQueryClient{schema: []bigQueryField{{Name: "name", Type: "STRING"}}, rows: []map[string]any{{"name": "content"}}}
+ connector := newFakeBigQueryConnector(t, map[string]any{
+ "project_id": "my-proj",
+ "dataset_id": "ds",
+ "table_id": "tbl",
+ "content_columns": "name",
+ "timestamp_column": "updated_at",
+ "credentials": map[string]any{
+ "service_account_json": `{"type":"service_account","project_id":"my-proj"}`,
+ },
+ }, fake)
+ start := mustTime(t, "2026-01-01T00:00:00Z")
+ end := mustTime(t, "2026-01-02T00:00:00Z")
+ if _, err := connector.OpenSync(context.Background(), SyncRequest{
+ WindowStart: &start,
+ WindowEnd: end,
+ }); err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ if len(fake.queries) != 1 {
+ t.Fatalf("query count = %d", len(fake.queries))
+ }
+ query := fake.queries[0]
+ if !strings.Contains(query.text, "ragflow_src.updated_at >= @start_cursor") ||
+ !strings.Contains(query.text, "ragflow_src.updated_at <= @end_cursor") {
+ t.Fatalf("query = %q", query.text)
+ }
+ if len(query.parameters) != 2 || query.parameters[0].Name != "start_cursor" || query.parameters[1].Name != "end_cursor" {
+ t.Fatalf("parameters = %+v", query.parameters)
+ }
+ if !strings.Contains(query.text, "ORDER BY ragflow_src.updated_at ASC") {
+ t.Fatalf("incremental query = %q, want stable ordering", query.text)
+ }
+}
+
+func TestBigQueryConnectorMD5FallbackID(t *testing.T) {
+ fake := &fakeBigQueryClient{
+ schema: []bigQueryField{{Name: "name", Type: "STRING"}},
+ rows: []map[string]any{{"name": "content"}},
+ }
+ connector := newFakeBigQueryConnector(t, map[string]any{
+ "project_id": "my-proj",
+ "dataset_id": "ds",
+ "table_id": "tbl",
+ "query": "SELECT * FROM `my-proj.ds.tbl`",
+ "content_columns": "name",
+ "credentials": map[string]any{
+ "service_account_json": `{"type":"service_account","project_id":"my-proj"}`,
+ },
+ }, fake)
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch failed: %v", err)
+ }
+ sum := md5.Sum([]byte("【name】:\ncontent"))
+ wantPrefix := "bigquery:my-proj:query:" + hex.EncodeToString(sum[:])
+ if batch.Documents[0].SourceID != wantPrefix {
+ t.Fatalf("source id = %q, want %q", batch.Documents[0].SourceID, wantPrefix)
+ }
+}
+
+func TestBigQueryConnectorResume(t *testing.T) {
+ fake := &fakeBigQueryClient{schema: bigQuerySchema(), rows: []map[string]any{
+ {"id": int64(1), "name": "one"},
+ {"id": int64(2), "name": "two"},
+ {"id": int64(3), "name": "three"},
+ }}
+ connector := newFakeBigQueryConnector(t, nil, fake)
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ FromBeginning: true,
+ Resume: &SyncCheckpoint{SourceID: "bigquery:my-proj:ds.tbl:2"},
+ })
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch failed: %v", err)
+ }
+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != "bigquery:my-proj:ds.tbl:3" {
+ t.Fatalf("documents = %+v", batch.Documents)
+ }
+}
+
+func TestBigQueryConnectorResumeMissingAnchor(t *testing.T) {
+ fake := &fakeBigQueryClient{schema: bigQuerySchema(), rows: []map[string]any{{"id": int64(1), "name": "one"}}}
+ connector := newFakeBigQueryConnector(t, nil, fake)
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ FromBeginning: true,
+ Resume: &SyncCheckpoint{SourceID: "bigquery:my-proj:ds.tbl:9"},
+ })
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ if _, err := session.NextBatch(context.Background()); !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("NextBatch error = %v, want ErrSyncResumeInvalid", err)
+ }
+}
+
+func TestBigQueryConnectorResumeCursorOnly(t *testing.T) {
+ fake := &fakeBigQueryClient{schema: bigQuerySchema(), rows: []map[string]any{
+ {"id": int64(2), "name": "two"},
+ {"id": int64(3), "name": "three"},
+ }}
+ connector := newFakeBigQueryConnector(t, nil, fake)
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ FromBeginning: true,
+ Resume: &SyncCheckpoint{Cursor: "bigquery:my-proj:ds.tbl:2"},
+ })
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch failed: %v", err)
+ }
+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != "bigquery:my-proj:ds.tbl:3" {
+ t.Fatalf("documents = %+v", batch.Documents)
+ }
+}
+
+func TestBigQueryConnectorResumeRequiresStableOrder(t *testing.T) {
+ fake := &fakeBigQueryClient{
+ schema: []bigQueryField{{Name: "name", Type: "STRING"}},
+ rows: []map[string]any{{"name": "content"}},
+ }
+ connector := newFakeBigQueryConnector(t, map[string]any{
+ "project_id": "my-proj",
+ "dataset_id": "ds",
+ "table_id": "tbl",
+ "content_columns": "name",
+ "credentials": map[string]any{
+ "service_account_json": `{"type":"service_account","project_id":"my-proj"}`,
+ },
+ }, fake)
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ FromBeginning: true,
+ Resume: &SyncCheckpoint{SourceID: "bigquery:my-proj:ds.tbl:anchor"},
+ })
+ if err == nil || !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("OpenSync error = %v, want ErrSyncResumeInvalid", err)
+ }
+ if session != nil {
+ t.Fatal("OpenSync returned a session after resume validation failure")
+ }
+ if strings.Contains(fake.queries[0].text, "ORDER BY") {
+ t.Fatalf("query = %q, want no synthetic ordering without stable columns", fake.queries[0].text)
+ }
+}
+
+func TestBigQueryConnectorOpenPrune(t *testing.T) {
+ fake := &fakeBigQueryClient{
+ schema: []bigQueryField{{Name: "id", Type: "INT64"}},
+ rows: []map[string]any{{"id": int64(3)}, {"id": int64(4)}},
+ }
+ connector := newFakeBigQueryConnector(t, map[string]any{
+ "project_id": "my-proj",
+ "dataset_id": "ds",
+ "table_id": "tbl",
+ "query": "SELECT * FROM `my-proj.ds.tbl`",
+ "content_columns": "name",
+ "id_column": "id",
+ "credentials": map[string]any{
+ "service_account_json": `{"type":"service_account","project_id":"my-proj"}`,
+ },
+ }, fake)
+ session, err := connector.OpenPrune(context.Background(), PruneRequest{})
+ if err != nil {
+ t.Fatalf("OpenPrune failed: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch failed: %v", err)
+ }
+ if len(batch.Documents) != 2 ||
+ batch.Documents[0].SourceID != "bigquery:my-proj:query:3" ||
+ batch.Documents[1].SourceID != "bigquery:my-proj:query:4" {
+ t.Fatalf("slim documents = %+v", batch.Documents)
+ }
+ if !strings.Contains(fake.queries[0].text, "ORDER BY ragflow_src.id ASC") {
+ t.Fatalf("prune query = %q, want stable ordering", fake.queries[0].text)
+ }
+}
+
+func TestBigQueryConnectorRegistry(t *testing.T) {
+ registry := NewRegistry()
+ RegisterBuiltIns(registry)
+ connector, err := registry.OpenFromConfig("bigquery", map[string]any{
+ "project_id": "my-proj",
+ "dataset_id": "ds",
+ "table_id": "tbl",
+ "content_columns": "name",
+ "credentials": map[string]any{
+ "service_account_json": `{"type":"service_account","project_id":"my-proj"}`,
+ },
+ })
+ if err != nil {
+ t.Fatalf("OpenFromConfig failed: %v", err)
+ }
+ if connector == nil {
+ t.Fatal("connector is nil")
+ }
+}
+
+func TestBigQueryConnectorInvalidProjectID(t *testing.T) {
+ _, err := NewBigQueryConnector(map[string]any{
+ "project_id": "bad project",
+ "content_columns": "name",
+ })
+ if err == nil || !strings.Contains(err.Error(), "project_id") {
+ t.Fatalf("NewBigQueryConnector error = %v, want project_id validation", err)
+ }
+}
+
+func TestBigQueryCursorSerialization(t *testing.T) {
+ now := mustTime(t, "2026-01-02T03:04:05Z")
+ serialized := serializeBigQueryCursor(now)
+ if _, ok := serialized.(map[string]any); !ok {
+ t.Fatalf("serialized cursor = %#v", serialized)
+ }
+ restored := deserializeBigQueryCursor(serialized)
+ if !restored.(time.Time).Equal(now) {
+ t.Fatalf("restored cursor = %v", restored)
+ }
+ if got := serializeBigQueryCursor(int64(42)); got != int64(42) {
+ t.Fatalf("numeric cursor = %#v", got)
+ }
+ if got := deserializeBigQueryCursor("plain"); got != "plain" {
+ t.Fatalf("plain cursor = %#v", got)
+ }
+ clock, err := civil.ParseTime("03:04:05")
+ if err != nil {
+ t.Fatalf("parse time cursor: %v", err)
+ }
+ restoredTime := deserializeBigQueryCursor(serializeBigQueryCursor(clock))
+ if restoredTime != clock {
+ t.Fatalf("restored time cursor = %#v", restoredTime)
+ }
+}
diff --git a/internal/syncer/connector/builtin.go b/internal/syncer/connector/builtin.go
index f87d394cee..2f68bf45c4 100644
--- a/internal/syncer/connector/builtin.go
+++ b/internal/syncer/connector/builtin.go
@@ -64,6 +64,7 @@ func RegisterBuiltIns(registry *Registry) {
registerBuiltIn(registry, "discord", NewDiscordConnector)
registerBuiltIn(registry, "webdav", NewWebDAVConnector)
registerBuiltIn(registry, "seafile", NewSeaFileConnector)
+ registerBuiltIn(registry, "bigquery", NewBigQueryConnector)
}
func registerBuiltIn[T Connector](registry *Registry, source string, factory func(map[string]any) (T, error)) {