RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(go-syncer): add BigQuery connector (#19106)
mkaaadc63c2374801c
diff --git a/go.mod b/go.modindex d07769e6e6..3a9f622f18 100644--- a/go.mod+++ b/go.mod@@ -3,6 +3,7 @@ module ragflowgo 1.26.4require (+ cloud.google.com/go/bigquery v1.82.0cloud.google.com/go/storage v1.63.0github.com/AkmalOt/gomsg v0.0.0-20260407083308-985c3a1a76b7github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0@@ -69,12 +70,12 @@ require (go.opentelemetry.io/otel/sdk v1.44.0go.opentelemetry.io/otel/trace v1.44.0go.uber.org/zap v1.27.1- golang.org/x/crypto v0.53.0+ golang.org/x/crypto v0.54.0golang.org/x/image v0.45.0- golang.org/x/net v0.56.0+ golang.org/x/net v0.57.0golang.org/x/oauth2 v0.36.0golang.org/x/sync v0.22.0- golang.org/x/term v0.44.0+ golang.org/x/term v0.45.0golang.org/x/text v0.41.0gonum.org/v1/gonum v0.17.0google.golang.org/api v0.287.1@@ -88,13 +89,20 @@ require (require (github.com/PuerkitoBio/goquery v1.9.2 // indirectgithub.com/andybalholm/cascadia v1.3.2 // indirect+ github.com/apache/arrow/go/v15 v15.0.2 // indirectgithub.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 // indirect+ github.com/google/flatbuffers v23.5.26+incompatible // indirect+ github.com/pierrec/lz4/v4 v4.1.18 // indirect+ golang.org/x/mod v0.38.0 // indirect+ golang.org/x/telemetry v0.0.0-20260708182218-49f421fb7959 // indirect+ golang.org/x/tools v0.48.0 // indirect+ golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect)require (buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20260415201107-50325440f8f2.1 // indirectcel.dev/expr v0.25.1 // indirect- cloud.google.com/go v0.123.0 // indirect+ cloud.google.com/go v0.123.0cloud.google.com/go/auth v0.21.0 // indirectcloud.google.com/go/auth/oauth2adapt v0.2.8 // indirectcloud.google.com/go/compute/metadata v0.9.0 // indirect@@ -232,13 +240,13 @@ require (go.uber.org/multierr v1.11.0 // indirectgo.yaml.in/yaml/v3 v3.0.4 // indirectgolang.org/x/arch v0.27.0 // indirect- golang.org/x/exp v0.0.0-20231226003508-02704c960a9b // indirect+ golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirectgolang.org/x/sys v0.47.0 // indirectgolang.org/x/time v0.15.0 // indirectgoogle.golang.org/genproto v0.0.0-20260519071638-aa98bba5eb94 // indirectgoogle.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 // indirectgoogle.golang.org/genproto/googleapis/rpc v0.0.0-20260630182238-925bb5da69e7 // indirect- google.golang.org/grpc v1.82.0 // indirect+ google.golang.org/grpc v1.82.1 // indirectgoogle.golang.org/protobuf v1.36.11 // indirectgopkg.in/ini.v1 v1.67.0 // indirectmodernc.org/libc v1.22.5 // indirectdiff --git a/go.sum b/go.sumindex 4c7646356f..e3a0399e75 100644--- a/go.sum+++ b/go.sum@@ -9,14 +9,18 @@ cloud.google.com/go/auth v0.21.0 h1:g/QwYfYb2Ai6HH8oomAOyBaIHLbscZ4+T/F/f5JZHkE=cloud.google.com/go/auth v0.21.0/go.mod h1:M9o2Oz+YI2jAfxewJgb1vyI3vceHF+eohmxyzmrl+9s=cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=+cloud.google.com/go/bigquery v1.82.0 h1:NcnkqYN8jjtn6B1o7W/aUNl7DjJiNqgO0xnDNGxW+2s=+cloud.google.com/go/bigquery v1.82.0/go.mod h1:cc0XscySNQNuHBxuZSg5yyxFsg/ZHAfViAG49gJbWew=cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs=cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10=+cloud.google.com/go/datacatalog v1.32.0 h1:fyYn8ODkGil5y3zTIqgIhOfzTu1ACaU2o+C750CO6Ac=+cloud.google.com/go/datacatalog v1.32.0/go.mod h1:DE272tynQUwheJeQAyVfV+nO8yrdkuDyOgH2LtOrkWM=cloud.google.com/go/iam v1.11.0 h1:KieQ9Pb+LLPak1O3Rv3GgCxhnmkYf7Xyh0P5HfF1jFM=cloud.google.com/go/iam v1.11.0/go.mod h1:KP+nKGugNJW4LcLx1uEZcq1ok5sQHFaQehQNl4QDgV4=cloud.google.com/go/logging v1.18.0 h1:KhzZq+1cSkPH9YUaKLLhLtQxIHitVayBmk0sGfoM9+k=cloud.google.com/go/logging v1.18.0/go.mod h1:ZGKnpBaURITh+g/uom2VhbiFoFWvejcrHPDhxFtU/gI=-cloud.google.com/go/longrunning v1.0.0 h1:lwzWEYD8+NkYV7dhexOz6kmlvajZA70+bW/xMhRVVdY=-cloud.google.com/go/longrunning v1.0.0/go.mod h1:8nqFBPOO1U/XkhWl0I19AMZEphrHi73VNABIpKYaTwM=+cloud.google.com/go/longrunning v1.2.0 h1:WjYH3YHBGCxGJP9M4dWGHBfXr/cFIjMkNgWcJj7/iMM=+cloud.google.com/go/longrunning v1.2.0/go.mod h1:5KMQALFGOCtFoi2xSOA1u3H7WKlhmckgiyFw7+LGQp0=cloud.google.com/go/monitoring v1.29.0 h1:AHhDsFaSax1/4k+qlIDX/SDGe6hggnfXJ9dkgD9qBPY=cloud.google.com/go/monitoring v1.29.0/go.mod h1:72NOVjJXHY/HBfoLT0+qlCZBT059+9VXLeAnL2PeeVM=cloud.google.com/go/storage v1.63.0 h1:hvXF2xfg9I32bjujggxgkEZn/Ej6sJ9pieFgeueBLrQ=@@ -111,6 +115,8 @@ github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsVigithub.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=+github.com/apache/arrow/go/v15 v15.0.2 h1:60IliRbiyTWCWjERBCkO1W4Qun9svcYoZrSLcyOsMLE=+github.com/apache/arrow/go/v15 v15.0.2/go.mod h1:DGXsR3ajT524njufqf95822i+KTh+yea1jass9YXgjA=github.com/apache/thrift v0.23.0 h1:wKR6YnefQSEnxpEfmgTPuJibNG4bF0p2TK34tHLWi3s=github.com/apache/thrift v0.23.0/go.mod h1:zPt6WxgvTOM6hF92y8C+MkEM5LMxZuk4JcQOiU4Esvs=github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ=@@ -299,6 +305,8 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWekgithub.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=github.com/gomarkdown/markdown v0.0.0-20260614204949-e08cff860f76 h1:Ltt9ldIaSYEsjA7sPY2c8r9dOmnKM1vlzhh3dxlhBHM=github.com/gomarkdown/markdown v0.0.0-20260614204949-e08cff860f76/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA=+github.com/google/flatbuffers v23.5.26+incompatible h1:M9dgRyhJemaM4Sw8+66GHBu8ioaQmyPLg1b8VwK5WJg=+github.com/google/flatbuffers v23.5.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=@@ -443,6 +451,8 @@ github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=github.com/phpdave11/gofpdi v1.0.14-0.20211212211723-1f10f9844311 h1:zyWXQ6vu27ETMpYsEMAsisQ+GqJ4e1TPvSNfdOPF0no=github.com/phpdave11/gofpdi v1.0.14-0.20211212211723-1f10f9844311/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI=+github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ=+github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA=github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=@@ -572,8 +582,6 @@ github.com/xuri/excelize/v2 v2.11.0 h1:HxaEFl6sRN2+8J5a8HaKq+0M4FsjBGMnWWtjOCPSGgithub.com/xuri/excelize/v2 v2.11.0/go.mod h1:jxFLbzaIwGQ5ufFNvYfUOHqXhfPaNmP14KWfmNz2Uak=github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9 h1:+C0TIdyyYmzadGaL/HBLbf3WdLgC29pgyhTjAT/0nuE=github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ=-github.com/yalue/onnxruntime_go v1.23.0 h1:Hin0mFphwGOeT7xEQrAIi/p2O6ngmSy4uz0yXkC9yCw=-github.com/yalue/onnxruntime_go v1.23.0/go.mod h1:b4X26A8pekNb1ACJ58wAXgNKeUCGEAQ9dmACut9Sm/4=github.com/yargevad/filepathx v1.0.0 h1:SYcT+N3tYGi+NvazubCNlvgIPbzAk7i7y2dwg3I5FYc=github.com/yargevad/filepathx v1.0.0/go.mod h1:BprfX/gpYNJHJfc35GjRRpVcwWXS89gGulUIU5tK3tA=github.com/yfedoseev/office_oxide/go v0.1.8 h1:Q6aQrrs0pN/B6fBoWN6PN95QSQ7EKGMb5HExh04mJJc=@@ -648,11 +656,11 @@ golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMgolang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=-golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto=-golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio=+golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw=+golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk=golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=-golang.org/x/exp v0.0.0-20231226003508-02704c960a9b h1:kLiC65FbiHWFAOu+lxwNPujcsl8VYyTYYEZnsOO1WK4=-golang.org/x/exp v0.0.0-20231226003508-02704c960a9b/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI=+golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=+golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=golang.org/x/image v0.45.0 h1:FMb1nTbH5H9vF55SriQHgFw5GnNL9Jg6L25BwXKzhB0=golang.org/x/image v0.45.0/go.mod h1:n62x/7RqlwXDvGsSU4u6IUTUf6KghUZ9Bt7cG/T9Fx4=golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=@@ -665,6 +673,8 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=+golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk=+golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40=golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=@@ -692,8 +702,8 @@ golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=-golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=-golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec=+golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=+golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=@@ -739,6 +749,8 @@ golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=+golang.org/x/telemetry v0.0.0-20260708182218-49f421fb7959 h1:RJhm5l6Fo4rmEIcndxDllNhhf/fAx8qIm4t6A7vpm2A=+golang.org/x/telemetry v0.0.0-20260708182218-49f421fb7959/go.mod h1:LV7u5Oco+Z/g6XI7PqN+EUUUGGkEcmB1uj2ceI0fOVg=golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=@@ -752,8 +764,8 @@ golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0=-golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc=-golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y=+golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=+golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=@@ -783,10 +795,14 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbcgolang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=+golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE=+golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk=golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=+golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY=+golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=google.golang.org/api v0.287.1 h1:LiyJx32VU3cwQfLchn/513qKhc25hq0pEANYJoWNnnI=@@ -807,8 +823,8 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZigoogle.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=-google.golang.org/grpc v1.82.0 h1:vguDnZUPjE26w09A63VoxZPnvPjB5Riyc0mkXPFmAIU=-google.golang.org/grpc v1.82.0/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA=+google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE=+google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA=google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=diff --git a/internal/syncer/connector/bigquery.go b/internal/syncer/connector/bigquery.gonew file mode 100644index 0000000000..f8e8aa0e84--- /dev/null+++ b/internal/syncer/connector/bigquery.go@@ -0,0 +1,1104 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package connector++import (+ "context"+ "crypto/md5"+ "encoding/base64"+ "encoding/hex"+ "encoding/json"+ "errors"+ "fmt"+ "io"+ "math/big"+ "regexp"+ "sort"+ "strconv"+ "strings"+ "time"++ gcpbigquery "cloud.google.com/go/bigquery"+ "cloud.google.com/go/civil"+ "google.golang.org/api/iterator"+ "google.golang.org/api/option"+)++const (+ defaultBigQueryBatchSize = 2+ defaultBigQueryPageSize = 1000+ defaultBigQueryMaxBytes = int64(1024 * 1024 * 1024)+ bigQueryCursorTypeKey = "__ragflow_bq_cursor_type__"+ bigQueryIdentifierPattern = `^[A-Za-z_][A-Za-z0-9_]*$`+ bigQueryProjectIDPattern = `^[A-Za-z][A-Za-z0-9_-]*$`+)++var bigQueryIdentifierRE = regexp.MustCompile(bigQueryIdentifierPattern)+var bigQueryProjectIDRE = regexp.MustCompile(bigQueryProjectIDPattern)++// BigQueryConnector imports rows from Google BigQuery tables or custom queries.+//+// It mirrors the Python BigQueryConnector: selected content columns form the+// document body, metadata columns are copied into document metadata, an+// optional id column gives stable document ids, and an optional timestamp+// column drives cursor-based incremental sync and pruning.+type BigQueryConnector struct {+ projectID string+ datasetID string+ tableID string+ location string+ query string+ contentColumns []string+ metadataColumns []string+ idColumn string+ timestampColumn string+ batchSize int+ pageSize int+ maximumBytesBilled int64+ jobTimeout time.Duration+ useQueryCache bool+ credentialsJSON []byte+ newClient func(context.Context) (bigQueryClient, error)+}++type bigQueryClient interface {+ Query(query string) bigQueryQuery+ Close() error+}++type bigQueryQuery interface {+ Run(ctx context.Context) (bigQueryJob, error)+ Read(ctx context.Context) (bigQueryRowIterator, error)+ SetDryRun(bool)+ SetDisableQueryCache(bool)+ SetMaxBytesBilled(int64)+ SetJobTimeout(time.Duration)+ SetParameters([]gcpbigquery.QueryParameter)+}++type bigQueryJob interface {+ Status(ctx context.Context) (bigQueryJobStatus, error)+ Wait(ctx context.Context) (bigQueryJobStatus, error)+ Read(ctx context.Context) (bigQueryRowIterator, error)+}++type bigQueryJobStatus interface {+ Done() bool+ Err() error+ Statistics() bigQueryJobStatistics+}++type bigQueryJobStatistics interface {+ QuerySchema() []bigQueryField+ TotalBytesProcessed() int64+}++type bigQueryRowIterator interface {+ Next(dst any) error+ Schema() []bigQueryField+ SetPageSize(int)+}++type bigQueryField struct {+ Name string+ Type string+}++type gcpBigQueryClient struct {+ client *gcpbigquery.Client+}++func (c *gcpBigQueryClient) Query(query string) bigQueryQuery {+ return &gcpBigQueryQuery{q: c.client.Query(query)}+}++func (c *gcpBigQueryClient) Close() error { return c.client.Close() }++type gcpBigQueryQuery struct {+ q *gcpbigquery.Query+}++func (q *gcpBigQueryQuery) Run(ctx context.Context) (bigQueryJob, error) {+ job, err := q.q.Run(ctx)+ if err != nil {+ return nil, err+ }+ return &gcpBigQueryJob{j: job}, nil+}++func (q *gcpBigQueryQuery) Read(ctx context.Context) (bigQueryRowIterator, error) {+ it, err := q.q.Read(ctx)+ if err != nil {+ return nil, err+ }+ return &gcpBigQueryRowIterator{it: it}, nil+}++func (q *gcpBigQueryQuery) SetDryRun(value bool) { q.q.DryRun = value }+func (q *gcpBigQueryQuery) SetDisableQueryCache(value bool) { q.q.DisableQueryCache = value }+func (q *gcpBigQueryQuery) SetMaxBytesBilled(value int64) { q.q.MaxBytesBilled = value }+func (q *gcpBigQueryQuery) SetJobTimeout(value time.Duration) { q.q.JobTimeout = value }+func (q *gcpBigQueryQuery) SetParameters(parameters []gcpbigquery.QueryParameter) {+ q.q.Parameters = append([]gcpbigquery.QueryParameter(nil), parameters...)+}++type gcpBigQueryJob struct{ j *gcpbigquery.Job }++func (j *gcpBigQueryJob) Status(ctx context.Context) (bigQueryJobStatus, error) {+ status, err := j.j.Status(ctx)+ if err != nil {+ return nil, err+ }+ return &gcpBigQueryJobStatus{status: status}, nil+}++func (j *gcpBigQueryJob) Wait(ctx context.Context) (bigQueryJobStatus, error) {+ status, err := j.j.Wait(ctx)+ if err != nil {+ return nil, err+ }+ return &gcpBigQueryJobStatus{status: status}, nil+}++func (j *gcpBigQueryJob) Read(ctx context.Context) (bigQueryRowIterator, error) {+ it, err := j.j.Read(ctx)+ if err != nil {+ return nil, err+ }+ return &gcpBigQueryRowIterator{it: it}, nil+}++type gcpBigQueryJobStatus struct{ status *gcpbigquery.JobStatus }++func (s *gcpBigQueryJobStatus) Done() bool { return s.status.Done() }+func (s *gcpBigQueryJobStatus) Err() error { return s.status.Err() }+func (s *gcpBigQueryJobStatus) Statistics() bigQueryJobStatistics {+ return &gcpBigQueryJobStatistics{stats: s.status.Statistics}+}++type gcpBigQueryJobStatistics struct{ stats *gcpbigquery.JobStatistics }++func (s *gcpBigQueryJobStatistics) QuerySchema() []bigQueryField {+ if s.stats == nil || s.stats.Details == nil {+ return nil+ }+ queryStats, ok := s.stats.Details.(*gcpbigquery.QueryStatistics)+ if !ok {+ return nil+ }+ return toBigQueryFields(queryStats.Schema)+}++func (s *gcpBigQueryJobStatistics) TotalBytesProcessed() int64 {+ if s.stats == nil {+ return 0+ }+ return s.stats.TotalBytesProcessed+}++type gcpBigQueryRowIterator struct{ it *gcpbigquery.RowIterator }++func (i *gcpBigQueryRowIterator) Next(dst any) error {+ err := i.it.Next(dst)+ if errors.Is(err, iterator.Done) {+ return io.EOF+ }+ return err+}++func (i *gcpBigQueryRowIterator) Schema() []bigQueryField { return toBigQueryFields(i.it.Schema) }++func (i *gcpBigQueryRowIterator) SetPageSize(size int) {+ if size <= 0 || i.it == nil || i.it.PageInfo() == nil {+ return+ }+ i.it.PageInfo().MaxSize = size+}++// NewBigQueryConnector creates a BigQuery connector from Python-compatible config.+func NewBigQueryConnector(config map[string]any) (*BigQueryConnector, error) {+ credentials := configAnyMap(config["credentials"])+ rawCredential := credentials["service_account_json"]+ var credentialBytes []byte+ switch value := rawCredential.(type) {+ case string:+ credentialBytes = []byte(strings.TrimSpace(value))+ case map[string]any:+ data, err := json.Marshal(value)+ if err != nil {+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("BigQuery: service_account_json is not valid JSON: %v", err)}+ }+ credentialBytes = data+ default:+ if rawCredential != nil {+ return nil, &ConnectorValidationError{Message: "BigQuery: service_account_json must be a JSON string or object"}+ }+ }++ connector := &BigQueryConnector{+ projectID: strings.TrimSpace(stringConfig(config["project_id"])),+ datasetID: strings.TrimSpace(stringConfig(config["dataset_id"])),+ tableID: strings.TrimSpace(stringConfig(config["table_id"])),+ location: strings.TrimSpace(stringConfig(config["location"])),+ query: sanitizeBigQueryQuery(stringConfig(config["query"])),+ idColumn: strings.TrimSpace(stringConfig(config["id_column"])),+ timestampColumn: strings.TrimSpace(stringConfig(config["timestamp_column"])),+ batchSize: configInt(config["batch_size"], defaultBigQueryBatchSize),+ pageSize: configInt(config["page_size"], defaultBigQueryPageSize),+ maximumBytesBilled: configInt64(config["maximum_bytes_billed"], defaultBigQueryMaxBytes),+ useQueryCache: configBoolDefault(config["use_query_cache"], true),+ credentialsJSON: credentialBytes,+ }+ if connector.maximumBytesBilled <= 0 {+ connector.maximumBytesBilled = defaultBigQueryMaxBytes+ }+ if value := config["job_timeout_ms"]; value != nil {+ connector.jobTimeout = time.Duration(configInt(value, 0)) * time.Millisecond+ }+ connector.contentColumns = connector.splitColumns(config["content_columns"])+ connector.metadataColumns = connector.splitColumns(config["metadata_columns"])+ if err := connector.validateIdentifiers(); err != nil {+ return nil, err+ }+ return connector, nil+}++// Validate validates BigQuery settings and credentials.+func (c *BigQueryConnector) Validate(ctx context.Context) error {+ if err := c.validateStatic(); err != nil {+ return err+ }+ client, err := c.openClient(ctx)+ if err != nil {+ return err+ }+ defer client.Close()++ connectivity := c.configuredQuery(client, "SELECT 1")+ if err := waitBigQueryJob(ctx, connectivity); err != nil {+ return &ConnectorValidationError{Message: fmt.Sprintf("BigQuery validation failed: %v", err)}+ }++ dryRun := c.configuredQuery(client, c.wrapQuery(c.buildBaseQuery(), "*"))+ dryRun.SetDryRun(true)+ dryRun.SetDisableQueryCache(true)+ job, err := dryRun.Run(ctx)+ if err != nil {+ return &ConnectorValidationError{Message: fmt.Sprintf("BigQuery validation failed: %v", err)}+ }+ status, err := job.Wait(ctx)+ if err != nil {+ return &ConnectorValidationError{Message: fmt.Sprintf("BigQuery validation failed: %v", err)}+ }+ if !status.Done() {+ return &ConnectorValidationError{Message: "BigQuery validation dry-run did not complete"}+ }+ if err := status.Err(); err != nil {+ return &ConnectorValidationError{Message: fmt.Sprintf("BigQuery validation failed: %v", err)}+ }+ schema := status.Statistics().QuerySchema()+ if len(schema) == 0 {+ return &ConnectorValidationError{Message: "BigQuery validation dry-run returned no schema"}+ }+ return c.validateSchema(schema)+}++// ValidateConnectorSetting validates an unsaved BigQuery config.+func (c *BigQueryConnector) ValidateConnectorSetting(ctx context.Context, request map[string]any) error {+ ctx, cancel := context.WithTimeout(ctx, connectorSettingValidationTimeout)+ defer cancel()+ candidate, err := NewBigQueryConnector(request)+ if err != nil {+ return err+ }+ return candidate.Validate(ctx)+}++// OpenSync opens one BigQuery sync session.+func (c *BigQueryConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {+ if err := c.validateStatic(); err != nil {+ return nil, err+ }+ client, err := c.openClient(ctx)+ if err != nil {+ return nil, err+ }+ queryText, parameters := c.buildSyncQuery(request)+ query := c.configuredQuery(client, queryText)+ query.SetParameters(parameters)+ iterator, err := query.Read(ctx)+ if err != nil {+ client.Close()+ return nil, err+ }+ iterator.SetPageSize(c.effectivePageSize())+ session := &bigQuerySyncSession{+ connector: c,+ client: client,+ rows: iterator,+ batchSize: c.effectiveBatchSize(),+ }+ if err := session.applyResume(request.Resume); err != nil {+ client.Close()+ return nil, err+ }+ return session, nil+}++// OpenPrune opens one complete BigQuery prune snapshot session.+func (c *BigQueryConnector) OpenPrune(ctx context.Context, request PruneRequest) (PruneSession, error) {+ if err := c.validateStatic(); err != nil {+ return nil, err+ }+ client, err := c.openClient(ctx)+ if err != nil {+ return nil, err+ }+ query := c.configuredQuery(client, c.buildSlimQuery(c.buildBaseQuery()))+ iterator, err := query.Read(ctx)+ if err != nil {+ client.Close()+ return nil, err+ }+ iterator.SetPageSize(c.effectivePageSize())+ return &bigQueryPruneSession{+ connector: c,+ client: client,+ rows: iterator,+ batchSize: c.effectiveBatchSize(),+ }, nil+}++func (c *BigQueryConnector) openClient(ctx context.Context) (bigQueryClient, error) {+ if c.newClient != nil {+ return c.newClient(ctx)+ }+ if len(c.credentialsJSON) == 0 {+ return nil, &ConnectorMissingCredentialError{Message: "BigQuery: missing service_account_json"}+ }+ if !json.Valid(c.credentialsJSON) {+ return nil, &ConnectorValidationError{Message: "BigQuery: service_account_json is not valid JSON"}+ }+ options := []option.ClientOption{option.WithCredentialsJSON(c.credentialsJSON)}+ client, err := gcpbigquery.NewClient(ctx, c.projectID, options...)+ if err != nil {+ return nil, fmt.Errorf("BigQuery: failed to create client: %w", err)+ }+ if c.location != "" {+ client.Location = c.location+ }+ return &gcpBigQueryClient{client: client}, nil+}++func (c *BigQueryConnector) configuredQuery(client bigQueryClient, query string) bigQueryQuery {+ q := client.Query(query)+ q.SetMaxBytesBilled(c.maximumBytesBilled)+ q.SetJobTimeout(c.jobTimeout)+ q.SetDisableQueryCache(!c.useQueryCache)+ return q+}++func (c *BigQueryConnector) validateStatic() error {+ if c == nil {+ return &ConnectorValidationError{Message: "BigQuery connector is nil"}+ }+ if c.projectID == "" {+ return &ConnectorValidationError{Message: "BigQuery project_id is required."}+ }+ if len(c.contentColumns) == 0 {+ return &ConnectorValidationError{Message: "At least one content column must be specified."}+ }+ if c.query == "" && (c.datasetID == "" || c.tableID == "") {+ return &ConnectorValidationError{Message: "BigQuery requires either a custom query or both dataset_id and table_id."}+ }+ if c.batchSize <= 0 {+ return &ConnectorValidationError{Message: "BigQuery batch_size must be a positive integer"}+ }+ if c.pageSize <= 0 {+ return &ConnectorValidationError{Message: "BigQuery page_size must be a positive integer"}+ }+ if len(c.credentialsJSON) == 0 {+ return &ConnectorMissingCredentialError{Message: "BigQuery: missing service_account_json"}+ }+ return nil+}++func (c *BigQueryConnector) validateIdentifiers() error {+ if c.projectID != "" && !bigQueryProjectIDRE.MatchString(c.projectID) {+ return &ConnectorValidationError{Message: fmt.Sprintf("Invalid BigQuery identifier for %q", "project_id")}+ }+ for _, column := range c.contentColumns {+ if err := validateBigQueryIdentifier(column, fmt.Sprintf("content_columns[%q]", column)); err != nil {+ return err+ }+ }+ for _, column := range c.metadataColumns {+ if err := validateBigQueryIdentifier(column, fmt.Sprintf("metadata_columns[%q]", column)); err != nil {+ return err+ }+ }+ for name, column := range map[string]string{"dataset_id": c.datasetID, "table_id": c.tableID} {+ if column != "" {+ if err := validateBigQueryIdentifier(column, name); err != nil {+ return err+ }+ }+ }+ if c.idColumn != "" {+ if err := validateBigQueryIdentifier(c.idColumn, "id_column"); err != nil {+ return err+ }+ }+ if c.timestampColumn != "" {+ if err := validateBigQueryIdentifier(c.timestampColumn, "timestamp_column"); err != nil {+ return err+ }+ }+ return nil+}++func validateBigQueryIdentifier(value, name string) error {+ if value == "" {+ return nil+ }+ if !bigQueryIdentifierRE.MatchString(value) {+ return &ConnectorValidationError{Message: fmt.Sprintf("Invalid BigQuery identifier for %q", name)}+ }+ return nil+}++func (c *BigQueryConnector) effectiveBatchSize() int {+ if c.batchSize > 0 {+ return c.batchSize+ }+ return defaultBigQueryBatchSize+}++func (c *BigQueryConnector) effectivePageSize() int {+ if c.pageSize > 0 {+ return c.pageSize+ }+ return defaultBigQueryPageSize+}++func (c *BigQueryConnector) buildBaseQuery() string {+ if c.query != "" {+ return c.query+ }+ return fmt.Sprintf("SELECT * FROM `%s.%s.%s`", c.projectID, c.datasetID, c.tableID)+}++func (c *BigQueryConnector) usesTableMode() bool { return c.query == "" }++func (c *BigQueryConnector) wrapQuery(base, selectClause string) string {+ return c.wrapFilteredQuery(base, selectClause, "")+}++func (c *BigQueryConnector) wrapFilteredQuery(base, selectClause, where string) string {+ query := fmt.Sprintf("SELECT %s FROM (%s) AS ragflow_src", selectClause, base)+ if where != "" {+ query += " WHERE " + where+ }+ return query + c.stableOrderClause()+}++func (c *BigQueryConnector) stableOrderClause() string {+ columns := make([]string, 0, 2)+ if c.idColumn != "" {+ columns = append(columns, c.idColumn)+ }+ if c.timestampColumn != "" && c.timestampColumn != c.idColumn {+ columns = append(columns, c.timestampColumn)+ }+ if len(columns) == 0 {+ return ""+ }+ order := make([]string, 0, len(columns))+ for _, column := range columns {+ order = append(order, fmt.Sprintf("ragflow_src.%s ASC", column))+ }+ return " ORDER BY " + strings.Join(order, ", ")+}++func (c *BigQueryConnector) validateSchema(schema []bigQueryField) error {+ names := map[string]struct{}{}+ for _, field := range schema {+ names[field.Name] = struct{}{}+ }+ required := map[string]struct{}{}+ for _, column := range c.contentColumns {+ required[column] = struct{}{}+ }+ optional := map[string]struct{}{}+ for _, column := range c.metadataColumns {+ optional[column] = struct{}{}+ }+ if c.idColumn != "" {+ optional[c.idColumn] = struct{}{}+ }+ if c.timestampColumn != "" {+ optional[c.timestampColumn] = struct{}{}+ }+ var missing []string+ for column := range required {+ if _, ok := names[column]; !ok {+ missing = append(missing, column)+ }+ }+ for column := range optional {+ if _, ok := names[column]; !ok {+ missing = append(missing, column)+ }+ }+ sort.Strings(missing)+ if len(missing) > 0 {+ return &ConnectorValidationError{Message: fmt.Sprintf("BigQuery configured columns not found in schema: %s", strings.Join(missing, ", "))}+ }+ if c.timestampColumn != "" {+ if _, ok := names[c.timestampColumn]; !ok {+ return &ConnectorValidationError{Message: fmt.Sprintf("BigQuery timestamp column '%s' was not found in the schema.", c.timestampColumn)}+ }+ }+ return nil+}++// buildSyncQuery returns the wrapped query and cursor parameters.+func (c *BigQueryConnector) buildSyncQuery(request SyncRequest) (string, []gcpbigquery.QueryParameter) {+ base := c.buildBaseQuery()+ if c.timestampColumn == "" || request.FromBeginning {+ return c.wrapQuery(base, "*"), nil+ }+ conditions := []string{}+ parameters := []gcpbigquery.QueryParameter{}+ if request.WindowStart != nil {+ conditions = append(conditions, fmt.Sprintf("ragflow_src.%s >= @start_cursor", c.timestampColumn))+ parameters = append(parameters, gcpbigquery.QueryParameter{Name: "start_cursor", Value: request.WindowStart.UTC()})+ }+ if !request.WindowEnd.IsZero() {+ conditions = append(conditions, fmt.Sprintf("ragflow_src.%s <= @end_cursor", c.timestampColumn))+ parameters = append(parameters, gcpbigquery.QueryParameter{Name: "end_cursor", Value: request.WindowEnd.UTC()})+ }+ if len(conditions) == 0 {+ return c.wrapQuery(base, "*"), nil+ }+ return c.wrapFilteredQuery(base, "*", strings.Join(conditions, " AND ")), parameters+}++func (c *BigQueryConnector) buildSlimQuery(base string) string {+ columns := c.contentColumns+ if c.idColumn != "" {+ columns = []string{c.idColumn}+ }+ if len(columns) == 0 {+ return c.wrapQuery(base, "*")+ }+ selects := make([]string, 0, len(columns))+ for _, column := range columns {+ selects = append(selects, fmt.Sprintf("ragflow_src.%s", column))+ }+ return c.wrapQuery(base, strings.Join(selects, ", "))+}++func (c *BigQueryConnector) splitColumns(value any) []string {+ switch typed := value.(type) {+ case string:+ parts := strings.Split(typed, ",")+ columns := make([]string, 0, len(parts))+ for _, part := range parts {+ if column := strings.TrimSpace(part); column != "" {+ columns = append(columns, column)+ }+ }+ return columns+ case []any:+ columns := make([]string, 0, len(typed))+ for _, item := range typed {+ if column := strings.TrimSpace(stringConfig(item)); column != "" {+ columns = append(columns, column)+ }+ }+ return columns+ }+ return nil+}++func (c *BigQueryConnector) contentColumnsForRow(row map[string]any, schema []bigQueryField) []string {+ if len(c.contentColumns) > 0 {+ return c.contentColumns+ }+ excluded := map[string]bool{}+ if c.idColumn != "" {+ excluded[c.idColumn] = true+ }+ if c.timestampColumn != "" {+ excluded[c.timestampColumn] = true+ }+ columns := make([]string, 0, len(schema))+ for _, field := range schema {+ if _, ok := row[field.Name]; ok && !excluded[field.Name] {+ columns = append(columns, field.Name)+ }+ }+ return columns+}++func (c *BigQueryConnector) buildContent(row map[string]any, columns []string) string {+ parts := []string{}+ for _, column := range columns {+ value, ok := row[column]+ if !ok || value == nil {+ continue+ }+ rendered := renderBigQueryContentValue(value)+ if rendered == "" {+ continue+ }+ parts = append(parts, fmt.Sprintf("【%s】:\n%s", column, rendered))+ }+ return strings.Join(parts, "\n\n")+}++func (c *BigQueryConnector) buildDocumentID(row map[string]any) string {+ prefix := c.idPrefix()+ if c.idColumn != "" {+ if value, ok := row[c.idColumn]; ok && value != nil {+ return prefix + ":" + bigQueryValueString(value)+ }+ }+ content := c.buildContent(row, c.contentColumns)+ sum := md5.Sum([]byte(content))+ return prefix + ":" + hex.EncodeToString(sum[:])+}++func (c *BigQueryConnector) idPrefix() string {+ if c.usesTableMode() {+ return fmt.Sprintf("bigquery:%s:%s.%s", c.projectID, c.datasetID, c.tableID)+ }+ return fmt.Sprintf("bigquery:%s:query", c.projectID)+}++func (c *BigQueryConnector) rowToSourceDocument(row map[string]any, schema []bigQueryField) (SourceDocument, bool) {+ contentColumns := c.contentColumnsForRow(row, schema)+ content := c.buildContent(row, contentColumns)++ metadata := map[string]any{}+ for _, column := range c.metadataColumns {+ value, ok := row[column]+ if !ok || value == nil {+ continue+ }+ metadata[column] = renderBigQueryMetadataValue(value)+ }++ updatedAt := time.Now().UTC()+ if c.timestampColumn != "" {+ if value, ok := row[c.timestampColumn]; ok && value != nil {+ updatedAt = parseBigQueryUpdatedAt(value)+ }+ }++ semanticID := "bigquery_record"+ if len(contentColumns) > 0 {+ if value, ok := row[contentColumns[0]]; ok && value != nil {+ semanticID = strings.TrimSpace(strings.ReplaceAll(strings.ReplaceAll(bigQueryValueString(value), "\n", " "), "\r", " "))+ if semanticID == "" {+ semanticID = "bigquery_record"+ } else if len(semanticID) > 100 {+ semanticID = semanticID[:100]+ }+ }+ }++ sourceID := c.buildDocumentID(row)+ blob := []byte(content)+ return SourceDocument{+ SourceID: sourceID,+ SemanticIdentifier: semanticID,+ Extension: ".txt",+ Blob: blob,+ UpdatedAt: updatedAt,+ SizeBytes: int64(len(blob)),+ Metadata: metadata,+ Fingerprint: stableFingerprint(map[string]any{+ "id": sourceID,+ "content": content,+ "metadata": metadata,+ }),+ }, true+}++func renderBigQueryContentValue(value any) string {+ switch typed := value.(type) {+ case []byte:+ return ""+ case time.Time:+ return typed.Format(time.RFC3339)+ case civil.Date:+ return typed.String()+ case civil.Time:+ return typed.String()+ case civil.DateTime:+ return typed.String()+ case *big.Rat:+ return typed.RatString()+ case map[string]any:+ data, err := json.Marshal(typed)+ if err != nil {+ return fmt.Sprint(value)+ }+ return string(data)+ case []any:+ data, err := json.Marshal(typed)+ if err != nil {+ return fmt.Sprint(value)+ }+ return string(data)+ default:+ return fmt.Sprint(value)+ }+}++func renderBigQueryMetadataValue(value any) string {+ switch typed := value.(type) {+ case []byte:+ return base64.StdEncoding.EncodeToString(typed)+ case time.Time:+ return typed.Format(time.RFC3339)+ case civil.Date:+ return typed.String()+ case civil.Time:+ return typed.String()+ case civil.DateTime:+ return typed.String()+ case *big.Rat:+ return typed.RatString()+ case map[string]any:+ data, err := json.Marshal(typed)+ if err != nil {+ return fmt.Sprint(value)+ }+ return string(data)+ case []any:+ data, err := json.Marshal(typed)+ if err != nil {+ return fmt.Sprint(value)+ }+ return string(data)+ default:+ return fmt.Sprint(value)+ }+}++func parseBigQueryUpdatedAt(value any) time.Time {+ switch typed := value.(type) {+ case time.Time:+ return typed.UTC()+ case civil.DateTime:+ return typed.In(time.UTC)+ case civil.Date:+ return time.Date(typed.Year, typed.Month, typed.Day, 0, 0, 0, 0, time.UTC)+ }+ return time.Now().UTC()+}++func bigQueryValueString(value any) string {+ switch typed := value.(type) {+ case []byte:+ return string(typed)+ case time.Time:+ return typed.Format(time.RFC3339)+ case civil.Date:+ return typed.String()+ case civil.Time:+ return typed.String()+ case civil.DateTime:+ return typed.String()+ case *big.Rat:+ return typed.RatString()+ case map[string]any:+ data, err := json.Marshal(typed)+ if err != nil {+ return fmt.Sprint(value)+ }+ return string(data)+ case []any:+ data, err := json.Marshal(typed)+ if err != nil {+ return fmt.Sprint(value)+ }+ return string(data)+ default:+ return fmt.Sprint(value)+ }+}++func waitBigQueryJob(ctx context.Context, query bigQueryQuery) error {+ job, err := query.Run(ctx)+ if err != nil {+ return err+ }+ status, err := job.Wait(ctx)+ if err != nil {+ return err+ }+ if !status.Done() {+ return fmt.Errorf("BigQuery query did not complete")+ }+ if err := status.Err(); err != nil {+ return err+ }+ return nil+}++func toBigQueryFields(schema gcpbigquery.Schema) []bigQueryField {+ if len(schema) == 0 {+ return nil+ }+ fields := make([]bigQueryField, 0, len(schema))+ for _, field := range schema {+ if field == nil {+ continue+ }+ fields = append(fields, bigQueryField{Name: field.Name, Type: string(field.Type)})+ }+ return fields+}++func sanitizeBigQueryQuery(raw string) string {+ query := strings.TrimSpace(raw)+ if query == "" {+ return ""+ }+ if strings.HasPrefix(query, "```") {+ query = strings.TrimPrefix(query, "```")+ if strings.HasSuffix(query, "```") {+ query = strings.TrimSuffix(query, "```")+ }+ if head, tail, ok := strings.Cut(query, "\n"); ok {+ head = strings.TrimSpace(head)+ if head == "sql" || head == "google-sql" || head == "bigquery" {+ head = ""+ }+ query = strings.TrimSpace(head) + "\n" + strings.TrimSpace(tail)+ }+ }+ return strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(query), ";"))+}++// serializeBigQueryCursor mirrors Python's cursor serialization used for+// connector config persistence. Only temporal values need a marker wrapper.+func serializeBigQueryCursor(value any) any {+ switch typed := value.(type) {+ case time.Time:+ return map[string]any{bigQueryCursorTypeKey: "datetime", "value": typed.Format(time.RFC3339)}+ case civil.Date:+ return map[string]any{bigQueryCursorTypeKey: "date", "value": typed.String()}+ case civil.Time:+ return map[string]any{bigQueryCursorTypeKey: "time", "value": typed.String()}+ case civil.DateTime:+ return map[string]any{bigQueryCursorTypeKey: "datetime", "value": typed.String()}+ case *big.Rat:+ return map[string]any{bigQueryCursorTypeKey: "decimal", "value": typed.RatString()}+ default:+ return value+ }+}++func deserializeBigQueryCursor(value any) any {+ typed, ok := value.(map[string]any)+ if !ok {+ return value+ }+ raw, ok := typed[bigQueryCursorTypeKey].(string)+ if !ok {+ return value+ }+ switch raw {+ case "datetime":+ if parsed, err := time.Parse(time.RFC3339, fmt.Sprint(typed["value"])); err == nil {+ return parsed+ }+ if parsed, err := time.Parse("2006-01-02 15:04:05", fmt.Sprint(typed["value"])); err == nil {+ return parsed+ }+ case "date":+ if parsed, err := time.Parse("2006-01-02", fmt.Sprint(typed["value"])); err == nil {+ return parsed+ }+ case "time":+ if parsed, err := civil.ParseTime(fmt.Sprint(typed["value"])); err == nil {+ return parsed+ }+ case "decimal":+ if parsed, ok := new(big.Rat).SetString(fmt.Sprint(typed["value"])); ok {+ return parsed+ }+ }+ return value+}++func configInt64(value any, fallback int64) int64 {+ if value == nil {+ return fallback+ }+ switch typed := value.(type) {+ case int:+ return int64(typed)+ case int64:+ return typed+ case float64:+ return int64(typed)+ case json.Number:+ if parsed, err := typed.Int64(); err == nil {+ return parsed+ }+ case string:+ parsed, err := strconv.ParseInt(strings.TrimSpace(typed), 10, 64)+ if err == nil {+ return parsed+ }+ }+ return fallback+}++type bigQuerySyncSession struct {+ connector *BigQueryConnector+ client bigQueryClient+ rows bigQueryRowIterator+ batchSize int+ resume *SyncCheckpoint+ closed bool+}++func (s *bigQuerySyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {+ if s.closed {+ return SyncBatch{}, io.EOF+ }+ documents := make([]SourceDocument, 0, s.batchSize)+ for len(documents) < s.batchSize {+ var row map[string]any+ err := s.rows.Next(&row)+ if errors.Is(err, io.EOF) {+ if len(documents) == 0 {+ if s.resume != nil {+ return SyncBatch{}, fmt.Errorf("bigquery resume anchor %q was not found in the current result: %w", s.resume.SourceID, ErrSyncResumeInvalid)+ }+ return SyncBatch{}, io.EOF+ }+ break+ }+ if err != nil {+ return SyncBatch{}, err+ }+ doc, ok := s.connector.rowToSourceDocument(row, s.rows.Schema())+ if !ok {+ continue+ }+ if !s.includeResumed(doc) {+ if s.resume != nil && doc.SourceID == s.resume.SourceID {+ // The anchor was found; continue emitting rows after it.+ }+ continue+ }+ documents = append(documents, doc)+ }+ if len(documents) == 0 {+ return SyncBatch{}, io.EOF+ }+ return SyncBatch{Documents: documents, Checkpoint: bigQuerySyncCheckpoint(documents[len(documents)-1])}, nil+}++func (s *bigQuerySyncSession) Close() error {+ if s.closed {+ return nil+ }+ s.closed = true+ return s.client.Close()+}++func (s *bigQuerySyncSession) includeResumed(doc SourceDocument) bool {+ if s.resume == nil || s.resume.SourceID == "" {+ return true+ }+ if doc.SourceID == s.resume.SourceID {+ s.resume = nil+ return false+ }+ return false+}++func (s *bigQuerySyncSession) applyResume(checkpoint *SyncCheckpoint) error {+ if checkpoint == nil {+ return nil+ }+ sourceID := firstNonEmpty(checkpoint.SourceID, checkpoint.Cursor)+ if sourceID == "" {+ return fmt.Errorf("bigquery sync checkpoint has no source anchor: %w", ErrSyncResumeInvalid)+ }+ if s.connector.stableOrderClause() == "" {+ return fmt.Errorf("bigquery sync resume requires a stable ordering: %w", ErrSyncResumeInvalid)+ }+ s.resume = &SyncCheckpoint{SourceID: sourceID}+ return nil+}++func bigQuerySyncCheckpoint(document SourceDocument) *SyncCheckpoint {+ updatedAt := document.UpdatedAt+ return &SyncCheckpoint{+ Cursor: document.SourceID,+ SourceID: document.SourceID,+ UpdatedAt: &updatedAt,+ }+}++type bigQueryPruneSession struct {+ connector *BigQueryConnector+ client bigQueryClient+ rows bigQueryRowIterator+ batchSize int+ closed bool+}++func (s *bigQueryPruneSession) NextBatch(ctx context.Context) (PruneBatch, error) {+ if s.closed {+ return PruneBatch{}, io.EOF+ }+ documents := make([]SlimDocument, 0, s.batchSize)+ for len(documents) < s.batchSize {+ var row map[string]any+ err := s.rows.Next(&row)+ if errors.Is(err, io.EOF) {+ break+ }+ if err != nil {+ return PruneBatch{}, err+ }+ documents = append(documents, SlimDocument{SourceID: s.connector.buildDocumentID(row)})+ }+ if len(documents) == 0 {+ return PruneBatch{}, io.EOF+ }+ return PruneBatch{Documents: documents}, nil+}++func (s *bigQueryPruneSession) Close() error {+ if s.closed {+ return nil+ }+ s.closed = true+ return s.client.Close()+}diff --git a/internal/syncer/connector/bigquery_test.go b/internal/syncer/connector/bigquery_test.gonew file mode 100644index 0000000000..83ef77888f--- /dev/null+++ b/internal/syncer/connector/bigquery_test.go@@ -0,0 +1,536 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package connector++import (+ "context"+ "crypto/md5"+ "encoding/hex"+ "errors"+ "io"+ "strings"+ "testing"+ "time"++ gcpbigquery "cloud.google.com/go/bigquery"+ "cloud.google.com/go/civil"+)++type fakeBigQueryClient struct {+ queries []*fakeBigQueryQuery+ rows []map[string]any+ schema []bigQueryField+ closed bool+}++func (f *fakeBigQueryClient) Query(query string) bigQueryQuery {+ q := &fakeBigQueryQuery{+ text: query,+ readIterator: &fakeBigQueryRowIterator{+ rows: f.rows,+ schema: f.schema,+ },+ }+ if strings.Contains(query, "FROM (") {+ q.runJob = &fakeBigQueryJob{status: &fakeBigQueryJobStatus{done: true, schema: f.schema}}+ }+ f.queries = append(f.queries, q)+ return q+}++func (f *fakeBigQueryClient) Close() error {+ f.closed = true+ return nil+}++type fakeBigQueryQuery struct {+ text string+ dryRun bool+ disableCache bool+ maxBytes int64+ jobTimeout time.Duration+ parameters []gcpbigquery.QueryParameter+ runJob *fakeBigQueryJob+ readIterator *fakeBigQueryRowIterator+ runErr, readErr error+}++func (f *fakeBigQueryQuery) Run(context.Context) (bigQueryJob, error) {+ if f.runErr != nil {+ return nil, f.runErr+ }+ if f.runJob == nil {+ return &fakeBigQueryJob{status: &fakeBigQueryJobStatus{done: true}}, nil+ }+ return f.runJob, nil+}++func (f *fakeBigQueryQuery) Read(context.Context) (bigQueryRowIterator, error) {+ if f.readErr != nil {+ return nil, f.readErr+ }+ if f.readIterator == nil {+ return &fakeBigQueryRowIterator{}, nil+ }+ return f.readIterator, nil+}++func (f *fakeBigQueryQuery) SetDryRun(v bool) { f.dryRun = v }+func (f *fakeBigQueryQuery) SetDisableQueryCache(v bool) { f.disableCache = v }+func (f *fakeBigQueryQuery) SetMaxBytesBilled(v int64) { f.maxBytes = v }+func (f *fakeBigQueryQuery) SetJobTimeout(v time.Duration) { f.jobTimeout = v }+func (f *fakeBigQueryQuery) SetParameters(v []gcpbigquery.QueryParameter) {+ f.parameters = append([]gcpbigquery.QueryParameter(nil), v...)+}++type fakeBigQueryJob struct {+ status *fakeBigQueryJobStatus+}++func (f *fakeBigQueryJob) Status(context.Context) (bigQueryJobStatus, error) {+ if f.status == nil {+ return &fakeBigQueryJobStatus{done: true}, nil+ }+ return f.status, nil+}++func (f *fakeBigQueryJob) Wait(context.Context) (bigQueryJobStatus, error) {+ return f.Status(context.Background())+}++func (f *fakeBigQueryJob) Read(context.Context) (bigQueryRowIterator, error) {+ return &fakeBigQueryRowIterator{}, nil+}++type fakeBigQueryJobStatus struct {+ done bool+ err error+ schema []bigQueryField+ bytes int64+}++func (f *fakeBigQueryJobStatus) Done() bool { return f.done }+func (f *fakeBigQueryJobStatus) Err() error { return f.err }+func (f *fakeBigQueryJobStatus) Statistics() bigQueryJobStatistics {+ return &fakeBigQueryJobStatistics{schema: f.schema, bytes: f.bytes}+}++type fakeBigQueryJobStatistics struct {+ schema []bigQueryField+ bytes int64+}++func (f *fakeBigQueryJobStatistics) QuerySchema() []bigQueryField { return f.schema }+func (f *fakeBigQueryJobStatistics) TotalBytesProcessed() int64 { return f.bytes }++type fakeBigQueryRowIterator struct {+ rows []map[string]any+ schema []bigQueryField+ index int+ page int+}++func (f *fakeBigQueryRowIterator) Next(dst any) error {+ if f.index >= len(f.rows) {+ return io.EOF+ }+ row := f.rows[f.index]+ f.index+++ target, ok := dst.(*map[string]any)+ if !ok {+ return errors.New("fake iterator expects *map[string]any")+ }+ *target = row+ return nil+}++func (f *fakeBigQueryRowIterator) Schema() []bigQueryField { return f.schema }++func (f *fakeBigQueryRowIterator) SetPageSize(size int) { f.page = size }++func newFakeBigQueryConnector(t *testing.T, config map[string]any, fake *fakeBigQueryClient) *BigQueryConnector {+ t.Helper()+ if config == nil {+ config = map[string]any{+ "project_id": "my-proj",+ "dataset_id": "ds",+ "table_id": "tbl",+ "content_columns": "name,description",+ "id_column": "id",+ "credentials": map[string]any{+ "service_account_json": `{"type":"service_account","project_id":"my-proj"}`,+ },+ }+ }+ connector, err := NewBigQueryConnector(config)+ if err != nil {+ t.Fatalf("NewBigQueryConnector failed: %v", err)+ }+ connector.newClient = func(context.Context) (bigQueryClient, error) {+ return fake, nil+ }+ return connector+}++func bigQuerySchema() []bigQueryField {+ return []bigQueryField{+ {Name: "id", Type: "INT64"},+ {Name: "name", Type: "STRING"},+ {Name: "description", Type: "STRING"},+ {Name: "category", Type: "STRING"},+ {Name: "updated_at", Type: "TIMESTAMP"},+ }+}++func TestBigQueryConnectorValidate(t *testing.T) {+ fake := &fakeBigQueryClient{schema: bigQuerySchema()}+ connector := newFakeBigQueryConnector(t, nil, fake)+ if err := connector.Validate(context.Background()); err != nil {+ t.Fatalf("Validate failed: %v", err)+ }+ if len(fake.queries) != 2 {+ t.Fatalf("query count = %d, want 2", len(fake.queries))+ }+ if fake.queries[0].text != "SELECT 1" {+ t.Fatalf("connectivity query = %q", fake.queries[0].text)+ }+ if !fake.queries[1].dryRun || !fake.queries[1].disableCache {+ t.Fatalf("dry-run flags = dry:%v cache:%v", fake.queries[1].dryRun, fake.queries[1].disableCache)+ }+}++func TestBigQueryConnectorValidateMissingColumn(t *testing.T) {+ fake := &fakeBigQueryClient{schema: []bigQueryField{+ {Name: "id", Type: "INT64"},+ {Name: "name", Type: "STRING"},+ }}+ connector := newFakeBigQueryConnector(t, nil, fake)+ err := connector.Validate(context.Background())+ if err == nil || !strings.Contains(err.Error(), "description") {+ t.Fatalf("Validate error = %v, want missing description", err)+ }+}++func TestBigQueryConnectorOpenSyncFull(t *testing.T) {+ updatedAt := mustTime(t, "2026-01-02T03:04:05Z")+ fake := &fakeBigQueryClient{schema: bigQuerySchema(), rows: []map[string]any{+ {"id": int64(7), "name": "Hello/World", "description": "Some body", "category": "news", "updated_at": updatedAt},+ }}+ connector := newFakeBigQueryConnector(t, map[string]any{+ "project_id": "my-proj",+ "dataset_id": "ds",+ "table_id": "tbl",+ "content_columns": "name,description",+ "metadata_columns": "id,category,updated_at",+ "id_column": "id",+ "timestamp_column": "updated_at",+ "credentials": map[string]any{+ "service_account_json": `{"type":"service_account","project_id":"my-proj"}`,+ },+ }, fake)+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch failed: %v", err)+ }+ if len(batch.Documents) != 1 {+ t.Fatalf("documents len = %d", len(batch.Documents))+ }+ doc := batch.Documents[0]+ if doc.SourceID != "bigquery:my-proj:ds.tbl:7" {+ t.Fatalf("source id = %q", doc.SourceID)+ }+ if doc.SemanticIdentifier != "Hello/World" {+ t.Fatalf("semantic id = %q", doc.SemanticIdentifier)+ }+ if doc.Extension != ".txt" {+ t.Fatalf("extension = %q", doc.Extension)+ }+ blob := string(doc.Blob)+ if !strings.Contains(blob, "【name】:\nHello/World") || !strings.Contains(blob, "【description】:\nSome body") {+ t.Fatalf("blob = %q", blob)+ }+ if fake.queries[0].readIterator.page != defaultBigQueryPageSize {+ t.Fatalf("page size = %d, want %d", fake.queries[0].readIterator.page, defaultBigQueryPageSize)+ }+ if !strings.Contains(fake.queries[0].text, "ORDER BY ragflow_src.id ASC, ragflow_src.updated_at ASC") {+ t.Fatalf("sync query = %q, want stable ordering", fake.queries[0].text)+ }+ if !doc.UpdatedAt.Equal(updatedAt) {+ t.Fatalf("updated at = %s", doc.UpdatedAt)+ }+ if doc.Metadata["category"] != "news" || doc.Metadata["id"] != "7" {+ t.Fatalf("metadata = %v", doc.Metadata)+ }+ if doc.Metadata["updated_at"] != updatedAt.Format(time.RFC3339) {+ t.Fatalf("metadata updated_at = %v", doc.Metadata["updated_at"])+ }+ if doc.Fingerprint == "" {+ t.Fatalf("fingerprint is empty")+ }+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("NextBatch EOF = %v", err)+ }+}++func TestBigQueryConnectorIncrementalParameters(t *testing.T) {+ fake := &fakeBigQueryClient{schema: []bigQueryField{{Name: "name", Type: "STRING"}}, rows: []map[string]any{{"name": "content"}}}+ connector := newFakeBigQueryConnector(t, map[string]any{+ "project_id": "my-proj",+ "dataset_id": "ds",+ "table_id": "tbl",+ "content_columns": "name",+ "timestamp_column": "updated_at",+ "credentials": map[string]any{+ "service_account_json": `{"type":"service_account","project_id":"my-proj"}`,+ },+ }, fake)+ start := mustTime(t, "2026-01-01T00:00:00Z")+ end := mustTime(t, "2026-01-02T00:00:00Z")+ if _, err := connector.OpenSync(context.Background(), SyncRequest{+ WindowStart: &start,+ WindowEnd: end,+ }); err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ if len(fake.queries) != 1 {+ t.Fatalf("query count = %d", len(fake.queries))+ }+ query := fake.queries[0]+ if !strings.Contains(query.text, "ragflow_src.updated_at >= @start_cursor") ||+ !strings.Contains(query.text, "ragflow_src.updated_at <= @end_cursor") {+ t.Fatalf("query = %q", query.text)+ }+ if len(query.parameters) != 2 || query.parameters[0].Name != "start_cursor" || query.parameters[1].Name != "end_cursor" {+ t.Fatalf("parameters = %+v", query.parameters)+ }+ if !strings.Contains(query.text, "ORDER BY ragflow_src.updated_at ASC") {+ t.Fatalf("incremental query = %q, want stable ordering", query.text)+ }+}++func TestBigQueryConnectorMD5FallbackID(t *testing.T) {+ fake := &fakeBigQueryClient{+ schema: []bigQueryField{{Name: "name", Type: "STRING"}},+ rows: []map[string]any{{"name": "content"}},+ }+ connector := newFakeBigQueryConnector(t, map[string]any{+ "project_id": "my-proj",+ "dataset_id": "ds",+ "table_id": "tbl",+ "query": "SELECT * FROM `my-proj.ds.tbl`",+ "content_columns": "name",+ "credentials": map[string]any{+ "service_account_json": `{"type":"service_account","project_id":"my-proj"}`,+ },+ }, fake)+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch failed: %v", err)+ }+ sum := md5.Sum([]byte("【name】:\ncontent"))+ wantPrefix := "bigquery:my-proj:query:" + hex.EncodeToString(sum[:])+ if batch.Documents[0].SourceID != wantPrefix {+ t.Fatalf("source id = %q, want %q", batch.Documents[0].SourceID, wantPrefix)+ }+}++func TestBigQueryConnectorResume(t *testing.T) {+ fake := &fakeBigQueryClient{schema: bigQuerySchema(), rows: []map[string]any{+ {"id": int64(1), "name": "one"},+ {"id": int64(2), "name": "two"},+ {"id": int64(3), "name": "three"},+ }}+ connector := newFakeBigQueryConnector(t, nil, fake)+ session, err := connector.OpenSync(context.Background(), SyncRequest{+ FromBeginning: true,+ Resume: &SyncCheckpoint{SourceID: "bigquery:my-proj:ds.tbl:2"},+ })+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch failed: %v", err)+ }+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != "bigquery:my-proj:ds.tbl:3" {+ t.Fatalf("documents = %+v", batch.Documents)+ }+}++func TestBigQueryConnectorResumeMissingAnchor(t *testing.T) {+ fake := &fakeBigQueryClient{schema: bigQuerySchema(), rows: []map[string]any{{"id": int64(1), "name": "one"}}}+ connector := newFakeBigQueryConnector(t, nil, fake)+ session, err := connector.OpenSync(context.Background(), SyncRequest{+ FromBeginning: true,+ Resume: &SyncCheckpoint{SourceID: "bigquery:my-proj:ds.tbl:9"},+ })+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ if _, err := session.NextBatch(context.Background()); !errors.Is(err, ErrSyncResumeInvalid) {+ t.Fatalf("NextBatch error = %v, want ErrSyncResumeInvalid", err)+ }+}++func TestBigQueryConnectorResumeCursorOnly(t *testing.T) {+ fake := &fakeBigQueryClient{schema: bigQuerySchema(), rows: []map[string]any{+ {"id": int64(2), "name": "two"},+ {"id": int64(3), "name": "three"},+ }}+ connector := newFakeBigQueryConnector(t, nil, fake)+ session, err := connector.OpenSync(context.Background(), SyncRequest{+ FromBeginning: true,+ Resume: &SyncCheckpoint{Cursor: "bigquery:my-proj:ds.tbl:2"},+ })+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch failed: %v", err)+ }+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != "bigquery:my-proj:ds.tbl:3" {+ t.Fatalf("documents = %+v", batch.Documents)+ }+}++func TestBigQueryConnectorResumeRequiresStableOrder(t *testing.T) {+ fake := &fakeBigQueryClient{+ schema: []bigQueryField{{Name: "name", Type: "STRING"}},+ rows: []map[string]any{{"name": "content"}},+ }+ connector := newFakeBigQueryConnector(t, map[string]any{+ "project_id": "my-proj",+ "dataset_id": "ds",+ "table_id": "tbl",+ "content_columns": "name",+ "credentials": map[string]any{+ "service_account_json": `{"type":"service_account","project_id":"my-proj"}`,+ },+ }, fake)+ session, err := connector.OpenSync(context.Background(), SyncRequest{+ FromBeginning: true,+ Resume: &SyncCheckpoint{SourceID: "bigquery:my-proj:ds.tbl:anchor"},+ })+ if err == nil || !errors.Is(err, ErrSyncResumeInvalid) {+ t.Fatalf("OpenSync error = %v, want ErrSyncResumeInvalid", err)+ }+ if session != nil {+ t.Fatal("OpenSync returned a session after resume validation failure")+ }+ if strings.Contains(fake.queries[0].text, "ORDER BY") {+ t.Fatalf("query = %q, want no synthetic ordering without stable columns", fake.queries[0].text)+ }+}++func TestBigQueryConnectorOpenPrune(t *testing.T) {+ fake := &fakeBigQueryClient{+ schema: []bigQueryField{{Name: "id", Type: "INT64"}},+ rows: []map[string]any{{"id": int64(3)}, {"id": int64(4)}},+ }+ connector := newFakeBigQueryConnector(t, map[string]any{+ "project_id": "my-proj",+ "dataset_id": "ds",+ "table_id": "tbl",+ "query": "SELECT * FROM `my-proj.ds.tbl`",+ "content_columns": "name",+ "id_column": "id",+ "credentials": map[string]any{+ "service_account_json": `{"type":"service_account","project_id":"my-proj"}`,+ },+ }, fake)+ session, err := connector.OpenPrune(context.Background(), PruneRequest{})+ if err != nil {+ t.Fatalf("OpenPrune failed: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch failed: %v", err)+ }+ if len(batch.Documents) != 2 ||+ batch.Documents[0].SourceID != "bigquery:my-proj:query:3" ||+ batch.Documents[1].SourceID != "bigquery:my-proj:query:4" {+ t.Fatalf("slim documents = %+v", batch.Documents)+ }+ if !strings.Contains(fake.queries[0].text, "ORDER BY ragflow_src.id ASC") {+ t.Fatalf("prune query = %q, want stable ordering", fake.queries[0].text)+ }+}++func TestBigQueryConnectorRegistry(t *testing.T) {+ registry := NewRegistry()+ RegisterBuiltIns(registry)+ connector, err := registry.OpenFromConfig("bigquery", map[string]any{+ "project_id": "my-proj",+ "dataset_id": "ds",+ "table_id": "tbl",+ "content_columns": "name",+ "credentials": map[string]any{+ "service_account_json": `{"type":"service_account","project_id":"my-proj"}`,+ },+ })+ if err != nil {+ t.Fatalf("OpenFromConfig failed: %v", err)+ }+ if connector == nil {+ t.Fatal("connector is nil")+ }+}++func TestBigQueryConnectorInvalidProjectID(t *testing.T) {+ _, err := NewBigQueryConnector(map[string]any{+ "project_id": "bad project",+ "content_columns": "name",+ })+ if err == nil || !strings.Contains(err.Error(), "project_id") {+ t.Fatalf("NewBigQueryConnector error = %v, want project_id validation", err)+ }+}++func TestBigQueryCursorSerialization(t *testing.T) {+ now := mustTime(t, "2026-01-02T03:04:05Z")+ serialized := serializeBigQueryCursor(now)+ if _, ok := serialized.(map[string]any); !ok {+ t.Fatalf("serialized cursor = %#v", serialized)+ }+ restored := deserializeBigQueryCursor(serialized)+ if !restored.(time.Time).Equal(now) {+ t.Fatalf("restored cursor = %v", restored)+ }+ if got := serializeBigQueryCursor(int64(42)); got != int64(42) {+ t.Fatalf("numeric cursor = %#v", got)+ }+ if got := deserializeBigQueryCursor("plain"); got != "plain" {+ t.Fatalf("plain cursor = %#v", got)+ }+ clock, err := civil.ParseTime("03:04:05")+ if err != nil {+ t.Fatalf("parse time cursor: %v", err)+ }+ restoredTime := deserializeBigQueryCursor(serializeBigQueryCursor(clock))+ if restoredTime != clock {+ t.Fatalf("restored time cursor = %#v", restoredTime)+ }+}diff --git a/internal/syncer/connector/builtin.go b/internal/syncer/connector/builtin.goindex f87d394cee..2f68bf45c4 100644--- a/internal/syncer/connector/builtin.go+++ b/internal/syncer/connector/builtin.go@@ -64,6 +64,7 @@ func RegisterBuiltIns(registry *Registry) {registerBuiltIn(registry, "discord", NewDiscordConnector)registerBuiltIn(registry, "webdav", NewWebDAVConnector)registerBuiltIn(registry, "seafile", NewSeaFileConnector)+ registerBuiltIn(registry, "bigquery", NewBigQueryConnector)}func registerBuiltIn[T Connector](registry *Registry, source string, factory func(map[string]any) (T, error)) {