Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat(syncer): add Moodle connector with checkpoint resume (#18554)

mkaaadcab70fdac304
diff --git a/go.mod b/go.mod
index d57708d43f..64284133f0 100644
--- a/go.mod
+++ b/go.mod
@@ -6,6 +6,7 @@ require (
cloud.google.com/go/storage v1.63.0
github.com/AkmalOt/gomsg v0.0.0-20260407083308-985c3a1a76b7
github.com/DATA-DOG/go-sqlmock v1.5.2
+ github.com/JohannesKaufmann/html-to-markdown v1.6.0
github.com/LuxorLabs/tenki-sdk-go/sandbox v0.7.0
github.com/alibabacloud-go/agentrun-20250910/v5 v5.8.4
github.com/alibabacloud-go/darabonba-openapi/v2 v2.2.1
@@ -58,7 +59,7 @@ require (
github.com/xuri/excelize/v2 v2.11.0
github.com/yfedoseev/office_oxide/go v0.1.8
github.com/yfedoseev/pdf_oxide/go v0.3.67
- github.com/yuin/goldmark v1.4.13
+ github.com/yuin/goldmark v1.7.1
github.com/zeebo/xxh3 v1.0.2
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0
go.opentelemetry.io/otel v1.44.0
@@ -80,7 +81,11 @@ require (
gorm.io/gorm v1.25.7
)
-require github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 // indirect
+require (
+ github.com/PuerkitoBio/goquery v1.9.2 // indirect
+ github.com/andybalholm/cascadia v1.3.2 // indirect
+ github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 // indirect
+)
require (
buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20260415201107-50325440f8f2.1 // indirect
diff --git a/go.sum b/go.sum
index 9dd5340ba8..dfaeba4f1b 100644
--- a/go.sum
+++ b/go.sum
@@ -39,8 +39,12 @@ github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.57.0/go.mod h1:dzcEjy1WJ0Q4u9twNR3LcLhNoYMRCrMCMafpxa0TjPQ=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0 h1:RoO5+d7uCmDqovLrHCr2/BuViUXvdcrNxyNM1pN9dDQ=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0/go.mod h1:YqwkQPrWSC7+byyc1VlKbWLBF5JsW5IoL6xUkemYSXk=
+github.com/JohannesKaufmann/html-to-markdown v1.6.0 h1:04VXMiE50YYfCfLboJCLcgqF5x+rHJnb1ssNmqpLH/k=
+github.com/JohannesKaufmann/html-to-markdown v1.6.0/go.mod h1:NUI78lGg/a7vpEJTz/0uOcYMaibytE4BUOQS8k78yPQ=
github.com/LuxorLabs/tenki-sdk-go/sandbox v0.7.0 h1:PBQ+ad8fZiLH/1ij1hoynR9aAPVmI3EzT7DiLNmUdBM=
github.com/LuxorLabs/tenki-sdk-go/sandbox v0.7.0/go.mod h1:rBypesed6hSrj+OgpJwyMVBF76PaQ1iOqc2RGUL4RmI=
+github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE=
+github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk=
github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk=
github.com/airbrake/gobrake v3.6.1+incompatible/go.mod h1:wM4gu3Cn0W0K7GUuVWnlXZU11AGBXMILnrdOU8Kn00o=
github.com/alibabacloud-go/agentrun-20250910/v5 v5.8.4 h1:hiAsm9pz6aICOPLI1FC54vga10xwd/XxfNj06ow5jVM=
@@ -91,6 +95,8 @@ github.com/aliyun/credentials-go v1.3.1/go.mod h1:8jKYhQuDawt8x2+fusqa1Y6mPxemTs
github.com/aliyun/credentials-go v1.3.6/go.mod h1:1LxUuX7L5YrZUWzBrRyk0SwSdH4OmPrib8NVePL3fxM=
github.com/aliyun/credentials-go v1.4.5 h1:O76WYKgdy1oQYYiJkERjlA2dxGuvLRrzuO2ScrtGWSk=
github.com/aliyun/credentials-go v1.4.5/go.mod h1:Jm6d+xIgwJVLVWT561vy67ZRP4lPTQxMbEYRuT2Ti1U=
+github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
+github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/apache/thrift v0.23.0 h1:wKR6YnefQSEnxpEfmgTPuJibNG4bF0p2TK34tHLWi3s=
@@ -458,6 +464,11 @@ github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6ke
github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4=
github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=
github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=
+github.com/sebdah/goldie/v2 v2.5.3 h1:9ES/mNN+HNUbNWpVAlrzuZ7jE+Nrczbj8uFRjM7624Y=
+github.com/sebdah/goldie/v2 v2.5.3/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI=
+github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
+github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=
github.com/signintech/gopdf v0.36.1 h1:cGpvEKvvqCV+ZXB9R2SQoWgouW91JpwsgoQEhLxIdp0=
github.com/signintech/gopdf v0.36.1/go.mod h1:d23eO35GpEliSrF22eJ4bsM3wVeQJTjXTHq5x5qGKjA=
github.com/siongui/gojianfan v0.0.0-20210926212422-2f175ac615de h1:1/P9CcR8iENN9ybbSRWohRd3rsPp9tEWlTS/7ygvjHE=
@@ -495,6 +506,7 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
@@ -549,8 +561,9 @@ github.com/yfedoseev/pdf_oxide/go v0.3.67/go.mod h1:QbJ/nLbez0al2EnqEdEPIlGflFpr
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.30/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.4.13 h1:fVcFKWvrslecOb/tg+Cc05dkeYx540o0FuFt3nUVDoE=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+github.com/yuin/goldmark v1.7.1 h1:3bajkSilaCbjdKVsKdZjZCLBNPL9pYzrCakKaf4U49U=
+github.com/yuin/goldmark v1.7.1/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
@@ -610,6 +623,7 @@ golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf
golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
+golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto=
@@ -646,12 +660,14 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
+golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=
@@ -688,12 +704,14 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
@@ -702,12 +720,14 @@ golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXct
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
+golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0=
golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc=
@@ -776,6 +796,7 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
@@ -789,6 +810,7 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
diff --git a/internal/syncer/connector/builtin.go b/internal/syncer/connector/builtin.go
index 1d5e9f2278..c15e0c04bf 100644
--- a/internal/syncer/connector/builtin.go
+++ b/internal/syncer/connector/builtin.go
@@ -42,6 +42,7 @@ func RegisterBuiltIns(registry *Registry) {
registerBuiltIn(registry, "outlook", NewOutlookConnector)
registerBuiltIn(registry, "notion", NewNotionConnector)
registerBuiltIn(registry, "rest_api", NewRestAPIConnector)
+ registerBuiltIn(registry, "moodle", NewMoodleConnector)
registerBuiltIn(registry, "mysql", NewMySQLConnector)
registerBuiltIn(registry, "postgresql", NewPostgreSQLConnector)
registerBuiltIn(registry, "discord", NewDiscordConnector)
diff --git a/internal/syncer/connector/moodle.go b/internal/syncer/connector/moodle.go
new file mode 100644
index 0000000000..0deafe3038
--- /dev/null
+++ b/internal/syncer/connector/moodle.go
@@ -0,0 +1,1041 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package connector
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net"
+ "net/http"
+ "net/url"
+ "path"
+ "path/filepath"
+ "sort"
+ "strconv"
+ "strings"
+ "time"
+
+ md "github.com/JohannesKaufmann/html-to-markdown"
+)
+
+const (
+ defaultMoodleBatchSize = 2
+ moodleRequestTimeout = 60 * time.Second
+ moodleMaxRedirects = 10
+ moodleRESTPath = "/webservice/rest/server.php"
+)
+
+// Retry/backoff knobs for Moodle web service calls. They are package
+// variables so tests can shrink the delays.
+var (
+ moodleRetryTries = 3
+ moodleRetryBaseDelay = time.Second
+ moodleRetryBackoff = 2
+)
+var (
+ // moodleMaxResponseSize caps REST JSON responses. Package variable so
+ // tests can shrink it.
+ moodleMaxResponseSize int64 = 32 * 1024 * 1024
+ // moodleMaxDownloadSize caps file downloads. Package variable so
+ // tests can shrink it.
+ moodleMaxDownloadSize int64 = 100 * 1024 * 1024
+)
+
+// MoodleConnector reads course content from a Moodle LMS through its REST web
+// service API.
+type MoodleConnector struct {
+ moodleURL string
+ token string
+ batchSize int
+
+ // restCall is a test hook that replaces the live web service transport.
+ restCall func(ctx context.Context, function string, params map[string]any, out any) error
+}
+
+// NewMoodleConnector creates a Moodle connector from connector config.
+func NewMoodleConnector(config map[string]any) (*MoodleConnector, error) {
+ credentials := configAnyMap(config["credentials"])
+ batchSize := configInt(firstNonEmpty(stringConfig(config["sync_batch_size"]), stringConfig(config["batch_size"])), defaultMoodleBatchSize)
+ return &MoodleConnector{
+ moodleURL: strings.TrimRight(strings.TrimSpace(stringConfig(config["moodle_url"])), "/"),
+ token: strings.TrimSpace(stringConfig(credentials["moodle_token"])),
+ batchSize: batchSize,
+ }, nil
+}
+
+// Validate validates Moodle connector settings, credentials, and connectivity.
+func (c *MoodleConnector) Validate(ctx context.Context) error {
+ if c == nil {
+ return fmt.Errorf("Moodle connector is nil")
+ }
+ if c.moodleURL == "" {
+ return fmt.Errorf("No Moodle URL was provided in connector settings.")
+ }
+ if c.token == "" {
+ return &ConnectorMissingCredentialError{Message: "Moodle API token is required"}
+ }
+ if c.batchSize <= 0 {
+ return fmt.Errorf("batch_size must be a positive integer")
+ }
+ if err := validateMoodleURLForSSRF(c.moodleURL); err != nil {
+ return err
+ }
+ var siteInfo moodleSiteInfo
+ if err := c.callREST(ctx, "core_webservice_get_site_info", nil, &siteInfo); err != nil {
+ return err
+ }
+ if siteInfo.SiteName == "" {
+ return &ConnectorValidationError{Message: "Invalid Moodle API response"}
+ }
+ return nil
+}
+
+// ValidateConnectorSetting validates Moodle settings from an unsaved config.
+func (c *MoodleConnector) ValidateConnectorSetting(ctx context.Context, request map[string]any) error {
+ ctx, cancel := context.WithTimeout(ctx, connectorSettingValidationTimeout)
+ defer cancel()
+ return c.Validate(ctx)
+}
+
+// OpenSync opens one Moodle sync session.
+func (c *MoodleConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {
+ if err := c.Validate(ctx); err != nil {
+ return nil, err
+ }
+ session := &moodleSyncSession{
+ connector: c,
+ request: request,
+ batchSize: c.batchSize,
+ }
+ if request.Resume != nil {
+ session.applyResume(request.Resume)
+ }
+ return session, nil
+}
+
+// OpenPrune opens one complete Moodle prune snapshot session.
+func (c *MoodleConnector) OpenPrune(ctx context.Context, request PruneRequest) (PruneSession, error) {
+ if err := c.Validate(ctx); err != nil {
+ return nil, err
+ }
+ courses, err := c.getCourses(ctx)
+ if err != nil {
+ return nil, err
+ }
+ var documents []SlimDocument
+ for _, course := range courses {
+ sections, err := c.getCourseContents(ctx, course.ID)
+ if err != nil {
+ return nil, err
+ }
+ for _, section := range sections {
+ for _, module := range section.Modules {
+ slimID := moodleSlimDocIDForModule(module)
+ if slimID == "" {
+ continue
+ }
+ documents = append(documents, SlimDocument{SourceID: slimID})
+ }
+ }
+ }
+ return &moodlePruneSession{documents: documents, batchSize: c.batchSize}, nil
+}
+
+// ---------------------------------------------------------------------------
+// Moodle REST web service client
+// ---------------------------------------------------------------------------
+
+type moodleSiteInfo struct {
+ SiteName string `json:"sitename"`
+}
+
+type moodleCourse struct {
+ ID int64 `json:"id"`
+ FullName string `json:"fullname"`
+ ShortName string `json:"shortname"`
+}
+
+type moodleSection struct {
+ ID int64 `json:"id"`
+ Name string `json:"name"`
+ Section *int64 `json:"section"`
+ Modules []moodleModule `json:"modules"`
+}
+
+type moodleModule struct {
+ ID int64 `json:"id"`
+ Name string `json:"name"`
+ ModName string `json:"modname"`
+ Instance *int64 `json:"instance"`
+ Description string `json:"description"`
+ Visible *int64 `json:"visible"`
+ GroupMode *int64 `json:"groupmode"`
+ TimeCreated *int64 `json:"timecreated"`
+ TimeModified *int64 `json:"timemodified"`
+ Added *int64 `json:"added"`
+ Contents []moodleContent `json:"contents"`
+}
+
+type moodleContent struct {
+ Filename string `json:"filename"`
+ FileURL string `json:"fileurl"`
+ FileSize *int64 `json:"filesize"`
+ MIMEType string `json:"mimetype"`
+ ChapterID *int64 `json:"chapterid"`
+ Title string `json:"title"`
+ TimeCreated *int64 `json:"timecreated"`
+ TimeModified *int64 `json:"timemodified"`
+}
+
+type moodleDiscussion struct {
+ ID int64 `json:"id"`
+ Name string `json:"name"`
+ Message string `json:"message"`
+ UserID *int64 `json:"userid"`
+ UserFullName string `json:"userfullname"`
+ TimeCreated *int64 `json:"timecreated"`
+ TimeModified *int64 `json:"timemodified"`
+}
+
+func (c *MoodleConnector) callREST(ctx context.Context, function string, params map[string]any, out any) error {
+ if c.restCall != nil {
+ return c.restCall(ctx, function, params, out)
+ }
+ endpoint := c.moodleURL + moodleRESTPath
+ form := url.Values{}
+ form.Set("wstoken", c.token)
+ form.Set("wsfunction", function)
+ form.Set("moodlewsrestformat", "json")
+ for key, value := range params {
+ switch typed := value.(type) {
+ case int64:
+ form.Set(key, strconv.FormatInt(typed, 10))
+ case int:
+ form.Set(key, strconv.Itoa(typed))
+ case string:
+ form.Set(key, typed)
+ }
+ }
+ resp, err := c.moodleHTTPDo(ctx, http.MethodPost, endpoint, []byte(form.Encode()), map[string]string{"Content-Type": "application/x-www-form-urlencoded"})
+ if err != nil {
+ return err
+ }
+ defer resp.Body.Close()
+ body, err := io.ReadAll(io.LimitReader(resp.Body, moodleMaxResponseSize+1))
+ if err != nil {
+ return err
+ }
+ if int64(len(body)) > moodleMaxResponseSize {
+ return fmt.Errorf("Moodle REST response exceeds maximum size of %d bytes", moodleMaxResponseSize)
+ }
+ if resp.StatusCode != http.StatusOK {
+ return moodleRESTError(body, resp.StatusCode)
+ }
+ var probe map[string]json.RawMessage
+ if err := json.Unmarshal(body, &probe); err == nil {
+ if _, ok := probe["exception"]; ok {
+ return moodleRESTError(body, resp.StatusCode)
+ }
+ }
+ return json.Unmarshal(body, out)
+}
+
+func moodleRESTError(body []byte, statusCode int) error {
+ var apiError struct {
+ ErrorCode string `json:"errorcode"`
+ Message string `json:"message"`
+ }
+ if err := json.Unmarshal(body, &apiError); err == nil && apiError.Message != "" {
+ return classifyMoodleAPIError(apiError.ErrorCode, apiError.Message)
+ }
+ return fmt.Errorf("Moodle web service request failed with status %d", statusCode)
+}
+
+func classifyMoodleAPIError(errorCode, message string) error {
+ combined := strings.ToLower(errorCode + " " + message)
+ switch {
+ case strings.Contains(combined, "invalidtoken"):
+ return &ConnectorMissingCredentialError{Message: "Moodle token is invalid or expired"}
+ case strings.Contains(combined, "accessexception"):
+ return &ConnectorValidationError{Message: "Insufficient permissions. Ensure web services are enabled and permissions are correct."}
+ default:
+ return &ConnectorValidationError{Message: fmt.Sprintf("Moodle validation error: %s", message)}
+ }
+}
+
+func retryMoodle(ctx context.Context, fn func() error) error {
+ var lastErr error
+ for attempt := 0; attempt < moodleRetryTries; attempt++ {
+ if err := fn(); err == nil {
+ return nil
+ } else {
+ lastErr = err
+ }
+ if attempt+1 >= moodleRetryTries {
+ break
+ }
+ delay := moodleRetryBaseDelay
+ for i := 0; i < attempt; i++ {
+ delay *= time.Duration(moodleRetryBackoff)
+ }
+ timer := time.NewTimer(delay)
+ select {
+ case <-ctx.Done():
+ timer.Stop()
+ return ctx.Err()
+ case <-timer.C:
+ }
+ }
+ return lastErr
+}
+
+func (c *MoodleConnector) getCourses(ctx context.Context) ([]moodleCourse, error) {
+ var courses []moodleCourse
+ if err := retryMoodle(ctx, func() error {
+ return c.callREST(ctx, "core_course_get_courses", nil, &courses)
+ }); err != nil {
+ return nil, err
+ }
+ sort.SliceStable(courses, func(i, j int) bool {
+ return courses[i].ID < courses[j].ID
+ })
+ return courses, nil
+}
+
+func (c *MoodleConnector) getCourseContents(ctx context.Context, courseID int64) ([]moodleSection, error) {
+ var sections []moodleSection
+ if err := retryMoodle(ctx, func() error {
+ return c.callREST(ctx, "core_course_get_contents", map[string]any{"courseid": courseID}, &sections)
+ }); err != nil {
+ return nil, fmt.Errorf("Moodle course %d contents failed: %w", courseID, err)
+ }
+ return sections, nil
+}
+
+func (c *MoodleConnector) getForumDiscussions(ctx context.Context, forumID int64) ([]moodleDiscussion, error) {
+ var result struct {
+ Discussions []moodleDiscussion `json:"discussions"`
+ }
+ if err := c.callREST(ctx, "mod_forum_get_forum_discussions", map[string]any{"forumid": forumID}, &result); err != nil {
+ return nil, err
+ }
+ return result.Discussions, nil
+}
+
+func (c *MoodleConnector) downloadFile(ctx context.Context, fileURL string) ([]byte, error) {
+ downloadURL := addMoodleToken(fileURL, c.token)
+ resp, err := c.moodleHTTPDo(ctx, http.MethodGet, downloadURL, nil, nil)
+ if err != nil {
+ return nil, err
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
+ return nil, fmt.Errorf("Moodle file download failed with status %d for %s", resp.StatusCode, moodleRedactedURL(fileURL))
+ }
+ blob, err := io.ReadAll(io.LimitReader(resp.Body, moodleMaxDownloadSize+1))
+ if err != nil {
+ return nil, err
+ }
+ if int64(len(blob)) > moodleMaxDownloadSize {
+ return nil, fmt.Errorf("Moodle file download exceeds maximum size of %d bytes", moodleMaxDownloadSize)
+ }
+ return blob, nil
+}
+
+func addMoodleToken(fileURL, token string) string {
+ if token == "" {
+ return fileURL
+ }
+ if strings.Contains(strings.ToLower(fileURL), "token=") {
+ return fileURL
+ }
+ delimiter := "?"
+ if strings.Contains(fileURL, "?") {
+ delimiter = "&"
+ }
+ return fileURL + delimiter + "token=" + token
+}
+
+func moodleHTMLToMarkdown(html string) (string, error) {
+ converter := md.NewConverter("", true, &md.Options{EmDelimiter: "*"})
+ out, err := converter.ConvertString(html)
+ if err != nil {
+ return "", fmt.Errorf("Moodle HTML to Markdown conversion failed: %w", err)
+ }
+ return out, nil
+}
+
+// ---------------------------------------------------------------------------
+// Document building
+// ---------------------------------------------------------------------------
+
+func (c *MoodleConnector) courseDocuments(ctx context.Context, request SyncRequest, course moodleCourse, sections []moodleSection) ([]SourceDocument, error) {
+ var documents []SourceDocument
+ for _, section := range sections {
+ for _, module := range section.Modules {
+ docs, err := c.moduleDocuments(ctx, request, course, section, module)
+ if err != nil {
+ return nil, err
+ }
+ documents = append(documents, docs...)
+ }
+ }
+ return documents, nil
+}
+
+func (c *MoodleConnector) moduleDocuments(ctx context.Context, request SyncRequest, course moodleCourse, section moodleSection, module moodleModule) ([]SourceDocument, error) {
+ var document *SourceDocument
+ var err error
+ switch module.ModName {
+ case "label", "url":
+ return nil, nil
+ case "resource":
+ document, err = c.resourceDocument(ctx, request, course, section, module)
+ case "page":
+ document, err = c.pageDocument(ctx, request, course, section, module)
+ case "forum":
+ document, err = c.forumDocument(ctx, request, course, section, module)
+ case "assign", "quiz":
+ document, err = c.activityDocument(ctx, request, course, section, module)
+ case "book":
+ document, err = c.bookDocument(ctx, request, course, section, module)
+ }
+ if err != nil {
+ return nil, err
+ }
+ if document == nil {
+ return nil, nil
+ }
+ return []SourceDocument{*document}, nil
+}
+
+func (c *MoodleConnector) resourceDocument(ctx context.Context, request SyncRequest, course moodleCourse, section moodleSection, module moodleModule) (*SourceDocument, error) {
+ if len(module.Contents) == 0 || module.Contents[0].FileURL == "" {
+ return nil, nil
+ }
+ fileInfo := module.Contents[0]
+ fileName := path.Base(fileInfo.Filename)
+ ts := moodleMaxTimestamp(module.TimeCreated, module.TimeModified, fileInfo.TimeModified)
+ if !includeMoodleModule(request, ts) {
+ return nil, nil
+ }
+ blob, err := c.downloadFile(ctx, fileInfo.FileURL)
+ if err != nil {
+ return nil, err
+ }
+ extension := filepath.Ext(fileName)
+ if extension == "" {
+ extension = ".bin"
+ }
+ fileSize := int64(len(blob))
+ if fileInfo.FileSize != nil {
+ fileSize = *fileInfo.FileSize
+ }
+ return &SourceDocument{
+ SourceID: fmt.Sprintf("moodle_resource_%d", module.ID),
+ SemanticIdentifier: fmt.Sprintf("%s / %s / %s", course.FullName, section.Name, fileName),
+ Extension: extension,
+ Blob: blob,
+ UpdatedAt: moodleTime(ts),
+ SizeBytes: int64(len(blob)),
+ Metadata: c.fileModuleMetadata(course, section, module, fileInfo, fileSize, fileName),
+ }, nil
+}
+
+func (c *MoodleConnector) pageDocument(ctx context.Context, request SyncRequest, course moodleCourse, section moodleSection, module moodleModule) (*SourceDocument, error) {
+ if len(module.Contents) == 0 || module.Contents[0].FileURL == "" {
+ return nil, nil
+ }
+ fileInfo := module.Contents[0]
+ fileName := path.Base(fileInfo.Filename)
+ ts := moodleMaxTimestamp(module.TimeCreated, module.TimeModified, fileInfo.TimeModified)
+ if !includeMoodleModule(request, ts) {
+ return nil, nil
+ }
+ blob, err := c.downloadFile(ctx, fileInfo.FileURL)
+ if err != nil {
+ return nil, err
+ }
+ extension := filepath.Ext(fileName)
+ if extension == "" {
+ extension = ".html"
+ }
+ fileSize := int64(len(blob))
+ if fileInfo.FileSize != nil {
+ fileSize = *fileInfo.FileSize
+ }
+ return &SourceDocument{
+ SourceID: fmt.Sprintf("moodle_page_%d", module.ID),
+ SemanticIdentifier: fmt.Sprintf("%s / %s / %s", course.FullName, section.Name, module.Name),
+ Extension: extension,
+ Blob: blob,
+ UpdatedAt: moodleTime(ts),
+ SizeBytes: int64(len(blob)),
+ Metadata: c.fileModuleMetadata(course, section, module, fileInfo, fileSize, fileName),
+ }, nil
+}
+
+func (c *MoodleConnector) forumDocument(ctx context.Context, request SyncRequest, course moodleCourse, section moodleSection, module moodleModule) (*SourceDocument, error) {
+ if module.Instance == nil {
+ return nil, nil
+ }
+ if !includeMoodleModule(request, module.windowTimestamp()) {
+ return nil, nil
+ }
+ discussions, err := c.getForumDiscussions(ctx, *module.Instance)
+ if err != nil {
+ return nil, err
+ }
+ if len(discussions) == 0 {
+ return nil, nil
+ }
+ markdown := []string{"# " + module.Name + "\n"}
+ latest := moodleMaxTimestamp(module.TimeCreated, module.TimeModified)
+ discussionMetadata := make([]map[string]any, 0, len(discussions))
+ for _, discussion := range discussions {
+ body, err := moodleHTMLToMarkdown(discussion.Message)
+ if err != nil {
+ return nil, err
+ }
+ markdown = append(markdown, fmt.Sprintf("## %s\n\n%s\n\n---\n", discussion.Name, body))
+ latest = maxInt64(latest, derefInt64(discussion.TimeModified))
+ discussionMetadata = append(discussionMetadata, map[string]any{
+ "id": discussion.ID,
+ "name": discussion.Name,
+ "user_id": discussion.UserID,
+ "user_fullname": discussion.UserFullName,
+ "time_created": discussion.TimeCreated,
+ "time_modified": discussion.TimeModified,
+ })
+ }
+ metadata := moodleBaseMetadata(c, course, section, module)
+ metadata["forum_id"] = module.Instance
+ metadata["discussion_count"] = len(discussions)
+ metadata["discussions"] = discussionMetadata
+ blob := []byte(strings.Join(markdown, "\n"))
+ return &SourceDocument{
+ SourceID: fmt.Sprintf("moodle_forum_%d", module.ID),
+ SemanticIdentifier: fmt.Sprintf("%s / %s / %s", course.FullName, section.Name, module.Name),
+ Extension: ".md",
+ Blob: blob,
+ UpdatedAt: moodleTime(latest),
+ SizeBytes: int64(len(blob)),
+ Metadata: metadata,
+ }, nil
+}
+
+func (c *MoodleConnector) activityDocument(ctx context.Context, request SyncRequest, course moodleCourse, section moodleSection, module moodleModule) (*SourceDocument, error) {
+ if module.Description == "" {
+ return nil, nil
+ }
+ if !includeMoodleModule(request, module.windowTimestamp()) {
+ return nil, nil
+ }
+ body, err := moodleHTMLToMarkdown(module.Description)
+ if err != nil {
+ return nil, err
+ }
+ markdown := fmt.Sprintf("# %s\n\n**Type:** %s\n\n%s", module.Name, capitalizeMoodleType(module.ModName), body)
+ ts := moodleMaxTimestamp(module.TimeCreated, module.TimeModified, module.Added)
+ metadata := moodleBaseMetadata(c, course, section, module)
+ metadata["activity_type"] = module.ModName
+ metadata["activity_instance"] = module.Instance
+ metadata["description"] = module.Description
+ metadata["added"] = module.Added
+ blob := []byte(markdown)
+ return &SourceDocument{
+ SourceID: fmt.Sprintf("moodle_%s_%d", module.ModName, module.ID),
+ SemanticIdentifier: fmt.Sprintf("%s / %s / %s", course.FullName, section.Name, module.Name),
+ Extension: ".md",
+ Blob: blob,
+ UpdatedAt: moodleTime(ts),
+ SizeBytes: int64(len(blob)),
+ Metadata: metadata,
+ }, nil
+}
+
+func (c *MoodleConnector) bookDocument(ctx context.Context, request SyncRequest, course moodleCourse, section moodleSection, module moodleModule) (*SourceDocument, error) {
+ if len(module.Contents) == 0 {
+ return nil, nil
+ }
+ var chapters []moodleContent
+ for _, content := range module.Contents {
+ if content.FileURL != "" && path.Base(content.Filename) == "index.html" {
+ chapters = append(chapters, content)
+ }
+ }
+ if len(chapters) == 0 {
+ return nil, nil
+ }
+ latest := moodleMaxTimestamp(module.TimeCreated, module.TimeModified)
+ for _, content := range module.Contents {
+ latest = maxInt64(latest, derefInt64(content.TimeCreated))
+ latest = maxInt64(latest, derefInt64(content.TimeModified))
+ }
+ if !includeMoodleModule(request, latest) {
+ return nil, nil
+ }
+ markdownParts := []string{"# " + module.Name + "\n"}
+ chapterMetadata := make([]map[string]any, 0, len(chapters))
+ for _, chapter := range chapters {
+ blob, err := c.downloadFile(ctx, chapter.FileURL)
+ if err != nil {
+ return nil, err
+ }
+ body, err := moodleHTMLToMarkdown(string(blob))
+ if err != nil {
+ return nil, err
+ }
+ markdownParts = append(markdownParts, body+"\n\n---\n")
+ chapterMetadata = append(chapterMetadata, map[string]any{
+ "chapter_id": chapter.ChapterID,
+ "title": chapter.Title,
+ "filename": chapter.Filename,
+ "fileurl": moodleRedactedURL(chapter.FileURL),
+ "time_created": chapter.TimeCreated,
+ "time_modified": chapter.TimeModified,
+ "size": chapter.FileSize,
+ })
+ }
+ metadata := moodleBaseMetadata(c, course, section, module)
+ metadata["book_id"] = module.Instance
+ metadata["chapter_count"] = len(chapters)
+ metadata["chapters"] = chapterMetadata
+ blob := []byte(strings.Join(markdownParts, "\n"))
+ return &SourceDocument{
+ SourceID: fmt.Sprintf("moodle_book_%d", module.ID),
+ SemanticIdentifier: fmt.Sprintf("%s / %s / %s", course.FullName, section.Name, module.Name),
+ Extension: ".md",
+ Blob: blob,
+ UpdatedAt: moodleTime(latest),
+ SizeBytes: int64(len(blob)),
+ Metadata: metadata,
+ }, nil
+}
+
+func (c *MoodleConnector) fileModuleMetadata(course moodleCourse, section moodleSection, module moodleModule, fileInfo moodleContent, fileSize int64, fileName string) map[string]any {
+ metadata := moodleBaseMetadata(c, course, section, module)
+ metadata["module_instance"] = module.Instance
+ metadata["file_url"] = moodleRedactedURL(fileInfo.FileURL)
+ metadata["file_name"] = fileName
+ metadata["file_size"] = fileSize
+ metadata["file_type"] = fileInfo.MIMEType
+ return metadata
+}
+
+func moodleBaseMetadata(c *MoodleConnector, course moodleCourse, section moodleSection, module moodleModule) map[string]any {
+ return map[string]any{
+ "moodle_url": c.moodleURL,
+ "course_id": course.ID,
+ "course_name": course.FullName,
+ "course_shortname": course.ShortName,
+ "section_id": section.ID,
+ "section_name": section.Name,
+ "section_number": section.Section,
+ "module_id": module.ID,
+ "module_name": module.Name,
+ "module_type": module.ModName,
+ "time_created": module.TimeCreated,
+ "time_modified": module.TimeModified,
+ "visible": module.Visible,
+ "groupmode": module.GroupMode,
+ }
+}
+
+// ---------------------------------------------------------------------------
+// Sync session with checkpoint resume
+// ---------------------------------------------------------------------------
+
+type moodleSyncSession struct {
+ connector *MoodleConnector
+ request SyncRequest
+ batchSize int
+ resumeAfterCourseID int64
+
+ coursesLoaded bool
+ courses []moodleCourse
+ courseIndex int
+ pending []SourceDocument
+ pendingCheckpoint *SyncCheckpoint
+ done bool
+}
+
+// NextBatch returns the next Moodle document batch.
+func (s *moodleSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {
+ for {
+ if s.done {
+ return SyncBatch{}, io.EOF
+ }
+ if len(s.pending) == 0 {
+ if !s.coursesLoaded {
+ courses, err := s.connector.getCourses(ctx)
+ if err != nil {
+ return SyncBatch{}, err
+ }
+ s.courses = courses
+ s.coursesLoaded = true
+ }
+ if s.courseIndex >= len(s.courses) {
+ s.done = true
+ return SyncBatch{}, io.EOF
+ }
+ course := s.courses[s.courseIndex]
+ s.courseIndex++
+ if course.ID <= s.resumeAfterCourseID {
+ continue
+ }
+ sections, err := s.connector.getCourseContents(ctx, course.ID)
+ if err != nil {
+ return SyncBatch{}, err
+ }
+ documents, err := s.connector.courseDocuments(ctx, s.request, course, sections)
+ if err != nil {
+ return SyncBatch{}, err
+ }
+ if len(documents) == 0 {
+ continue
+ }
+ s.pending = documents
+ updatedAt := documents[len(documents)-1].UpdatedAt
+ checkpoint := &SyncCheckpoint{
+ Cursor: fmt.Sprintf("moodle_course_%d", course.ID),
+ SourceID: fmt.Sprintf("moodle_course_%d", course.ID),
+ UpdatedAt: &updatedAt,
+ }
+ // The checkpoint only advances once the whole course is flushed so
+ // an interrupted run re-processes a partially committed course.
+ s.pendingCheckpoint = checkpoint
+ }
+ n := s.batchSize
+ if n > len(s.pending) {
+ n = len(s.pending)
+ }
+ chunk := s.pending[:n]
+ s.pending = s.pending[n:]
+ var checkpoint *SyncCheckpoint
+ if len(s.pending) == 0 && s.pendingCheckpoint != nil {
+ checkpoint = s.pendingCheckpoint
+ s.pendingCheckpoint = nil
+ }
+ return SyncBatch{Documents: chunk, Checkpoint: checkpoint}, nil
+ }
+}
+
+// Close closes the Moodle sync session.
+func (s *moodleSyncSession) Close() error {
+ return nil
+}
+
+func (s *moodleSyncSession) applyResume(checkpoint *SyncCheckpoint) {
+ sourceID := firstNonEmpty(checkpoint.SourceID, checkpoint.Cursor)
+ const prefix = "moodle_course_"
+ if !strings.HasPrefix(sourceID, prefix) {
+ return
+ }
+ courseID, err := strconv.ParseInt(strings.TrimPrefix(sourceID, prefix), 10, 64)
+ if err != nil {
+ return
+ }
+ s.resumeAfterCourseID = courseID
+}
+
+type moodlePruneSession struct {
+ documents []SlimDocument
+ batchSize int
+ batchIndex int
+}
+
+// NextBatch returns the next Moodle prune snapshot batch.
+func (s *moodlePruneSession) NextBatch(ctx context.Context) (PruneBatch, error) {
+ if s.batchIndex >= len(s.documents) {
+ return PruneBatch{}, io.EOF
+ }
+ end := s.batchIndex + s.batchSize
+ if end > len(s.documents) {
+ end = len(s.documents)
+ }
+ documents := s.documents[s.batchIndex:end]
+ s.batchIndex = end
+ return PruneBatch{Documents: documents}, nil
+}
+
+// Close closes the Moodle prune session.
+func (s *moodlePruneSession) Close() error {
+ return nil
+}
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+func includeMoodleModule(request SyncRequest, ts int64) bool {
+ if request.FromBeginning {
+ return true
+ }
+ if ts <= 0 {
+ return false
+ }
+ updatedAt := time.Unix(ts, 0).UTC()
+ return !beforeOrAtWindowStart(updatedAt, request.WindowStart) && !afterWindowEnd(updatedAt, request.WindowEnd)
+}
+
+func moodleSlimDocIDForModule(module moodleModule) string {
+ switch module.ModName {
+ case "label", "url":
+ return ""
+ case "resource":
+ return fmt.Sprintf("moodle_resource_%d", module.ID)
+ case "forum":
+ return fmt.Sprintf("moodle_forum_%d", module.ID)
+ case "page":
+ return fmt.Sprintf("moodle_page_%d", module.ID)
+ case "book":
+ return fmt.Sprintf("moodle_book_%d", module.ID)
+ case "assign", "quiz":
+ return fmt.Sprintf("moodle_%s_%d", module.ModName, module.ID)
+ default:
+ return ""
+ }
+}
+
+func (m moodleModule) windowTimestamp() int64 {
+ ts := moodleMaxTimestamp(m.TimeCreated, m.TimeModified)
+ for _, content := range m.Contents {
+ ts = maxInt64(ts, derefInt64(content.TimeModified))
+ }
+ return ts
+}
+
+func moodleMaxTimestamp(values ...*int64) int64 {
+ var latest int64
+ for _, value := range values {
+ if value != nil && *value > latest {
+ latest = *value
+ }
+ }
+ return latest
+}
+
+func maxInt64(a, b int64) int64 {
+ if a > b {
+ return a
+ }
+ return b
+}
+
+func derefInt64(value *int64) int64 {
+ if value == nil {
+ return 0
+ }
+ return *value
+}
+
+func moodleTime(ts int64) time.Time {
+ if ts <= 0 {
+ return time.Time{}
+ }
+ return time.Unix(ts, 0).UTC()
+}
+
+func capitalizeMoodleType(mtype string) string {
+ if mtype == "" {
+ return mtype
+ }
+ return strings.ToUpper(mtype[:1]) + strings.ToLower(mtype[1:])
+}
+
+// ---------------------------------------------------------------------------
+// SSRF protection
+// ---------------------------------------------------------------------------
+
+func validateMoodleURLForSSRF(rawURL string) error {
+ parsed, err := url.Parse(rawURL)
+ if err != nil {
+ return &ConnectorValidationError{Message: "Moodle URL must include a hostname."}
+ }
+ if parsed.Scheme != "http" && parsed.Scheme != "https" {
+ return &ConnectorValidationError{Message: fmt.Sprintf("Unsupported URL scheme for Moodle connector: %q. Only http/https are allowed.", parsed.Scheme)}
+ }
+ hostname := parsed.Hostname()
+ if hostname == "" {
+ return &ConnectorValidationError{Message: "Moodle URL must include a hostname."}
+ }
+ if strings.EqualFold(hostname, "localhost") {
+ return &ConnectorValidationError{Message: fmt.Sprintf("Moodle URL hostname %q is not allowed (localhost is blocked).", hostname)}
+ }
+ addrs, err := net.LookupIP(hostname)
+ if err != nil {
+ // Resolution failure is not an SSRF condition by itself; the
+ // per-request check surfaces it if it matters.
+ return nil
+ }
+ if restAPISSRFAllowLoopback {
+ allLoopback := true
+ for _, addr := range addrs {
+ if !addr.IsLoopback() {
+ allLoopback = false
+ break
+ }
+ }
+ if allLoopback {
+ return nil
+ }
+ // Not all loopback — fall through to normal validation.
+ }
+ for _, addr := range addrs {
+ if !restAPIIPIsGlobal(restAPIEffectiveIP(addr)) {
+ return &ConnectorValidationError{Message: fmt.Sprintf(
+ "Moodle URL %q resolves to disallowed address %s (localhost, private, link-local, reserved, or multicast addresses are blocked).",
+ rawURL, addr)}
+ }
+ }
+ return nil
+}
+
+// moodleAssertURLSafe validates a per-request URL for SSRF, HTTPS, and
+// same-origin policy. It returns the hostname and first validated IP so the
+// caller can pin DNS for the actual dial.
+func moodleAssertURLSafe(ctx context.Context, rawURL, originURL string) (string, net.IP, error) {
+ parsed, err := url.Parse(rawURL)
+ if err != nil {
+ return "", nil, fmt.Errorf("Moodle URL is missing a host.")
+ }
+ if parsed.Scheme != "http" && parsed.Scheme != "https" {
+ return "", nil, fmt.Errorf("Disallowed URL scheme: %q. Only [http https] are allowed.", parsed.Scheme)
+ }
+ hostname := parsed.Hostname()
+ if hostname == "" {
+ return "", nil, fmt.Errorf("Moodle URL is missing a host.")
+ }
+ // Reject cross-origin targets: every request must go to the configured
+ // Moodle host.
+ if originURL != "" {
+ if origin, err := url.Parse(originURL); err == nil && origin.Hostname() != "" {
+ if !strings.EqualFold(hostname, origin.Hostname()) {
+ return "", nil, fmt.Errorf("Moodle URL host %q does not match configured origin %q.", hostname, origin.Hostname())
+ }
+ }
+ }
+ addrs, err := net.DefaultResolver.LookupIPAddr(ctx, hostname)
+ if err != nil {
+ return "", nil, fmt.Errorf("Could not resolve hostname %q: %v", hostname, err)
+ }
+ if len(addrs) == 0 {
+ return "", nil, fmt.Errorf("Hostname %q resolved to no addresses.", hostname)
+ }
+ if restAPISSRFAllowLoopback {
+ allLoopback := true
+ for _, addr := range addrs {
+ if !addr.IP.IsLoopback() {
+ allLoopback = false
+ break
+ }
+ }
+ if allLoopback {
+ return hostname, addrs[0].IP, nil
+ }
+ // Not all loopback — fall through to normal validation.
+ }
+ var first net.IP
+ for _, addr := range addrs {
+ if !restAPIIPIsGlobal(restAPIEffectiveIP(addr.IP)) {
+ return "", nil, fmt.Errorf("Moodle URL resolves to a non-public address (%s), which is not allowed.", addr.IP)
+ }
+ if first == nil {
+ first = addr.IP
+ }
+ }
+ if first == nil {
+ return "", nil, fmt.Errorf("Hostname %q resolved to no addresses.", hostname)
+ }
+ return hostname, first, nil
+}
+
+// moodleHTTPDo sends an HTTP request with DNS-pinned transport and manual
+// redirect handling. Each hop is independently validated for SSRF, HTTPS, and
+// same-origin policy, preventing DNS rebinding and redirect-based bypasses.
+func (c *MoodleConnector) moodleHTTPDo(ctx context.Context, method, rawURL string, body []byte, headers map[string]string) (*http.Response, error) {
+ currentURL := rawURL
+ currentMethod := method
+ currentBody := body
+ for hop := 0; hop <= moodleMaxRedirects; hop++ {
+ hostname, pinIP, err := moodleAssertURLSafe(ctx, currentURL, c.moodleURL)
+ if err != nil {
+ return nil, err
+ }
+ transport := newRestAPIPinnedTransport(hostname, pinIP)
+ client := &http.Client{
+ Transport: transport,
+ Timeout: moodleRequestTimeout,
+ CheckRedirect: func(req *http.Request, via []*http.Request) error {
+ return http.ErrUseLastResponse
+ },
+ }
+ var bodyReader io.Reader
+ if currentBody != nil {
+ bodyReader = bytes.NewReader(currentBody)
+ }
+ req, err := http.NewRequestWithContext(ctx, currentMethod, currentURL, bodyReader)
+ if err != nil {
+ transport.CloseIdleConnections()
+ return nil, err
+ }
+ for k, v := range headers {
+ req.Header.Set(k, v)
+ }
+ resp, err := client.Do(req)
+ if err != nil {
+ transport.CloseIdleConnections()
+ return nil, err
+ }
+ if !restAPIIsRedirect(resp.StatusCode) {
+ resp.Body = &restAPICloseIdleBody{body: resp.Body, transport: transport}
+ return resp, nil
+ }
+ location := resp.Header.Get("Location")
+ resp.Body.Close()
+ transport.CloseIdleConnections()
+ if location == "" {
+ return nil, fmt.Errorf("Moodle redirect with empty Location header")
+ }
+ nextURL, err := restAPIResolveURL(currentURL, location)
+ if err != nil {
+ return nil, err
+ }
+ currentURL = nextURL
+ if resp.StatusCode == http.StatusMovedPermanently || resp.StatusCode == http.StatusFound || resp.StatusCode == http.StatusSeeOther {
+ currentMethod = http.MethodGet
+ currentBody = nil
+ }
+ }
+ return nil, fmt.Errorf("Moodle request stopped after %d redirects", moodleMaxRedirects)
+}
+
+// moodleRedactedURL strips query and fragment from a URL so that tokens or
+// other sensitive query parameters are not persisted in document metadata.
+func moodleRedactedURL(rawURL string) string {
+ parsed, err := url.Parse(rawURL)
+ if err != nil {
+ return rawURL
+ }
+ parsed.RawQuery = ""
+ parsed.Fragment = ""
+ return parsed.String()
+}
diff --git a/internal/syncer/connector/moodle_test.go b/internal/syncer/connector/moodle_test.go
new file mode 100644
index 0000000000..bc80d12602
--- /dev/null
+++ b/internal/syncer/connector/moodle_test.go
@@ -0,0 +1,689 @@
+package connector
+
+import (
+ "context"
+ "errors"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+)
+
+func withMoodleTestHooks(t *testing.T) {
+ t.Helper()
+ origLoopback := restAPISSRFAllowLoopback
+ origTries := moodleRetryTries
+ origBaseDelay := moodleRetryBaseDelay
+ origBackoff := moodleRetryBackoff
+ restAPISSRFAllowLoopback = true
+ moodleRetryTries = 2
+ moodleRetryBaseDelay = time.Millisecond
+ moodleRetryBackoff = 2
+ t.Cleanup(func() {
+ restAPISSRFAllowLoopback = origLoopback
+ moodleRetryTries = origTries
+ moodleRetryBaseDelay = origBaseDelay
+ moodleRetryBackoff = origBackoff
+ })
+}
+
+func mustMoodleConnector(t *testing.T, moodleURL string) *MoodleConnector {
+ t.Helper()
+ connector, err := NewMoodleConnector(map[string]any{
+ "moodle_url": moodleURL,
+ "batch_size": 2,
+ "credentials": map[string]any{
+ "moodle_token": "testtoken",
+ },
+ })
+ if err != nil {
+ t.Fatalf("NewMoodleConnector: %v", err)
+ }
+ return connector
+}
+
+type moodleTestFixtures struct {
+ courses string
+ contents map[string]string
+ forumDiscussions map[string]string
+ files map[string]string
+ siteInfoBody string
+ courseListCount *atomic.Int64
+ forumCallCount *atomic.Int64
+}
+
+func newTestMoodleServer(t *testing.T, buildFixtures func(serverURL string) moodleTestFixtures) *httptest.Server {
+ t.Helper()
+ mux := http.NewServeMux()
+ server := httptest.NewServer(mux)
+ t.Cleanup(server.Close)
+ fixtures := buildFixtures(server.URL)
+ siteInfoBody := fixtures.siteInfoBody
+ if siteInfoBody == "" {
+ siteInfoBody = `{"sitename":"Test Moodle","siteurl":"https://example.com"}`
+ }
+ mux.HandleFunc("/webservice/rest/server.php", func(w http.ResponseWriter, r *http.Request) {
+ if err := r.ParseForm(); err != nil {
+ http.Error(w, "bad form", http.StatusBadRequest)
+ return
+ }
+ switch r.Form.Get("wsfunction") {
+ case "core_webservice_get_site_info":
+ w.Write([]byte(siteInfoBody))
+ case "core_course_get_courses":
+ if fixtures.courseListCount != nil {
+ fixtures.courseListCount.Add(1)
+ }
+ w.Write([]byte(fixtures.courses))
+ case "core_course_get_contents":
+ body, ok := fixtures.contents[r.Form.Get("courseid")]
+ if !ok {
+ http.NotFound(w, r)
+ return
+ }
+ w.Write([]byte(body))
+ case "mod_forum_get_forum_discussions":
+ if fixtures.forumCallCount != nil {
+ fixtures.forumCallCount.Add(1)
+ }
+ body, ok := fixtures.forumDiscussions[r.Form.Get("forumid")]
+ if !ok {
+ http.NotFound(w, r)
+ return
+ }
+ w.Write([]byte(body))
+ default:
+ http.Error(w, "unknown function", http.StatusBadRequest)
+ }
+ })
+ mux.HandleFunc("/pluginfile.php/", func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Query().Get("token") != "testtoken" {
+ http.Error(w, "missing token", http.StatusForbidden)
+ return
+ }
+ body, ok := fixtures.files[r.URL.Path]
+ if !ok {
+ http.NotFound(w, r)
+ return
+ }
+ w.Write([]byte(body))
+ })
+ return server
+}
+
+func fullSyncMoodleFixtures(serverURL string) moodleTestFixtures {
+ return moodleTestFixtures{
+ courses: `[{"id":1,"fullname":"Course One","shortname":"c1"},{"id":2,"fullname":"Course Two","shortname":"c2"}]`,
+ contents: map[string]string{
+ "1": `[{"id":11,"name":"Week 1","section":0,"modules":[` +
+ `{"id":100,"name":"Guide","modname":"resource","instance":7,"visible":1,"groupmode":0,"timecreated":1000000000,"timemodified":1700000000,"contents":[{"type":"file","filename":"guide.pdf","filepath":"/","filesize":123,"fileurl":"` + serverURL + `/pluginfile.php/guides/guide.pdf","mimetype":"application/pdf","timemodified":1700000000}]},` +
+ `{"id":101,"name":"About","modname":"page","instance":8,"visible":1,"groupmode":0,"timecreated":1000000000,"timemodified":1700000000,"contents":[{"type":"file","filename":"about.html","filepath":"/","filesize":456,"fileurl":"` + serverURL + `/pluginfile.php/pages/about.html","mimetype":"text/html","timemodified":1700000000}]},` +
+ `{"id":102,"name":"Announcement Label","modname":"label","instance":9,"visible":1,"groupmode":0,"timecreated":1000000000,"timemodified":1700000000,"contents":[]},` +
+ `{"id":103,"name":"External Link","modname":"url","instance":10,"visible":1,"groupmode":0,"timecreated":1000000000,"timemodified":1700000000,"contents":[]}` +
+ `]}]`,
+ "2": `[{"id":12,"name":"Week 2","section":1,"modules":[` +
+ `{"id":200,"name":"Announcements","modname":"forum","instance":50,"visible":1,"groupmode":0,"timecreated":1000000000,"timemodified":1700000000,"contents":[]},` +
+ `{"id":201,"name":"Assignment 1","modname":"assign","instance":51,"visible":1,"groupmode":0,"timecreated":1000000000,"timemodified":1700000000,"added":1600000000,"description":"<p>Write a <em>report</em>.</p>","contents":[]},` +
+ `{"id":202,"name":"Quiz 1","modname":"quiz","instance":52,"visible":1,"groupmode":0,"timecreated":1000000000,"timemodified":1700000000,"added":1600000000,"description":"<p>Answer the questions.</p>","contents":[]},` +
+ `{"id":203,"name":"Book Title","modname":"book","instance":53,"visible":1,"groupmode":0,"timecreated":1000000000,"timemodified":1700000000,"contents":[` +
+ `{"type":"file","filename":"index.html","filepath":"/chapter1/","filesize":100,"fileurl":"` + serverURL + `/pluginfile.php/books/chapter1/index.html","chapterid":1001,"title":"Chapter 1","timecreated":1000000000,"timemodified":1700000000},` +
+ `{"type":"file","filename":"index.html","filepath":"/chapter2/","filesize":100,"fileurl":"` + serverURL + `/pluginfile.php/books/chapter2/index.html","chapterid":1002,"title":"Chapter 2","timecreated":1000000000,"timemodified":1700000000},` +
+ `{"type":"file","filename":"cover.png","filepath":"/","filesize":50,"fileurl":"` + serverURL + `/pluginfile.php/books/cover.png"}` +
+ `]}` +
+ `]}]`,
+ },
+ forumDiscussions: map[string]string{
+ "50": `{"discussions":[{"id":10,"name":"Intro","message":"<p>Hello <strong>world</strong></p>","userid":5,"userfullname":"Alice","timecreated":100,"timemodified":200}]}`,
+ },
+ files: map[string]string{
+ "/pluginfile.php/guides/guide.pdf": "%PDF-1.4 fake",
+ "/pluginfile.php/pages/about.html": "<html><body><h1>About</h1><p>Hello</p></body></html>",
+ "/pluginfile.php/books/chapter1/index.html": "<h1>Chapter 1</h1><p>First</p>",
+ "/pluginfile.php/books/chapter2/index.html": "<h1>Chapter 2</h1><p>Second</p>",
+ },
+ }
+}
+
+func collectMoodleDocuments(t *testing.T, session SyncSession) []SourceDocument {
+ t.Helper()
+ var documents []SourceDocument
+ for {
+ batch, err := session.NextBatch(context.Background())
+ if errors.Is(err, io.EOF) {
+ return documents
+ }
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ documents = append(documents, batch.Documents...)
+ }
+}
+
+func TestMoodleConnectorValidate(t *testing.T) {
+ withMoodleTestHooks(t)
+ server := newTestMoodleServer(t, fullSyncMoodleFixtures)
+ connector := mustMoodleConnector(t, server.URL)
+ if err := connector.Validate(context.Background()); err != nil {
+ t.Fatalf("Validate: %v", err)
+ }
+}
+
+func TestMoodleConnectorValidateMissingURLAndToken(t *testing.T) {
+ withMoodleTestHooks(t)
+ connector, err := NewMoodleConnector(map[string]any{"credentials": map[string]any{}})
+ if err != nil {
+ t.Fatalf("NewMoodleConnector: %v", err)
+ }
+ if err := connector.Validate(context.Background()); err == nil {
+ t.Fatalf("expected missing URL validation error")
+ }
+ connector.moodleURL = "https://example.com"
+ err = connector.Validate(context.Background())
+ if err == nil {
+ t.Fatalf("expected missing token validation error")
+ }
+ var missingCred *ConnectorMissingCredentialError
+ if !errors.As(err, &missingCred) {
+ t.Fatalf("missing token error type = %T, want ConnectorMissingCredentialError", err)
+ }
+}
+
+func TestMoodleConnectorValidateInvalidToken(t *testing.T) {
+ withMoodleTestHooks(t)
+ server := newTestMoodleServer(t, func(serverURL string) moodleTestFixtures {
+ fixtures := fullSyncMoodleFixtures(serverURL)
+ fixtures.siteInfoBody = `{"exception":"moodle_exception","errorcode":"invalidtoken","message":"Invalid token - token has expired or is invalid"}`
+ return fixtures
+ })
+ connector := mustMoodleConnector(t, server.URL)
+ err := connector.Validate(context.Background())
+ if err == nil {
+ t.Fatalf("expected invalid token validation error")
+ }
+ var missingCred *ConnectorMissingCredentialError
+ if !errors.As(err, &missingCred) {
+ t.Fatalf("invalid token error type = %T, want ConnectorMissingCredentialError", err)
+ }
+}
+
+func TestMoodleConnectorOpenSyncBuildsAllModuleTypes(t *testing.T) {
+ withMoodleTestHooks(t)
+ server := newTestMoodleServer(t, fullSyncMoodleFixtures)
+ connector := mustMoodleConnector(t, server.URL)
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ documents := collectMoodleDocuments(t, session)
+ if len(documents) != 6 {
+ t.Fatalf("documents len = %d, want 6", len(documents))
+ }
+ byID := map[string]SourceDocument{}
+ for _, document := range documents {
+ byID[document.SourceID] = document
+ }
+ for _, sourceID := range []string{
+ "moodle_resource_100", "moodle_page_101", "moodle_forum_200",
+ "moodle_assign_201", "moodle_quiz_202", "moodle_book_203",
+ } {
+ if _, ok := byID[sourceID]; !ok {
+ t.Fatalf("missing document %q", sourceID)
+ }
+ }
+ resource := byID["moodle_resource_100"]
+ if resource.SemanticIdentifier != "Course One / Week 1 / guide.pdf" {
+ t.Fatalf("resource semantic = %q", resource.SemanticIdentifier)
+ }
+ if resource.Extension != ".pdf" {
+ t.Fatalf("resource extension = %q", resource.Extension)
+ }
+ if string(resource.Blob) != "%PDF-1.4 fake" {
+ t.Fatalf("resource blob = %q", resource.Blob)
+ }
+ if resource.Metadata["file_name"] != "guide.pdf" || resource.Metadata["file_size"] != int64(123) {
+ t.Fatalf("resource metadata = %+v", resource.Metadata)
+ }
+ if resource.Metadata["course_name"] != "Course One" || resource.Metadata["section_name"] != "Week 1" {
+ t.Fatalf("resource metadata = %+v", resource.Metadata)
+ }
+
+ page := byID["moodle_page_101"]
+ if page.Extension != ".html" || string(page.Blob) != "<html><body><h1>About</h1><p>Hello</p></body></html>" {
+ t.Fatalf("page = %+v", page)
+ }
+
+ forum := byID["moodle_forum_200"]
+ if forum.Extension != ".md" {
+ t.Fatalf("forum extension = %q", forum.Extension)
+ }
+ forumBlob := string(forum.Blob)
+ if !strings.Contains(forumBlob, "# Announcements") || !strings.Contains(forumBlob, "## Intro") || !strings.Contains(forumBlob, "Hello **world**") {
+ t.Fatalf("forum blob = %q", forumBlob)
+ }
+ if forum.Metadata["discussion_count"] != 1 {
+ t.Fatalf("forum metadata = %+v", forum.Metadata)
+ }
+ forumID, ok := forum.Metadata["forum_id"].(*int64)
+ if !ok || forumID == nil || *forumID != 50 {
+ t.Fatalf("forum metadata = %+v", forum.Metadata)
+ }
+
+ assign := byID["moodle_assign_201"]
+ if !strings.Contains(string(assign.Blob), "**Type:** Assign") || !strings.Contains(string(assign.Blob), "Write a *report*.") {
+ t.Fatalf("assign blob = %q", assign.Blob)
+ }
+ if assign.Metadata["description"] != "<p>Write a <em>report</em>.</p>" {
+ t.Fatalf("assign metadata = %+v", assign.Metadata)
+ }
+
+ quiz := byID["moodle_quiz_202"]
+ if !strings.Contains(string(quiz.Blob), "**Type:** Quiz") {
+ t.Fatalf("quiz blob = %q", quiz.Blob)
+ }
+
+ book := byID["moodle_book_203"]
+ bookBlob := string(book.Blob)
+ if !strings.Contains(bookBlob, "# Book Title") || !strings.Contains(bookBlob, "# Chapter 1") || !strings.Contains(bookBlob, "# Chapter 2") {
+ t.Fatalf("book blob = %q", bookBlob)
+ }
+ if book.Metadata["chapter_count"] != 2 {
+ t.Fatalf("book metadata = %+v", book.Metadata)
+ }
+
+ if _, ok := byID["moodle_label_102"]; ok {
+ t.Fatalf("label module must be skipped")
+ }
+ if _, ok := byID["moodle_url_103"]; ok {
+ t.Fatalf("url module must be skipped")
+ }
+}
+
+func TestMoodleConnectorOpenSyncWindowFilter(t *testing.T) {
+ withMoodleTestHooks(t)
+ server := newTestMoodleServer(t, func(serverURL string) moodleTestFixtures {
+ return moodleTestFixtures{
+ courses: `[{"id":1,"fullname":"Course One","shortname":"c1"}]`,
+ contents: map[string]string{
+ "1": `[{"id":11,"name":"Week 1","section":0,"modules":[` +
+ `{"id":100,"name":"Before","modname":"resource","instance":7,"visible":1,"groupmode":0,"timemodified":1704067200,"contents":[{"type":"file","filename":"before.pdf","filepath":"/","fileurl":"` + serverURL + `/pluginfile.php/before.pdf","timemodified":1704067200}]},` +
+ `{"id":101,"name":"Inside","modname":"resource","instance":8,"visible":1,"groupmode":0,"timemodified":1704153600,"contents":[{"type":"file","filename":"inside.pdf","filepath":"/","fileurl":"` + serverURL + `/pluginfile.php/inside.pdf","timemodified":1704153600}]},` +
+ `{"id":102,"name":"After","modname":"resource","instance":9,"visible":1,"groupmode":0,"timemodified":1709251200,"contents":[{"type":"file","filename":"after.pdf","filepath":"/","fileurl":"` + serverURL + `/pluginfile.php/after.pdf","timemodified":1709251200}]}` +
+ `]}]`,
+ },
+ files: map[string]string{
+ "/pluginfile.php/before.pdf": "before",
+ "/pluginfile.php/inside.pdf": "inside",
+ "/pluginfile.php/after.pdf": "after",
+ },
+ }
+ })
+ connector := mustMoodleConnector(t, server.URL)
+ start := mustTime(t, "2024-01-01T00:00:00Z")
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ WindowStart: &start,
+ WindowEnd: mustTime(t, "2024-02-01T00:00:00Z"),
+ })
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ documents := collectMoodleDocuments(t, session)
+ if len(documents) != 1 {
+ t.Fatalf("documents len = %d, want 1", len(documents))
+ }
+ if documents[0].SourceID != "moodle_resource_101" {
+ t.Fatalf("window filtered source id = %q, want moodle_resource_101", documents[0].SourceID)
+ }
+}
+
+func TestMoodleConnectorOpenSyncResumesFromCheckpoint(t *testing.T) {
+ withMoodleTestHooks(t)
+ server := newTestMoodleServer(t, fullSyncMoodleFixtures)
+ connector := mustMoodleConnector(t, server.URL)
+ updatedAt := mustTime(t, "2023-11-14T22:13:20Z")
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ FromBeginning: true,
+ Resume: &SyncCheckpoint{
+ Cursor: "moodle_course_1",
+ SourceID: "moodle_course_1",
+ UpdatedAt: &updatedAt,
+ },
+ })
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ documents := collectMoodleDocuments(t, session)
+ if len(documents) != 4 {
+ t.Fatalf("documents len = %d, want 4 (course 1 skipped)", len(documents))
+ }
+ for _, document := range documents {
+ if strings.HasPrefix(document.SourceID, "moodle_resource_") || strings.HasPrefix(document.SourceID, "moodle_page_") {
+ t.Fatalf("resume must skip course 1, got %q", document.SourceID)
+ }
+ }
+}
+
+func TestMoodleConnectorOpenSyncDeferredCoursesUntilNextBatch(t *testing.T) {
+ withMoodleTestHooks(t)
+ var courseListCount atomic.Int64
+ server := newTestMoodleServer(t, func(serverURL string) moodleTestFixtures {
+ fixtures := fullSyncMoodleFixtures(serverURL)
+ fixtures.courseListCount = &courseListCount
+ return fixtures
+ })
+ connector := mustMoodleConnector(t, server.URL)
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ if courseListCount.Load() != 0 {
+ t.Fatalf("course list calls after OpenSync = %d, want 0", courseListCount.Load())
+ }
+ if _, err := session.NextBatch(context.Background()); err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if courseListCount.Load() != 1 {
+ t.Fatalf("course list calls after NextBatch = %d, want 1", courseListCount.Load())
+ }
+}
+
+func TestMoodleConnectorOpenPruneReturnsSlimSnapshot(t *testing.T) {
+ withMoodleTestHooks(t)
+ server := newTestMoodleServer(t, fullSyncMoodleFixtures)
+ connector := mustMoodleConnector(t, server.URL)
+ session, err := connector.OpenPrune(context.Background(), PruneRequest{})
+ if err != nil {
+ t.Fatalf("OpenPrune: %v", err)
+ }
+ expected := map[string]bool{
+ "moodle_resource_100": false,
+ "moodle_page_101": false,
+ "moodle_forum_200": false,
+ "moodle_assign_201": false,
+ "moodle_quiz_202": false,
+ "moodle_book_203": false,
+ }
+ var total int
+ for {
+ batch, err := session.NextBatch(context.Background())
+ if errors.Is(err, io.EOF) {
+ break
+ }
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ for _, document := range batch.Documents {
+ total++
+ if _, ok := expected[document.SourceID]; !ok {
+ t.Fatalf("unexpected slim document %q", document.SourceID)
+ }
+ expected[document.SourceID] = true
+ }
+ }
+ if total != 6 {
+ t.Fatalf("slim documents total = %d, want 6", total)
+ }
+ for sourceID, seen := range expected {
+ if !seen {
+ t.Fatalf("missing slim document %q", sourceID)
+ }
+ }
+}
+
+func TestMoodleConnectorOpenSyncCheckpointAdvancesPerCourse(t *testing.T) {
+ withMoodleTestHooks(t)
+ server := newTestMoodleServer(t, fullSyncMoodleFixtures)
+ connector := mustMoodleConnector(t, server.URL)
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ var checkpoints []string
+ for {
+ batch, err := session.NextBatch(context.Background())
+ if errors.Is(err, io.EOF) {
+ break
+ }
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if batch.Checkpoint != nil {
+ checkpoints = append(checkpoints, batch.Checkpoint.Cursor)
+ }
+ }
+ // Course 1 has one batch (2 docs) and course 2 has two batches (4 docs);
+ // the checkpoint advances only on the last batch of each course.
+ if len(checkpoints) != 2 || checkpoints[0] != "moodle_course_1" || checkpoints[1] != "moodle_course_2" {
+ t.Fatalf("checkpoints = %v, want [moodle_course_1 moodle_course_2]", checkpoints)
+ }
+}
+
+func TestMoodleHTMLToMarkdown(t *testing.T) {
+ out, err := moodleHTMLToMarkdown("<p>Hello <strong>world</strong> and <em>you</em></p>")
+ if err != nil {
+ t.Fatalf("moodleHTMLToMarkdown: %v", err)
+ }
+ if !strings.Contains(out, "**world**") || !strings.Contains(out, "*you*") {
+ t.Fatalf("markdown = %q", out)
+ }
+ out, err = moodleHTMLToMarkdown("<h1>Title</h1><p>Body</p>")
+ if err != nil {
+ t.Fatalf("moodleHTMLToMarkdown: %v", err)
+ }
+ if !strings.Contains(out, "# Title") || !strings.Contains(out, "Body") {
+ t.Fatalf("markdown = %q", out)
+ }
+}
+
+func TestAddMoodleToken(t *testing.T) {
+ if got := addMoodleToken("https://example.com/file.pdf", "tok"); got != "https://example.com/file.pdf?token=tok" {
+ t.Fatalf("plain url = %q", got)
+ }
+ if got := addMoodleToken("https://example.com/file.pdf?x=1", "tok"); got != "https://example.com/file.pdf?x=1&token=tok" {
+ t.Fatalf("query url = %q", got)
+ }
+ if got := addMoodleToken("https://example.com/file.pdf?token=old", "tok"); got != "https://example.com/file.pdf?token=old" {
+ t.Fatalf("existing token url = %q", got)
+ }
+}
+
+func TestValidateMoodleURLForSSRF(t *testing.T) {
+ withMoodleTestHooks(t)
+ origLoopback := restAPISSRFAllowLoopback
+ restAPISSRFAllowLoopback = false
+ defer func() { restAPISSRFAllowLoopback = origLoopback }()
+
+ if err := validateMoodleURLForSSRF("ftp://example.com"); err == nil {
+ t.Fatalf("expected scheme rejection")
+ }
+ if err := validateMoodleURLForSSRF("localhost"); err == nil {
+ t.Fatalf("expected missing host rejection")
+ }
+ if err := validateMoodleURLForSSRF("https://localhost/moodle"); err == nil {
+ t.Fatalf("expected localhost rejection")
+ }
+ if err := validateMoodleURLForSSRF("https://127.0.0.1/moodle"); err == nil {
+ t.Fatalf("expected loopback rejection")
+ }
+}
+
+func TestMoodleConnectorFileDownloadRequiresToken(t *testing.T) {
+ withMoodleTestHooks(t)
+ server := newTestMoodleServer(t, fullSyncMoodleFixtures)
+ connector := mustMoodleConnector(t, server.URL)
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ documents := collectMoodleDocuments(t, session)
+ for _, document := range documents {
+ if strings.HasPrefix(document.SourceID, "moodle_resource_") && string(document.Blob) != "%PDF-1.4 fake" {
+ t.Fatalf("resource blob = %q", document.Blob)
+ }
+ }
+}
+
+func TestMoodleConnectorValidateConnectorSetting(t *testing.T) {
+ withMoodleTestHooks(t)
+ server := newTestMoodleServer(t, fullSyncMoodleFixtures)
+ connector := mustMoodleConnector(t, server.URL)
+ if err := connector.ValidateConnectorSetting(context.Background(), nil); err != nil {
+ t.Fatalf("ValidateConnectorSetting: %v", err)
+ }
+}
+
+func TestMoodleRedactedURL(t *testing.T) {
+ original := "https://moodle.example.com/pluginfile.php/guides/guide.pdf?token=secret&forcedownload=1"
+ redacted := moodleRedactedURL(original)
+ if strings.Contains(redacted, "token") || strings.Contains(redacted, "?") {
+ t.Fatalf("redacted URL still contains query params: %q", redacted)
+ }
+ want := "https://moodle.example.com/pluginfile.php/guides/guide.pdf"
+ if redacted != want {
+ t.Fatalf("redacted URL = %q, want %q", redacted, want)
+ }
+}
+
+func TestMoodleConnectorDownloadErrorFailsCourse(t *testing.T) {
+ withMoodleTestHooks(t)
+ server := newTestMoodleServer(t, func(serverURL string) moodleTestFixtures {
+ return moodleTestFixtures{
+ courses: `[{"id":1,"fullname":"Course One","shortname":"c1"}]`,
+ contents: map[string]string{
+ "1": `[{"id":11,"name":"Week 1","section":0,"modules":[` +
+ `{"id":100,"name":"Guide","modname":"resource","instance":7,"visible":1,"groupmode":0,"timecreated":1000000000,"timemodified":1700000000,"contents":[{"type":"file","filename":"guide.pdf","filepath":"/","filesize":123,"fileurl":"` + serverURL + `/pluginfile.php/nonexistent/guide.pdf","mimetype":"application/pdf","timemodified":1700000000}]}` +
+ `]}]`,
+ },
+ files: map[string]string{},
+ }
+ })
+ connector := mustMoodleConnector(t, server.URL)
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ _, err = session.NextBatch(context.Background())
+ if err == nil {
+ t.Fatalf("expected NextBatch to fail when a download fails")
+ }
+}
+
+func TestMoodleConnectorMetadataRedactsFileURL(t *testing.T) {
+ withMoodleTestHooks(t)
+ server := newTestMoodleServer(t, func(serverURL string) moodleTestFixtures {
+ return moodleTestFixtures{
+ courses: `[{"id":1,"fullname":"Course One","shortname":"c1"}]`,
+ contents: map[string]string{
+ "1": `[{"id":11,"name":"Week 1","section":0,"modules":[` +
+ `{"id":100,"name":"Guide","modname":"resource","instance":7,"visible":1,"groupmode":0,"timecreated":1000000000,"timemodified":1700000000,"contents":[{"type":"file","filename":"guide.pdf","filepath":"/","filesize":123,"fileurl":"` + serverURL + `/pluginfile.php/guides/guide.pdf?forcedownload=1","mimetype":"application/pdf","timemodified":1700000000}]}` +
+ `]}]`,
+ },
+ files: map[string]string{
+ "/pluginfile.php/guides/guide.pdf": "%PDF-1.4 fake",
+ },
+ }
+ })
+ connector := mustMoodleConnector(t, server.URL)
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ documents := collectMoodleDocuments(t, session)
+ for _, doc := range documents {
+ if doc.SourceID != "moodle_resource_100" {
+ continue
+ }
+ fileURL, ok := doc.Metadata["file_url"].(string)
+ if !ok {
+ t.Fatalf("file_url not a string: %T", doc.Metadata["file_url"])
+ }
+ if strings.Contains(fileURL, "?") {
+ t.Fatalf("file_url should be redacted (no query): %q", fileURL)
+ }
+ }
+}
+
+func TestMoodleAssertURLSafeRejectsCrossOrigin(t *testing.T) {
+ origLoopback := restAPISSRFAllowLoopback
+ restAPISSRFAllowLoopback = false
+ defer func() { restAPISSRFAllowLoopback = origLoopback }()
+
+ _, _, err := moodleAssertURLSafe(context.Background(), "https://evil.com/api", "https://example.com")
+ if err == nil {
+ t.Fatalf("expected cross-origin rejection")
+ }
+}
+
+func TestMoodleAssertURLSafeLoopbackAllAddresses(t *testing.T) {
+ origLoopback := restAPISSRFAllowLoopback
+ restAPISSRFAllowLoopback = true
+ defer func() { restAPISSRFAllowLoopback = origLoopback }()
+
+ // All loopback → allowed.
+ _, _, err := moodleAssertURLSafe(context.Background(), "http://127.0.0.1/path", "http://127.0.0.1")
+ if err != nil {
+ t.Fatalf("expected loopback to be allowed: %v", err)
+ }
+
+ // Not all loopback (private address) → rejected even with loopback flag.
+ _, _, err = moodleAssertURLSafe(context.Background(), "http://10.0.0.1/path", "http://10.0.0.1")
+ if err == nil {
+ t.Fatalf("expected non-loopback private address to be rejected even with loopback flag")
+ }
+}
+
+func TestMoodleConnectorRESTResponseSizeCap(t *testing.T) {
+ withMoodleTestHooks(t)
+ origMax := moodleMaxResponseSize
+ moodleMaxResponseSize = 10
+ defer func() { moodleMaxResponseSize = origMax }()
+
+ server := newTestMoodleServer(t, func(serverURL string) moodleTestFixtures {
+ fixtures := fullSyncMoodleFixtures(serverURL)
+ fixtures.siteInfoBody = `{"sitename":"Test Moodle with a very long name that exceeds the cap"}`
+ return fixtures
+ })
+ connector := mustMoodleConnector(t, server.URL)
+ err := connector.Validate(context.Background())
+ if err == nil {
+ t.Fatalf("expected oversize response error")
+ }
+ if !strings.Contains(err.Error(), "exceeds maximum size") {
+ t.Fatalf("expected oversize error, got: %v", err)
+ }
+}
+
+func TestMoodleConnectorDownloadSizeCap(t *testing.T) {
+ withMoodleTestHooks(t)
+ origMax := moodleMaxDownloadSize
+ moodleMaxDownloadSize = 10
+ defer func() { moodleMaxDownloadSize = origMax }()
+
+ server := newTestMoodleServer(t, func(serverURL string) moodleTestFixtures {
+ return moodleTestFixtures{
+ courses: `[{"id":1,"fullname":"Course One","shortname":"c1"}]`,
+ contents: map[string]string{
+ "1": `[{"id":11,"name":"Week 1","section":0,"modules":[` +
+ `{"id":100,"name":"Guide","modname":"resource","instance":7,"visible":1,"groupmode":0,"timecreated":1000000000,"timemodified":1700000000,"contents":[{"type":"file","filename":"guide.pdf","filepath":"/","filesize":123,"fileurl":"` + serverURL + `/pluginfile.php/guides/guide.pdf","mimetype":"application/pdf","timemodified":1700000000}]}` +
+ `]}]`,
+ },
+ files: map[string]string{
+ "/pluginfile.php/guides/guide.pdf": "this content is longer than ten bytes",
+ },
+ }
+ })
+ connector := mustMoodleConnector(t, server.URL)
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ _, err = session.NextBatch(context.Background())
+ if err == nil {
+ t.Fatalf("expected oversize download error")
+ }
+ if !strings.Contains(err.Error(), "exceeds maximum size") {
+ t.Fatalf("expected oversize error, got: %v", err)
+ }
+}