RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(syncer): add Moodle connector with checkpoint resume (#18554)
mkaaadcab70fdac304
diff --git a/go.mod b/go.modindex d57708d43f..64284133f0 100644--- a/go.mod+++ b/go.mod@@ -6,6 +6,7 @@ require (cloud.google.com/go/storage v1.63.0github.com/AkmalOt/gomsg v0.0.0-20260407083308-985c3a1a76b7github.com/DATA-DOG/go-sqlmock v1.5.2+ github.com/JohannesKaufmann/html-to-markdown v1.6.0github.com/LuxorLabs/tenki-sdk-go/sandbox v0.7.0github.com/alibabacloud-go/agentrun-20250910/v5 v5.8.4github.com/alibabacloud-go/darabonba-openapi/v2 v2.2.1@@ -58,7 +59,7 @@ require (github.com/xuri/excelize/v2 v2.11.0github.com/yfedoseev/office_oxide/go v0.1.8github.com/yfedoseev/pdf_oxide/go v0.3.67- github.com/yuin/goldmark v1.4.13+ github.com/yuin/goldmark v1.7.1github.com/zeebo/xxh3 v1.0.2go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0go.opentelemetry.io/otel v1.44.0@@ -80,7 +81,11 @@ require (gorm.io/gorm v1.25.7)-require github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 // indirect+require (+ github.com/PuerkitoBio/goquery v1.9.2 // indirect+ github.com/andybalholm/cascadia v1.3.2 // indirect+ github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 // indirect+)require (buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20260415201107-50325440f8f2.1 // indirectdiff --git a/go.sum b/go.sumindex 9dd5340ba8..dfaeba4f1b 100644--- a/go.sum+++ b/go.sum@@ -39,8 +39,12 @@ github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.57.0/go.mod h1:dzcEjy1WJ0Q4u9twNR3LcLhNoYMRCrMCMafpxa0TjPQ=github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0 h1:RoO5+d7uCmDqovLrHCr2/BuViUXvdcrNxyNM1pN9dDQ=github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0/go.mod h1:YqwkQPrWSC7+byyc1VlKbWLBF5JsW5IoL6xUkemYSXk=+github.com/JohannesKaufmann/html-to-markdown v1.6.0 h1:04VXMiE50YYfCfLboJCLcgqF5x+rHJnb1ssNmqpLH/k=+github.com/JohannesKaufmann/html-to-markdown v1.6.0/go.mod h1:NUI78lGg/a7vpEJTz/0uOcYMaibytE4BUOQS8k78yPQ=github.com/LuxorLabs/tenki-sdk-go/sandbox v0.7.0 h1:PBQ+ad8fZiLH/1ij1hoynR9aAPVmI3EzT7DiLNmUdBM=github.com/LuxorLabs/tenki-sdk-go/sandbox v0.7.0/go.mod h1:rBypesed6hSrj+OgpJwyMVBF76PaQ1iOqc2RGUL4RmI=+github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE=+github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk=github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk=github.com/airbrake/gobrake v3.6.1+incompatible/go.mod h1:wM4gu3Cn0W0K7GUuVWnlXZU11AGBXMILnrdOU8Kn00o=github.com/alibabacloud-go/agentrun-20250910/v5 v5.8.4 h1:hiAsm9pz6aICOPLI1FC54vga10xwd/XxfNj06ow5jVM=@@ -91,6 +95,8 @@ github.com/aliyun/credentials-go v1.3.1/go.mod h1:8jKYhQuDawt8x2+fusqa1Y6mPxemTsgithub.com/aliyun/credentials-go v1.3.6/go.mod h1:1LxUuX7L5YrZUWzBrRyk0SwSdH4OmPrib8NVePL3fxM=github.com/aliyun/credentials-go v1.4.5 h1:O76WYKgdy1oQYYiJkERjlA2dxGuvLRrzuO2ScrtGWSk=github.com/aliyun/credentials-go v1.4.5/go.mod h1:Jm6d+xIgwJVLVWT561vy67ZRP4lPTQxMbEYRuT2Ti1U=+github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=+github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=github.com/apache/thrift v0.23.0 h1:wKR6YnefQSEnxpEfmgTPuJibNG4bF0p2TK34tHLWi3s=@@ -458,6 +464,11 @@ github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6kegithub.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4=github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=+github.com/sebdah/goldie/v2 v2.5.3 h1:9ES/mNN+HNUbNWpVAlrzuZ7jE+Nrczbj8uFRjM7624Y=+github.com/sebdah/goldie/v2 v2.5.3/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI=+github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=+github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=+github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=github.com/signintech/gopdf v0.36.1 h1:cGpvEKvvqCV+ZXB9R2SQoWgouW91JpwsgoQEhLxIdp0=github.com/signintech/gopdf v0.36.1/go.mod h1:d23eO35GpEliSrF22eJ4bsM3wVeQJTjXTHq5x5qGKjA=github.com/siongui/gojianfan v0.0.0-20210926212422-2f175ac615de h1:1/P9CcR8iENN9ybbSRWohRd3rsPp9tEWlTS/7ygvjHE=@@ -495,6 +506,7 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEgithub.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=@@ -549,8 +561,9 @@ github.com/yfedoseev/pdf_oxide/go v0.3.67/go.mod h1:QbJ/nLbez0al2EnqEdEPIlGflFprgithub.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=github.com/yuin/goldmark v1.1.30/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=-github.com/yuin/goldmark v1.4.13 h1:fVcFKWvrslecOb/tg+Cc05dkeYx540o0FuFt3nUVDoE=github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=+github.com/yuin/goldmark v1.7.1 h1:3bajkSilaCbjdKVsKdZjZCLBNPL9pYzrCakKaf4U49U=+github.com/yuin/goldmark v1.7.1/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=@@ -610,6 +623,7 @@ golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQfgolang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=+golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto=@@ -646,12 +660,14 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxgolang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=+golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=+golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=@@ -688,12 +704,14 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBcgolang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=+golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=+golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=@@ -702,12 +720,14 @@ golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctgolang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=+golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY=golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=+golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0=golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc=@@ -776,6 +796,7 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=@@ -789,6 +810,7 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDgopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=diff --git a/internal/syncer/connector/builtin.go b/internal/syncer/connector/builtin.goindex 1d5e9f2278..c15e0c04bf 100644--- a/internal/syncer/connector/builtin.go+++ b/internal/syncer/connector/builtin.go@@ -42,6 +42,7 @@ func RegisterBuiltIns(registry *Registry) {registerBuiltIn(registry, "outlook", NewOutlookConnector)registerBuiltIn(registry, "notion", NewNotionConnector)registerBuiltIn(registry, "rest_api", NewRestAPIConnector)+ registerBuiltIn(registry, "moodle", NewMoodleConnector)registerBuiltIn(registry, "mysql", NewMySQLConnector)registerBuiltIn(registry, "postgresql", NewPostgreSQLConnector)registerBuiltIn(registry, "discord", NewDiscordConnector)diff --git a/internal/syncer/connector/moodle.go b/internal/syncer/connector/moodle.gonew file mode 100644index 0000000000..0deafe3038--- /dev/null+++ b/internal/syncer/connector/moodle.go@@ -0,0 +1,1041 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package connector++import (+ "bytes"+ "context"+ "encoding/json"+ "fmt"+ "io"+ "net"+ "net/http"+ "net/url"+ "path"+ "path/filepath"+ "sort"+ "strconv"+ "strings"+ "time"++ md "github.com/JohannesKaufmann/html-to-markdown"+)++const (+ defaultMoodleBatchSize = 2+ moodleRequestTimeout = 60 * time.Second+ moodleMaxRedirects = 10+ moodleRESTPath = "/webservice/rest/server.php"+)++// Retry/backoff knobs for Moodle web service calls. They are package+// variables so tests can shrink the delays.+var (+ moodleRetryTries = 3+ moodleRetryBaseDelay = time.Second+ moodleRetryBackoff = 2+)+var (+ // moodleMaxResponseSize caps REST JSON responses. Package variable so+ // tests can shrink it.+ moodleMaxResponseSize int64 = 32 * 1024 * 1024+ // moodleMaxDownloadSize caps file downloads. Package variable so+ // tests can shrink it.+ moodleMaxDownloadSize int64 = 100 * 1024 * 1024+)++// MoodleConnector reads course content from a Moodle LMS through its REST web+// service API.+type MoodleConnector struct {+ moodleURL string+ token string+ batchSize int++ // restCall is a test hook that replaces the live web service transport.+ restCall func(ctx context.Context, function string, params map[string]any, out any) error+}++// NewMoodleConnector creates a Moodle connector from connector config.+func NewMoodleConnector(config map[string]any) (*MoodleConnector, error) {+ credentials := configAnyMap(config["credentials"])+ batchSize := configInt(firstNonEmpty(stringConfig(config["sync_batch_size"]), stringConfig(config["batch_size"])), defaultMoodleBatchSize)+ return &MoodleConnector{+ moodleURL: strings.TrimRight(strings.TrimSpace(stringConfig(config["moodle_url"])), "/"),+ token: strings.TrimSpace(stringConfig(credentials["moodle_token"])),+ batchSize: batchSize,+ }, nil+}++// Validate validates Moodle connector settings, credentials, and connectivity.+func (c *MoodleConnector) Validate(ctx context.Context) error {+ if c == nil {+ return fmt.Errorf("Moodle connector is nil")+ }+ if c.moodleURL == "" {+ return fmt.Errorf("No Moodle URL was provided in connector settings.")+ }+ if c.token == "" {+ return &ConnectorMissingCredentialError{Message: "Moodle API token is required"}+ }+ if c.batchSize <= 0 {+ return fmt.Errorf("batch_size must be a positive integer")+ }+ if err := validateMoodleURLForSSRF(c.moodleURL); err != nil {+ return err+ }+ var siteInfo moodleSiteInfo+ if err := c.callREST(ctx, "core_webservice_get_site_info", nil, &siteInfo); err != nil {+ return err+ }+ if siteInfo.SiteName == "" {+ return &ConnectorValidationError{Message: "Invalid Moodle API response"}+ }+ return nil+}++// ValidateConnectorSetting validates Moodle settings from an unsaved config.+func (c *MoodleConnector) ValidateConnectorSetting(ctx context.Context, request map[string]any) error {+ ctx, cancel := context.WithTimeout(ctx, connectorSettingValidationTimeout)+ defer cancel()+ return c.Validate(ctx)+}++// OpenSync opens one Moodle sync session.+func (c *MoodleConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {+ if err := c.Validate(ctx); err != nil {+ return nil, err+ }+ session := &moodleSyncSession{+ connector: c,+ request: request,+ batchSize: c.batchSize,+ }+ if request.Resume != nil {+ session.applyResume(request.Resume)+ }+ return session, nil+}++// OpenPrune opens one complete Moodle prune snapshot session.+func (c *MoodleConnector) OpenPrune(ctx context.Context, request PruneRequest) (PruneSession, error) {+ if err := c.Validate(ctx); err != nil {+ return nil, err+ }+ courses, err := c.getCourses(ctx)+ if err != nil {+ return nil, err+ }+ var documents []SlimDocument+ for _, course := range courses {+ sections, err := c.getCourseContents(ctx, course.ID)+ if err != nil {+ return nil, err+ }+ for _, section := range sections {+ for _, module := range section.Modules {+ slimID := moodleSlimDocIDForModule(module)+ if slimID == "" {+ continue+ }+ documents = append(documents, SlimDocument{SourceID: slimID})+ }+ }+ }+ return &moodlePruneSession{documents: documents, batchSize: c.batchSize}, nil+}++// ---------------------------------------------------------------------------+// Moodle REST web service client+// ---------------------------------------------------------------------------++type moodleSiteInfo struct {+ SiteName string `json:"sitename"`+}++type moodleCourse struct {+ ID int64 `json:"id"`+ FullName string `json:"fullname"`+ ShortName string `json:"shortname"`+}++type moodleSection struct {+ ID int64 `json:"id"`+ Name string `json:"name"`+ Section *int64 `json:"section"`+ Modules []moodleModule `json:"modules"`+}++type moodleModule struct {+ ID int64 `json:"id"`+ Name string `json:"name"`+ ModName string `json:"modname"`+ Instance *int64 `json:"instance"`+ Description string `json:"description"`+ Visible *int64 `json:"visible"`+ GroupMode *int64 `json:"groupmode"`+ TimeCreated *int64 `json:"timecreated"`+ TimeModified *int64 `json:"timemodified"`+ Added *int64 `json:"added"`+ Contents []moodleContent `json:"contents"`+}++type moodleContent struct {+ Filename string `json:"filename"`+ FileURL string `json:"fileurl"`+ FileSize *int64 `json:"filesize"`+ MIMEType string `json:"mimetype"`+ ChapterID *int64 `json:"chapterid"`+ Title string `json:"title"`+ TimeCreated *int64 `json:"timecreated"`+ TimeModified *int64 `json:"timemodified"`+}++type moodleDiscussion struct {+ ID int64 `json:"id"`+ Name string `json:"name"`+ Message string `json:"message"`+ UserID *int64 `json:"userid"`+ UserFullName string `json:"userfullname"`+ TimeCreated *int64 `json:"timecreated"`+ TimeModified *int64 `json:"timemodified"`+}++func (c *MoodleConnector) callREST(ctx context.Context, function string, params map[string]any, out any) error {+ if c.restCall != nil {+ return c.restCall(ctx, function, params, out)+ }+ endpoint := c.moodleURL + moodleRESTPath+ form := url.Values{}+ form.Set("wstoken", c.token)+ form.Set("wsfunction", function)+ form.Set("moodlewsrestformat", "json")+ for key, value := range params {+ switch typed := value.(type) {+ case int64:+ form.Set(key, strconv.FormatInt(typed, 10))+ case int:+ form.Set(key, strconv.Itoa(typed))+ case string:+ form.Set(key, typed)+ }+ }+ resp, err := c.moodleHTTPDo(ctx, http.MethodPost, endpoint, []byte(form.Encode()), map[string]string{"Content-Type": "application/x-www-form-urlencoded"})+ if err != nil {+ return err+ }+ defer resp.Body.Close()+ body, err := io.ReadAll(io.LimitReader(resp.Body, moodleMaxResponseSize+1))+ if err != nil {+ return err+ }+ if int64(len(body)) > moodleMaxResponseSize {+ return fmt.Errorf("Moodle REST response exceeds maximum size of %d bytes", moodleMaxResponseSize)+ }+ if resp.StatusCode != http.StatusOK {+ return moodleRESTError(body, resp.StatusCode)+ }+ var probe map[string]json.RawMessage+ if err := json.Unmarshal(body, &probe); err == nil {+ if _, ok := probe["exception"]; ok {+ return moodleRESTError(body, resp.StatusCode)+ }+ }+ return json.Unmarshal(body, out)+}++func moodleRESTError(body []byte, statusCode int) error {+ var apiError struct {+ ErrorCode string `json:"errorcode"`+ Message string `json:"message"`+ }+ if err := json.Unmarshal(body, &apiError); err == nil && apiError.Message != "" {+ return classifyMoodleAPIError(apiError.ErrorCode, apiError.Message)+ }+ return fmt.Errorf("Moodle web service request failed with status %d", statusCode)+}++func classifyMoodleAPIError(errorCode, message string) error {+ combined := strings.ToLower(errorCode + " " + message)+ switch {+ case strings.Contains(combined, "invalidtoken"):+ return &ConnectorMissingCredentialError{Message: "Moodle token is invalid or expired"}+ case strings.Contains(combined, "accessexception"):+ return &ConnectorValidationError{Message: "Insufficient permissions. Ensure web services are enabled and permissions are correct."}+ default:+ return &ConnectorValidationError{Message: fmt.Sprintf("Moodle validation error: %s", message)}+ }+}++func retryMoodle(ctx context.Context, fn func() error) error {+ var lastErr error+ for attempt := 0; attempt < moodleRetryTries; attempt++ {+ if err := fn(); err == nil {+ return nil+ } else {+ lastErr = err+ }+ if attempt+1 >= moodleRetryTries {+ break+ }+ delay := moodleRetryBaseDelay+ for i := 0; i < attempt; i++ {+ delay *= time.Duration(moodleRetryBackoff)+ }+ timer := time.NewTimer(delay)+ select {+ case <-ctx.Done():+ timer.Stop()+ return ctx.Err()+ case <-timer.C:+ }+ }+ return lastErr+}++func (c *MoodleConnector) getCourses(ctx context.Context) ([]moodleCourse, error) {+ var courses []moodleCourse+ if err := retryMoodle(ctx, func() error {+ return c.callREST(ctx, "core_course_get_courses", nil, &courses)+ }); err != nil {+ return nil, err+ }+ sort.SliceStable(courses, func(i, j int) bool {+ return courses[i].ID < courses[j].ID+ })+ return courses, nil+}++func (c *MoodleConnector) getCourseContents(ctx context.Context, courseID int64) ([]moodleSection, error) {+ var sections []moodleSection+ if err := retryMoodle(ctx, func() error {+ return c.callREST(ctx, "core_course_get_contents", map[string]any{"courseid": courseID}, §ions)+ }); err != nil {+ return nil, fmt.Errorf("Moodle course %d contents failed: %w", courseID, err)+ }+ return sections, nil+}++func (c *MoodleConnector) getForumDiscussions(ctx context.Context, forumID int64) ([]moodleDiscussion, error) {+ var result struct {+ Discussions []moodleDiscussion `json:"discussions"`+ }+ if err := c.callREST(ctx, "mod_forum_get_forum_discussions", map[string]any{"forumid": forumID}, &result); err != nil {+ return nil, err+ }+ return result.Discussions, nil+}++func (c *MoodleConnector) downloadFile(ctx context.Context, fileURL string) ([]byte, error) {+ downloadURL := addMoodleToken(fileURL, c.token)+ resp, err := c.moodleHTTPDo(ctx, http.MethodGet, downloadURL, nil, nil)+ if err != nil {+ return nil, err+ }+ defer resp.Body.Close()+ if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {+ return nil, fmt.Errorf("Moodle file download failed with status %d for %s", resp.StatusCode, moodleRedactedURL(fileURL))+ }+ blob, err := io.ReadAll(io.LimitReader(resp.Body, moodleMaxDownloadSize+1))+ if err != nil {+ return nil, err+ }+ if int64(len(blob)) > moodleMaxDownloadSize {+ return nil, fmt.Errorf("Moodle file download exceeds maximum size of %d bytes", moodleMaxDownloadSize)+ }+ return blob, nil+}++func addMoodleToken(fileURL, token string) string {+ if token == "" {+ return fileURL+ }+ if strings.Contains(strings.ToLower(fileURL), "token=") {+ return fileURL+ }+ delimiter := "?"+ if strings.Contains(fileURL, "?") {+ delimiter = "&"+ }+ return fileURL + delimiter + "token=" + token+}++func moodleHTMLToMarkdown(html string) (string, error) {+ converter := md.NewConverter("", true, &md.Options{EmDelimiter: "*"})+ out, err := converter.ConvertString(html)+ if err != nil {+ return "", fmt.Errorf("Moodle HTML to Markdown conversion failed: %w", err)+ }+ return out, nil+}++// ---------------------------------------------------------------------------+// Document building+// ---------------------------------------------------------------------------++func (c *MoodleConnector) courseDocuments(ctx context.Context, request SyncRequest, course moodleCourse, sections []moodleSection) ([]SourceDocument, error) {+ var documents []SourceDocument+ for _, section := range sections {+ for _, module := range section.Modules {+ docs, err := c.moduleDocuments(ctx, request, course, section, module)+ if err != nil {+ return nil, err+ }+ documents = append(documents, docs...)+ }+ }+ return documents, nil+}++func (c *MoodleConnector) moduleDocuments(ctx context.Context, request SyncRequest, course moodleCourse, section moodleSection, module moodleModule) ([]SourceDocument, error) {+ var document *SourceDocument+ var err error+ switch module.ModName {+ case "label", "url":+ return nil, nil+ case "resource":+ document, err = c.resourceDocument(ctx, request, course, section, module)+ case "page":+ document, err = c.pageDocument(ctx, request, course, section, module)+ case "forum":+ document, err = c.forumDocument(ctx, request, course, section, module)+ case "assign", "quiz":+ document, err = c.activityDocument(ctx, request, course, section, module)+ case "book":+ document, err = c.bookDocument(ctx, request, course, section, module)+ }+ if err != nil {+ return nil, err+ }+ if document == nil {+ return nil, nil+ }+ return []SourceDocument{*document}, nil+}++func (c *MoodleConnector) resourceDocument(ctx context.Context, request SyncRequest, course moodleCourse, section moodleSection, module moodleModule) (*SourceDocument, error) {+ if len(module.Contents) == 0 || module.Contents[0].FileURL == "" {+ return nil, nil+ }+ fileInfo := module.Contents[0]+ fileName := path.Base(fileInfo.Filename)+ ts := moodleMaxTimestamp(module.TimeCreated, module.TimeModified, fileInfo.TimeModified)+ if !includeMoodleModule(request, ts) {+ return nil, nil+ }+ blob, err := c.downloadFile(ctx, fileInfo.FileURL)+ if err != nil {+ return nil, err+ }+ extension := filepath.Ext(fileName)+ if extension == "" {+ extension = ".bin"+ }+ fileSize := int64(len(blob))+ if fileInfo.FileSize != nil {+ fileSize = *fileInfo.FileSize+ }+ return &SourceDocument{+ SourceID: fmt.Sprintf("moodle_resource_%d", module.ID),+ SemanticIdentifier: fmt.Sprintf("%s / %s / %s", course.FullName, section.Name, fileName),+ Extension: extension,+ Blob: blob,+ UpdatedAt: moodleTime(ts),+ SizeBytes: int64(len(blob)),+ Metadata: c.fileModuleMetadata(course, section, module, fileInfo, fileSize, fileName),+ }, nil+}++func (c *MoodleConnector) pageDocument(ctx context.Context, request SyncRequest, course moodleCourse, section moodleSection, module moodleModule) (*SourceDocument, error) {+ if len(module.Contents) == 0 || module.Contents[0].FileURL == "" {+ return nil, nil+ }+ fileInfo := module.Contents[0]+ fileName := path.Base(fileInfo.Filename)+ ts := moodleMaxTimestamp(module.TimeCreated, module.TimeModified, fileInfo.TimeModified)+ if !includeMoodleModule(request, ts) {+ return nil, nil+ }+ blob, err := c.downloadFile(ctx, fileInfo.FileURL)+ if err != nil {+ return nil, err+ }+ extension := filepath.Ext(fileName)+ if extension == "" {+ extension = ".html"+ }+ fileSize := int64(len(blob))+ if fileInfo.FileSize != nil {+ fileSize = *fileInfo.FileSize+ }+ return &SourceDocument{+ SourceID: fmt.Sprintf("moodle_page_%d", module.ID),+ SemanticIdentifier: fmt.Sprintf("%s / %s / %s", course.FullName, section.Name, module.Name),+ Extension: extension,+ Blob: blob,+ UpdatedAt: moodleTime(ts),+ SizeBytes: int64(len(blob)),+ Metadata: c.fileModuleMetadata(course, section, module, fileInfo, fileSize, fileName),+ }, nil+}++func (c *MoodleConnector) forumDocument(ctx context.Context, request SyncRequest, course moodleCourse, section moodleSection, module moodleModule) (*SourceDocument, error) {+ if module.Instance == nil {+ return nil, nil+ }+ if !includeMoodleModule(request, module.windowTimestamp()) {+ return nil, nil+ }+ discussions, err := c.getForumDiscussions(ctx, *module.Instance)+ if err != nil {+ return nil, err+ }+ if len(discussions) == 0 {+ return nil, nil+ }+ markdown := []string{"# " + module.Name + "\n"}+ latest := moodleMaxTimestamp(module.TimeCreated, module.TimeModified)+ discussionMetadata := make([]map[string]any, 0, len(discussions))+ for _, discussion := range discussions {+ body, err := moodleHTMLToMarkdown(discussion.Message)+ if err != nil {+ return nil, err+ }+ markdown = append(markdown, fmt.Sprintf("## %s\n\n%s\n\n---\n", discussion.Name, body))+ latest = maxInt64(latest, derefInt64(discussion.TimeModified))+ discussionMetadata = append(discussionMetadata, map[string]any{+ "id": discussion.ID,+ "name": discussion.Name,+ "user_id": discussion.UserID,+ "user_fullname": discussion.UserFullName,+ "time_created": discussion.TimeCreated,+ "time_modified": discussion.TimeModified,+ })+ }+ metadata := moodleBaseMetadata(c, course, section, module)+ metadata["forum_id"] = module.Instance+ metadata["discussion_count"] = len(discussions)+ metadata["discussions"] = discussionMetadata+ blob := []byte(strings.Join(markdown, "\n"))+ return &SourceDocument{+ SourceID: fmt.Sprintf("moodle_forum_%d", module.ID),+ SemanticIdentifier: fmt.Sprintf("%s / %s / %s", course.FullName, section.Name, module.Name),+ Extension: ".md",+ Blob: blob,+ UpdatedAt: moodleTime(latest),+ SizeBytes: int64(len(blob)),+ Metadata: metadata,+ }, nil+}++func (c *MoodleConnector) activityDocument(ctx context.Context, request SyncRequest, course moodleCourse, section moodleSection, module moodleModule) (*SourceDocument, error) {+ if module.Description == "" {+ return nil, nil+ }+ if !includeMoodleModule(request, module.windowTimestamp()) {+ return nil, nil+ }+ body, err := moodleHTMLToMarkdown(module.Description)+ if err != nil {+ return nil, err+ }+ markdown := fmt.Sprintf("# %s\n\n**Type:** %s\n\n%s", module.Name, capitalizeMoodleType(module.ModName), body)+ ts := moodleMaxTimestamp(module.TimeCreated, module.TimeModified, module.Added)+ metadata := moodleBaseMetadata(c, course, section, module)+ metadata["activity_type"] = module.ModName+ metadata["activity_instance"] = module.Instance+ metadata["description"] = module.Description+ metadata["added"] = module.Added+ blob := []byte(markdown)+ return &SourceDocument{+ SourceID: fmt.Sprintf("moodle_%s_%d", module.ModName, module.ID),+ SemanticIdentifier: fmt.Sprintf("%s / %s / %s", course.FullName, section.Name, module.Name),+ Extension: ".md",+ Blob: blob,+ UpdatedAt: moodleTime(ts),+ SizeBytes: int64(len(blob)),+ Metadata: metadata,+ }, nil+}++func (c *MoodleConnector) bookDocument(ctx context.Context, request SyncRequest, course moodleCourse, section moodleSection, module moodleModule) (*SourceDocument, error) {+ if len(module.Contents) == 0 {+ return nil, nil+ }+ var chapters []moodleContent+ for _, content := range module.Contents {+ if content.FileURL != "" && path.Base(content.Filename) == "index.html" {+ chapters = append(chapters, content)+ }+ }+ if len(chapters) == 0 {+ return nil, nil+ }+ latest := moodleMaxTimestamp(module.TimeCreated, module.TimeModified)+ for _, content := range module.Contents {+ latest = maxInt64(latest, derefInt64(content.TimeCreated))+ latest = maxInt64(latest, derefInt64(content.TimeModified))+ }+ if !includeMoodleModule(request, latest) {+ return nil, nil+ }+ markdownParts := []string{"# " + module.Name + "\n"}+ chapterMetadata := make([]map[string]any, 0, len(chapters))+ for _, chapter := range chapters {+ blob, err := c.downloadFile(ctx, chapter.FileURL)+ if err != nil {+ return nil, err+ }+ body, err := moodleHTMLToMarkdown(string(blob))+ if err != nil {+ return nil, err+ }+ markdownParts = append(markdownParts, body+"\n\n---\n")+ chapterMetadata = append(chapterMetadata, map[string]any{+ "chapter_id": chapter.ChapterID,+ "title": chapter.Title,+ "filename": chapter.Filename,+ "fileurl": moodleRedactedURL(chapter.FileURL),+ "time_created": chapter.TimeCreated,+ "time_modified": chapter.TimeModified,+ "size": chapter.FileSize,+ })+ }+ metadata := moodleBaseMetadata(c, course, section, module)+ metadata["book_id"] = module.Instance+ metadata["chapter_count"] = len(chapters)+ metadata["chapters"] = chapterMetadata+ blob := []byte(strings.Join(markdownParts, "\n"))+ return &SourceDocument{+ SourceID: fmt.Sprintf("moodle_book_%d", module.ID),+ SemanticIdentifier: fmt.Sprintf("%s / %s / %s", course.FullName, section.Name, module.Name),+ Extension: ".md",+ Blob: blob,+ UpdatedAt: moodleTime(latest),+ SizeBytes: int64(len(blob)),+ Metadata: metadata,+ }, nil+}++func (c *MoodleConnector) fileModuleMetadata(course moodleCourse, section moodleSection, module moodleModule, fileInfo moodleContent, fileSize int64, fileName string) map[string]any {+ metadata := moodleBaseMetadata(c, course, section, module)+ metadata["module_instance"] = module.Instance+ metadata["file_url"] = moodleRedactedURL(fileInfo.FileURL)+ metadata["file_name"] = fileName+ metadata["file_size"] = fileSize+ metadata["file_type"] = fileInfo.MIMEType+ return metadata+}++func moodleBaseMetadata(c *MoodleConnector, course moodleCourse, section moodleSection, module moodleModule) map[string]any {+ return map[string]any{+ "moodle_url": c.moodleURL,+ "course_id": course.ID,+ "course_name": course.FullName,+ "course_shortname": course.ShortName,+ "section_id": section.ID,+ "section_name": section.Name,+ "section_number": section.Section,+ "module_id": module.ID,+ "module_name": module.Name,+ "module_type": module.ModName,+ "time_created": module.TimeCreated,+ "time_modified": module.TimeModified,+ "visible": module.Visible,+ "groupmode": module.GroupMode,+ }+}++// ---------------------------------------------------------------------------+// Sync session with checkpoint resume+// ---------------------------------------------------------------------------++type moodleSyncSession struct {+ connector *MoodleConnector+ request SyncRequest+ batchSize int+ resumeAfterCourseID int64++ coursesLoaded bool+ courses []moodleCourse+ courseIndex int+ pending []SourceDocument+ pendingCheckpoint *SyncCheckpoint+ done bool+}++// NextBatch returns the next Moodle document batch.+func (s *moodleSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {+ for {+ if s.done {+ return SyncBatch{}, io.EOF+ }+ if len(s.pending) == 0 {+ if !s.coursesLoaded {+ courses, err := s.connector.getCourses(ctx)+ if err != nil {+ return SyncBatch{}, err+ }+ s.courses = courses+ s.coursesLoaded = true+ }+ if s.courseIndex >= len(s.courses) {+ s.done = true+ return SyncBatch{}, io.EOF+ }+ course := s.courses[s.courseIndex]+ s.courseIndex+++ if course.ID <= s.resumeAfterCourseID {+ continue+ }+ sections, err := s.connector.getCourseContents(ctx, course.ID)+ if err != nil {+ return SyncBatch{}, err+ }+ documents, err := s.connector.courseDocuments(ctx, s.request, course, sections)+ if err != nil {+ return SyncBatch{}, err+ }+ if len(documents) == 0 {+ continue+ }+ s.pending = documents+ updatedAt := documents[len(documents)-1].UpdatedAt+ checkpoint := &SyncCheckpoint{+ Cursor: fmt.Sprintf("moodle_course_%d", course.ID),+ SourceID: fmt.Sprintf("moodle_course_%d", course.ID),+ UpdatedAt: &updatedAt,+ }+ // The checkpoint only advances once the whole course is flushed so+ // an interrupted run re-processes a partially committed course.+ s.pendingCheckpoint = checkpoint+ }+ n := s.batchSize+ if n > len(s.pending) {+ n = len(s.pending)+ }+ chunk := s.pending[:n]+ s.pending = s.pending[n:]+ var checkpoint *SyncCheckpoint+ if len(s.pending) == 0 && s.pendingCheckpoint != nil {+ checkpoint = s.pendingCheckpoint+ s.pendingCheckpoint = nil+ }+ return SyncBatch{Documents: chunk, Checkpoint: checkpoint}, nil+ }+}++// Close closes the Moodle sync session.+func (s *moodleSyncSession) Close() error {+ return nil+}++func (s *moodleSyncSession) applyResume(checkpoint *SyncCheckpoint) {+ sourceID := firstNonEmpty(checkpoint.SourceID, checkpoint.Cursor)+ const prefix = "moodle_course_"+ if !strings.HasPrefix(sourceID, prefix) {+ return+ }+ courseID, err := strconv.ParseInt(strings.TrimPrefix(sourceID, prefix), 10, 64)+ if err != nil {+ return+ }+ s.resumeAfterCourseID = courseID+}++type moodlePruneSession struct {+ documents []SlimDocument+ batchSize int+ batchIndex int+}++// NextBatch returns the next Moodle prune snapshot batch.+func (s *moodlePruneSession) NextBatch(ctx context.Context) (PruneBatch, error) {+ if s.batchIndex >= len(s.documents) {+ return PruneBatch{}, io.EOF+ }+ end := s.batchIndex + s.batchSize+ if end > len(s.documents) {+ end = len(s.documents)+ }+ documents := s.documents[s.batchIndex:end]+ s.batchIndex = end+ return PruneBatch{Documents: documents}, nil+}++// Close closes the Moodle prune session.+func (s *moodlePruneSession) Close() error {+ return nil+}++// ---------------------------------------------------------------------------+// Helpers+// ---------------------------------------------------------------------------++func includeMoodleModule(request SyncRequest, ts int64) bool {+ if request.FromBeginning {+ return true+ }+ if ts <= 0 {+ return false+ }+ updatedAt := time.Unix(ts, 0).UTC()+ return !beforeOrAtWindowStart(updatedAt, request.WindowStart) && !afterWindowEnd(updatedAt, request.WindowEnd)+}++func moodleSlimDocIDForModule(module moodleModule) string {+ switch module.ModName {+ case "label", "url":+ return ""+ case "resource":+ return fmt.Sprintf("moodle_resource_%d", module.ID)+ case "forum":+ return fmt.Sprintf("moodle_forum_%d", module.ID)+ case "page":+ return fmt.Sprintf("moodle_page_%d", module.ID)+ case "book":+ return fmt.Sprintf("moodle_book_%d", module.ID)+ case "assign", "quiz":+ return fmt.Sprintf("moodle_%s_%d", module.ModName, module.ID)+ default:+ return ""+ }+}++func (m moodleModule) windowTimestamp() int64 {+ ts := moodleMaxTimestamp(m.TimeCreated, m.TimeModified)+ for _, content := range m.Contents {+ ts = maxInt64(ts, derefInt64(content.TimeModified))+ }+ return ts+}++func moodleMaxTimestamp(values ...*int64) int64 {+ var latest int64+ for _, value := range values {+ if value != nil && *value > latest {+ latest = *value+ }+ }+ return latest+}++func maxInt64(a, b int64) int64 {+ if a > b {+ return a+ }+ return b+}++func derefInt64(value *int64) int64 {+ if value == nil {+ return 0+ }+ return *value+}++func moodleTime(ts int64) time.Time {+ if ts <= 0 {+ return time.Time{}+ }+ return time.Unix(ts, 0).UTC()+}++func capitalizeMoodleType(mtype string) string {+ if mtype == "" {+ return mtype+ }+ return strings.ToUpper(mtype[:1]) + strings.ToLower(mtype[1:])+}++// ---------------------------------------------------------------------------+// SSRF protection+// ---------------------------------------------------------------------------++func validateMoodleURLForSSRF(rawURL string) error {+ parsed, err := url.Parse(rawURL)+ if err != nil {+ return &ConnectorValidationError{Message: "Moodle URL must include a hostname."}+ }+ if parsed.Scheme != "http" && parsed.Scheme != "https" {+ return &ConnectorValidationError{Message: fmt.Sprintf("Unsupported URL scheme for Moodle connector: %q. Only http/https are allowed.", parsed.Scheme)}+ }+ hostname := parsed.Hostname()+ if hostname == "" {+ return &ConnectorValidationError{Message: "Moodle URL must include a hostname."}+ }+ if strings.EqualFold(hostname, "localhost") {+ return &ConnectorValidationError{Message: fmt.Sprintf("Moodle URL hostname %q is not allowed (localhost is blocked).", hostname)}+ }+ addrs, err := net.LookupIP(hostname)+ if err != nil {+ // Resolution failure is not an SSRF condition by itself; the+ // per-request check surfaces it if it matters.+ return nil+ }+ if restAPISSRFAllowLoopback {+ allLoopback := true+ for _, addr := range addrs {+ if !addr.IsLoopback() {+ allLoopback = false+ break+ }+ }+ if allLoopback {+ return nil+ }+ // Not all loopback — fall through to normal validation.+ }+ for _, addr := range addrs {+ if !restAPIIPIsGlobal(restAPIEffectiveIP(addr)) {+ return &ConnectorValidationError{Message: fmt.Sprintf(+ "Moodle URL %q resolves to disallowed address %s (localhost, private, link-local, reserved, or multicast addresses are blocked).",+ rawURL, addr)}+ }+ }+ return nil+}++// moodleAssertURLSafe validates a per-request URL for SSRF, HTTPS, and+// same-origin policy. It returns the hostname and first validated IP so the+// caller can pin DNS for the actual dial.+func moodleAssertURLSafe(ctx context.Context, rawURL, originURL string) (string, net.IP, error) {+ parsed, err := url.Parse(rawURL)+ if err != nil {+ return "", nil, fmt.Errorf("Moodle URL is missing a host.")+ }+ if parsed.Scheme != "http" && parsed.Scheme != "https" {+ return "", nil, fmt.Errorf("Disallowed URL scheme: %q. Only [http https] are allowed.", parsed.Scheme)+ }+ hostname := parsed.Hostname()+ if hostname == "" {+ return "", nil, fmt.Errorf("Moodle URL is missing a host.")+ }+ // Reject cross-origin targets: every request must go to the configured+ // Moodle host.+ if originURL != "" {+ if origin, err := url.Parse(originURL); err == nil && origin.Hostname() != "" {+ if !strings.EqualFold(hostname, origin.Hostname()) {+ return "", nil, fmt.Errorf("Moodle URL host %q does not match configured origin %q.", hostname, origin.Hostname())+ }+ }+ }+ addrs, err := net.DefaultResolver.LookupIPAddr(ctx, hostname)+ if err != nil {+ return "", nil, fmt.Errorf("Could not resolve hostname %q: %v", hostname, err)+ }+ if len(addrs) == 0 {+ return "", nil, fmt.Errorf("Hostname %q resolved to no addresses.", hostname)+ }+ if restAPISSRFAllowLoopback {+ allLoopback := true+ for _, addr := range addrs {+ if !addr.IP.IsLoopback() {+ allLoopback = false+ break+ }+ }+ if allLoopback {+ return hostname, addrs[0].IP, nil+ }+ // Not all loopback — fall through to normal validation.+ }+ var first net.IP+ for _, addr := range addrs {+ if !restAPIIPIsGlobal(restAPIEffectiveIP(addr.IP)) {+ return "", nil, fmt.Errorf("Moodle URL resolves to a non-public address (%s), which is not allowed.", addr.IP)+ }+ if first == nil {+ first = addr.IP+ }+ }+ if first == nil {+ return "", nil, fmt.Errorf("Hostname %q resolved to no addresses.", hostname)+ }+ return hostname, first, nil+}++// moodleHTTPDo sends an HTTP request with DNS-pinned transport and manual+// redirect handling. Each hop is independently validated for SSRF, HTTPS, and+// same-origin policy, preventing DNS rebinding and redirect-based bypasses.+func (c *MoodleConnector) moodleHTTPDo(ctx context.Context, method, rawURL string, body []byte, headers map[string]string) (*http.Response, error) {+ currentURL := rawURL+ currentMethod := method+ currentBody := body+ for hop := 0; hop <= moodleMaxRedirects; hop++ {+ hostname, pinIP, err := moodleAssertURLSafe(ctx, currentURL, c.moodleURL)+ if err != nil {+ return nil, err+ }+ transport := newRestAPIPinnedTransport(hostname, pinIP)+ client := &http.Client{+ Transport: transport,+ Timeout: moodleRequestTimeout,+ CheckRedirect: func(req *http.Request, via []*http.Request) error {+ return http.ErrUseLastResponse+ },+ }+ var bodyReader io.Reader+ if currentBody != nil {+ bodyReader = bytes.NewReader(currentBody)+ }+ req, err := http.NewRequestWithContext(ctx, currentMethod, currentURL, bodyReader)+ if err != nil {+ transport.CloseIdleConnections()+ return nil, err+ }+ for k, v := range headers {+ req.Header.Set(k, v)+ }+ resp, err := client.Do(req)+ if err != nil {+ transport.CloseIdleConnections()+ return nil, err+ }+ if !restAPIIsRedirect(resp.StatusCode) {+ resp.Body = &restAPICloseIdleBody{body: resp.Body, transport: transport}+ return resp, nil+ }+ location := resp.Header.Get("Location")+ resp.Body.Close()+ transport.CloseIdleConnections()+ if location == "" {+ return nil, fmt.Errorf("Moodle redirect with empty Location header")+ }+ nextURL, err := restAPIResolveURL(currentURL, location)+ if err != nil {+ return nil, err+ }+ currentURL = nextURL+ if resp.StatusCode == http.StatusMovedPermanently || resp.StatusCode == http.StatusFound || resp.StatusCode == http.StatusSeeOther {+ currentMethod = http.MethodGet+ currentBody = nil+ }+ }+ return nil, fmt.Errorf("Moodle request stopped after %d redirects", moodleMaxRedirects)+}++// moodleRedactedURL strips query and fragment from a URL so that tokens or+// other sensitive query parameters are not persisted in document metadata.+func moodleRedactedURL(rawURL string) string {+ parsed, err := url.Parse(rawURL)+ if err != nil {+ return rawURL+ }+ parsed.RawQuery = ""+ parsed.Fragment = ""+ return parsed.String()+}diff --git a/internal/syncer/connector/moodle_test.go b/internal/syncer/connector/moodle_test.gonew file mode 100644index 0000000000..bc80d12602--- /dev/null+++ b/internal/syncer/connector/moodle_test.go@@ -0,0 +1,689 @@+package connector++import (+ "context"+ "errors"+ "io"+ "net/http"+ "net/http/httptest"+ "strings"+ "sync/atomic"+ "testing"+ "time"+)++func withMoodleTestHooks(t *testing.T) {+ t.Helper()+ origLoopback := restAPISSRFAllowLoopback+ origTries := moodleRetryTries+ origBaseDelay := moodleRetryBaseDelay+ origBackoff := moodleRetryBackoff+ restAPISSRFAllowLoopback = true+ moodleRetryTries = 2+ moodleRetryBaseDelay = time.Millisecond+ moodleRetryBackoff = 2+ t.Cleanup(func() {+ restAPISSRFAllowLoopback = origLoopback+ moodleRetryTries = origTries+ moodleRetryBaseDelay = origBaseDelay+ moodleRetryBackoff = origBackoff+ })+}++func mustMoodleConnector(t *testing.T, moodleURL string) *MoodleConnector {+ t.Helper()+ connector, err := NewMoodleConnector(map[string]any{+ "moodle_url": moodleURL,+ "batch_size": 2,+ "credentials": map[string]any{+ "moodle_token": "testtoken",+ },+ })+ if err != nil {+ t.Fatalf("NewMoodleConnector: %v", err)+ }+ return connector+}++type moodleTestFixtures struct {+ courses string+ contents map[string]string+ forumDiscussions map[string]string+ files map[string]string+ siteInfoBody string+ courseListCount *atomic.Int64+ forumCallCount *atomic.Int64+}++func newTestMoodleServer(t *testing.T, buildFixtures func(serverURL string) moodleTestFixtures) *httptest.Server {+ t.Helper()+ mux := http.NewServeMux()+ server := httptest.NewServer(mux)+ t.Cleanup(server.Close)+ fixtures := buildFixtures(server.URL)+ siteInfoBody := fixtures.siteInfoBody+ if siteInfoBody == "" {+ siteInfoBody = `{"sitename":"Test Moodle","siteurl":"https://example.com"}`+ }+ mux.HandleFunc("/webservice/rest/server.php", func(w http.ResponseWriter, r *http.Request) {+ if err := r.ParseForm(); err != nil {+ http.Error(w, "bad form", http.StatusBadRequest)+ return+ }+ switch r.Form.Get("wsfunction") {+ case "core_webservice_get_site_info":+ w.Write([]byte(siteInfoBody))+ case "core_course_get_courses":+ if fixtures.courseListCount != nil {+ fixtures.courseListCount.Add(1)+ }+ w.Write([]byte(fixtures.courses))+ case "core_course_get_contents":+ body, ok := fixtures.contents[r.Form.Get("courseid")]+ if !ok {+ http.NotFound(w, r)+ return+ }+ w.Write([]byte(body))+ case "mod_forum_get_forum_discussions":+ if fixtures.forumCallCount != nil {+ fixtures.forumCallCount.Add(1)+ }+ body, ok := fixtures.forumDiscussions[r.Form.Get("forumid")]+ if !ok {+ http.NotFound(w, r)+ return+ }+ w.Write([]byte(body))+ default:+ http.Error(w, "unknown function", http.StatusBadRequest)+ }+ })+ mux.HandleFunc("/pluginfile.php/", func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Query().Get("token") != "testtoken" {+ http.Error(w, "missing token", http.StatusForbidden)+ return+ }+ body, ok := fixtures.files[r.URL.Path]+ if !ok {+ http.NotFound(w, r)+ return+ }+ w.Write([]byte(body))+ })+ return server+}++func fullSyncMoodleFixtures(serverURL string) moodleTestFixtures {+ return moodleTestFixtures{+ courses: `[{"id":1,"fullname":"Course One","shortname":"c1"},{"id":2,"fullname":"Course Two","shortname":"c2"}]`,+ contents: map[string]string{+ "1": `[{"id":11,"name":"Week 1","section":0,"modules":[` ++ `{"id":100,"name":"Guide","modname":"resource","instance":7,"visible":1,"groupmode":0,"timecreated":1000000000,"timemodified":1700000000,"contents":[{"type":"file","filename":"guide.pdf","filepath":"/","filesize":123,"fileurl":"` + serverURL + `/pluginfile.php/guides/guide.pdf","mimetype":"application/pdf","timemodified":1700000000}]},` ++ `{"id":101,"name":"About","modname":"page","instance":8,"visible":1,"groupmode":0,"timecreated":1000000000,"timemodified":1700000000,"contents":[{"type":"file","filename":"about.html","filepath":"/","filesize":456,"fileurl":"` + serverURL + `/pluginfile.php/pages/about.html","mimetype":"text/html","timemodified":1700000000}]},` ++ `{"id":102,"name":"Announcement Label","modname":"label","instance":9,"visible":1,"groupmode":0,"timecreated":1000000000,"timemodified":1700000000,"contents":[]},` ++ `{"id":103,"name":"External Link","modname":"url","instance":10,"visible":1,"groupmode":0,"timecreated":1000000000,"timemodified":1700000000,"contents":[]}` ++ `]}]`,+ "2": `[{"id":12,"name":"Week 2","section":1,"modules":[` ++ `{"id":200,"name":"Announcements","modname":"forum","instance":50,"visible":1,"groupmode":0,"timecreated":1000000000,"timemodified":1700000000,"contents":[]},` ++ `{"id":201,"name":"Assignment 1","modname":"assign","instance":51,"visible":1,"groupmode":0,"timecreated":1000000000,"timemodified":1700000000,"added":1600000000,"description":"<p>Write a <em>report</em>.</p>","contents":[]},` ++ `{"id":202,"name":"Quiz 1","modname":"quiz","instance":52,"visible":1,"groupmode":0,"timecreated":1000000000,"timemodified":1700000000,"added":1600000000,"description":"<p>Answer the questions.</p>","contents":[]},` ++ `{"id":203,"name":"Book Title","modname":"book","instance":53,"visible":1,"groupmode":0,"timecreated":1000000000,"timemodified":1700000000,"contents":[` ++ `{"type":"file","filename":"index.html","filepath":"/chapter1/","filesize":100,"fileurl":"` + serverURL + `/pluginfile.php/books/chapter1/index.html","chapterid":1001,"title":"Chapter 1","timecreated":1000000000,"timemodified":1700000000},` ++ `{"type":"file","filename":"index.html","filepath":"/chapter2/","filesize":100,"fileurl":"` + serverURL + `/pluginfile.php/books/chapter2/index.html","chapterid":1002,"title":"Chapter 2","timecreated":1000000000,"timemodified":1700000000},` ++ `{"type":"file","filename":"cover.png","filepath":"/","filesize":50,"fileurl":"` + serverURL + `/pluginfile.php/books/cover.png"}` ++ `]}` ++ `]}]`,+ },+ forumDiscussions: map[string]string{+ "50": `{"discussions":[{"id":10,"name":"Intro","message":"<p>Hello <strong>world</strong></p>","userid":5,"userfullname":"Alice","timecreated":100,"timemodified":200}]}`,+ },+ files: map[string]string{+ "/pluginfile.php/guides/guide.pdf": "%PDF-1.4 fake",+ "/pluginfile.php/pages/about.html": "<html><body><h1>About</h1><p>Hello</p></body></html>",+ "/pluginfile.php/books/chapter1/index.html": "<h1>Chapter 1</h1><p>First</p>",+ "/pluginfile.php/books/chapter2/index.html": "<h1>Chapter 2</h1><p>Second</p>",+ },+ }+}++func collectMoodleDocuments(t *testing.T, session SyncSession) []SourceDocument {+ t.Helper()+ var documents []SourceDocument+ for {+ batch, err := session.NextBatch(context.Background())+ if errors.Is(err, io.EOF) {+ return documents+ }+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ documents = append(documents, batch.Documents...)+ }+}++func TestMoodleConnectorValidate(t *testing.T) {+ withMoodleTestHooks(t)+ server := newTestMoodleServer(t, fullSyncMoodleFixtures)+ connector := mustMoodleConnector(t, server.URL)+ if err := connector.Validate(context.Background()); err != nil {+ t.Fatalf("Validate: %v", err)+ }+}++func TestMoodleConnectorValidateMissingURLAndToken(t *testing.T) {+ withMoodleTestHooks(t)+ connector, err := NewMoodleConnector(map[string]any{"credentials": map[string]any{}})+ if err != nil {+ t.Fatalf("NewMoodleConnector: %v", err)+ }+ if err := connector.Validate(context.Background()); err == nil {+ t.Fatalf("expected missing URL validation error")+ }+ connector.moodleURL = "https://example.com"+ err = connector.Validate(context.Background())+ if err == nil {+ t.Fatalf("expected missing token validation error")+ }+ var missingCred *ConnectorMissingCredentialError+ if !errors.As(err, &missingCred) {+ t.Fatalf("missing token error type = %T, want ConnectorMissingCredentialError", err)+ }+}++func TestMoodleConnectorValidateInvalidToken(t *testing.T) {+ withMoodleTestHooks(t)+ server := newTestMoodleServer(t, func(serverURL string) moodleTestFixtures {+ fixtures := fullSyncMoodleFixtures(serverURL)+ fixtures.siteInfoBody = `{"exception":"moodle_exception","errorcode":"invalidtoken","message":"Invalid token - token has expired or is invalid"}`+ return fixtures+ })+ connector := mustMoodleConnector(t, server.URL)+ err := connector.Validate(context.Background())+ if err == nil {+ t.Fatalf("expected invalid token validation error")+ }+ var missingCred *ConnectorMissingCredentialError+ if !errors.As(err, &missingCred) {+ t.Fatalf("invalid token error type = %T, want ConnectorMissingCredentialError", err)+ }+}++func TestMoodleConnectorOpenSyncBuildsAllModuleTypes(t *testing.T) {+ withMoodleTestHooks(t)+ server := newTestMoodleServer(t, fullSyncMoodleFixtures)+ connector := mustMoodleConnector(t, server.URL)+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ documents := collectMoodleDocuments(t, session)+ if len(documents) != 6 {+ t.Fatalf("documents len = %d, want 6", len(documents))+ }+ byID := map[string]SourceDocument{}+ for _, document := range documents {+ byID[document.SourceID] = document+ }+ for _, sourceID := range []string{+ "moodle_resource_100", "moodle_page_101", "moodle_forum_200",+ "moodle_assign_201", "moodle_quiz_202", "moodle_book_203",+ } {+ if _, ok := byID[sourceID]; !ok {+ t.Fatalf("missing document %q", sourceID)+ }+ }+ resource := byID["moodle_resource_100"]+ if resource.SemanticIdentifier != "Course One / Week 1 / guide.pdf" {+ t.Fatalf("resource semantic = %q", resource.SemanticIdentifier)+ }+ if resource.Extension != ".pdf" {+ t.Fatalf("resource extension = %q", resource.Extension)+ }+ if string(resource.Blob) != "%PDF-1.4 fake" {+ t.Fatalf("resource blob = %q", resource.Blob)+ }+ if resource.Metadata["file_name"] != "guide.pdf" || resource.Metadata["file_size"] != int64(123) {+ t.Fatalf("resource metadata = %+v", resource.Metadata)+ }+ if resource.Metadata["course_name"] != "Course One" || resource.Metadata["section_name"] != "Week 1" {+ t.Fatalf("resource metadata = %+v", resource.Metadata)+ }++ page := byID["moodle_page_101"]+ if page.Extension != ".html" || string(page.Blob) != "<html><body><h1>About</h1><p>Hello</p></body></html>" {+ t.Fatalf("page = %+v", page)+ }++ forum := byID["moodle_forum_200"]+ if forum.Extension != ".md" {+ t.Fatalf("forum extension = %q", forum.Extension)+ }+ forumBlob := string(forum.Blob)+ if !strings.Contains(forumBlob, "# Announcements") || !strings.Contains(forumBlob, "## Intro") || !strings.Contains(forumBlob, "Hello **world**") {+ t.Fatalf("forum blob = %q", forumBlob)+ }+ if forum.Metadata["discussion_count"] != 1 {+ t.Fatalf("forum metadata = %+v", forum.Metadata)+ }+ forumID, ok := forum.Metadata["forum_id"].(*int64)+ if !ok || forumID == nil || *forumID != 50 {+ t.Fatalf("forum metadata = %+v", forum.Metadata)+ }++ assign := byID["moodle_assign_201"]+ if !strings.Contains(string(assign.Blob), "**Type:** Assign") || !strings.Contains(string(assign.Blob), "Write a *report*.") {+ t.Fatalf("assign blob = %q", assign.Blob)+ }+ if assign.Metadata["description"] != "<p>Write a <em>report</em>.</p>" {+ t.Fatalf("assign metadata = %+v", assign.Metadata)+ }++ quiz := byID["moodle_quiz_202"]+ if !strings.Contains(string(quiz.Blob), "**Type:** Quiz") {+ t.Fatalf("quiz blob = %q", quiz.Blob)+ }++ book := byID["moodle_book_203"]+ bookBlob := string(book.Blob)+ if !strings.Contains(bookBlob, "# Book Title") || !strings.Contains(bookBlob, "# Chapter 1") || !strings.Contains(bookBlob, "# Chapter 2") {+ t.Fatalf("book blob = %q", bookBlob)+ }+ if book.Metadata["chapter_count"] != 2 {+ t.Fatalf("book metadata = %+v", book.Metadata)+ }++ if _, ok := byID["moodle_label_102"]; ok {+ t.Fatalf("label module must be skipped")+ }+ if _, ok := byID["moodle_url_103"]; ok {+ t.Fatalf("url module must be skipped")+ }+}++func TestMoodleConnectorOpenSyncWindowFilter(t *testing.T) {+ withMoodleTestHooks(t)+ server := newTestMoodleServer(t, func(serverURL string) moodleTestFixtures {+ return moodleTestFixtures{+ courses: `[{"id":1,"fullname":"Course One","shortname":"c1"}]`,+ contents: map[string]string{+ "1": `[{"id":11,"name":"Week 1","section":0,"modules":[` ++ `{"id":100,"name":"Before","modname":"resource","instance":7,"visible":1,"groupmode":0,"timemodified":1704067200,"contents":[{"type":"file","filename":"before.pdf","filepath":"/","fileurl":"` + serverURL + `/pluginfile.php/before.pdf","timemodified":1704067200}]},` ++ `{"id":101,"name":"Inside","modname":"resource","instance":8,"visible":1,"groupmode":0,"timemodified":1704153600,"contents":[{"type":"file","filename":"inside.pdf","filepath":"/","fileurl":"` + serverURL + `/pluginfile.php/inside.pdf","timemodified":1704153600}]},` ++ `{"id":102,"name":"After","modname":"resource","instance":9,"visible":1,"groupmode":0,"timemodified":1709251200,"contents":[{"type":"file","filename":"after.pdf","filepath":"/","fileurl":"` + serverURL + `/pluginfile.php/after.pdf","timemodified":1709251200}]}` ++ `]}]`,+ },+ files: map[string]string{+ "/pluginfile.php/before.pdf": "before",+ "/pluginfile.php/inside.pdf": "inside",+ "/pluginfile.php/after.pdf": "after",+ },+ }+ })+ connector := mustMoodleConnector(t, server.URL)+ start := mustTime(t, "2024-01-01T00:00:00Z")+ session, err := connector.OpenSync(context.Background(), SyncRequest{+ WindowStart: &start,+ WindowEnd: mustTime(t, "2024-02-01T00:00:00Z"),+ })+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ documents := collectMoodleDocuments(t, session)+ if len(documents) != 1 {+ t.Fatalf("documents len = %d, want 1", len(documents))+ }+ if documents[0].SourceID != "moodle_resource_101" {+ t.Fatalf("window filtered source id = %q, want moodle_resource_101", documents[0].SourceID)+ }+}++func TestMoodleConnectorOpenSyncResumesFromCheckpoint(t *testing.T) {+ withMoodleTestHooks(t)+ server := newTestMoodleServer(t, fullSyncMoodleFixtures)+ connector := mustMoodleConnector(t, server.URL)+ updatedAt := mustTime(t, "2023-11-14T22:13:20Z")+ session, err := connector.OpenSync(context.Background(), SyncRequest{+ FromBeginning: true,+ Resume: &SyncCheckpoint{+ Cursor: "moodle_course_1",+ SourceID: "moodle_course_1",+ UpdatedAt: &updatedAt,+ },+ })+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ documents := collectMoodleDocuments(t, session)+ if len(documents) != 4 {+ t.Fatalf("documents len = %d, want 4 (course 1 skipped)", len(documents))+ }+ for _, document := range documents {+ if strings.HasPrefix(document.SourceID, "moodle_resource_") || strings.HasPrefix(document.SourceID, "moodle_page_") {+ t.Fatalf("resume must skip course 1, got %q", document.SourceID)+ }+ }+}++func TestMoodleConnectorOpenSyncDeferredCoursesUntilNextBatch(t *testing.T) {+ withMoodleTestHooks(t)+ var courseListCount atomic.Int64+ server := newTestMoodleServer(t, func(serverURL string) moodleTestFixtures {+ fixtures := fullSyncMoodleFixtures(serverURL)+ fixtures.courseListCount = &courseListCount+ return fixtures+ })+ connector := mustMoodleConnector(t, server.URL)+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ if courseListCount.Load() != 0 {+ t.Fatalf("course list calls after OpenSync = %d, want 0", courseListCount.Load())+ }+ if _, err := session.NextBatch(context.Background()); err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if courseListCount.Load() != 1 {+ t.Fatalf("course list calls after NextBatch = %d, want 1", courseListCount.Load())+ }+}++func TestMoodleConnectorOpenPruneReturnsSlimSnapshot(t *testing.T) {+ withMoodleTestHooks(t)+ server := newTestMoodleServer(t, fullSyncMoodleFixtures)+ connector := mustMoodleConnector(t, server.URL)+ session, err := connector.OpenPrune(context.Background(), PruneRequest{})+ if err != nil {+ t.Fatalf("OpenPrune: %v", err)+ }+ expected := map[string]bool{+ "moodle_resource_100": false,+ "moodle_page_101": false,+ "moodle_forum_200": false,+ "moodle_assign_201": false,+ "moodle_quiz_202": false,+ "moodle_book_203": false,+ }+ var total int+ for {+ batch, err := session.NextBatch(context.Background())+ if errors.Is(err, io.EOF) {+ break+ }+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ for _, document := range batch.Documents {+ total+++ if _, ok := expected[document.SourceID]; !ok {+ t.Fatalf("unexpected slim document %q", document.SourceID)+ }+ expected[document.SourceID] = true+ }+ }+ if total != 6 {+ t.Fatalf("slim documents total = %d, want 6", total)+ }+ for sourceID, seen := range expected {+ if !seen {+ t.Fatalf("missing slim document %q", sourceID)+ }+ }+}++func TestMoodleConnectorOpenSyncCheckpointAdvancesPerCourse(t *testing.T) {+ withMoodleTestHooks(t)+ server := newTestMoodleServer(t, fullSyncMoodleFixtures)+ connector := mustMoodleConnector(t, server.URL)+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ var checkpoints []string+ for {+ batch, err := session.NextBatch(context.Background())+ if errors.Is(err, io.EOF) {+ break+ }+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if batch.Checkpoint != nil {+ checkpoints = append(checkpoints, batch.Checkpoint.Cursor)+ }+ }+ // Course 1 has one batch (2 docs) and course 2 has two batches (4 docs);+ // the checkpoint advances only on the last batch of each course.+ if len(checkpoints) != 2 || checkpoints[0] != "moodle_course_1" || checkpoints[1] != "moodle_course_2" {+ t.Fatalf("checkpoints = %v, want [moodle_course_1 moodle_course_2]", checkpoints)+ }+}++func TestMoodleHTMLToMarkdown(t *testing.T) {+ out, err := moodleHTMLToMarkdown("<p>Hello <strong>world</strong> and <em>you</em></p>")+ if err != nil {+ t.Fatalf("moodleHTMLToMarkdown: %v", err)+ }+ if !strings.Contains(out, "**world**") || !strings.Contains(out, "*you*") {+ t.Fatalf("markdown = %q", out)+ }+ out, err = moodleHTMLToMarkdown("<h1>Title</h1><p>Body</p>")+ if err != nil {+ t.Fatalf("moodleHTMLToMarkdown: %v", err)+ }+ if !strings.Contains(out, "# Title") || !strings.Contains(out, "Body") {+ t.Fatalf("markdown = %q", out)+ }+}++func TestAddMoodleToken(t *testing.T) {+ if got := addMoodleToken("https://example.com/file.pdf", "tok"); got != "https://example.com/file.pdf?token=tok" {+ t.Fatalf("plain url = %q", got)+ }+ if got := addMoodleToken("https://example.com/file.pdf?x=1", "tok"); got != "https://example.com/file.pdf?x=1&token=tok" {+ t.Fatalf("query url = %q", got)+ }+ if got := addMoodleToken("https://example.com/file.pdf?token=old", "tok"); got != "https://example.com/file.pdf?token=old" {+ t.Fatalf("existing token url = %q", got)+ }+}++func TestValidateMoodleURLForSSRF(t *testing.T) {+ withMoodleTestHooks(t)+ origLoopback := restAPISSRFAllowLoopback+ restAPISSRFAllowLoopback = false+ defer func() { restAPISSRFAllowLoopback = origLoopback }()++ if err := validateMoodleURLForSSRF("ftp://example.com"); err == nil {+ t.Fatalf("expected scheme rejection")+ }+ if err := validateMoodleURLForSSRF("localhost"); err == nil {+ t.Fatalf("expected missing host rejection")+ }+ if err := validateMoodleURLForSSRF("https://localhost/moodle"); err == nil {+ t.Fatalf("expected localhost rejection")+ }+ if err := validateMoodleURLForSSRF("https://127.0.0.1/moodle"); err == nil {+ t.Fatalf("expected loopback rejection")+ }+}++func TestMoodleConnectorFileDownloadRequiresToken(t *testing.T) {+ withMoodleTestHooks(t)+ server := newTestMoodleServer(t, fullSyncMoodleFixtures)+ connector := mustMoodleConnector(t, server.URL)+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ documents := collectMoodleDocuments(t, session)+ for _, document := range documents {+ if strings.HasPrefix(document.SourceID, "moodle_resource_") && string(document.Blob) != "%PDF-1.4 fake" {+ t.Fatalf("resource blob = %q", document.Blob)+ }+ }+}++func TestMoodleConnectorValidateConnectorSetting(t *testing.T) {+ withMoodleTestHooks(t)+ server := newTestMoodleServer(t, fullSyncMoodleFixtures)+ connector := mustMoodleConnector(t, server.URL)+ if err := connector.ValidateConnectorSetting(context.Background(), nil); err != nil {+ t.Fatalf("ValidateConnectorSetting: %v", err)+ }+}++func TestMoodleRedactedURL(t *testing.T) {+ original := "https://moodle.example.com/pluginfile.php/guides/guide.pdf?token=secret&forcedownload=1"+ redacted := moodleRedactedURL(original)+ if strings.Contains(redacted, "token") || strings.Contains(redacted, "?") {+ t.Fatalf("redacted URL still contains query params: %q", redacted)+ }+ want := "https://moodle.example.com/pluginfile.php/guides/guide.pdf"+ if redacted != want {+ t.Fatalf("redacted URL = %q, want %q", redacted, want)+ }+}++func TestMoodleConnectorDownloadErrorFailsCourse(t *testing.T) {+ withMoodleTestHooks(t)+ server := newTestMoodleServer(t, func(serverURL string) moodleTestFixtures {+ return moodleTestFixtures{+ courses: `[{"id":1,"fullname":"Course One","shortname":"c1"}]`,+ contents: map[string]string{+ "1": `[{"id":11,"name":"Week 1","section":0,"modules":[` ++ `{"id":100,"name":"Guide","modname":"resource","instance":7,"visible":1,"groupmode":0,"timecreated":1000000000,"timemodified":1700000000,"contents":[{"type":"file","filename":"guide.pdf","filepath":"/","filesize":123,"fileurl":"` + serverURL + `/pluginfile.php/nonexistent/guide.pdf","mimetype":"application/pdf","timemodified":1700000000}]}` ++ `]}]`,+ },+ files: map[string]string{},+ }+ })+ connector := mustMoodleConnector(t, server.URL)+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ _, err = session.NextBatch(context.Background())+ if err == nil {+ t.Fatalf("expected NextBatch to fail when a download fails")+ }+}++func TestMoodleConnectorMetadataRedactsFileURL(t *testing.T) {+ withMoodleTestHooks(t)+ server := newTestMoodleServer(t, func(serverURL string) moodleTestFixtures {+ return moodleTestFixtures{+ courses: `[{"id":1,"fullname":"Course One","shortname":"c1"}]`,+ contents: map[string]string{+ "1": `[{"id":11,"name":"Week 1","section":0,"modules":[` ++ `{"id":100,"name":"Guide","modname":"resource","instance":7,"visible":1,"groupmode":0,"timecreated":1000000000,"timemodified":1700000000,"contents":[{"type":"file","filename":"guide.pdf","filepath":"/","filesize":123,"fileurl":"` + serverURL + `/pluginfile.php/guides/guide.pdf?forcedownload=1","mimetype":"application/pdf","timemodified":1700000000}]}` ++ `]}]`,+ },+ files: map[string]string{+ "/pluginfile.php/guides/guide.pdf": "%PDF-1.4 fake",+ },+ }+ })+ connector := mustMoodleConnector(t, server.URL)+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ documents := collectMoodleDocuments(t, session)+ for _, doc := range documents {+ if doc.SourceID != "moodle_resource_100" {+ continue+ }+ fileURL, ok := doc.Metadata["file_url"].(string)+ if !ok {+ t.Fatalf("file_url not a string: %T", doc.Metadata["file_url"])+ }+ if strings.Contains(fileURL, "?") {+ t.Fatalf("file_url should be redacted (no query): %q", fileURL)+ }+ }+}++func TestMoodleAssertURLSafeRejectsCrossOrigin(t *testing.T) {+ origLoopback := restAPISSRFAllowLoopback+ restAPISSRFAllowLoopback = false+ defer func() { restAPISSRFAllowLoopback = origLoopback }()++ _, _, err := moodleAssertURLSafe(context.Background(), "https://evil.com/api", "https://example.com")+ if err == nil {+ t.Fatalf("expected cross-origin rejection")+ }+}++func TestMoodleAssertURLSafeLoopbackAllAddresses(t *testing.T) {+ origLoopback := restAPISSRFAllowLoopback+ restAPISSRFAllowLoopback = true+ defer func() { restAPISSRFAllowLoopback = origLoopback }()++ // All loopback → allowed.+ _, _, err := moodleAssertURLSafe(context.Background(), "http://127.0.0.1/path", "http://127.0.0.1")+ if err != nil {+ t.Fatalf("expected loopback to be allowed: %v", err)+ }++ // Not all loopback (private address) → rejected even with loopback flag.+ _, _, err = moodleAssertURLSafe(context.Background(), "http://10.0.0.1/path", "http://10.0.0.1")+ if err == nil {+ t.Fatalf("expected non-loopback private address to be rejected even with loopback flag")+ }+}++func TestMoodleConnectorRESTResponseSizeCap(t *testing.T) {+ withMoodleTestHooks(t)+ origMax := moodleMaxResponseSize+ moodleMaxResponseSize = 10+ defer func() { moodleMaxResponseSize = origMax }()++ server := newTestMoodleServer(t, func(serverURL string) moodleTestFixtures {+ fixtures := fullSyncMoodleFixtures(serverURL)+ fixtures.siteInfoBody = `{"sitename":"Test Moodle with a very long name that exceeds the cap"}`+ return fixtures+ })+ connector := mustMoodleConnector(t, server.URL)+ err := connector.Validate(context.Background())+ if err == nil {+ t.Fatalf("expected oversize response error")+ }+ if !strings.Contains(err.Error(), "exceeds maximum size") {+ t.Fatalf("expected oversize error, got: %v", err)+ }+}++func TestMoodleConnectorDownloadSizeCap(t *testing.T) {+ withMoodleTestHooks(t)+ origMax := moodleMaxDownloadSize+ moodleMaxDownloadSize = 10+ defer func() { moodleMaxDownloadSize = origMax }()++ server := newTestMoodleServer(t, func(serverURL string) moodleTestFixtures {+ return moodleTestFixtures{+ courses: `[{"id":1,"fullname":"Course One","shortname":"c1"}]`,+ contents: map[string]string{+ "1": `[{"id":11,"name":"Week 1","section":0,"modules":[` ++ `{"id":100,"name":"Guide","modname":"resource","instance":7,"visible":1,"groupmode":0,"timecreated":1000000000,"timemodified":1700000000,"contents":[{"type":"file","filename":"guide.pdf","filepath":"/","filesize":123,"fileurl":"` + serverURL + `/pluginfile.php/guides/guide.pdf","mimetype":"application/pdf","timemodified":1700000000}]}` ++ `]}]`,+ },+ files: map[string]string{+ "/pluginfile.php/guides/guide.pdf": "this content is longer than ten bytes",+ },+ }+ })+ connector := mustMoodleConnector(t, server.URL)+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ _, err = session.NextBatch(context.Background())+ if err == nil {+ t.Fatalf("expected oversize download error")+ }+ if !strings.Contains(err.Error(), "exceeds maximum size") {+ t.Fatalf("expected oversize error, got: %v", err)+ }+}