Skip to content

Building 70 Projects ranging from beginner to advanced so anyone can — learn from, build upon, use as a reference, or even copy directly. Owned by my cybersecurity company 👇

feat(nadezhda): M1 ingestion core (fetch, parse, normalize, dedup)

CarterPerez-devb41276004fb2
diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/scrape.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/scrape.go
new file mode 100644
index 00000000..6d3c5587
--- /dev/null
+++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/scrape.go
@@ -0,0 +1,127 @@
+// ©AngelaMos | 2026
+// scrape.go
+
+package main
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "os/signal"
+ "time"
+
+ "github.com/spf13/cobra"
+
+ "github.com/CarterPerez-dev/nadezhda/internal/fetch"
+ "github.com/CarterPerez-dev/nadezhda/internal/ingest"
+ "github.com/CarterPerez-dev/nadezhda/internal/source"
+ "github.com/CarterPerez-dev/nadezhda/internal/store"
+)
+
+const (
+ statusNotModified = "304"
+ statusError = "error"
+ statusOK = "ok"
+ dash = "-"
+)
+
+var scrapeSource string
+
+var scrapeCmd = &cobra.Command{
+ Use: "scrape",
+ Short: "Ingest all enabled sources once",
+ RunE: runScrape,
+}
+
+func init() {
+ scrapeCmd.Flags().StringVar(&scrapeSource, "source", "", "ingest only this source by name")
+ rootCmd.AddCommand(scrapeCmd)
+}
+
+func runScrape(cmd *cobra.Command, args []string) error {
+ cfg, err := loadConfig()
+ if err != nil {
+ return err
+ }
+ srcs, err := source.Load(cfg.SourcesPath)
+ if err != nil {
+ return err
+ }
+
+ targets, err := selectTargets(srcs, scrapeSource)
+ if err != nil {
+ return err
+ }
+
+ st, err := store.Open(cfg.DBPath)
+ if err != nil {
+ return err
+ }
+ defer st.Close()
+
+ fc := fetch.New(fetch.Options{
+ UserAgent: cfg.Fetch.UserAgent,
+ PerHostRate: cfg.Fetch.PerHostRate,
+ PerHostBurst: cfg.Fetch.PerHostBurst,
+ Timeout: time.Duration(cfg.Fetch.TimeoutSeconds) * time.Second,
+ MaxRetries: cfg.Fetch.MaxRetries,
+ })
+
+ ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt)
+ defer stop()
+
+ summary, err := ingest.Run(ctx, fc, st, cfg, targets, time.Now())
+ if err != nil {
+ return err
+ }
+
+ printSummary(cmd, summary)
+ return nil
+}
+
+func selectTargets(srcs []source.Source, only string) ([]source.Source, error) {
+ if only != "" {
+ for _, s := range srcs {
+ if s.Name == only {
+ return []source.Source{s}, nil
+ }
+ }
+ return nil, fmt.Errorf("scrape: unknown source %q", only)
+ }
+ return source.Enabled(srcs), nil
+}
+
+func printSummary(cmd *cobra.Command, summary ingest.Summary) {
+ out := cmd.OutOrStdout()
+ fmt.Fprintf(out, "%-18s %-8s %-8s %-5s %-5s %-5s\n", "SOURCE", "STATUS", "PARSED", "NEW", "DUP", "ERR")
+ for _, r := range summary.Results {
+ fmt.Fprintf(out, "%-18s %-8s %-8s %-5s %-5s %-5s\n",
+ r.Name, status(r), count(r, r.Parsed), count(r, r.New), count(r, r.Duplicates), count(r, r.ItemErrors))
+ }
+ newArticles, duplicates, failed := summary.Totals()
+ fmt.Fprintf(out, "\n%d new, %d duplicate across %d sources (%d failed)\n",
+ newArticles, duplicates, len(summary.Results), failed)
+ for _, r := range summary.Results {
+ if r.Err != nil {
+ fmt.Fprintf(out, " %s: %v\n", r.Name, r.Err)
+ }
+ }
+}
+
+func status(r ingest.SourceResult) string {
+ switch {
+ case r.Err != nil:
+ return statusError
+ case r.NotModified:
+ return statusNotModified
+ default:
+ return statusOK
+ }
+}
+
+func count(r ingest.SourceResult, n int) string {
+ if r.Err != nil || r.NotModified {
+ return dash
+ }
+ return fmt.Sprintf("%d", n)
+}
diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go
index 2f9eb03e..3b9f1431 100644
--- a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go
+++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go
@@ -21,7 +21,6 @@ func init() {
short string
milestone string
}{
- {"scrape", "Ingest all enabled sources once", "milestone M1"},
{"list", "List stored articles with filters", "milestone M3"},
{"cve", "Show an enriched CVE and articles mentioning it", "milestone M3"},
{"digest", "Render a ranked digest to Markdown or JSON", "milestone M4"},
diff --git a/PROJECTS/intermediate/security-news-scraper/go.mod b/PROJECTS/intermediate/security-news-scraper/go.mod
index 0f7c5876..7a74c19b 100644
--- a/PROJECTS/intermediate/security-news-scraper/go.mod
+++ b/PROJECTS/intermediate/security-news-scraper/go.mod
@@ -3,20 +3,32 @@ module github.com/CarterPerez-dev/nadezhda
go 1.25.0
require (
+ github.com/PuerkitoBio/goquery v1.12.0
+ github.com/mmcdole/gofeed v1.3.0
github.com/spf13/cobra v1.10.2
+ github.com/temoto/robotstxt v1.1.2
+ golang.org/x/sync v0.21.0
+ golang.org/x/time v0.15.0
gopkg.in/yaml.v3 v3.0.1
modernc.org/sqlite v1.53.0
)
require (
+ github.com/andybalholm/cascadia v1.3.3 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
+ github.com/json-iterator/go v1.1.12 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
+ github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 // indirect
+ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+ github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/ncruces/go-strftime v1.0.0 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/spf13/pflag v1.0.9 // indirect
+ golang.org/x/net v0.52.0 // indirect
golang.org/x/sys v0.44.0 // indirect
+ golang.org/x/text v0.35.0 // indirect
modernc.org/libc v1.73.4 // indirect
modernc.org/mathutil v1.7.1 // indirect
modernc.org/memory v1.11.0 // indirect
diff --git a/PROJECTS/intermediate/security-news-scraper/go.sum b/PROJECTS/intermediate/security-news-scraper/go.sum
index 7c4811a2..d61e7395 100644
--- a/PROJECTS/intermediate/security-news-scraper/go.sum
+++ b/PROJECTS/intermediate/security-news-scraper/go.sum
@@ -1,6 +1,15 @@
+github.com/PuerkitoBio/goquery v1.12.0 h1:pAcL4g3WRXekcB9AU/y1mbKez2dbY2AajVhtkO8RIBo=
+github.com/PuerkitoBio/goquery v1.12.0/go.mod h1:802ej+gV2y7bbIhOIoPY5sT183ZW0YFofScC4q/hIpQ=
+github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
+github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs=
github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
@@ -9,10 +18,23 @@ github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mmcdole/gofeed v1.3.0 h1:5yn+HeqlcvjMeAI4gu6T+crm7d0anY85+M+v6fIFNG4=
+github.com/mmcdole/gofeed v1.3.0/go.mod h1:9TGv2LcJhdXePDzxiuMnukhV2/zb6VtnZt1mS+SjkLE=
+github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 h1:Zr92CAlFhy2gL+V1F+EyIuzbQNbSgP4xhTODZtrXUtk=
+github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
@@ -20,16 +42,92 @@ github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY=
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg=
+github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
+golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
+golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
+golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4=
golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ=
-golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
-golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
+golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
+golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
+golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
+golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
+golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ=
golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
+golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
+golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
+golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
+golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
+golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
+golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
+golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
+golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8=
golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
diff --git a/PROJECTS/intermediate/security-news-scraper/internal/config/config.go b/PROJECTS/intermediate/security-news-scraper/internal/config/config.go
index cbe18b30..a8692f5f 100644
--- a/PROJECTS/intermediate/security-news-scraper/internal/config/config.go
+++ b/PROJECTS/intermediate/security-news-scraper/internal/config/config.go
@@ -13,12 +13,13 @@ import (
const (
defaultDBPath = "nadezhda.db"
- defaultUserAgent = "nadezhda/0.1 (+https://github.com/CarterPerez-dev/nadezhda)"
- defaultPerHostRate = 0.5
- defaultPerHostBurst = 1
- defaultTimeoutSeconds = 25
- defaultWorkers = 8
- defaultMaxRetries = 3
+ defaultUserAgent = "nadezhda/0.1 (+https://github.com/CarterPerez-dev/nadezhda)"
+ defaultPerHostRate = 0.5
+ defaultPerHostBurst = 1
+ defaultTimeoutSeconds = 25
+ defaultSourceTimeoutSeconds = 90
+ defaultWorkers = 8
+ defaultMaxRetries = 3
defaultCacheTTLHours = 24
defaultNegativeTTLHours = 3
@@ -26,6 +27,8 @@ const (
defaultTitleJaccard = 0.6
defaultWindowHours = 72
+ trackingUTMPrefix = "utm_*"
+
defaultHalfLifeHours = 48
defaultVelocityNorm = 0.5
@@ -48,13 +51,18 @@ const (
defaultClaudeModel = "claude-opus-4-8"
)
+var defaultTrackingParams = []string{
+ trackingUTMPrefix, "gclid", "fbclid", "ref", "mc_cid", "mc_eid",
+}
+
type Fetch struct {
- UserAgent string `yaml:"user_agent"`
- PerHostRate float64 `yaml:"per_host_rate"`
- PerHostBurst int `yaml:"per_host_burst"`
- TimeoutSeconds int `yaml:"timeout_seconds"`
- Workers int `yaml:"workers"`
- MaxRetries int `yaml:"max_retries"`
+ UserAgent string `yaml:"user_agent"`
+ PerHostRate float64 `yaml:"per_host_rate"`
+ PerHostBurst int `yaml:"per_host_burst"`
+ TimeoutSeconds int `yaml:"timeout_seconds"`
+ SourceTimeoutSeconds int `yaml:"source_timeout_seconds"`
+ Workers int `yaml:"workers"`
+ MaxRetries int `yaml:"max_retries"`
}
type Enrich struct {
@@ -63,8 +71,9 @@ type Enrich struct {
}
type Cluster struct {
- TitleJaccard float64 `yaml:"title_jaccard_threshold"`
- WindowHours int `yaml:"window_hours"`
+ TitleJaccard float64 `yaml:"title_jaccard_threshold"`
+ WindowHours int `yaml:"window_hours"`
+ TrackingParams []string `yaml:"tracking_params"`
}
type Weights struct {
@@ -112,20 +121,22 @@ func Default() Config {
return Config{
DBPath: defaultDBPath,
Fetch: Fetch{
- UserAgent: defaultUserAgent,
- PerHostRate: defaultPerHostRate,
- PerHostBurst: defaultPerHostBurst,
- TimeoutSeconds: defaultTimeoutSeconds,
- Workers: defaultWorkers,
- MaxRetries: defaultMaxRetries,
+ UserAgent: defaultUserAgent,
+ PerHostRate: defaultPerHostRate,
+ PerHostBurst: defaultPerHostBurst,
+ TimeoutSeconds: defaultTimeoutSeconds,
+ SourceTimeoutSeconds: defaultSourceTimeoutSeconds,
+ Workers: defaultWorkers,
+ MaxRetries: defaultMaxRetries,
},
Enrich: Enrich{
CacheTTLHours: defaultCacheTTLHours,
NegativeTTLHours: defaultNegativeTTLHours,
},
Cluster: Cluster{
- TitleJaccard: defaultTitleJaccard,
- WindowHours: defaultWindowHours,
+ TitleJaccard: defaultTitleJaccard,
+ WindowHours: defaultWindowHours,
+ TrackingParams: defaultTrackingParams,
},
Rank: Rank{
HalfLifeHours: defaultHalfLifeHours,
@@ -185,6 +196,12 @@ func (c Config) validate() error {
if c.Fetch.PerHostBurst < 1 {
return fmt.Errorf("config: fetch.per_host_burst must be >= 1, got %d", c.Fetch.PerHostBurst)
}
+ if c.Fetch.TimeoutSeconds < 1 {
+ return fmt.Errorf("config: fetch.timeout_seconds must be >= 1, got %d", c.Fetch.TimeoutSeconds)
+ }
+ if c.Fetch.SourceTimeoutSeconds < 1 {
+ return fmt.Errorf("config: fetch.source_timeout_seconds must be >= 1, got %d", c.Fetch.SourceTimeoutSeconds)
+ }
if c.Enrich.CacheTTLHours < 0 || c.Enrich.NegativeTTLHours < 0 {
return fmt.Errorf("config: enrich TTLs must be >= 0, got cache=%d negative=%d", c.Enrich.CacheTTLHours, c.Enrich.NegativeTTLHours)
}
diff --git a/PROJECTS/intermediate/security-news-scraper/internal/fetch/fetch.go b/PROJECTS/intermediate/security-news-scraper/internal/fetch/fetch.go
new file mode 100644
index 00000000..42419123
--- /dev/null
+++ b/PROJECTS/intermediate/security-news-scraper/internal/fetch/fetch.go
@@ -0,0 +1,235 @@
+// ©AngelaMos | 2026
+// fetch.go
+
+package fetch
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "net/url"
+ "strconv"
+ "sync"
+ "time"
+
+ "golang.org/x/time/rate"
+)
+
+const (
+ headerUserAgent = "User-Agent"
+ headerAccept = "Accept"
+ headerIfNoneMatch = "If-None-Match"
+ headerIfModifiedSince = "If-Modified-Since"
+ headerETag = "ETag"
+ headerLastModified = "Last-Modified"
+ headerRetryAfter = "Retry-After"
+
+ acceptFeed = "application/rss+xml, application/atom+xml, application/xml;q=0.9, */*;q=0.8"
+
+ defaultBackoffBase = 500 * time.Millisecond
+ maxBodyBytes = 16 << 20
+ maxRetryAfter = 60 * time.Second
+)
+
+type Options struct {
+ UserAgent string
+ PerHostRate float64
+ PerHostBurst int
+ Timeout time.Duration
+ MaxRetries int
+}
+
+type Request struct {
+ URL string
+ ETag string
+ LastModified string
+}
+
+type Result struct {
+ Status int
+ Body []byte
+ ETag string
+ LastModified string
+ NotModified bool
+}
+
+type Client struct {
+ http *http.Client
+ ua string
+ rate rate.Limit
+ burst int
+ maxRetries int
+ backoffBase time.Duration
+
+ mu sync.Mutex
+ limiters map[string]*rate.Limiter
+
+ robots *robotsCache
+}
+
+func New(opts Options) *Client {
+ c := &Client{
+ http: &http.Client{Timeout: opts.Timeout},
+ ua: opts.UserAgent,
+ rate: rate.Limit(opts.PerHostRate),
+ burst: opts.PerHostBurst,
+ maxRetries: opts.MaxRetries,
+ backoffBase: defaultBackoffBase,
+ limiters: make(map[string]*rate.Limiter),
+ }
+ c.robots = newRobotsCache(c)
+ return c
+}
+
+func (c *Client) limiterFor(host string) *rate.Limiter {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ l, ok := c.limiters[host]
+ if !ok {
+ l = rate.NewLimiter(c.rate, c.burst)
+ c.limiters[host] = l
+ }
+ return l
+}
+
+func (c *Client) Fetch(ctx context.Context, req Request) (Result, error) {
+ u, err := url.Parse(req.URL)
+ if err != nil {
+ return Result{}, fmt.Errorf("fetch: parse url %q: %w", req.URL, err)
+ }
+
+ var lastErr error
+ var override time.Duration
+ for attempt := 0; attempt <= c.maxRetries; attempt++ {
+ if attempt > 0 {
+ delay := c.backoff(attempt)
+ if override > 0 {
+ delay = override
+ }
+ if err := sleep(ctx, delay); err != nil {
+ return Result{}, err
+ }
+ }
+ if err := c.limiterFor(u.Host).Wait(ctx); err != nil {
+ return Result{}, err
+ }
+ res, retryAfter, retry, err := c.do(ctx, req)
+ if err == nil {
+ return res, nil
+ }
+ lastErr = err
+ override = retryAfter
+ if !retry {
+ return Result{}, err
+ }
+ }
+ return Result{}, fmt.Errorf("fetch %s: exhausted %d retries: %w", req.URL, c.maxRetries, lastErr)
+}
+
+func (c *Client) Allowed(ctx context.Context, rawURL string) (bool, error) {
+ u, err := url.Parse(rawURL)
+ if err != nil {
+ return false, fmt.Errorf("fetch: parse url %q: %w", rawURL, err)
+ }
+ return c.robots.allowed(ctx, u)
+}
+
+func (c *Client) do(ctx context.Context, req Request) (Result, time.Duration, bool, error) {
+ httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, req.URL, nil)
+ if err != nil {
+ return Result{}, 0, false, fmt.Errorf("fetch %s: build request: %w", req.URL, err)
+ }
+ httpReq.Header.Set(headerUserAgent, c.ua)
+ httpReq.Header.Set(headerAccept, acceptFeed)
+ if req.ETag != "" {
+ httpReq.Header.Set(headerIfNoneMatch, req.ETag)
+ }
+ if req.LastModified != "" {
+ httpReq.Header.Set(headerIfModifiedSince, req.LastModified)
+ }
+
+ resp, err := c.http.Do(httpReq)
+ if err != nil {
+ if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
+ return Result{}, 0, false, fmt.Errorf("fetch %s: %w", req.URL, err)
+ }
+ return Result{}, 0, true, fmt.Errorf("fetch %s: %w", req.URL, err)
+ }
+ defer resp.Body.Close()
+
+ switch {
+ case resp.StatusCode == http.StatusNotModified:
+ return Result{
+ Status: resp.StatusCode,
+ NotModified: true,
+ ETag: req.ETag,
+ LastModified: req.LastModified,
+ }, 0, false, nil
+ case resp.StatusCode == http.StatusOK:
+ body, err := io.ReadAll(io.LimitReader(resp.Body, maxBodyBytes))
+ if err != nil {
+ return Result{}, 0, true, fmt.Errorf("fetch %s: read body: %w", req.URL, err)
+ }
+ return Result{
+ Status: resp.StatusCode,
+ Body: body,
+ ETag: resp.Header.Get(headerETag),
+ LastModified: resp.Header.Get(headerLastModified),
+ }, 0, false, nil
+ case resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode >= http.StatusInternalServerError:
+ drain(resp.Body)
+ return Result{}, retryAfter(resp), true, fmt.Errorf("fetch %s: server status %d", req.URL, resp.StatusCode)
+ default:
+ drain(resp.Body)
+ return Result{}, 0, false, fmt.Errorf("fetch %s: status %d", req.URL, resp.StatusCode)
+ }
+}
+
+func drain(body io.Reader) {
+ _, _ = io.Copy(io.Discard, io.LimitReader(body, maxBodyBytes))
+}
+
+func retryAfter(resp *http.Response) time.Duration {
+ v := resp.Header.Get(headerRetryAfter)
+ if v == "" {
+ return 0
+ }
+ if secs, err := strconv.Atoi(v); err == nil {
+ d := time.Duration(secs) * time.Second
+ if d > maxRetryAfter {
+ return maxRetryAfter
+ }
+ if d < 0 {
+ return 0
+ }
+ return d
+ }
+ if t, err := http.ParseTime(v); err == nil {
+ d := time.Until(t)
+ if d <= 0 {
+ return 0
+ }
+ if d > maxRetryAfter {
+ return maxRetryAfter
+ }
+ return d
+ }
+ return 0
+}
+
+func (c *Client) backoff(attempt int) time.Duration {
+ return c.backoffBase * time.Duration(1<<(attempt-1))
+}
+
+func sleep(ctx context.Context, d time.Duration) error {
+ timer := time.NewTimer(d)
+ defer timer.Stop()
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case <-timer.C:
+ return nil
+ }
+}
diff --git a/PROJECTS/intermediate/security-news-scraper/internal/fetch/fetch_test.go b/PROJECTS/intermediate/security-news-scraper/internal/fetch/fetch_test.go
new file mode 100644
index 00000000..a94beefa
--- /dev/null
+++ b/PROJECTS/intermediate/security-news-scraper/internal/fetch/fetch_test.go
@@ -0,0 +1,244 @@
+// ©AngelaMos | 2026
+// fetch_test.go
+
+package fetch
+
+import (
+ "context"
+ "net/http"
+ "net/http/httptest"
+ "sync/atomic"
+ "testing"
+ "time"
+)
+
+func testClient() *Client {
+ c := New(Options{
+ UserAgent: "nadezhda-test/1.0",
+ PerHostRate: 1e6,
+ PerHostBurst: 1,
+ Timeout: 5 * time.Second,
+ MaxRetries: 3,
+ })
+ c.backoffBase = time.Millisecond
+ return c
+}
+
+func TestFetchOKCapturesValidators(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set(headerETag, `"v1"`)
+ w.Header().Set(headerLastModified, "Wed, 01 Jul 2026 00:00:00 GMT")
+ _, _ = w.Write([]byte("<rss></rss>"))
+ }))
+ defer srv.Close()
+
+ res, err := testClient().Fetch(context.Background(), Request{URL: srv.URL + "/feed"})
+ if err != nil {
+ t.Fatalf("Fetch: %v", err)
+ }
+ if res.Status != http.StatusOK {
+ t.Errorf("status = %d, want 200", res.Status)
+ }
+ if string(res.Body) != "<rss></rss>" {
+ t.Errorf("body = %q", res.Body)
+ }
+ if res.ETag != `"v1"` {
+ t.Errorf("etag = %q, want \"v1\"", res.ETag)
+ }
+ if res.LastModified == "" {
+ t.Error("last-modified not captured")
+ }
+}
+
+func TestFetchConditionalGET(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Header.Get(headerIfNoneMatch) == `"v1"` {
+ w.WriteHeader(http.StatusNotModified)
+ return
+ }
+ w.Header().Set(headerETag, `"v1"`)
+ _, _ = w.Write([]byte("body"))
+ }))
+ defer srv.Close()
+
+ c := testClient()
+ first, err := c.Fetch(context.Background(), Request{URL: srv.URL + "/feed"})
+ if err != nil || first.Status != http.StatusOK {
+ t.Fatalf("first fetch: status=%d err=%v", first.Status, err)
+ }
+
+ second, err := c.Fetch(context.Background(), Request{URL: srv.URL + "/feed", ETag: first.ETag})
+ if err != nil {
+ t.Fatalf("second fetch: %v", err)
+ }
+ if !second.NotModified {
+ t.Error("expected NotModified on matching ETag")
+ }
+ if second.ETag != `"v1"` {
+ t.Errorf("304 should retain etag, got %q", second.ETag)
+ }
+}
+
+func TestFetchRetriesServerError(t *testing.T) {
+ var calls atomic.Int32
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if calls.Add(1) == 1 {
+ w.WriteHeader(http.StatusInternalServerError)
+ return
+ }
+ _, _ = w.Write([]byte("ok"))
+ }))
+ defer srv.Close()
+
+ res, err := testClient().Fetch(context.Background(), Request{URL: srv.URL + "/feed"})
+ if err != nil {
+ t.Fatalf("Fetch: %v", err)
+ }
+ if string(res.Body) != "ok" {
+ t.Errorf("body = %q, want ok", res.Body)
+ }
+ if calls.Load() != 2 {
+ t.Errorf("calls = %d, want 2 (one 500, one 200)", calls.Load())
+ }
+}
+
+func TestFetchDoesNotRetryClientError(t *testing.T) {
+ var calls atomic.Int32
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ calls.Add(1)
+ w.WriteHeader(http.StatusNotFound)
+ }))
+ defer srv.Close()
+
+ _, err := testClient().Fetch(context.Background(), Request{URL: srv.URL + "/feed"})
+ if err == nil {
+ t.Fatal("expected error on 404")
+ }
+ if calls.Load() != 1 {
+ t.Errorf("calls = %d, want 1 (no retry on 4xx)", calls.Load())
+ }
+}
+
+func TestFetch429IsRetried(t *testing.T) {
+ var calls atomic.Int32
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if calls.Add(1) == 1 {
+ w.WriteHeader(http.StatusTooManyRequests)
+ return
+ }
+ _, _ = w.Write([]byte("ok"))
+ }))
+ defer srv.Close()
+
+ res, err := testClient().Fetch(context.Background(), Request{URL: srv.URL + "/feed"})
+ if err != nil {
+ t.Fatalf("Fetch: %v", err)
+ }
+ if string(res.Body) != "ok" || calls.Load() != 2 {
+ t.Errorf("body=%q calls=%d, want ok/2", res.Body, calls.Load())
+ }
+}
+
+func TestFetchTimeoutNotRetried(t *testing.T) {
+ var calls atomic.Int32
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ calls.Add(1)
+ time.Sleep(100 * time.Millisecond)
+ }))
+ defer srv.Close()
+
+ c := New(Options{
+ UserAgent: "nadezhda-test/1.0", PerHostRate: 1e6, PerHostBurst: 1,
+ Timeout: 15 * time.Millisecond, MaxRetries: 3,
+ })
+ c.backoffBase = time.Millisecond
+
+ _, err := c.Fetch(context.Background(), Request{URL: srv.URL + "/feed"})
+ if err == nil {
+ t.Fatal("expected timeout error")
+ }
+ if calls.Load() != 1 {
+ t.Errorf("calls = %d, want 1 (timeout must not be retried)", calls.Load())
+ }
+}
+
+func TestRetryAfterParsing(t *testing.T) {
+ cases := []struct {
+ header string
+ want time.Duration
+ }{
+ {"", 0},
+ {"5", 5 * time.Second},
+ {"0", 0},
+ {"-3", 0},
+ {"9999", maxRetryAfter},
+ {"garbage", 0},
+ }
+ for _, tc := range cases {
+ resp := &http.Response{Header: http.Header{}}
+ if tc.header != "" {
+ resp.Header.Set(headerRetryAfter, tc.header)
+ }
+ if got := retryAfter(resp); got != tc.want {
+ t.Errorf("retryAfter(%q) = %s, want %s", tc.header, got, tc.want)
+ }
+ }
+}
+
+func TestFeedFetchIgnoresRobots(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path == robotsPath {
+ t.Error("Fetch must not request robots.txt on the feed path")
+ _, _ = w.Write([]byte("User-agent: *\nDisallow: /\n"))
+ return
+ }
+ _, _ = w.Write([]byte("feed body"))
+ }))
+ defer srv.Close()
+
+ res, err := testClient().Fetch(context.Background(), Request{URL: srv.URL + "/feed"})
+ if err != nil {
+ t.Fatalf("Fetch: %v", err)
+ }
+ if string(res.Body) != "feed body" {
+ t.Errorf("body = %q, want feed body", res.Body)
+ }
+}
+
+func TestAllowedRespectsDisallow(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path == robotsPath {
+ _, _ = w.Write([]byte("User-agent: *\nDisallow: /article\n"))
+ return
+ }
+ w.WriteHeader(http.StatusNotFound)
+ }))
+ defer srv.Close()
+
+ ok, err := testClient().Allowed(context.Background(), srv.URL+"/article/123")
+ if err != nil {
+ t.Fatalf("Allowed: %v", err)
+ }
+ if ok {
+ t.Error("expected disallowed for /article path")
+ }
+}
+
+func TestAllowedPermitsOtherPaths(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path == robotsPath {
+ _, _ = w.Write([]byte("User-agent: *\nDisallow: /admin\n"))
+ return
+ }
+ w.WriteHeader(http.StatusNotFound)
+ }))
+ defer srv.Close()
+
+ ok, err := testClient().Allowed(context.Background(), srv.URL+"/article/123")
+ if err != nil {
+ t.Fatalf("Allowed: %v", err)
+ }
+ if !ok {
+ t.Error("expected allowed for non-admin path")
+ }
+}
diff --git a/PROJECTS/intermediate/security-news-scraper/internal/fetch/robots.go b/PROJECTS/intermediate/security-news-scraper/internal/fetch/robots.go
new file mode 100644
index 00000000..f8aee1be
--- /dev/null
+++ b/PROJECTS/intermediate/security-news-scraper/internal/fetch/robots.go
@@ -0,0 +1,88 @@
+// ©AngelaMos | 2026
+// robots.go
+
+package fetch
+
+import (
+ "context"
+ "io"
+ "net/http"
+ "net/url"
+ "sync"
+
+ "github.com/temoto/robotstxt"
+)
+
+const (
+ robotsPath = "/robots.txt"
+ maxRobotsBytes = 512 << 10
+ robotsRootPath = "/"
+)
+
+type robotsEntry struct {
+ once sync.Once
+ data *robotstxt.RobotsData
+}
+
+type robotsCache struct {
+ client *Client
+
+ mu sync.Mutex
+ entries map[string]*robotsEntry
+}
+
+func newRobotsCache(client *Client) *robotsCache {
+ return &robotsCache{
+ client: client,
+ entries: make(map[string]*robotsEntry),
+ }
+}
+
+func (rc *robotsCache) allowed(ctx context.Context, u *url.URL) (bool, error) {
+ rc.mu.Lock()
+ e, ok := rc.entries[u.Host]
+ if !ok {
+ e = &robotsEntry{}
+ rc.entries[u.Host] = e
+ }
+ rc.mu.Unlock()
+
+ e.once.Do(func() {
+ data, err := rc.load(ctx, u)
+ if err != nil {
+ data, _ = robotstxt.FromStatusAndBytes(http.StatusOK, nil)
+ }
+ e.data = data
+ })
+
+ path := u.EscapedPath()
+ if path == "" {
+ path = robotsRootPath
+ }
+ return e.data.FindGroup(rc.client.ua).Test(path), nil
+}
+
+func (rc *robotsCache) load(ctx context.Context, u *url.URL) (*robotstxt.RobotsData, error) {
+ robotsURL := u.Scheme + "://" + u.Host + robotsPath
+ if err := rc.client.limiterFor(u.Host).Wait(ctx); err != nil {
+ return nil, err
+ }
+
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, robotsURL, nil)
+ if err != nil {
+ return nil, err
+ }
+ req.Header.Set(headerUserAgent, rc.client.ua)
+
+ resp, err := rc.client.http.Do(req)
+ if err != nil {
+ return nil, err
+ }
+ defer resp.Body.Close()
+
+ body, err := io.ReadAll(io.LimitReader(resp.Body, maxRobotsBytes))
+ if err != nil {
+ return nil, err
+ }
+ return robotstxt.FromStatusAndBytes(resp.StatusCode, body)
+}
diff --git a/PROJECTS/intermediate/security-news-scraper/internal/ingest/ingest.go b/PROJECTS/intermediate/security-news-scraper/internal/ingest/ingest.go
new file mode 100644
index 00000000..215dc79b
--- /dev/null
+++ b/PROJECTS/intermediate/security-news-scraper/internal/ingest/ingest.go
@@ -0,0 +1,162 @@
+// ©AngelaMos | 2026
+// ingest.go
+
+package ingest
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "time"
+
+ "golang.org/x/sync/errgroup"
+
+ "github.com/CarterPerez-dev/nadezhda/internal/config"
+ "github.com/CarterPerez-dev/nadezhda/internal/fetch"
+ "github.com/CarterPerez-dev/nadezhda/internal/normalize"
+ "github.com/CarterPerez-dev/nadezhda/internal/parse"
+ "github.com/CarterPerez-dev/nadezhda/internal/source"
+ "github.com/CarterPerez-dev/nadezhda/internal/store"
+)
+
+type SourceResult struct {
+ Name string
+ Parsed int
+ New int
+ Duplicates int
+ ItemErrors int
+ NotModified bool
+ Err error
+}
+
+type Summary struct {
+ Results []SourceResult
+}
+
+func (s Summary) Totals() (newArticles, duplicates, failed int) {
+ for _, r := range s.Results {
+ newArticles += r.New
+ duplicates += r.Duplicates
+ if r.Err != nil {
+ failed++
+ }
+ }
+ return newArticles, duplicates, failed
+}
+
+func Run(ctx context.Context, fc *fetch.Client, st *store.Store, cfg config.Config, targets []source.Source, now time.Time) (Summary, error) {
+ results := make([]SourceResult, len(targets))
+
+ workers := cfg.Fetch.Workers
+ if workers < 1 {
+ workers = 1
+ }
+
+ g, gctx := errgroup.WithContext(ctx)
+ g.SetLimit(workers)
+
+ for i, src := range targets {
+ results[i].Name = src.Name
+ g.Go(func() error {
+ process(gctx, fc, st, cfg, src, now, &results[i])
+ return nil
+ })
+ }
+ if err := g.Wait(); err != nil {
+ return Summary{Results: results}, err
+ }
+ return Summary{Results: results}, nil
+}
+
+func process(ctx context.Context, fc *fetch.Client, st *store.Store, cfg config.Config, src source.Source, now time.Time, out *SourceResult) {
+ ctx, cancel := context.WithTimeout(ctx, time.Duration(cfg.Fetch.SourceTimeoutSeconds)*time.Second)
+ defer cancel()
+
+ id, err := st.UpsertSource(store.SourceInput{
+ Name: src.Name, Title: src.Title, URL: src.URL, Type: string(src.Type),
+ Weight: src.Weight, Tags: src.Tags, Enabled: src.Enabled,
+ })
+ if err != nil {
+ out.Err = err
+ return
+ }
+
+ prev, _, err := st.GetFetchState(id)
+ if err != nil {
+ out.Err = err
+ return
+ }
+
+ res, err := fc.Fetch(ctx, fetch.Request{
+ URL: src.URL, ETag: prev.ETag, LastModified: prev.LastModified,
+ })
+ if err != nil {
+ out.Err = err
+ return
+ }
+
+ if res.NotModified {
+ out.NotModified = true
+ out.Err = st.UpsertFetchState(id, store.FetchState{
+ ETag: prev.ETag, LastModified: prev.LastModified,
+ LastFetched: now.Unix(), LastStatus: int64(res.Status),
+ })
+ return
+ }
+
+ items, err := parse.Feed(bytes.NewReader(res.Body))
+ if err != nil {
+ out.Err = err
+ return
+ }
+ out.Parsed = len(items)
+
+ for _, it := range items {
+ storeItem(st, cfg, id, it, now, out)
+ }
+
+ if err := st.UpsertFetchState(id, store.FetchState{
+ ETag: res.ETag, LastModified: res.LastModified,
+ LastFetched: now.Unix(), LastStatus: int64(res.Status),
+ }); err != nil && out.Err == nil {
+ out.Err = err
+ }
+}
+
+func storeItem(st *store.Store, cfg config.Config, sourceID int64, it parse.Item, now time.Time, out *SourceResult) {
+ if it.Link == "" {
+ out.ItemErrors++
+ return
+ }
+ canonical, err := normalize.CanonicalURL(it.Link, cfg.Cluster.TrackingParams)
+ if err != nil {
+ out.ItemErrors++
+ return
+ }
+
+ var publishedAt int64
+ if !it.Published.IsZero() {
+ publishedAt = it.Published.Unix()
+ }
+
+ _, err = st.InsertArticle(store.Article{
+ SourceID: sourceID,
+ CanonicalURL: canonical,
+ ContentHash: normalize.ContentHash(canonical),
+ TitleHash: normalize.TitleHash(normalize.NormalizeTitle(it.Title)),
+ Title: it.Title,
+ Summary: normalize.StripHTML(it.Summary),
+ Body: normalize.StripHTML(it.Body),
+ Author: it.Author,
+ PublishedAt: publishedAt,
+ FetchedAt: now.Unix(),
+ })
+ switch {
+ case err == nil:
+ out.New++
+ case errors.Is(err, store.ErrDuplicate):
+ out.Duplicates++
+ default:
+ out.ItemErrors++
+ }
+}
diff --git a/PROJECTS/intermediate/security-news-scraper/internal/ingest/ingest_test.go b/PROJECTS/intermediate/security-news-scraper/internal/ingest/ingest_test.go
new file mode 100644
index 00000000..0aa0f3e1
--- /dev/null
+++ b/PROJECTS/intermediate/security-news-scraper/internal/ingest/ingest_test.go
@@ -0,0 +1,199 @@
+// ©AngelaMos | 2026
+// ingest_test.go
+
+package ingest
+
+import (
+ "context"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+
+ "github.com/CarterPerez-dev/nadezhda/internal/config"
+ "github.com/CarterPerez-dev/nadezhda/internal/fetch"
+ "github.com/CarterPerez-dev/nadezhda/internal/normalize"
+ "github.com/CarterPerez-dev/nadezhda/internal/source"
+ "github.com/CarterPerez-dev/nadezhda/internal/store"
+)
+
+const fixture = "../../testdata/feeds/thehackernews.xml"
+
+func loadFixture(t *testing.T) []byte {
+ t.Helper()
+ b, err := os.ReadFile(fixture)
+ if err != nil {
+ t.Fatalf("read fixture: %v", err)
+ }
+ return b
+}
+
+func newStore(t *testing.T) *store.Store {
+ t.Helper()
+ st, err := store.Open(filepath.Join(t.TempDir(), "ingest.db"))
+ if err != nil {
+ t.Fatalf("open store: %v", err)
+ }
+ t.Cleanup(func() { _ = st.Close() })
+ return st
+}
+
+func newClient() *fetch.Client {
+ return fetch.New(fetch.Options{
+ UserAgent: "nadezhda-test/1.0",
+ PerHostRate: 1e6,
+ PerHostBurst: 1,
+ Timeout: 5 * time.Second,
+ MaxRetries: 2,
+ })
+}
+
+func target(url string) []source.Source {
+ return []source.Source{{
+ Name: "test", Title: "Test Feed", URL: url,
+ Type: source.KindRSS, Weight: 1.0, Tags: []string{"news"}, Enabled: true,
+ }}
+}
+
+func TestRunIngestsAndDedups(t *testing.T) {
+ body := loadFixture(t)
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ _, _ = w.Write(body)
+ }))
+ defer srv.Close()
+
+ st := newStore(t)
+ cfg := config.Default()
+ targets := target(srv.URL + "/feed")
+ now := time.Unix(1_800_000_000, 0)
+
+ first, err := Run(context.Background(), newClient(), st, cfg, targets, now)
+ if err != nil {
+ t.Fatalf("first Run: %v", err)
+ }
+ r0 := first.Results[0]
+ if r0.Err != nil {
+ t.Fatalf("source error: %v", r0.Err)
+ }
+ if r0.New != 3 || r0.Parsed != 3 {
+ t.Fatalf("first run: parsed=%d new=%d, want 3/3", r0.Parsed, r0.New)
+ }
+
+ count, err := st.CountArticles()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if count != 3 {
+ t.Fatalf("stored articles = %d, want 3", count)
+ }
+
+ var title, canonical, contentHash, titleHash string
+ if err := st.DB().QueryRow(
+ `SELECT title, canonical_url, content_hash, title_hash FROM articles LIMIT 1`,
+ ).Scan(&title, &canonical, &contentHash, &titleHash); err != nil {
+ t.Fatal(err)
+ }
+ if contentHash != normalize.ContentHash(canonical) {
+ t.Errorf("content_hash = %q, want sha256(canonical)", contentHash)
+ }
+ if titleHash != normalize.TitleHash(normalize.NormalizeTitle(title)) {
+ t.Errorf("title_hash = %q, want sha256(normalized title)", titleHash)
+ }
+
+ second, err := Run(context.Background(), newClient(), st, cfg, targets, now)
+ if err != nil {
+ t.Fatalf("second Run: %v", err)
+ }
+ r1 := second.Results[0]
+ if r1.New != 0 || r1.Duplicates != 3 {
+ t.Fatalf("second run: new=%d dup=%d, want 0/3", r1.New, r1.Duplicates)
+ }
+
+ count, _ = st.CountArticles()
+ if count != 3 {
+ t.Fatalf("after re-run stored = %d, want 3 (idempotent)", count)
+ }
+}
+
+func TestRunConditionalNotModified(t *testing.T) {
+ body := loadFixture(t)
+ const etag = `"abc123"`
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Header.Get("If-None-Match") == etag {
+ w.WriteHeader(http.StatusNotModified)
+ return
+ }
+ w.Header().Set("ETag", etag)
+ _, _ = w.Write(body)
+ }))
+ defer srv.Close()
+
+ st := newStore(t)
+ cfg := config.Default()
+ targets := target(srv.URL + "/feed")
+ now := time.Unix(1_800_000_000, 0)
+
+ first, err := Run(context.Background(), newClient(), st, cfg, targets, now)
+ if err != nil {
+ t.Fatalf("first Run: %v", err)
+ }
+ if first.Results[0].New != 3 {
+ t.Fatalf("first run new = %d, want 3", first.Results[0].New)
+ }
+ var storedETag string
+ if err := st.DB().QueryRow(`SELECT etag FROM fetch_state LIMIT 1`).Scan(&storedETag); err != nil {
+ t.Fatal(err)
+ }
+ if storedETag != etag {
+ t.Errorf("persisted etag = %q, want %q", storedETag, etag)
+ }
+
+ second, err := Run(context.Background(), newClient(), st, cfg, targets, now)
+ if err != nil {
+ t.Fatalf("second Run: %v", err)
+ }
+ r := second.Results[0]
+ if !r.NotModified {
+ t.Errorf("expected NotModified on second run, got %+v", r)
+ }
+ if r.New != 0 {
+ t.Errorf("new = %d on 304, want 0", r.New)
+ }
+}
+
+func TestRunFailsSoftOnBadSource(t *testing.T) {
+ body := loadFixture(t)
+ good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ _, _ = w.Write(body)
+ }))
+ defer good.Close()
+ bad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.WriteHeader(http.StatusInternalServerError)
+ }))
+ defer bad.Close()
+
+ st := newStore(t)
+ cfg := config.Default()
+ targets := []source.Source{
+ {Name: "bad", URL: bad.URL + "/feed", Type: source.KindRSS, Weight: 1, Enabled: true},
+ {Name: "good", URL: good.URL + "/feed", Type: source.KindRSS, Weight: 1, Enabled: true},
+ }
+ now := time.Unix(1_800_000_000, 0)
+
+ summary, err := Run(context.Background(), newClient(), st, cfg, targets, now)
+ if err != nil {
+ t.Fatalf("Run returned error, should fail soft: %v", err)
+ }
+ byName := map[string]SourceResult{}
+ for _, r := range summary.Results {
+ byName[r.Name] = r
+ }
+ if byName["bad"].Err == nil {
+ t.Error("bad source should record an error")
+ }
+ if byName["good"].New != 3 {
+ t.Errorf("good source new = %d, want 3 (bad source must not abort run)", byName["good"].New)
+ }
+}
diff --git a/PROJECTS/intermediate/security-news-scraper/internal/normalize/normalize.go b/PROJECTS/intermediate/security-news-scraper/internal/normalize/normalize.go
new file mode 100644
index 00000000..47fd8c7e
--- /dev/null
+++ b/PROJECTS/intermediate/security-news-scraper/internal/normalize/normalize.go
@@ -0,0 +1,111 @@
+// ©AngelaMos | 2026
+// normalize.go
+
+package normalize
+
+import (
+ "crypto/sha256"
+ "encoding/hex"
+ "fmt"
+ "net/url"
+ "strings"
+ "unicode"
+
+ "github.com/PuerkitoBio/goquery"
+)
+
+const (
+ wildcardSuffix = "*"
+ trailingSlash = "/"
+)
+
+func CanonicalURL(raw string, trackingParams []string) (string, error) {
+ u, err := url.Parse(strings.TrimSpace(raw))
+ if err != nil {
+ return "", fmt.Errorf("normalize: parse url %q: %w", raw, err)
+ }
+ if u.Host == "" {
+ return "", fmt.Errorf("normalize: url %q has no host", raw)
+ }
+
+ u.Scheme = strings.ToLower(u.Scheme)
+ u.Host = strings.ToLower(u.Host)
+ u.Fragment = ""
+ u.RawFragment = ""
+
+ if q := u.Query(); len(q) > 0 {
+ for key := range q {
+ if isTracking(key, trackingParams) {
+ q.Del(key)
+ }
+ }
+ u.RawQuery = q.Encode()
+ }
+
+ u.Path = strings.TrimRight(u.Path, trailingSlash)
+ if u.RawPath != "" {
+ u.RawPath = strings.TrimRight(u.RawPath, trailingSlash)
+ }
+
+ return u.String(), nil
+}
+
+func isTracking(key string, trackingParams []string) bool {
+ lowered := strings.ToLower(key)
+ for _, p := range trackingParams {
+ if strings.HasSuffix(p, wildcardSuffix) {
+ if strings.HasPrefix(lowered, strings.TrimSuffix(p, wildcardSuffix)) {
+ return true
+ }
+ continue
+ }
+ if lowered == p {
+ return true
+ }
+ }
+ return false
+}
+
+func NormalizeTitle(title string) string {
+ var b strings.Builder
+ b.Grow(len(title))
+ prevSpace := false
+ for _, r := range strings.ToLower(title) {
+ if unicode.IsLetter(r) || unicode.IsDigit(r) {
+ b.WriteRune(r)
+ prevSpace = false
+ continue
+ }
+ if !prevSpace {
+ b.WriteByte(' ')
+ prevSpace = true
+ }
+ }
+ return strings.TrimSpace(b.String())
+}
+
+func StripHTML(s string) string {
+ doc, err := goquery.NewDocumentFromReader(strings.NewReader(s))
+ if err != nil {
+ return collapseWhitespace(s)
+ }
+ doc.Find("script,style").Remove()
+ return collapseWhitespace(doc.Text())
+}
+
+func collapseWhitespace(s string) string {
+ return strings.Join(strings.Fields(s), " ")
+}
+
+func ContentHash(canonicalURL string) string {
+ return sha256Hex(canonicalURL)
+}
+
+func TitleHash(normalizedTitle string) string {
+ return sha256Hex(normalizedTitle)
+}
+
+func sha256Hex(s string) string {
+ sum := sha256.Sum256([]byte(s))
+ return hex.EncodeToString(sum[:])
+}
diff --git a/PROJECTS/intermediate/security-news-scraper/internal/normalize/normalize_test.go b/PROJECTS/intermediate/security-news-scraper/internal/normalize/normalize_test.go
new file mode 100644
index 00000000..364e5039
--- /dev/null
+++ b/PROJECTS/intermediate/security-news-scraper/internal/normalize/normalize_test.go
@@ -0,0 +1,111 @@
+// ©AngelaMos | 2026
+// normalize_test.go
+
+package normalize
+
+import "testing"
+
+var params = []string{"utm_*", "gclid", "fbclid", "ref", "mc_cid", "mc_eid"}
+
+func TestCanonicalURL(t *testing.T) {
+ cases := []struct {
+ name string
+ in string
+ want string
+ }{
+ {"lowercase host+scheme", "HTTPS://Example.COM/Path", "https://example.com/Path"},
+ {"drop fragment", "https://example.com/a#section", "https://example.com/a"},
+ {"strip utm params", "https://example.com/a?utm_source=x&utm_medium=y&id=7", "https://example.com/a?id=7"},
+ {"strip gclid fbclid ref", "https://example.com/a?gclid=1&fbclid=2&ref=z&keep=1", "https://example.com/a?keep=1"},
+ {"strip mailchimp", "https://example.com/a?mc_cid=1&mc_eid=2", "https://example.com/a"},
+ {"drop trailing slash", "https://example.com/a/b/", "https://example.com/a/b"},
+ {"root trailing slash", "https://example.com/", "https://example.com"},
+ {"sorted query", "https://example.com/a?b=2&a=1", "https://example.com/a?a=1&b=2"},
+ {"tracking case insensitive", "https://example.com/a?UTM_Source=x&id=1", "https://example.com/a?id=1"},
+ {"path preserved case", "https://example.com/Foo/Bar", "https://example.com/Foo/Bar"},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ got, err := CanonicalURL(tc.in, params)
+ if err != nil {
+ t.Fatalf("CanonicalURL(%q): %v", tc.in, err)
+ }
+ if got != tc.want {
+ t.Errorf("CanonicalURL(%q) = %q, want %q", tc.in, got, tc.want)
+ }
+ })
+ }
+}
+
+func TestCanonicalURLCollapsesTracking(t *testing.T) {
+ a, _ := CanonicalURL("https://example.com/story?utm_source=twitter", params)
+ b, _ := CanonicalURL("https://example.com/story", params)
+ if a != b {
+ t.Errorf("tracking variants did not collapse: %q vs %q", a, b)
+ }
+ if ContentHash(a) != ContentHash(b) {
+ t.Error("content hashes differ for tracking variants")
+ }
+}
+
+func TestCanonicalURLErrors(t *testing.T) {
+ for _, in := range []string{"", "not a url", "/relative/only"} {
+ if _, err := CanonicalURL(in, params); err == nil {
+ t.Errorf("CanonicalURL(%q) expected error", in)
+ }
+ }
+}
+
+func TestNormalizeTitle(t *testing.T) {
+ cases := []struct {
+ in string
+ want string
+ }{
+ {"CVE-2021-44228: Log4Shell!", "cve 2021 44228 log4shell"},
+ {" Multiple Spaces ", "multiple spaces"},
+ {"Punctuation, and; stuff.", "punctuation and stuff"},
+ {"Café déjà vu", "café déjà vu"},
+ {"", ""},
+ }
+ for _, tc := range cases {
+ if got := NormalizeTitle(tc.in); got != tc.want {
+ t.Errorf("NormalizeTitle(%q) = %q, want %q", tc.in, got, tc.want)
+ }
+ }
+}
+
+func TestNormalizeTitleStableHash(t *testing.T) {
+ a := TitleHash(NormalizeTitle("Breaking: Big Hack!!!"))
+ b := TitleHash(NormalizeTitle("breaking big hack"))
+ if a != b {
+ t.Error("normalized-title hashes differ for equivalent titles")
+ }
+}
+
+func TestStripHTML(t *testing.T) {
+ cases := []struct {
+ in string
+ want string
+ }{
+ {"<p>Hello <b>world</b></p>", "Hello world"},
+ {"<div>a</div>\n<div>b</div>", "a b"},
+ {"<script>bad()</script>visible", "visible"},
+ {"plain text", "plain text"},
+ {"<a href='x'>link</a> and text", "link and text"},
+ }
+ for _, tc := range cases {
+ if got := StripHTML(tc.in); got != tc.want {
+ t.Errorf("StripHTML(%q) = %q, want %q", tc.in, got, tc.want)
+ }
+ }
+}
+
+func TestContentHashDeterministic(t *testing.T) {
+ const u = "https://example.com/a"
+ if ContentHash(u) != ContentHash(u) {
+ t.Error("ContentHash not deterministic")
+ }
+ if len(ContentHash(u)) != 64 {
+ t.Errorf("ContentHash length = %d, want 64 hex chars", len(ContentHash(u)))
+ }
+}
diff --git a/PROJECTS/intermediate/security-news-scraper/internal/parse/parse.go b/PROJECTS/intermediate/security-news-scraper/internal/parse/parse.go
new file mode 100644
index 00000000..c8d02b8a
--- /dev/null
+++ b/PROJECTS/intermediate/security-news-scraper/internal/parse/parse.go
@@ -0,0 +1,74 @@
+// ©AngelaMos | 2026
+// parse.go
+
+package parse
+
+import (
+ "fmt"
+ "io"
+ "strings"
+ "time"
+
+ "github.com/mmcdole/gofeed"
+)
+
+type Item struct {
+ Title string
+ Link string
+ Summary string
+ Body string
+ Author string
+ Published time.Time
+ Categories []string
+}
+
+var timeLayouts = []string{time.RFC1123Z, time.RFC1123, time.RFC822Z, time.RFC822, time.RFC3339}
+
+func Feed(r io.Reader) ([]Item, error) {
+ feed, err := gofeed.NewParser().Parse(r)
+ if err != nil {
+ return nil, fmt.Errorf("parse: %w", err)
+ }
+ items := make([]Item, 0, len(feed.Items))
+ for _, it := range feed.Items {
+ items = append(items, Item{
+ Title: strings.TrimSpace(it.Title),
+ Link: strings.TrimSpace(it.Link),
+ Summary: it.Description,
+ Body: it.Content,
+ Author: author(it),
+ Published: published(it),
+ Categories: it.Categories,
+ })
+ }
+ return items, nil
+}
+
+func author(it *gofeed.Item) string {
+ if it.Author != nil && it.Author.Name != "" {
+ return it.Author.Name
+ }
+ if len(it.Authors) > 0 {
+ return it.Authors[0].Name
+ }
+ return ""
+}
+
+func published(it *gofeed.Item) time.Time {
+ if it.PublishedParsed != nil {
+ return it.PublishedParsed.UTC()
+ }
+ if it.UpdatedParsed != nil {
+ return it.UpdatedParsed.UTC()
+ }
+ raw := strings.TrimSpace(it.Published)
+ if raw == "" {
+ raw = strings.TrimSpace(it.Updated)
+ }
+ for _, layout := range timeLayouts {
+ if t, err := time.Parse(layout, raw); err == nil {
+ return t.UTC()
+ }
+ }
+ return time.Time{}
+}
diff --git a/PROJECTS/intermediate/security-news-scraper/internal/parse/parse_test.go b/PROJECTS/intermediate/security-news-scraper/internal/parse/parse_test.go
new file mode 100644
index 00000000..4c21c79e
--- /dev/null
+++ b/PROJECTS/intermediate/security-news-scraper/internal/parse/parse_test.go
@@ -0,0 +1,105 @@
+// ©AngelaMos | 2026
+// parse_test.go
+
+package parse
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+)
+
+const feedsDir = "../../testdata/feeds"
+
+func loadFeed(t *testing.T, name string) []Item {
+ t.Helper()
+ f, err := os.Open(filepath.Join(feedsDir, name))
+ if err != nil {
+ t.Fatalf("open fixture %s: %v", name, err)
+ }
+ t.Cleanup(func() { _ = f.Close() })
+ items, err := Feed(f)
+ if err != nil {
+ t.Fatalf("Feed(%s): %v", name, err)
+ }
+ return items
+}
+
+func TestFeedGolden(t *testing.T) {
+ cases := []struct {
+ file string
+ wantItems int
+ fullBody bool
+ }{
+ {"krebs.xml", 3, true},
+ {"theregister.xml", 3, true},
+ {"thehackernews.xml", 3, false},
+ {"bleepingcomputer.xml", 3, false},
+ {"securityweek.xml", 3, false},
+ {"darkreading.xml", 3, false},
+ {"cisa.xml", 3, false},
+ }
+ for _, tc := range cases {
+ t.Run(tc.file, func(t *testing.T) {
+ items := loadFeed(t, tc.file)
+ if len(items) != tc.wantItems {
+ t.Fatalf("items = %d, want %d", len(items), tc.wantItems)
+ }
+ for i, it := range items {
+ if it.Title == "" {
+ t.Errorf("item %d: empty title", i)
+ }
+ if it.Link == "" {
+ t.Errorf("item %d: empty link", i)
+ }
+ if it.Published.IsZero() {
+ t.Errorf("item %d (%q): published time did not parse", i, it.Title)
+ }
+ if tc.fullBody && it.Body == "" {
+ t.Errorf("item %d (%q): expected full body, got empty", i, it.Title)
+ }
+ }
+ })
+ }
+}
+
+func TestFeedExactPublishedTime(t *testing.T) {
+ cases := []struct {
+ file string
+ raw string
+ }{
+ {"thehackernews.xml", "Sat, 04 Jul 2026 18:17:53 +0530"},
+ {"krebs.xml", "Thu, 02 Jul 2026 19:27:33 +0000"},
+ }
+ for _, tc := range cases {
+ t.Run(tc.file, func(t *testing.T) {
+ want, err := time.Parse(time.RFC1123Z, tc.raw)
+ if err != nil {
+ t.Fatalf("parse expected time: %v", err)
+ }
+ items := loadFeed(t, tc.file)
+ if !items[0].Published.Equal(want) {
+ t.Errorf("published = %s, want %s", items[0].Published, want.UTC())
+ }
+ })
+ }
+}
+
+func TestFeedExactTitles(t *testing.T) {
+ cases := []struct {
+ file string
+ title string
+ }{
+ {"krebs.xml", "FBI Seizes NetNut Proxy Platform, Popa Botnet"},
+ {"thehackernews.xml", "U.S. Government Entity Paid Kairos $1 Million in Data-Theft Extortion Case"},
+ }
+ for _, tc := range cases {
+ t.Run(tc.file, func(t *testing.T) {
+ items := loadFeed(t, tc.file)
+ if items[0].Title != tc.title {
+ t.Errorf("first title = %q, want %q", items[0].Title, tc.title)
+ }
+ })
+ }
+}
diff --git a/PROJECTS/intermediate/security-news-scraper/internal/store/fetch_state_test.go b/PROJECTS/intermediate/security-news-scraper/internal/store/fetch_state_test.go
new file mode 100644
index 00000000..1e89e2f1
--- /dev/null
+++ b/PROJECTS/intermediate/security-news-scraper/internal/store/fetch_state_test.go
@@ -0,0 +1,81 @@
+// ©AngelaMos | 2026
+// fetch_state_test.go
+
+package store
+
+import "testing"
+
+func seedSource(t *testing.T, s *Store) int64 {
+ t.Helper()
+ id, err := s.UpsertSource(SourceInput{
+ Name: "krebs", Title: "Krebs", URL: "https://krebsonsecurity.com/feed/",
+ Type: "rss", Weight: 1.0, Tags: []string{"news"}, Enabled: true,
+ })
+ if err != nil {
+ t.Fatalf("UpsertSource: %v", err)
+ }
+ return id
+}
+
+func TestFetchStateMissing(t *testing.T) {
+ s := openTemp(t)
+ id := seedSource(t, s)
+ fs, ok, err := s.GetFetchState(id)
+ if err != nil {
+ t.Fatalf("GetFetchState: %v", err)
+ }
+ if ok {
+ t.Error("expected no fetch_state for fresh source")
+ }
+ if fs != (FetchState{}) {
+ t.Errorf("expected zero FetchState, got %+v", fs)
+ }
+}
+
+func TestFetchStateRoundTrip(t *testing.T) {
+ s := openTemp(t)
+ id := seedSource(t, s)
+
+ want := FetchState{ETag: `"v1"`, LastModified: "Wed, 01 Jul 2026 00:00:00 GMT", LastFetched: 1700, LastStatus: 200}
+ if err := s.UpsertFetchState(id, want); err != nil {
+ t.Fatalf("UpsertFetchState: %v", err)
+ }
+
+ got, ok, err := s.GetFetchState(id)
+ if err != nil {
+ t.Fatalf("GetFetchState: %v", err)
+ }
+ if !ok {
+ t.Fatal("expected fetch_state to exist")
+ }
+ if got != want {
+ t.Errorf("round-trip = %+v, want %+v", got, want)
+ }
+
+ updated := FetchState{ETag: `"v2"`, LastModified: "Thu, 02 Jul 2026 00:00:00 GMT", LastFetched: 1800, LastStatus: 304}
+ if err := s.UpsertFetchState(id, updated); err != nil {
+ t.Fatalf("second UpsertFetchState: %v", err)
+ }
+ got, _, _ = s.GetFetchState(id)
+ if got != updated {
+ t.Errorf("after update = %+v, want %+v", got, updated)
+ }
+}
+
+func TestInsertArticleStoresTitleHash(t *testing.T) {
+ s := openTemp(t)
+ id := seedSource(t, s)
+ if _, err := s.InsertArticle(Article{
+ SourceID: id, CanonicalURL: "https://example.com/a", ContentHash: "hash1",
+ TitleHash: "thash1", Title: "A",
+ }); err != nil {
+ t.Fatalf("InsertArticle: %v", err)
+ }
+ var th string
+ if err := s.DB().QueryRow(`SELECT title_hash FROM articles WHERE content_hash = ?`, "hash1").Scan(&th); err != nil {
+ t.Fatal(err)
+ }
+ if th != "thash1" {
+ t.Errorf("title_hash = %q, want thash1", th)
+ }
+}
diff --git a/PROJECTS/intermediate/security-news-scraper/internal/store/migrations/0002_article_title_hash.sql b/PROJECTS/intermediate/security-news-scraper/internal/store/migrations/0002_article_title_hash.sql
new file mode 100644
index 00000000..85d0c17f
--- /dev/null
+++ b/PROJECTS/intermediate/security-news-scraper/internal/store/migrations/0002_article_title_hash.sql
@@ -0,0 +1,6 @@
+-- ©AngelaMos | 2026
+-- 0002_article_title_hash.sql
+
+ALTER TABLE articles ADD COLUMN title_hash TEXT NOT NULL DEFAULT '';
+
+CREATE INDEX idx_articles_title_hash ON articles(title_hash);
diff --git a/PROJECTS/intermediate/security-news-scraper/internal/store/store.go b/PROJECTS/intermediate/security-news-scraper/internal/store/store.go
index dc874de7..a19dddc7 100644
--- a/PROJECTS/intermediate/security-news-scraper/internal/store/store.go
+++ b/PROJECTS/intermediate/security-news-scraper/internal/store/store.go
@@ -45,6 +45,7 @@ type Article struct {
SourceID int64
CanonicalURL string
ContentHash string
+ TitleHash string
Title string
Summary string
Body string
@@ -53,6 +54,13 @@ type Article struct {
FetchedAt int64
}
+type FetchState struct {
+ ETag string
+ LastModified string
+ LastFetched int64
+ LastStatus int64
+}
+
func Open(path string) (*Store, error) {
dsn := fmt.Sprintf("file:%s?_pragma=busy_timeout(5000)&_pragma=journal_mode(WAL)&_pragma=foreign_keys(1)", path)
db, err := sql.Open("sqlite", dsn)
@@ -114,9 +122,9 @@ func (s *Store) GetSourceByName(name string) (SourceRow, error) {
func (s *Store) InsertArticle(a Article) (int64, error) {
res, err := s.db.Exec(`
INSERT INTO articles
- (source_id, canonical_url, content_hash, title, summary, body, author, published_at, fetched_at)
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
- a.SourceID, a.CanonicalURL, a.ContentHash, a.Title, a.Summary, a.Body,
+ (source_id, canonical_url, content_hash, title_hash, title, summary, body, author, published_at, fetched_at)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+ a.SourceID, a.CanonicalURL, a.ContentHash, a.TitleHash, a.Title, a.Summary, a.Body,
a.Author, a.PublishedAt, a.FetchedAt,
)
if err != nil {
@@ -141,6 +149,36 @@ func (s *Store) CountArticles() (int, error) {
return n, nil
}
+func (s *Store) GetFetchState(sourceID int64) (FetchState, bool, error) {
+ var fs FetchState
+ err := s.db.QueryRow(`
+ SELECT etag, last_modified, last_fetched, last_status
+ FROM fetch_state WHERE source_id = ?`, sourceID,
+ ).Scan(&fs.ETag, &fs.LastModified, &fs.LastFetched, &fs.LastStatus)
+ if errors.Is(err, sql.ErrNoRows) {
+ return FetchState{}, false, nil
+ }
+ if err != nil {
+ return FetchState{}, false, fmt.Errorf("get fetch_state %d: %w", sourceID, err)
+ }
+ return fs, true, nil
+}
+
+func (s *Store) UpsertFetchState(sourceID int64, fs FetchState) error {
+ _, err := s.db.Exec(`
+ INSERT INTO fetch_state (source_id, etag, last_modified, last_fetched, last_status)
+ VALUES (?, ?, ?, ?, ?)
+ ON CONFLICT(source_id) DO UPDATE SET
+ etag = excluded.etag, last_modified = excluded.last_modified,
+ last_fetched = excluded.last_fetched, last_status = excluded.last_status`,
+ sourceID, fs.ETag, fs.LastModified, fs.LastFetched, fs.LastStatus,
+ )
+ if err != nil {
+ return fmt.Errorf("upsert fetch_state %d: %w", sourceID, err)
+ }
+ return nil
+}
+
func boolToInt(b bool) int {
if b {
return 1
diff --git a/PROJECTS/intermediate/security-news-scraper/testdata/feeds/bleepingcomputer.xml b/PROJECTS/intermediate/security-news-scraper/testdata/feeds/bleepingcomputer.xml
new file mode 100644
index 00000000..311dc127
--- /dev/null
+++ b/PROJECTS/intermediate/security-news-scraper/testdata/feeds/bleepingcomputer.xml
@@ -0,0 +1,62 @@
+<?xml version="1.0" encoding="utf-8"?>
+<rss version="2.0"
+ xmlns:content="http://purl.org/rss/1.0/modules/content/"
+ xmlns:wfw="http://wellformedweb.org/CommentAPI/"
+ xmlns:dc="http://purl.org/dc/elements/1.1/"
+ xmlns:atom="http://www.w3.org/2005/Atom"
+ >
+
+<channel>
+ <title>BleepingComputer</title>
+
+ <link>https://www.bleepingcomputer.com/</link>
+ <description>BleepingComputer - All Stories</description>
+ <pubDate>Sun, 05 Jul 2026 16:55:00 GMT</pubDate>
+ <generator>https://www.bleepingcomputer.com/</generator>
+ <language>en</language>
+ <atom:link href="https://www.bleepingcomputer.com/feed/" rel="self" type="application/rss+xml" />
+
+ <item>
+ <title>Flipper Zero firmware development continues with community help</title>
+ <link>https://www.bleepingcomputer.com/news/security/flipper-zero-firmware-development-continues-with-community-help/</link>
+ <pubDate>Sun, 05 Jul 2026 10:14:52 -0400</pubDate>
+ <dc:creator>Bill Toulas</dc:creator>
+
+
+ <category><![CDATA[Security]]></category>
+
+
+ <guid>https://www.bleepingcomputer.com/news/security/flipper-zero-firmware-development-continues-with-community-help/</guid>
+ <description><![CDATA[Flipper Devices says development of the Flipper Zero firmware will continue, albeit with a smaller internal team and greater reliance on community contributions. [...]]]></description>
+ </item>
+<item>
+ <title>JadePuffer ransomware used AI agent to automate entire attack</title>
+ <link>https://www.bleepingcomputer.com/news/security/jadepuffer-ransomware-used-ai-agent-to-automate-entire-attack/</link>
+ <pubDate>Sat, 04 Jul 2026 10:16:38 -0400</pubDate>
+ <dc:creator>Bill Toulas</dc:creator>
+
+
+ <category><![CDATA[Security]]></category>
+
+ <category><![CDATA[Artificial Intelligence]]></category>
+
+
+ <guid>https://www.bleepingcomputer.com/news/security/jadepuffer-ransomware-used-ai-agent-to-automate-entire-attack/</guid>
+ <description><![CDATA[Researchers identified what they believe is the first documented case of a ransomware operation, JadePuffer, conducted entirely by a large language model (LLM) agent. [...]]]></description>
+ </item>
+<item>
+ <title>NetNut proxy network disrupted, 2 million infected devices cut off</title>
+ <link>https://www.bleepingcomputer.com/news/security/netnut-proxy-network-disrupted-2-million-infected-devices-cut-off/</link>
+ <pubDate>Fri, 03 Jul 2026 13:50:04 -0400</pubDate>
+ <dc:creator>Ionut Ilascu</dc:creator>
+
+
+ <category><![CDATA[Security]]></category>
+
+
+ <guid>https://www.bleepingcomputer.com/news/security/netnut-proxy-network-disrupted-2-million-infected-devices-cut-off/</guid>
+ <description><![CDATA[A joint operation involving Google has disrupted NetNut, a residential proxy network that gave access to millions of compromised Android devices, including smart TVs and streaming boxes. [...]]]></description>
+ </item>
+
+</channel>
+</rss>
diff --git a/PROJECTS/intermediate/security-news-scraper/testdata/feeds/cisa.xml b/PROJECTS/intermediate/security-news-scraper/testdata/feeds/cisa.xml
new file mode 100644
index 00000000..3c9971e4
--- /dev/null
+++ b/PROJECTS/intermediate/security-news-scraper/testdata/feeds/cisa.xml
@@ -0,0 +1,598 @@
+<?xml version="1.0" encoding="utf-8"?>
+<rss version="2.0" xml:base="https://www.cisa.gov/" xmlns:dc="http://purl.org/dc/elements/1.1/">
+ <channel>
+ <title>All CISA Advisories</title>
+ <link>https://www.cisa.gov/</link>
+ <description></description>
+ <language>en</language>
+
+ <item>
+ <title>CubeSpace CW0057 Reaction Wheel</title>
+ <link>https://www.cisa.gov/news-events/ics-advisories/icsa-26-183-02</link>
+ <description>&lt;p&gt;&lt;a href=&quot;https://github.com/cisagov/CSAF/blob/develop/csaf_files/OT/white/2026/icsa-26-183-02.json&quot;&gt;&lt;strong&gt;View CSAF&lt;/strong&gt;&lt;/a&gt;&lt;/p&gt;
+&lt;h2&gt;Summary&lt;/h2&gt;
+&lt;p&gt;&lt;strong&gt;Successful exploitation of this vulnerability could allow an attacker to upload arbitrary malicious firmware to the device.&lt;/strong&gt;&lt;/p&gt;
+&lt;p&gt;The following versions of CubeSpace CW0057 Reaction Wheel are affected:&lt;/p&gt;
+&lt;ul&gt;
+&lt;li&gt;CW0057 Reaction Wheel&lt;/li&gt;
+&lt;/ul&gt;
+&lt;div class=&quot;csaf-table&quot;&gt;
+&lt;table class=&quot;tablesaw tablesaw-stack&quot; data-tablesaw-mode=&quot;stack&quot; data-tablesaw-minimap&gt;
+&lt;thead&gt;
+&lt;tr&gt;
+&lt;th role=&quot;columnheader&quot; data-tablesaw-priority=&quot;persist&quot;&gt;CVSS&lt;/th&gt;
+&lt;th role=&quot;columnheader&quot;&gt;Vendor&lt;/th&gt;
+&lt;th role=&quot;columnheader&quot;&gt;Equipment&lt;/th&gt;
+&lt;th role=&quot;columnheader&quot;&gt;Vulnerabilities&lt;/th&gt;
+&lt;/tr&gt;
+&lt;/thead&gt;
+&lt;tbody&gt;
+&lt;tr&gt;
+&lt;td&gt;v3 6.1&lt;/td&gt;
+&lt;td&gt;CubeSpace&lt;/td&gt;
+&lt;td&gt;CubeSpace CW0057 Reaction Wheel&lt;/td&gt;
+&lt;td&gt;Improper Verification of Cryptographic Signature&lt;/td&gt;
+&lt;/tr&gt;
+&lt;/tbody&gt;
+&lt;/table&gt;
+&lt;/div&gt;
+&lt;h3&gt;Background&lt;/h3&gt;
+&lt;ul&gt;
+&lt;li&gt;&lt;strong&gt;Critical Infrastructure Sectors: &lt;/strong&gt;Communications&lt;/li&gt;
+&lt;li&gt;&lt;strong&gt;Countries/Areas Deployed: &lt;/strong&gt;Worldwide&lt;/li&gt;
+&lt;li&gt;&lt;strong&gt;Company Headquarters Location: &lt;/strong&gt;South Africa&lt;/li&gt;
+&lt;/ul&gt;
+&lt;hr&gt;
+&lt;h2&gt;Vulnerabilities&lt;/h2&gt;
+&lt;div class=&quot;csaf-accordion&quot;&gt;
+&lt;p&gt;&lt;a class=&quot;csaf-accordion-toggle-all&quot; href=&quot;#&quot;&gt;Expand All +&lt;/a&gt;&lt;/p&gt;
+&lt;div class=&quot;csaf-accordion-item&quot;&gt;
+&lt;h3&gt;&lt;a class=&quot;csaf-accordion-toggle&quot; href=&quot;#&quot;&gt;CVE-2026-13743&lt;/a&gt;&lt;/h3&gt;
+&lt;div class=&quot;csaf-accordion-content&quot;&gt;
+&lt;p&gt;CubeSpace CW0057 Reaction Wheel firmware versions prior to 5.0.20 are vulnerable to an Improper Verification of Cryptographic Signature vulnerability. This could allow an attacker with physical access to the product to upload arbitrary malicious firmware to the device without authentication.&lt;/p&gt;
+&lt;p&gt;&lt;a href=&quot;https://www.cve.org/CVERecord?id=CVE-2026-13743&quot;&gt;View CVE Details&lt;/a&gt;&lt;/p&gt;
+&lt;hr&gt;
+&lt;h4&gt;Affected Products&lt;/h4&gt;
+&lt;h5&gt;CubeSpace CW0057 Reaction Wheel&lt;/h5&gt;
+&lt;div class=&quot;ics-vendor-version-status&quot;&gt;
+&lt;div class=&quot;ics-vendor&quot;&gt;&lt;strong&gt;Vendor:&lt;/strong&gt;&lt;br&gt;CubeSpace&lt;/div&gt;
+&lt;div class=&quot;ics-version&quot;&gt;&lt;strong&gt;Product Version:&lt;/strong&gt;&lt;br&gt;CubeSpace CW0057 Reaction Wheel: &amp;lt;firmware_5.0.20&lt;/div&gt;
+&lt;div class=&quot;ics-status&quot;&gt;&lt;strong&gt;Product Status:&lt;/strong&gt;&lt;br&gt;known_affected&lt;/div&gt;
+&lt;/div&gt;
+&lt;div class=&quot;ics-remediations&quot;&gt;
+&lt;h6&gt;Remediations&lt;/h6&gt;
+&lt;p&gt;&lt;strong&gt;Vendor fix&lt;/strong&gt;&lt;br&gt;CubeSpace has released the following firmware versions for users to enable: Firmware version 5.0.20. Firmware version 5.0.20 introduces the capability for cryptographically verified secure boot; however, this protection is not enabled by default. Users must activate signed‑boot functionality, particularly the fully immutable mode, to achieve full security.&lt;/p&gt;
+&lt;p&gt;&lt;strong&gt;Mitigation&lt;/strong&gt;&lt;br&gt;CubeSpace acknowledges the finding. The CW0057 reaction wheel authenticates firmware updates with a CRC-32 integrity check, which confirms image integrity but does not verify the source of an image. Exploitation requires direct physical access to the device and is not exploitable remotely. A device affected by this method remains recoverable: the bootloader operates independently of the application firmware and can reload known-good, CubeSpace-supplied images, so an affected unit cannot be permanently disabled by this method. Starting with firmware version 5.0.20, CubeSpace offers optional cryptographic secure boot of varying security levels which customers can enable. Given the physical-access prerequisite and the availability of recovery, CubeSpace assesses the practical risk as low.&lt;/p&gt;
+&lt;/div&gt;
+&lt;p&gt;&lt;strong&gt;Relevant CWE:&lt;/strong&gt; &lt;a href=&quot;https://cwe.mitre.org/data/definitions/347.html&quot;&gt;CWE-347 Improper Verification of Cryptographic Signature&lt;/a&gt;&lt;/p&gt;
+&lt;hr&gt;
+&lt;h4&gt;Metrics&lt;/h4&gt;
+&lt;div class=&quot;csaf-table csaf-metrics-table&quot;&gt;
+&lt;table class=&quot;tablesaw tablesaw-stack&quot; data-tablesaw-mode=&quot;stack&quot; data-tablesaw-minimap&gt;
+&lt;thead&gt;
+&lt;tr&gt;
+&lt;th role=&quot;columnheader&quot; data-tablesaw-priority=&quot;persist&quot;&gt;CVSS Version&lt;/th&gt;
+&lt;th role=&quot;columnheader&quot;&gt;Base Score&lt;/th&gt;
+&lt;th role=&quot;columnheader&quot;&gt;Base Severity&lt;/th&gt;
+&lt;th role=&quot;columnheader&quot;&gt;Vector String&lt;/th&gt;
+&lt;/tr&gt;
+&lt;/thead&gt;
+&lt;tbody&gt;
+&lt;tr&gt;
+&lt;td&gt;3.1&lt;/td&gt;
+&lt;td&gt;6.1&lt;/td&gt;
+&lt;td&gt;MEDIUM&lt;/td&gt;
+&lt;td&gt;&lt;a href=&quot;https://www.first.org/cvss/calculator/3.1#CVSS:3.1/AV:P/AC:L/PR:N/UI:N/S:U/C:N/I:H/A:H&quot;&gt;CVSS:3.1/AV:P/AC:L/PR:N/UI:N/S:U/C:N/I:H/A:H&lt;/a&gt;&lt;/td&gt;
+&lt;/tr&gt;
+&lt;tr&gt;
+&lt;td&gt;4.0&lt;/td&gt;
+&lt;td&gt;3.3&lt;/td&gt;
+&lt;td&gt;LOW&lt;/td&gt;
+&lt;td&gt;&lt;a href=&quot;https://www.first.org/cvss/calculator/4.0#CVSS:4.0/AV:P/AC:L/AT:N/PR:N/UI:N/VC:N/VI:H/VA:H/SC:N/SI:N/SA:N/E:P&quot;&gt;CVSS:4.0/AV:P/AC:L/AT:N/PR:N/UI:N/VC:N/VI:H/VA:H/SC:N/SI:N/SA:N/E:P&lt;/a&gt;&lt;/td&gt;
+&lt;/tr&gt;
+&lt;/tbody&gt;
+&lt;/table&gt;
+&lt;/div&gt;
+&lt;/div&gt;
+&lt;/div&gt;
+&lt;/div&gt;
+&lt;hr&gt;
+&lt;h2&gt;Acknowledgments&lt;/h2&gt;
+&lt;ul&gt;
+&lt;li&gt;Anthony Rose reported this vulnerability to CISA&lt;/li&gt;
+&lt;/ul&gt;
+&lt;hr&gt;
+&lt;h2&gt;Legal Notice and Terms of Use&lt;/h2&gt;
+&lt;p&gt;This product is provided subject to this Notification (https://www.cisa.gov/notification) and this Privacy &amp;amp; Use policy (https://www.cisa.gov/privacy-policy).&lt;/p&gt;
+&lt;hr&gt;
+&lt;h2&gt;Recommended Practices&lt;/h2&gt;
+&lt;p&gt;CISA recommends users take defensive measures to minimize the risk of exploitation of this vulnerability.&lt;/p&gt;
+&lt;p&gt;Minimize network exposure for all control system devices and/or systems, ensuring they are not accessible from the internet.&lt;/p&gt;
+&lt;p&gt;Locate control system networks and remote devices behind firewalls and isolating them from business networks.&lt;/p&gt;
+&lt;p&gt;When remote access is required, use more secure methods, such as Virtual Private Networks (VPNs), recognizing VPNs may have vulnerabilities and should be updated to the most current version available. Also recognize VPN is only as secure as the connected devices.&lt;/p&gt;
+&lt;p&gt;CISA reminds organizations to perform proper impact analysis and risk assessment prior to deploying defensive measures.&lt;/p&gt;
+&lt;p&gt;CISA also provides a section for control systems security recommended practices on the ICS webpage on cisa.gov/ics. Several CISA products detailing cyber defense best practices are available for reading and download, including Improving Industrial Control Systems Cybersecurity with Defense-in-Depth Strategies.&lt;/p&gt;
+&lt;p&gt;CISA encourages organizations to implement recommended cybersecurity strategies for proactive defense of ICS assets.&lt;/p&gt;
+&lt;p&gt;Additional mitigation guidance and recommended practices are publicly available on the ICS webpage at cisa.gov/ics in the technical information paper, ICS-TIP-12-146-01B--Targeted Cyber Intrusion Detection and Mitigation Strategies.&lt;/p&gt;
+&lt;p&gt;Organizations observing suspected malicious activity should follow established internal procedures and report findings to CISA for tracking and correlation against other incidents.&lt;/p&gt;
+&lt;p&gt;CISA also recommends users take the following measures to protect themselves from social engineering attacks:&lt;/p&gt;
+&lt;p&gt;Do not click web links or open attachments in unsolicited email messages.&lt;/p&gt;
+&lt;p&gt;Refer to Recognizing and Avoiding Email Scams for more information on avoiding email scams.&lt;/p&gt;
+&lt;p&gt;Refer to Avoiding Social Engineering and Phishing Attacks for more information on social engineering attacks.&lt;/p&gt;
+&lt;p&gt;No known public exploitation specifically targeting this vulnerability has been reported to CISA at this time. This vulnerability is not exploitable remotely.&lt;/p&gt;
+&lt;hr&gt;
+&lt;h2&gt;Revision History&lt;/h2&gt;
+&lt;ul&gt;
+&lt;li&gt;&lt;strong&gt;Initial Release Date: &lt;/strong&gt;2026-07-02&lt;/li&gt;
+&lt;/ul&gt;
+&lt;table class=&quot;tablesaw tablesaw-stack&quot; data-tablesaw-mode=&quot;stack&quot; data-tablesaw-minimap&gt;
+&lt;thead&gt;
+&lt;tr&gt;
+&lt;th role=&quot;columnheader&quot; data-tablesaw-priority=&quot;persist&quot;&gt;Date&lt;/th&gt;
+&lt;th role=&quot;columnheader&quot;&gt;Revision&lt;/th&gt;
+&lt;th role=&quot;columnheader&quot;&gt;Summary&lt;/th&gt;
+&lt;/tr&gt;
+&lt;/thead&gt;
+&lt;tbody&gt;
+&lt;tr&gt;
+&lt;td&gt;2026-07-02&lt;/td&gt;
+&lt;td&gt;1&lt;/td&gt;
+&lt;td&gt;Initial Publication&lt;/td&gt;
+&lt;/tr&gt;
+&lt;/tbody&gt;
+&lt;/table&gt;
+&lt;hr&gt;
+&lt;h2&gt;Legal Notice and Terms of Use&lt;/h2&gt;
+</description>
+ <pubDate>Thu, 02 Jul 26 12:00:00 +0000</pubDate>
+ <dc:creator>CISA</dc:creator>
+ <guid isPermaLink="false">/node/25107</guid>
+ </item>
+<item>
+ <title>Gardyn IoT Hub</title>
+ <link>https://www.cisa.gov/news-events/ics-advisories/icsa-26-183-03</link>
+ <description>&lt;p&gt;&lt;a href=&quot;https://github.com/cisagov/CSAF/blob/develop/csaf_files/OT/white/2026/icsa-26-183-03.json&quot;&gt;&lt;strong&gt;View CSAF&lt;/strong&gt;&lt;/a&gt;&lt;/p&gt;
+&lt;h2&gt;Summary&lt;/h2&gt;
+&lt;p&gt;&lt;strong&gt;Successful exploitation of these vulnerabilities could allow unauthenticated users to access and control IoT Hub managed devices.&lt;/strong&gt;&lt;/p&gt;
+&lt;p&gt;The following versions of Gardyn IoT Hub are affected:&lt;/p&gt;
+&lt;ul&gt;
+&lt;li&gt;Home Firmware&lt;/li&gt;
+&lt;li&gt;Studio Firmware&lt;/li&gt;
+&lt;li&gt;Cloud API &amp;lt;2.12.2026 (CVE-2026-13768, CVE-2026-55726, CVE-2026-54477)&lt;/li&gt;
+&lt;/ul&gt;
+&lt;div class=&quot;csaf-table&quot;&gt;
+&lt;table class=&quot;tablesaw tablesaw-stack&quot; data-tablesaw-mode=&quot;stack&quot; data-tablesaw-minimap&gt;
+&lt;thead&gt;
+&lt;tr&gt;
+&lt;th role=&quot;columnheader&quot; data-tablesaw-priority=&quot;persist&quot;&gt;CVSS&lt;/th&gt;
+&lt;th role=&quot;columnheader&quot;&gt;Vendor&lt;/th&gt;
+&lt;th role=&quot;columnheader&quot;&gt;Equipment&lt;/th&gt;
+&lt;th role=&quot;columnheader&quot;&gt;Vulnerabilities&lt;/th&gt;
+&lt;/tr&gt;
+&lt;/thead&gt;
+&lt;tbody&gt;
+&lt;tr&gt;
+&lt;td&gt;v3 10&lt;/td&gt;
+&lt;td&gt;Gardyn&lt;/td&gt;
+&lt;td&gt;Gardyn IoT Hub&lt;/td&gt;
+&lt;td&gt;Use of Hard-coded Credentials, Exposure of Sensitive System Information to an Unauthorized Control Sphere, Improper Neutralization of HTTP Headers for Scripting Syntax&lt;/td&gt;
+&lt;/tr&gt;
+&lt;/tbody&gt;
+&lt;/table&gt;
+&lt;/div&gt;
+&lt;h3&gt;Background&lt;/h3&gt;
+&lt;ul&gt;
+&lt;li&gt;&lt;strong&gt;Critical Infrastructure Sectors: &lt;/strong&gt;Food and Agriculture&lt;/li&gt;
+&lt;li&gt;&lt;strong&gt;Countries/Areas Deployed: &lt;/strong&gt;United States&lt;/li&gt;
+&lt;li&gt;&lt;strong&gt;Company Headquarters Location: &lt;/strong&gt;United States&lt;/li&gt;
+&lt;/ul&gt;
+&lt;hr&gt;
+&lt;h2&gt;Vulnerabilities&lt;/h2&gt;
+&lt;div class=&quot;csaf-accordion&quot;&gt;
+&lt;p&gt;&lt;a class=&quot;csaf-accordion-toggle-all&quot; href=&quot;#&quot;&gt;Expand All +&lt;/a&gt;&lt;/p&gt;
+&lt;div class=&quot;csaf-accordion-item&quot;&gt;
+&lt;h3&gt;&lt;a class=&quot;csaf-accordion-toggle&quot; href=&quot;#&quot;&gt;CVE-2026-13768&lt;/a&gt;&lt;/h3&gt;
+&lt;div class=&quot;csaf-accordion-content&quot;&gt;
+&lt;p&gt;Gardyn devices expose a privileged iothubowner key. Access to this key will allow a malicious user to invoke an IoTHub Registry Manager function which returns connection information for all Gardyn Home Kit and Studio devices. Access to this key also allows a malicious user to execute arbitrary commands on a specific connected device and may allow the malicious user to pivot to other devices on the user&#039;s network.&lt;/p&gt;
+&lt;p&gt;&lt;a href=&quot;https://www.cve.org/CVERecord?id=CVE-2026-13768&quot;&gt;View CVE Details&lt;/a&gt;&lt;/p&gt;
+&lt;hr&gt;
+&lt;h4&gt;Affected Products&lt;/h4&gt;
+&lt;h5&gt;Gardyn IoT Hub&lt;/h5&gt;
+&lt;div class=&quot;ics-vendor-version-status&quot;&gt;
+&lt;div class=&quot;ics-vendor&quot;&gt;&lt;strong&gt;Vendor:&lt;/strong&gt;&lt;br&gt;Gardyn&lt;/div&gt;
+&lt;div class=&quot;ics-version&quot;&gt;&lt;strong&gt;Product Version:&lt;/strong&gt;&lt;br&gt;Gardyn Home Firmware: &amp;lt;master.627, Gardyn Studio Firmware: &amp;lt;master.627, Gardyn Cloud API: &amp;lt;2.12.2026&lt;/div&gt;
+&lt;div class=&quot;ics-status&quot;&gt;&lt;strong&gt;Product Status:&lt;/strong&gt;&lt;br&gt;known_affected&lt;/div&gt;
+&lt;/div&gt;
+&lt;div class=&quot;ics-remediations&quot;&gt;
+&lt;h6&gt;Remediations&lt;/h6&gt;
+&lt;p&gt;&lt;strong&gt;Mitigation&lt;/strong&gt;&lt;br&gt;Gardyn states that IoT Hub deployed infrastructure has been updated to fix the listed vulnerabilities.&lt;/p&gt;
+&lt;p&gt;&lt;strong&gt;Mitigation&lt;/strong&gt;&lt;br&gt;Gardyn requests that users ensure their devices have Internet connectivity in order to automatically download needed firmware updates. Unconnected devices will automatically update when configured with a working Internet connection. Gardyn also recommends that users update their mobile application to the most recent version. The current versions of the Gardyn App and the Gardyn Home firmware can be checked in the Gardyn App.&lt;/p&gt;
+&lt;p&gt;&lt;strong&gt;Mitigation&lt;/strong&gt;&lt;br&gt;Further information on Gardyn security can be found here: https://mygardyn.com/security/&lt;br&gt;&lt;a href=&quot;https://mygardyn.com/security/&quot;&gt;https://mygardyn.com/security/&lt;/a&gt;&lt;/p&gt;
+&lt;p&gt;&lt;strong&gt;Mitigation&lt;/strong&gt;&lt;br&gt;Further customer support can be obtained from Gardyn at: [email protected]&lt;br&gt;&lt;a href=&quot;mailto:[email protected]&quot;&gt;mailto:[email protected]&lt;/a&gt;&lt;/p&gt;
+&lt;/div&gt;
+&lt;p&gt;&lt;strong&gt;Relevant CWE:&lt;/strong&gt; &lt;a href=&quot;https://cwe.mitre.org/data/definitions/798.html&quot;&gt;CWE-798 Use of Hard-coded Credentials&lt;/a&gt;&lt;/p&gt;
+&lt;hr&gt;
+&lt;h4&gt;Metrics&lt;/h4&gt;
+&lt;div class=&quot;csaf-table csaf-metrics-table&quot;&gt;
+&lt;table class=&quot;tablesaw tablesaw-stack&quot; data-tablesaw-mode=&quot;stack&quot; data-tablesaw-minimap&gt;
+&lt;thead&gt;
+&lt;tr&gt;
+&lt;th role=&quot;columnheader&quot; data-tablesaw-priority=&quot;persist&quot;&gt;CVSS Version&lt;/th&gt;
+&lt;th role=&quot;columnheader&quot;&gt;Base Score&lt;/th&gt;
+&lt;th role=&quot;columnheader&quot;&gt;Base Severity&lt;/th&gt;
+&lt;th role=&quot;columnheader&quot;&gt;Vector String&lt;/th&gt;
+&lt;/tr&gt;
+&lt;/thead&gt;
+&lt;tbody&gt;
+&lt;tr&gt;
+&lt;td&gt;3.1&lt;/td&gt;
+&lt;td&gt;10&lt;/td&gt;
+&lt;td&gt;CRITICAL&lt;/td&gt;
+&lt;td&gt;&lt;a href=&quot;https://www.first.org/cvss/calculator/3.1#CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:L&quot;&gt;CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:L&lt;/a&gt;&lt;/td&gt;
+&lt;/tr&gt;
+&lt;tr&gt;
+&lt;td&gt;4.0&lt;/td&gt;
+&lt;td&gt;9.5&lt;/td&gt;
+&lt;td&gt;CRITICAL&lt;/td&gt;
+&lt;td&gt;&lt;a href=&quot;https://www.first.org/cvss/calculator/4.0#CVSS:4.0/AV:N/AC:L/AT:P/PR:N/UI:N/VC:H/VI:H/VA:L/SC:H/SI:H/SA:L&quot;&gt;CVSS:4.0/AV:N/AC:L/AT:P/PR:N/UI:N/VC:H/VI:H/VA:L/SC:H/SI:H/SA:L&lt;/a&gt;&lt;/td&gt;
+&lt;/tr&gt;
+&lt;/tbody&gt;
+&lt;/table&gt;
+&lt;/div&gt;
+&lt;/div&gt;
+&lt;/div&gt;
+&lt;div class=&quot;csaf-accordion-item&quot;&gt;
+&lt;h3&gt;&lt;a class=&quot;csaf-accordion-toggle&quot; href=&quot;#&quot;&gt;CVE-2026-55726&lt;/a&gt;&lt;/h3&gt;
+&lt;div class=&quot;csaf-accordion-content&quot;&gt;
+&lt;p&gt;The Azure Blob Storage container used for Gardyn device logs is publicly listable without authentication. A malicious user would be able to access any device log file available in the blob storage container.&lt;/p&gt;
+&lt;p&gt;&lt;a href=&quot;https://www.cve.org/CVERecord?id=CVE-2026-55726&quot;&gt;View CVE Details&lt;/a&gt;&lt;/p&gt;
+&lt;hr&gt;
+&lt;h4&gt;Affected Products&lt;/h4&gt;
+&lt;h5&gt;Gardyn IoT Hub&lt;/h5&gt;
+&lt;div class=&quot;ics-vendor-version-status&quot;&gt;
+&lt;div class=&quot;ics-vendor&quot;&gt;&lt;strong&gt;Vendor:&lt;/strong&gt;&lt;br&gt;Gardyn&lt;/div&gt;
+&lt;div class=&quot;ics-version&quot;&gt;&lt;strong&gt;Product Version:&lt;/strong&gt;&lt;br&gt;Gardyn Home Firmware: &amp;lt;master.627, Gardyn Studio Firmware: &amp;lt;master.627, Gardyn Cloud API: &amp;lt;2.12.2026&lt;/div&gt;
+&lt;div class=&quot;ics-status&quot;&gt;&lt;strong&gt;Product Status:&lt;/strong&gt;&lt;br&gt;known_affected&lt;/div&gt;
+&lt;/div&gt;
+&lt;div class=&quot;ics-remediations&quot;&gt;
+&lt;h6&gt;Remediations&lt;/h6&gt;
+&lt;p&gt;&lt;strong&gt;Mitigation&lt;/strong&gt;&lt;br&gt;Gardyn states that IoT Hub deployed infrastructure has been updated to fix the listed vulnerabilities.&lt;/p&gt;
+&lt;p&gt;&lt;strong&gt;Mitigation&lt;/strong&gt;&lt;br&gt;Gardyn requests that users ensure their devices have Internet connectivity in order to automatically download needed firmware updates. Unconnected devices will automatically update when configured with a working Internet connection. Gardyn also recommends that users update their mobile application to the most recent version. The current versions of the Gardyn App and the Gardyn Home firmware can be checked in the Gardyn App.&lt;/p&gt;
+&lt;p&gt;&lt;strong&gt;Mitigation&lt;/strong&gt;&lt;br&gt;Further information on Gardyn security can be found here: https://mygardyn.com/security/&lt;br&gt;&lt;a href=&quot;https://mygardyn.com/security/&quot;&gt;https://mygardyn.com/security/&lt;/a&gt;&lt;/p&gt;
+&lt;p&gt;&lt;strong&gt;Mitigation&lt;/strong&gt;&lt;br&gt;Further customer support can be obtained from Gardyn at: [email protected]&lt;br&gt;&lt;a href=&quot;mailto:[email protected]&quot;&gt;mailto:[email protected]&lt;/a&gt;&lt;/p&gt;
+&lt;/div&gt;
+&lt;p&gt;&lt;strong&gt;Relevant CWE:&lt;/strong&gt; &lt;a href=&quot;https://cwe.mitre.org/data/definitions/497.html&quot;&gt;CWE-497 Exposure of Sensitive System Information to an Unauthorized Control Sphere&lt;/a&gt;&lt;/p&gt;
+&lt;hr&gt;
+&lt;h4&gt;Metrics&lt;/h4&gt;
+&lt;div class=&quot;csaf-table csaf-metrics-table&quot;&gt;
+&lt;table class=&quot;tablesaw tablesaw-stack&quot; data-tablesaw-mode=&quot;stack&quot; data-tablesaw-minimap&gt;
+&lt;thead&gt;
+&lt;tr&gt;
+&lt;th role=&quot;columnheader&quot; data-tablesaw-priority=&quot;persist&quot;&gt;CVSS Version&lt;/th&gt;
+&lt;th role=&quot;columnheader&quot;&gt;Base Score&lt;/th&gt;
+&lt;th role=&quot;columnheader&quot;&gt;Base Severity&lt;/th&gt;
+&lt;th role=&quot;columnheader&quot;&gt;Vector String&lt;/th&gt;
+&lt;/tr&gt;
+&lt;/thead&gt;
+&lt;tbody&gt;
+&lt;tr&gt;
+&lt;td&gt;3.1&lt;/td&gt;
+&lt;td&gt;5.3&lt;/td&gt;
+&lt;td&gt;MEDIUM&lt;/td&gt;
+&lt;td&gt;&lt;a href=&quot;https://www.first.org/cvss/calculator/3.1#CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:L/I:N/A:N&quot;&gt;CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:L/I:N/A:N&lt;/a&gt;&lt;/td&gt;
+&lt;/tr&gt;
+&lt;tr&gt;
+&lt;td&gt;4.0&lt;/td&gt;
+&lt;td&gt;6.9&lt;/td&gt;
+&lt;td&gt;MEDIUM&lt;/td&gt;
+&lt;td&gt;&lt;a href=&quot;https://www.first.org/cvss/calculator/4.0#CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:N/VC:L/VI:N/VA:N/SC:N/SI:N/SA:N&quot;&gt;CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:N/VC:L/VI:N/VA:N/SC:N/SI:N/SA:N&lt;/a&gt;&lt;/td&gt;
+&lt;/tr&gt;
+&lt;/tbody&gt;
+&lt;/table&gt;
+&lt;/div&gt;
+&lt;/div&gt;
+&lt;/div&gt;
+&lt;div class=&quot;csaf-accordion-item&quot;&gt;
+&lt;h3&gt;&lt;a class=&quot;csaf-accordion-toggle&quot; href=&quot;#&quot;&gt;CVE-2026-54477&lt;/a&gt;&lt;/h3&gt;
+&lt;div class=&quot;csaf-accordion-content&quot;&gt;
+&lt;p&gt;The admin panel lacks standard security headers, enabling clickjacking and cross-site scripting attacks.&lt;/p&gt;
+&lt;p&gt;&lt;a href=&quot;https://www.cve.org/CVERecord?id=CVE-2026-54477&quot;&gt;View CVE Details&lt;/a&gt;&lt;/p&gt;
+&lt;hr&gt;
+&lt;h4&gt;Affected Products&lt;/h4&gt;
+&lt;h5&gt;Gardyn IoT Hub&lt;/h5&gt;
+&lt;div class=&quot;ics-vendor-version-status&quot;&gt;
+&lt;div class=&quot;ics-vendor&quot;&gt;&lt;strong&gt;Vendor:&lt;/strong&gt;&lt;br&gt;Gardyn&lt;/div&gt;
+&lt;div class=&quot;ics-version&quot;&gt;&lt;strong&gt;Product Version:&lt;/strong&gt;&lt;br&gt;Gardyn Home Firmware: &amp;lt;master.627, Gardyn Studio Firmware: &amp;lt;master.627, Gardyn Cloud API: &amp;lt;2.12.2026&lt;/div&gt;
+&lt;div class=&quot;ics-status&quot;&gt;&lt;strong&gt;Product Status:&lt;/strong&gt;&lt;br&gt;known_affected&lt;/div&gt;
+&lt;/div&gt;
+&lt;div class=&quot;ics-remediations&quot;&gt;
+&lt;h6&gt;Remediations&lt;/h6&gt;
+&lt;p&gt;&lt;strong&gt;Mitigation&lt;/strong&gt;&lt;br&gt;Gardyn states that IoT Hub deployed infrastructure has been updated to fix the listed vulnerabilities.&lt;/p&gt;
+&lt;p&gt;&lt;strong&gt;Mitigation&lt;/strong&gt;&lt;br&gt;Gardyn requests that users ensure their devices have Internet connectivity in order to automatically download needed firmware updates. Unconnected devices will automatically update when configured with a working Internet connection. Gardyn also recommends that users update their mobile application to the most recent version. The current versions of the Gardyn App and the Gardyn Home firmware can be checked in the Gardyn App.&lt;/p&gt;
+&lt;p&gt;&lt;strong&gt;Mitigation&lt;/strong&gt;&lt;br&gt;Further information on Gardyn security can be found here: https://mygardyn.com/security/&lt;br&gt;&lt;a href=&quot;https://mygardyn.com/security/&quot;&gt;https://mygardyn.com/security/&lt;/a&gt;&lt;/p&gt;
+&lt;p&gt;&lt;strong&gt;Mitigation&lt;/strong&gt;&lt;br&gt;Further customer support can be obtained from Gardyn at: [email protected]&lt;br&gt;&lt;a href=&quot;mailto:[email protected]&quot;&gt;mailto:[email protected]&lt;/a&gt;&lt;/p&gt;
+&lt;/div&gt;
+&lt;p&gt;&lt;strong&gt;Relevant CWE:&lt;/strong&gt; &lt;a href=&quot;https://cwe.mitre.org/data/definitions/644.html&quot;&gt;CWE-644 Improper Neutralization of HTTP Headers for Scripting Syntax&lt;/a&gt;&lt;/p&gt;
+&lt;hr&gt;
+&lt;h4&gt;Metrics&lt;/h4&gt;
+&lt;div class=&quot;csaf-table csaf-metrics-table&quot;&gt;
+&lt;table class=&quot;tablesaw tablesaw-stack&quot; data-tablesaw-mode=&quot;stack&quot; data-tablesaw-minimap&gt;
+&lt;thead&gt;
+&lt;tr&gt;
+&lt;th role=&quot;columnheader&quot; data-tablesaw-priority=&quot;persist&quot;&gt;CVSS Version&lt;/th&gt;
+&lt;th role=&quot;columnheader&quot;&gt;Base Score&lt;/th&gt;
+&lt;th role=&quot;columnheader&quot;&gt;Base Severity&lt;/th&gt;
+&lt;th role=&quot;columnheader&quot;&gt;Vector String&lt;/th&gt;
+&lt;/tr&gt;
+&lt;/thead&gt;
+&lt;tbody&gt;
+&lt;tr&gt;
+&lt;td&gt;3.1&lt;/td&gt;
+&lt;td&gt;5.4&lt;/td&gt;
+&lt;td&gt;MEDIUM&lt;/td&gt;
+&lt;td&gt;&lt;a href=&quot;https://www.first.org/cvss/calculator/3.1#CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:L/I:L/A:N&quot;&gt;CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:L/I:L/A:N&lt;/a&gt;&lt;/td&gt;
+&lt;/tr&gt;
+&lt;tr&gt;
+&lt;td&gt;4.0&lt;/td&gt;
+&lt;td&gt;5.1&lt;/td&gt;
+&lt;td&gt;MEDIUM&lt;/td&gt;
+&lt;td&gt;&lt;a href=&quot;https://www.first.org/cvss/calculator/4.0#CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:A/VC:L/VI:L/VA:N/SC:N/SI:N/SA:N&quot;&gt;CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:A/VC:L/VI:L/VA:N/SC:N/SI:N/SA:N&lt;/a&gt;&lt;/td&gt;
+&lt;/tr&gt;
+&lt;/tbody&gt;
+&lt;/table&gt;
+&lt;/div&gt;
+&lt;/div&gt;
+&lt;/div&gt;
+&lt;/div&gt;
+&lt;hr&gt;
+&lt;h2&gt;Acknowledgments&lt;/h2&gt;
+&lt;ul&gt;
+&lt;li&gt;Michael Groberman reported these vulnerabilities to CISA&lt;/li&gt;
+&lt;/ul&gt;
+&lt;hr&gt;
+&lt;h2&gt;Legal Notice and Terms of Use&lt;/h2&gt;
+&lt;p&gt;This product is provided subject to this Notification (https://www.cisa.gov/notification) and this Privacy &amp;amp; Use policy (https://www.cisa.gov/privacy-policy).&lt;/p&gt;
+&lt;hr&gt;
+&lt;h2&gt;Recommended Practices&lt;/h2&gt;
+&lt;p&gt;CISA recommends users take defensive measures to minimize the risk of exploitation of these vulnerabilities.&lt;/p&gt;
+&lt;p&gt;Minimize network exposure for all control system devices and/or systems, ensuring they are not accessible from the Internet.&lt;/p&gt;
+&lt;p&gt;Locate control system networks and remote devices behind firewalls and isolating them from business networks.&lt;/p&gt;
+&lt;p&gt;When remote access is required, use more secure methods, such as Virtual Private Networks (VPNs), recognizing VPNs may have vulnerabilities and should be updated to the most current version available. Also recognize VPN is only as secure as the connected devices.&lt;/p&gt;
+&lt;p&gt;CISA reminds organizations to perform proper impact analysis and risk assessment prior to deploying defensive measures.&lt;/p&gt;
+&lt;p&gt;CISA also provides a section for control systems security recommended practices on the ICS webpage on cisa.gov/ics. Several CISA products detailing cyber defense best practices are available for reading and download, including Improving Industrial Control Systems Cybersecurity with Defense-in-Depth Strategies.&lt;/p&gt;
+&lt;p&gt;CISA encourages organizations to implement recommended cybersecurity strategies for proactive defense of ICS assets.&lt;/p&gt;
+&lt;p&gt;Additional mitigation guidance and recommended practices are publicly available on the ICS webpage at cisa.gov/ics in the technical information paper, ICS-TIP-12-146-01B--Targeted Cyber Intrusion Detection and Mitigation Strategies.&lt;/p&gt;
+&lt;p&gt;Organizations observing suspected malicious activity should follow established internal procedures and report findings to CISA for tracking and correlation against other incidents.&lt;/p&gt;
+&lt;p&gt;No known public exploitation specifically targeting these vulnerabilities has been reported to CISA at this time.&lt;/p&gt;
+&lt;hr&gt;
+&lt;h2&gt;Revision History&lt;/h2&gt;
+&lt;ul&gt;
+&lt;li&gt;&lt;strong&gt;Initial Release Date: &lt;/strong&gt;2026-07-02&lt;/li&gt;
+&lt;/ul&gt;
+&lt;table class=&quot;tablesaw tablesaw-stack&quot; data-tablesaw-mode=&quot;stack&quot; data-tablesaw-minimap&gt;
+&lt;thead&gt;
+&lt;tr&gt;
+&lt;th role=&quot;columnheader&quot; data-tablesaw-priority=&quot;persist&quot;&gt;Date&lt;/th&gt;
+&lt;th role=&quot;columnheader&quot;&gt;Revision&lt;/th&gt;
+&lt;th role=&quot;columnheader&quot;&gt;Summary&lt;/th&gt;
+&lt;/tr&gt;
+&lt;/thead&gt;
+&lt;tbody&gt;
+&lt;tr&gt;
+&lt;td&gt;2026-07-02&lt;/td&gt;
+&lt;td&gt;1&lt;/td&gt;
+&lt;td&gt;Initial Publication&lt;/td&gt;
+&lt;/tr&gt;
+&lt;/tbody&gt;
+&lt;/table&gt;
+&lt;hr&gt;
+&lt;h2&gt;Legal Notice and Terms of Use&lt;/h2&gt;
+</description>
+ <pubDate>Thu, 02 Jul 26 12:00:00 +0000</pubDate>
+ <dc:creator>CISA</dc:creator>
+ <guid isPermaLink="false">/node/25108</guid>
+ </item>
+<item>
+ <title>ST Engineering iDirect iQ-Series Terminals</title>
+ <link>https://www.cisa.gov/news-events/ics-advisories/icsa-26-183-01</link>
+ <description>&lt;p&gt;&lt;a href=&quot;https://github.com/cisagov/CSAF/blob/develop/csaf_files/OT/white/2026/icsa-26-183-01.json&quot;&gt;&lt;strong&gt;View CSAF&lt;/strong&gt;&lt;/a&gt;&lt;/p&gt;
+&lt;h2&gt;Summary&lt;/h2&gt;
+&lt;p&gt;&lt;strong&gt;Successful exploitation of these vulnerabilities could allow an attacker to gain unauthorized access to device information or cause a denial-of-service condition.&lt;/strong&gt;&lt;/p&gt;
+&lt;p&gt;The following versions of ST Engineering iDirect iQ-Series Terminals are affected:&lt;/p&gt;
+&lt;ul&gt;
+&lt;li&gt;Evolution iQ‑Series terminals &amp;lt;=4.5.2.1 (CVE-2026-38059, CVE-2026-38057)&lt;/li&gt;
+&lt;li&gt;3315‑Series terminals &amp;lt;=4.5.2.1 (CVE-2026-38059, CVE-2026-38057)&lt;/li&gt;
+&lt;li&gt;9‑Series terminals &amp;lt;=4.5.2.1 (CVE-2026-38059, CVE-2026-38057)&lt;/li&gt;
+&lt;/ul&gt;
+&lt;div class=&quot;csaf-table&quot;&gt;
+&lt;table class=&quot;tablesaw tablesaw-stack&quot; data-tablesaw-mode=&quot;stack&quot; data-tablesaw-minimap&gt;
+&lt;thead&gt;
+&lt;tr&gt;
+&lt;th role=&quot;columnheader&quot; data-tablesaw-priority=&quot;persist&quot;&gt;CVSS&lt;/th&gt;
+&lt;th role=&quot;columnheader&quot;&gt;Vendor&lt;/th&gt;
+&lt;th role=&quot;columnheader&quot;&gt;Equipment&lt;/th&gt;
+&lt;th role=&quot;columnheader&quot;&gt;Vulnerabilities&lt;/th&gt;
+&lt;/tr&gt;
+&lt;/thead&gt;
+&lt;tbody&gt;
+&lt;tr&gt;
+&lt;td&gt;v3 8.1&lt;/td&gt;
+&lt;td&gt;ST Engineering iDirect&lt;/td&gt;
+&lt;td&gt;ST Engineering iDirect iQ-Series Terminals&lt;/td&gt;
+&lt;td&gt;Missing Authentication for Critical Function, Cross-Site Request Forgery (CSRF)&lt;/td&gt;
+&lt;/tr&gt;
+&lt;/tbody&gt;
+&lt;/table&gt;
+&lt;/div&gt;
+&lt;h3&gt;Background&lt;/h3&gt;
+&lt;ul&gt;
+&lt;li&gt;&lt;strong&gt;Critical Infrastructure Sectors: &lt;/strong&gt;Communications, Defense Industrial Base, Energy, Government Services and Facilities, Transportation Systems&lt;/li&gt;
+&lt;li&gt;&lt;strong&gt;Countries/Areas Deployed: &lt;/strong&gt;Worldwide&lt;/li&gt;
+&lt;li&gt;&lt;strong&gt;Company Headquarters Location: &lt;/strong&gt;United States&lt;/li&gt;
+&lt;/ul&gt;
+&lt;hr&gt;
+&lt;h2&gt;Vulnerabilities&lt;/h2&gt;
+&lt;div class=&quot;csaf-accordion&quot;&gt;
+&lt;p&gt;&lt;a class=&quot;csaf-accordion-toggle-all&quot; href=&quot;#&quot;&gt;Expand All +&lt;/a&gt;&lt;/p&gt;
+&lt;div class=&quot;csaf-accordion-item&quot;&gt;
+&lt;h3&gt;&lt;a class=&quot;csaf-accordion-toggle&quot; href=&quot;#&quot;&gt;CVE-2026-38059&lt;/a&gt;&lt;/h3&gt;
+&lt;div class=&quot;csaf-accordion-content&quot;&gt;
+&lt;p&gt;The iDirect iQ200 exposes the /api/identity and /api/ REST API endpoints without authentication. An unauthenticated attacker with network access can retrieve sensitive device information including the serial number, Device ID (DID), Terminal Private Key identifier (TPK), MAC address, and exact firmware version. The DID and TPK are used for satellite network authentication in the iDirect platform, potentially enabling terminal impersonation and network reconnaissance.&lt;/p&gt;
+&lt;p&gt;&lt;a href=&quot;https://www.cve.org/CVERecord?id=CVE-2026-38059&quot;&gt;View CVE Details&lt;/a&gt;&lt;/p&gt;
+&lt;hr&gt;
+&lt;h4&gt;Affected Products&lt;/h4&gt;
+&lt;h5&gt;ST Engineering iDirect iQ-Series Terminals&lt;/h5&gt;
+&lt;div class=&quot;ics-vendor-version-status&quot;&gt;
+&lt;div class=&quot;ics-vendor&quot;&gt;&lt;strong&gt;Vendor:&lt;/strong&gt;&lt;br&gt;ST Engineering iDirect&lt;/div&gt;
+&lt;div class=&quot;ics-version&quot;&gt;&lt;strong&gt;Product Version:&lt;/strong&gt;&lt;br&gt;ST Engineering iDirect Evolution iQ‑Series terminals: &amp;lt;=4.5.2.1, ST Engineering iDirect 3315‑Series terminals: &amp;lt;=4.5.2.1, ST Engineering iDirect 9‑Series terminals: &amp;lt;=4.5.2.1&lt;/div&gt;
+&lt;div class=&quot;ics-status&quot;&gt;&lt;strong&gt;Product Status:&lt;/strong&gt;&lt;br&gt;known_affected&lt;/div&gt;
+&lt;/div&gt;
+&lt;div class=&quot;ics-remediations&quot;&gt;
+&lt;h6&gt;Remediations&lt;/h6&gt;
+&lt;p&gt;&lt;strong&gt;Mitigation&lt;/strong&gt;&lt;br&gt;ST Engineering iDirect has fixed the vulnerabilities and recommend users update the software to version 4.5.2.2 or newer.&lt;/p&gt;
+&lt;p&gt;&lt;strong&gt;Mitigation&lt;/strong&gt;&lt;br&gt;Registered users are able to download patches from the iDirect Support Portal https://support.idirect.net/s/login.&lt;br&gt;&lt;a href=&quot;https://support.idirect.net/s/login&quot;&gt;https://support.idirect.net/s/login&lt;/a&gt;&lt;/p&gt;
+&lt;p&gt;&lt;strong&gt;Mitigation&lt;/strong&gt;&lt;br&gt;Restrict management interfaces to trusted networks (e.g., VPN, ACLs).&lt;/p&gt;
+&lt;p&gt;&lt;strong&gt;Mitigation&lt;/strong&gt;&lt;br&gt;Avoid exposing administrative APIs to the public internet.&lt;/p&gt;
+&lt;p&gt;&lt;strong&gt;Mitigation&lt;/strong&gt;&lt;br&gt;Enforce strong authentication practices.&lt;/p&gt;
+&lt;p&gt;&lt;strong&gt;Mitigation&lt;/strong&gt;&lt;br&gt;Monitor for anomalous API activity and unexpected device reboots.&lt;/p&gt;
+&lt;/div&gt;
+&lt;p&gt;&lt;strong&gt;Relevant CWE:&lt;/strong&gt; &lt;a href=&quot;https://cwe.mitre.org/data/definitions/306.html&quot;&gt;CWE-306 Missing Authentication for Critical Function&lt;/a&gt;&lt;/p&gt;
+&lt;hr&gt;
+&lt;h4&gt;Metrics&lt;/h4&gt;
+&lt;div class=&quot;csaf-table csaf-metrics-table&quot;&gt;
+&lt;table class=&quot;tablesaw tablesaw-stack&quot; data-tablesaw-mode=&quot;stack&quot; data-tablesaw-minimap&gt;
+&lt;thead&gt;
+&lt;tr&gt;
+&lt;th role=&quot;columnheader&quot; data-tablesaw-priority=&quot;persist&quot;&gt;CVSS Version&lt;/th&gt;
+&lt;th role=&quot;columnheader&quot;&gt;Base Score&lt;/th&gt;
+&lt;th role=&quot;columnheader&quot;&gt;Base Severity&lt;/th&gt;
+&lt;th role=&quot;columnheader&quot;&gt;Vector String&lt;/th&gt;
+&lt;/tr&gt;
+&lt;/thead&gt;
+&lt;tbody&gt;
+&lt;tr&gt;
+&lt;td&gt;3.1&lt;/td&gt;
+&lt;td&gt;7.5&lt;/td&gt;
+&lt;td&gt;HIGH&lt;/td&gt;
+&lt;td&gt;&lt;a href=&quot;https://www.first.org/cvss/calculator/3.1#CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:N/A:N&quot;&gt;CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:N/A:N&lt;/a&gt;&lt;/td&gt;
+&lt;/tr&gt;
+&lt;tr&gt;
+&lt;td&gt;4.0&lt;/td&gt;
+&lt;td&gt;8.7&lt;/td&gt;
+&lt;td&gt;HIGH&lt;/td&gt;
+&lt;td&gt;&lt;a href=&quot;https://www.first.org/cvss/calculator/4.0#CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:N/VC:H/VI:N/VA:N/SC:N/SI:N/SA:N&quot;&gt;CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:N/VC:H/VI:N/VA:N/SC:N/SI:N/SA:N&lt;/a&gt;&lt;/td&gt;
+&lt;/tr&gt;
+&lt;/tbody&gt;
+&lt;/table&gt;
+&lt;/div&gt;
+&lt;/div&gt;
+&lt;/div&gt;
+&lt;div class=&quot;csaf-accordion-item&quot;&gt;
+&lt;h3&gt;&lt;a class=&quot;csaf-accordion-toggle&quot; href=&quot;#&quot;&gt;CVE-2026-38057&lt;/a&gt;&lt;/h3&gt;
+&lt;div class=&quot;csaf-accordion-content&quot;&gt;
+&lt;p&gt;The iDirect iQ200 does not validate CSRF tokens on state-changing API endpoints after authentication. The /api/reboot endpoint accepts POST requests authenticated solely by a session cookie that lacks the SameSite attribute. A remote attacker can host a malicious web page that, when visited by an authenticated administrator, automatically submits a cross-site POST request causing an immediate device reboot and satellite link loss. Repeated attacks can sustain a denial-of-service condition.&lt;/p&gt;
+&lt;p&gt;&lt;a href=&quot;https://www.cve.org/CVERecord?id=CVE-2026-38057&quot;&gt;View CVE Details&lt;/a&gt;&lt;/p&gt;
+&lt;hr&gt;
+&lt;h4&gt;Affected Products&lt;/h4&gt;
+&lt;h5&gt;ST Engineering iDirect iQ-Series Terminals&lt;/h5&gt;
+&lt;div class=&quot;ics-vendor-version-status&quot;&gt;
+&lt;div class=&quot;ics-vendor&quot;&gt;&lt;strong&gt;Vendor:&lt;/strong&gt;&lt;br&gt;ST Engineering iDirect&lt;/div&gt;
+&lt;div class=&quot;ics-version&quot;&gt;&lt;strong&gt;Product Version:&lt;/strong&gt;&lt;br&gt;ST Engineering iDirect Evolution iQ‑Series terminals: &amp;lt;=4.5.2.1, ST Engineering iDirect 3315‑Series terminals: &amp;lt;=4.5.2.1, ST Engineering iDirect 9‑Series terminals: &amp;lt;=4.5.2.1&lt;/div&gt;
+&lt;div class=&quot;ics-status&quot;&gt;&lt;strong&gt;Product Status:&lt;/strong&gt;&lt;br&gt;known_affected&lt;/div&gt;
+&lt;/div&gt;
+&lt;div class=&quot;ics-remediations&quot;&gt;
+&lt;h6&gt;Remediations&lt;/h6&gt;
+&lt;p&gt;&lt;strong&gt;Mitigation&lt;/strong&gt;&lt;br&gt;ST Engineering iDirect has fixed the vulnerabilities and recommend users update the software to version 4.5.2.2 or newer.&lt;/p&gt;
+&lt;p&gt;&lt;strong&gt;Mitigation&lt;/strong&gt;&lt;br&gt;Registered users are able to download patches from the iDirect Support Portal https://support.idirect.net/s/login.&lt;br&gt;&lt;a href=&quot;https://support.idirect.net/s/login&quot;&gt;https://support.idirect.net/s/login&lt;/a&gt;&lt;/p&gt;
+&lt;p&gt;&lt;strong&gt;Mitigation&lt;/strong&gt;&lt;br&gt;Restrict management interfaces to trusted networks (e.g., VPN, ACLs).&lt;/p&gt;
+&lt;p&gt;&lt;strong&gt;Mitigation&lt;/strong&gt;&lt;br&gt;Avoid exposing administrative APIs to the public internet.&lt;/p&gt;
+&lt;p&gt;&lt;strong&gt;Mitigation&lt;/strong&gt;&lt;br&gt;Enforce strong authentication practices.&lt;/p&gt;
+&lt;p&gt;&lt;strong&gt;Mitigation&lt;/strong&gt;&lt;br&gt;Monitor for anomalous API activity and unexpected device reboots.&lt;/p&gt;
+&lt;/div&gt;
+&lt;p&gt;&lt;strong&gt;Relevant CWE:&lt;/strong&gt; &lt;a href=&quot;https://cwe.mitre.org/data/definitions/352.html&quot;&gt;CWE-352 Cross-Site Request Forgery (CSRF)&lt;/a&gt;&lt;/p&gt;
+&lt;hr&gt;
+&lt;h4&gt;Metrics&lt;/h4&gt;
+&lt;div class=&quot;csaf-table csaf-metrics-table&quot;&gt;
+&lt;table class=&quot;tablesaw tablesaw-stack&quot; data-tablesaw-mode=&quot;stack&quot; data-tablesaw-minimap&gt;
+&lt;thead&gt;
+&lt;tr&gt;
+&lt;th role=&quot;columnheader&quot; data-tablesaw-priority=&quot;persist&quot;&gt;CVSS Version&lt;/th&gt;
+&lt;th role=&quot;columnheader&quot;&gt;Base Score&lt;/th&gt;
+&lt;th role=&quot;columnheader&quot;&gt;Base Severity&lt;/th&gt;
+&lt;th role=&quot;columnheader&quot;&gt;Vector String&lt;/th&gt;
+&lt;/tr&gt;
+&lt;/thead&gt;
+&lt;tbody&gt;
+&lt;tr&gt;
+&lt;td&gt;3.1&lt;/td&gt;
+&lt;td&gt;8.1&lt;/td&gt;
+&lt;td&gt;HIGH&lt;/td&gt;
+&lt;td&gt;&lt;a href=&quot;https://www.first.org/cvss/calculator/3.1#CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:N/I:H/A:H&quot;&gt;CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:N/I:H/A:H&lt;/a&gt;&lt;/td&gt;
+&lt;/tr&gt;
+&lt;tr&gt;
+&lt;td&gt;4.0&lt;/td&gt;
+&lt;td&gt;7&lt;/td&gt;
+&lt;td&gt;HIGH&lt;/td&gt;
+&lt;td&gt;&lt;a href=&quot;https://www.first.org/cvss/calculator/4.0#CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:A/VC:N/VI:H/VA:H/SC:N/SI:N/SA:N&quot;&gt;CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:A/VC:N/VI:H/VA:H/SC:N/SI:N/SA:N&lt;/a&gt;&lt;/td&gt;
+&lt;/tr&gt;
+&lt;/tbody&gt;
+&lt;/table&gt;
+&lt;/div&gt;
+&lt;/div&gt;
+&lt;/div&gt;
+&lt;/div&gt;
+&lt;hr&gt;
+&lt;h2&gt;Acknowledgments&lt;/h2&gt;
+&lt;ul&gt;
+&lt;li&gt;Ahmed Alqahtani of Aramco reported these vulnerabilities to CISA&lt;/li&gt;
+&lt;/ul&gt;
+&lt;hr&gt;
+&lt;h2&gt;Legal Notice and Terms of Use&lt;/h2&gt;
+&lt;p&gt;This product is provided subject to this Notification (https://www.cisa.gov/notification) and this Privacy &amp;amp; Use policy (https://www.cisa.gov/privacy-policy).&lt;/p&gt;
+&lt;hr&gt;
+&lt;h2&gt;Recommended Practices&lt;/h2&gt;
+&lt;p&gt;CISA recommends users take defensive measures to minimize the risk of exploitation of these vulnerabilities.&lt;/p&gt;
+&lt;p&gt;Minimize network exposure for all control system devices and/or systems, ensuring they are not accessible from the internet.&lt;/p&gt;
+&lt;p&gt;Locate control system networks and remote devices behind firewalls and isolating them from business networks.&lt;/p&gt;
+&lt;p&gt;When remote access is required, use more secure methods, such as Virtual Private Networks (VPNs), recognizing VPNs may have vulnerabilities and should be updated to the most current version available. Also recognize VPN is only as secure as the connected devices.&lt;/p&gt;
+&lt;p&gt;CISA reminds organizations to perform proper impact analysis and risk assessment prior to deploying defensive measures.&lt;/p&gt;
+&lt;p&gt;CISA also provides a section for control systems security recommended practices on the ICS webpage on cisa.gov/ics. Several CISA products detailing cyber defense best practices are available for reading and download, including Improving Industrial Control Systems Cybersecurity with Defense-in-Depth Strategies.&lt;/p&gt;
+&lt;p&gt;CISA encourages organizations to implement recommended cybersecurity strategies for proactive defense of ICS assets.&lt;/p&gt;
+&lt;p&gt;Additional mitigation guidance and recommended practices are publicly available on the ICS webpage at cisa.gov/ics in the technical information paper, ICS-TIP-12-146-01B--Targeted Cyber Intrusion Detection and Mitigation Strategies.&lt;/p&gt;
+&lt;p&gt;Organizations observing suspected malicious activity should follow established internal procedures and report findings to CISA for tracking and correlation against other incidents.&lt;/p&gt;
+&lt;p&gt;CISA also recommends users take the following measures to protect themselves from social engineering attacks:&lt;/p&gt;
+&lt;p&gt;Do not click web links or open attachments in unsolicited email messages.&lt;/p&gt;
+&lt;p&gt;Refer to Recognizing and Avoiding Email Scams for more information on avoiding email scams.&lt;/p&gt;
+&lt;p&gt;Refer to Avoiding Social Engineering and Phishing Attacks for more information on social engineering attacks.&lt;/p&gt;
+&lt;p&gt;No known public exploitation specifically targeting these vulnerabilities has been reported to CISA at this time.&lt;/p&gt;
+&lt;hr&gt;
+&lt;h2&gt;Revision History&lt;/h2&gt;
+&lt;ul&gt;
+&lt;li&gt;&lt;strong&gt;Initial Release Date: &lt;/strong&gt;2026-07-02&lt;/li&gt;
+&lt;/ul&gt;
+&lt;table class=&quot;tablesaw tablesaw-stack&quot; data-tablesaw-mode=&quot;stack&quot; data-tablesaw-minimap&gt;
+&lt;thead&gt;
+&lt;tr&gt;
+&lt;th role=&quot;columnheader&quot; data-tablesaw-priority=&quot;persist&quot;&gt;Date&lt;/th&gt;
+&lt;th role=&quot;columnheader&quot;&gt;Revision&lt;/th&gt;
+&lt;th role=&quot;columnheader&quot;&gt;Summary&lt;/th&gt;
+&lt;/tr&gt;
+&lt;/thead&gt;
+&lt;tbody&gt;
+&lt;tr&gt;
+&lt;td&gt;2026-07-02&lt;/td&gt;
+&lt;td&gt;1&lt;/td&gt;
+&lt;td&gt;Initial Publication&lt;/td&gt;
+&lt;/tr&gt;
+&lt;/tbody&gt;
+&lt;/table&gt;
+&lt;hr&gt;
+&lt;h2&gt;Legal Notice and Terms of Use&lt;/h2&gt;
+</description>
+ <pubDate>Thu, 02 Jul 26 12:00:00 +0000</pubDate>
+ <dc:creator>CISA</dc:creator>
+ <guid isPermaLink="false">/node/25106</guid>
+ </item>
+
+ </channel>
+</rss>
diff --git a/PROJECTS/intermediate/security-news-scraper/testdata/feeds/darkreading.xml b/PROJECTS/intermediate/security-news-scraper/testdata/feeds/darkreading.xml
new file mode 100644
index 00000000..ddbc13c3
--- /dev/null
+++ b/PROJECTS/intermediate/security-news-scraper/testdata/feeds/darkreading.xml
@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="utf-8"?>
+<rss version="2.0" xml:base="https://www.darkreading.com"
+ xmlns:dc="http://purl.org/dc/elements/1.1/"
+ xmlns:atom="http://www.w3.org/2005/Atom"
+ xmlns:media="http://search.yahoo.com/mrss/"
+ xmlns:content="http://purl.org/rss/1.0/modules/content/">
+ <channel>
+ <title>darkreading</title>
+ <link>https://www.darkreading.com</link>
+ <description>Public RSS feed</description>
+ <language>en</language>
+ <atom:link href="https://www.darkreading.com/feeds/rss.xml" rel="self" type="application/rss+xml" />
+ <item>
+ <title><![CDATA[Chinese LLMs Broaden the Gap Between Attackers &amp; Defenders]]></title>
+ <link><![CDATA[https://www.darkreading.com/cyber-risk/chinese-llms-broaden-gap-between-attackers-and-defenders]]></link>
+ <description><![CDATA[Two new models from Chinese firms compete with top US mainstream and frontier models. Should cyber-defenders be worried?]]></description>
+ <pubDate>Fri, 03 Jul 2026 13:01:00 GMT</pubDate>
+ <dc:creator>Robert Lemos</dc:creator>
+ <guid isPermaLink="false"><![CDATA[https://www.darkreading.com/cyber-risk/chinese-llms-broaden-gap-between-attackers-and-defenders]]></guid>
+
+ <media:thumbnail url="https://eu-images.contentstack.com/v3/assets/blt6d90778a997de1cd/blt0617bd7899766864/6a46aac83cc4ed702aef33b2/castle-of-the-moors-Gabriela_Beres-shutterstock.jpg?width=720&amp;quality=80&amp;disable=upscale" />
+ <media:content url="https://eu-images.contentstack.com/v3/assets/blt6d90778a997de1cd/blt0617bd7899766864/6a46aac83cc4ed702aef33b2/castle-of-the-moors-Gabriela_Beres-shutterstock.jpg?width=720&amp;quality=80&amp;disable=upscale" medium="image">
+ <media:title type="html">castle-of-the-moors-Gabriela_Beres-shutterstock.jpg</media:title>
+ </media:content>
+ </item>
+<item>
+ <title><![CDATA[Aussies Face Reduced Cybercrime Risk, as Pressure Shifts to SMBs]]></title>
+ <link><![CDATA[https://www.darkreading.com/cybersecurity-analytics/aussies-face-reduced-cybercrime-risk-pressure-shifts-smbs]]></link>
+ <description><![CDATA[Improved institutional safeguards and stricter regulations have pushed the burdens of protection and risk reduction on to Australian businesses.]]></description>
+ <pubDate>Thu, 02 Jul 2026 23:01:00 GMT</pubDate>
+ <dc:creator>Nate Nelson</dc:creator>
+ <guid isPermaLink="false"><![CDATA[https://www.darkreading.com/cybersecurity-analytics/aussies-face-reduced-cybercrime-risk-pressure-shifts-smbs]]></guid>
+
+ <media:thumbnail url="https://eu-images.contentstack.com/v3/assets/blt6d90778a997de1cd/bltef5cf25552ad1b17/6a46744dec9caa31e05a16a8/Australia-imaginima-Getty.jpg?width=720&amp;quality=80&amp;disable=upscale" />
+ <media:content url="https://eu-images.contentstack.com/v3/assets/blt6d90778a997de1cd/bltef5cf25552ad1b17/6a46744dec9caa31e05a16a8/Australia-imaginima-Getty.jpg?width=720&amp;quality=80&amp;disable=upscale" medium="image">
+ <media:title type="html">Australia-imaginima-Getty.jpg</media:title>
+ </media:content>
+ </item>
+<item>
+ <title><![CDATA[Apple Reverses Age-Old Patch Policy to Keep Up With AI]]></title>
+ <link><![CDATA[https://www.darkreading.com/cybersecurity-operations/apple-patch-policy-ai]]></link>
+ <description><![CDATA[Expect more compressed patching cycles from Apple going forward, as attackers leverage artificial intelligence to reduce time to exploit.]]></description>
+ <pubDate>Thu, 02 Jul 2026 19:31:58 GMT</pubDate>
+ <dc:creator>Nate Nelson</dc:creator>
+ <guid isPermaLink="false"><![CDATA[https://www.darkreading.com/cybersecurity-operations/apple-patch-policy-ai]]></guid>
+
+ <media:thumbnail url="https://eu-images.contentstack.com/v3/assets/blt6d90778a997de1cd/bltb900b57271d6fc24/6a46a07ecef8a87304f980df/Apple_bandage-imtmphoto-Getty.jpg?width=720&amp;quality=80&amp;disable=upscale" />
+ <media:content url="https://eu-images.contentstack.com/v3/assets/blt6d90778a997de1cd/bltb900b57271d6fc24/6a46a07ecef8a87304f980df/Apple_bandage-imtmphoto-Getty.jpg?width=720&amp;quality=80&amp;disable=upscale" medium="image">
+ <media:title type="html">Apple_bandage-imtmphoto-Getty.jpg</media:title>
+ </media:content>
+ </item> </channel>
+</rss>
diff --git a/PROJECTS/intermediate/security-news-scraper/testdata/feeds/krebs.xml b/PROJECTS/intermediate/security-news-scraper/testdata/feeds/krebs.xml
new file mode 100644
index 00000000..4b80de7a
--- /dev/null
+++ b/PROJECTS/intermediate/security-news-scraper/testdata/feeds/krebs.xml
@@ -0,0 +1,245 @@
+<?xml version="1.0" encoding="UTF-8"?><rss version="2.0"
+ xmlns:content="http://purl.org/rss/1.0/modules/content/"
+ xmlns:wfw="http://wellformedweb.org/CommentAPI/"
+ xmlns:dc="http://purl.org/dc/elements/1.1/"
+ xmlns:atom="http://www.w3.org/2005/Atom"
+ xmlns:sy="http://purl.org/rss/1.0/modules/syndication/"
+ xmlns:slash="http://purl.org/rss/1.0/modules/slash/"
+ >
+
+<channel>
+ <title>Krebs on Security</title>
+ <atom:link href="https://krebsonsecurity.com/feed/?utm_source=chatgpt.com" rel="self" type="application/rss+xml" />
+ <link>https://krebsonsecurity.com</link>
+ <description>In-depth security news and investigation</description>
+ <lastBuildDate>Sat, 04 Jul 2026 16:16:19 +0000</lastBuildDate>
+ <language>en-US</language>
+ <sy:updatePeriod>
+ hourly </sy:updatePeriod>
+ <sy:updateFrequency>
+ 1 </sy:updateFrequency>
+ <generator>https://wordpress.org/?v=6.2.2</generator>
+ <item>
+ <title>FBI Seizes NetNut Proxy Platform, Popa Botnet</title>
+ <link>https://krebsonsecurity.com/2026/07/fbi-seizes-netnut-proxy-platform-popa-botnet/</link>
+ <comments>https://krebsonsecurity.com/2026/07/fbi-seizes-netnut-proxy-platform-popa-botnet/#comments</comments>
+
+ <dc:creator><![CDATA[BrianKrebs]]></dc:creator>
+ <pubDate>Thu, 02 Jul 2026 19:27:33 +0000</pubDate>
+ <category><![CDATA[A Little Sunshine]]></category>
+ <category><![CDATA[Ne'er-Do-Well News]]></category>
+ <category><![CDATA[Web Fraud 2.0]]></category>
+ <category><![CDATA[Alarum Technologies Ltd]]></category>
+ <category><![CDATA[Benjamin Brundage]]></category>
+ <category><![CDATA[fbi]]></category>
+ <category><![CDATA[google]]></category>
+ <category><![CDATA[Google Threat Intelligence Group]]></category>
+ <category><![CDATA[Internal Revenue Service Criminal Investigation]]></category>
+ <category><![CDATA[IPidea]]></category>
+ <category><![CDATA[LG]]></category>
+ <category><![CDATA[Lumen]]></category>
+ <category><![CDATA[NetNut]]></category>
+ <category><![CDATA[Popa botnet]]></category>
+ <category><![CDATA[Samsung]]></category>
+ <category><![CDATA[Shadowserver]]></category>
+ <category><![CDATA[Synthient]]></category>
+ <guid isPermaLink="false">https://krebsonsecurity.com/?p=73923</guid>
+
+ <description><![CDATA[The Federal Bureau of Investigation (FBI) said today it worked with industry partners to seize hundreds of domains associated with NetNut, a sprawling residential proxy service operated by the publicly-traded Israeli company Alarum Technologies [NASDAQ: ALAR]. The action comes roughly two weeks after KrebsOnSecurity published findings from multiple security firms connecting NetNut to the Popa botnet, a collection of at least two million devices that have been compromised by malicious software with little or no consent from victims.]]></description>
+ <content:encoded><![CDATA[<p>The <strong>Federal Bureau of Investigation</strong> (FBI) said today it worked with industry partners to seize hundreds of domains associated with <strong>NetNut</strong>, a sprawling residential proxy service operated by the publicly-traded Israeli company <strong>Alarum Technologies </strong>[NASDAQ: ALAR]. The action comes roughly two weeks after KrebsOnSecurity published findings from multiple security firms connecting NetNut to the <strong>Popa</strong> botnet, a collection of at least two million devices that have been compromised by malicious software with little or no consent from victims.</p>
+<div id="attachment_73926" style="width: 757px" class="wp-caption aligncenter"><img aria-describedby="caption-attachment-73926" decoding="async" class=" wp-image-73926" src="https://krebsonsecurity.com/wp-content/uploads/2026/07/netnutseizure.png" alt="" width="747" height="415" srcset="https://krebsonsecurity.com/wp-content/uploads/2026/07/netnutseizure.png 1407w, https://krebsonsecurity.com/wp-content/uploads/2026/07/netnutseizure-768x427.png 768w, https://krebsonsecurity.com/wp-content/uploads/2026/07/netnutseizure-782x435.png 782w" sizes="(max-width: 747px) 100vw, 747px" /><p id="caption-attachment-73926" class="wp-caption-text">The NetNut homepage today was replaced by this seizure banner from the FBI.</p></div>
+<p>On June 19, three different security firms <a href="https://krebsonsecurity.com/2026/06/popa-botnet-linked-to-publicly-traded-israeli-firm/" target="_blank" rel="noopener">issued similar findings</a>: That NetNut is a residential proxy network which populates a botnet called Popa, and distributes software for devices commonly found in homes, such as smart TVs and streaming boxes. NetNut&#8217;s software turns those systems into always-on residential proxy nodes that are rented to others, who predominantly use them to relay abusive and intrusive Internet traffic, such as mass content scraping, advertising fraud, and account takeover activity.</p>
+<p>Earlier today, NetNut&#8217;s homepage was replaced with a seizure notice from the FBI and the <strong>Internal Revenue Service Criminal Investigation</strong> division. The seizure notice thanked <strong>Google</strong>, <strong>Lumen</strong>, <strong>Shadowserver</strong> and other industry partners for their help in dismantling hundreds of domains tied to the Popa botnet, which experts say has long been synonymous with NetNut&#8217;s residential proxy infrastructure.</p>
+<p>In a blog post published today, the <strong>Google</strong> <strong>Threat Intelligence Group</strong> (GTIG) said NetNut&#8217;s proxy network is widely resold and white-labeled by a number of third-party proxy providers, and that its services are heavily sought out by cybercriminals seeking to obfuscate the source of their malicious traffic. The GTIG said that in a single week during June 2026, they observed 316 distinct clusters of threat actors using suspected NetNut exit nodes, including cybercriminal and espionage groups.</p>
+<p>&#8220;These bad actors can use NetNut to mask their origin IP address when accessing victim environments, accessing their own infrastructure, and conducting password spray attacks,&#8221; Google&#8217;s GTIG <a href="https://cloud.google.com/blog/topics/threat-intelligence/google-continued-disruption-residential-proxy-networks" target="_blank" rel="noopener">wrote</a>. &#8220;Furthermore, when a consumer device becomes an exit node, unauthorized network traffic passes through it. This means bad actors can access other private devices on the same home network, effectively exposing them to Internet threats.&#8221;</p>
+<p>Google said it disabled Google accounts and services used by NetNut for malware command and control, and that it shared technical intelligence on NetNut&#8217;s software development kits (SDKs) and backend infrastructure with platform providers, law enforcement and research firms. The company also disabled apps known to bundle NetNut&#8217;s various SDKs.</p>
+<p><strong>Omer Weiss</strong>, legal counsel for NetNut parent Alarum Technologies, said the company was aware of the FBI seizure and cooperating with investigators.</p>
+<p>&#8220;Alarum takes this matter seriously and will fully cooperate with law enforcement to ensure any misuse of its infrastructure is thoroughly investigated and those responsible are held to account,&#8221; Weiss said in a written statement.<br />
+<span id="more-73923"></span></p>
+<p><strong>Benjamin Brundage</strong> is founder of the proxy tracking service <strong>Synthient</strong>, one of the companies that <a href="https://synthient.com/blog/popa-from-sourcing-to-distribution" target="_blank" rel="noopener">published evidence last month</a> linking the Popa botnet to NetNut and Alarum Technologies. Brundage said the domain seizures appear to have disrupted both the Popa botnet and the NetNut proxy network that rides on top of it.</p>
+<p>Brundage said NetNut&#8217;s apparent demise is likely to be a great disadvantage for the cybercrime community, which was already reeling from <a href="https://cloud.google.com/blog/topics/threat-intelligence/disrupting-largest-residential-proxy-network" target="_blank" rel="noopener">legal actions by Google</a> earlier this year that seized infrastructure for NetNut&#8217;s biggest competitor &#8212; <a href="https://krebsonsecurity.com/tag/ipidea/" target="_blank" rel="noopener">IPIDEA</a>.</p>
+<p>&#8220;I think this takedown is going to have a big impact, because NetNut gained significant popularity after the IPIDEA takedown,&#8221; he said. &#8220;Also NetNut has been incredibly common among resellers, and they were on par with IPIDEA in terms of their daily traffic, quality, size, price per gigabyte, all of it.&#8221;</p>
+<div id="attachment_73938" style="width: 759px" class="wp-caption aligncenter"><img aria-describedby="caption-attachment-73938" decoding="async" loading="lazy" class=" wp-image-73938" src="https://krebsonsecurity.com/wp-content/uploads/2026/07/netnut-popa-blacklotus.png" alt="" width="749" height="374" srcset="https://krebsonsecurity.com/wp-content/uploads/2026/07/netnut-popa-blacklotus.png 1435w, https://krebsonsecurity.com/wp-content/uploads/2026/07/netnut-popa-blacklotus-768x384.png 768w, https://krebsonsecurity.com/wp-content/uploads/2026/07/netnut-popa-blacklotus-782x391.png 782w" sizes="(max-width: 749px) 100vw, 749px" /><p id="caption-attachment-73938" class="wp-caption-text">NetNut&#8217;s infrastructure, in a nutshell. Image: Black Lotus Labs, Lumen.</p></div>
+<p>The NetNut and Popa botnet takedown may have another added benefit, Brundage said: Lessening the impact of large distributed denial-of-service botnets that have been built on the backs of poorly configured residential proxy services. In January, Synthient <a href="https://krebsonsecurity.com/2026/01/the-kimwolf-botnet-is-stalking-your-local-network/" target="_blank" rel="noopener">revealed</a> how cybercriminals had built the world&#8217;s largest DDoS botnet (Kimwolf) by tunneling through IPIDEA proxy connections into the local networks of TV box owners, and infecting other Android-based devices behind the victim&#8217;s firewall.</p>
+<p>While many of the bigger proxy providers took steps to block this activity, resellers of the major proxy networks have been far slower to respond to the threat, Brundage said.</p>
+<p>&#8220;In terms of all these TV box devices getting compromised from the proxy network, it will have an impact on the DDoS botnets out there,&#8221; he said.</p>
+<p>For its part, Google reckons today&#8217;s actions have caused &#8220;significant degradation to NetNut’s proxy network and its business operations, reducing the available pool of devices for the proxy operator by millions.&#8221; But the company warns that proxy networks can rebuild themselves by effectively reselling other proxy services, as IPIDEA has done over the past few months.</p>
+<p>&#8220;Google has high confidence that many popular residential proxy brands are in fact whitelabeling the NetNut botnet,&#8221; the GTIG report concludes. &#8220;While we expect this disruption to have a larger ripple effect across the residential proxy ecosystem, observations after the disruption of IPIDEA proved that individual networks can appear resilient. What we have observed is that when faced with the degradation of their own botnet, proxy operators begin buying capacity from their competitors, effectively becoming a reseller. We recognize that creating a lasting disruption in this fluid ecosystem means we must scale our efforts to target the infrastructure of several interconnected providers.&#8221;</p>
+<p>As KrebsOnSecurity has warned repeatedly, most of the no-name TV streaming boxes for sale on the major e-commerce websites either come <a href="https://krebsonsecurity.com/2025/11/is-your-android-tv-streaming-box-part-of-a-botnet/" target="_blank" rel="noopener">pre-installed with residential proxy software</a>, or require the installation of proxy SDKs in order to use the device for its stated purpose (streaming pirated movies, sporting events and TV shows). Google&#8217;s advice here is sound: When it comes to TV boxes, stick to name brands from reputable manufacturers, and then be sparing and judicious with any apps you choose to install.</p>
+<p>The <a href="https://krebsonsecurity.com/2025/11/is-your-android-tv-streaming-box-part-of-a-botnet/" target="_blank" rel="noopener">sketchy TV boxes</a> that are being commandeered by the Popa botnet and other threats all come with or require the user to install unofficial Android operating systems that do not operate within the confines of Google&#8217;s Official Play Protect store. Google says consumers can confirm whether or not a device is built with the official Android TV OS and Play Protect certification by following <a href="https://support.google.com/googleplay/answer/7165974" target="_blank" rel="noopener">these instructions</a>.</p>
+<p>Even people without TV streaming boxes can find their smart TVs enrolled in residential proxy networks, just by installing one of thousands of apps available for download on <strong>Samsung</strong> and <strong>LG</strong> smart TVs. In <a href="https://spur.us/blog/smart-tv-apps-residential-proxy-sdks" target="_blank" rel="noopener">a report</a> released last month, the proxy tracking company <strong>Spur</strong> found 42 percent of apps available for download via the webOS operating system on LG smart TVs include SDKs that turn one&#8217;s television into an always-on residential proxy node. More than a quarter of the apps made for Samsung’s <strong>Tizen</strong> operating system had similar residential proxy components, Spur found.</p>
+<div id="attachment_73849" style="width: 758px" class="wp-caption aligncenter"><img aria-describedby="caption-attachment-73849" decoding="async" loading="lazy" class=" wp-image-73849" src="https://krebsonsecurity.com/wp-content/uploads/2026/06/spur-tvproxy.png" alt="" width="748" height="305" srcset="https://krebsonsecurity.com/wp-content/uploads/2026/06/spur-tvproxy.png 1272w, https://krebsonsecurity.com/wp-content/uploads/2026/06/spur-tvproxy-768x313.png 768w, https://krebsonsecurity.com/wp-content/uploads/2026/06/spur-tvproxy-782x318.png 782w" sizes="(max-width: 748px) 100vw, 748px" /><p id="caption-attachment-73849" class="wp-caption-text">Image: Spur.us.</p></div>
+<p><strong>Update, 4:24 p.m. ET:</strong> Included a statement shared post-publication from an attorney representing NetNut parent Alarum Technologies.</p>
+]]></content:encoded>
+
+ <wfw:commentRss>https://krebsonsecurity.com/2026/07/fbi-seizes-netnut-proxy-platform-popa-botnet/feed/</wfw:commentRss>
+ <slash:comments>17</slash:comments>
+
+
+ </item>
+<item>
+ <title>Scattered Spider Hackers Plead Guilty on Day 1 of Trial</title>
+ <link>https://krebsonsecurity.com/2026/06/scattered-spider-hackers-plead-guilty-on-day-1-of-trial/</link>
+ <comments>https://krebsonsecurity.com/2026/06/scattered-spider-hackers-plead-guilty-on-day-1-of-trial/#comments</comments>
+
+ <dc:creator><![CDATA[BrianKrebs]]></dc:creator>
+ <pubDate>Tue, 23 Jun 2026 16:12:49 +0000</pubDate>
+ <category><![CDATA[Ne'er-Do-Well News]]></category>
+ <category><![CDATA[Ransomware]]></category>
+ <category><![CDATA[BBC]]></category>
+ <category><![CDATA[Everlynn]]></category>
+ <category><![CDATA[Owen Flowers]]></category>
+ <category><![CDATA[Scattered Spider]]></category>
+ <category><![CDATA[Thalha Jubair]]></category>
+ <category><![CDATA[Transport for London]]></category>
+ <category><![CDATA[Tyler "Tylerb" Buchanan]]></category>
+ <guid isPermaLink="false">https://krebsonsecurity.com/?p=73876</guid>
+
+ <description><![CDATA[Two men pleaded guilty in the United Kingdom this week to criminal charges stemming from an August 2024 cyberattack that crippled Transport for London, the entity responsible for the public transport network in the Greater London area. The duo were key members of a prolific cybercrime group known as Scattered Spider, and their guilty pleas came on the first day of what was expected to be a six-week trial.]]></description>
+ <content:encoded><![CDATA[<p>Two men pleaded guilty in the United Kingdom this week to criminal charges stemming from an August 2024 cyberattack that crippled <strong>Transport for London</strong>, the entity responsible for the public transport network in the Greater London area. The duo were key members of a prolific cybercrime group known as <strong>Scattered Spider</strong>, and their guilty pleas came on the first day of what was expected to be a six-week trial.</p>
+<div id="attachment_73881" style="width: 760px" class="wp-caption aligncenter"><img aria-describedby="caption-attachment-73881" decoding="async" loading="lazy" class=" wp-image-73881" src="https://krebsonsecurity.com/wp-content/uploads/2026/06/flowers-jubair-nca.png" alt="" width="750" height="421" srcset="https://krebsonsecurity.com/wp-content/uploads/2026/06/flowers-jubair-nca.png 926w, https://krebsonsecurity.com/wp-content/uploads/2026/06/flowers-jubair-nca-768x431.png 768w, https://krebsonsecurity.com/wp-content/uploads/2026/06/flowers-jubair-nca-782x439.png 782w" sizes="(max-width: 750px) 100vw, 750px" /><p id="caption-attachment-73881" class="wp-caption-text">Owen Flowers (left) 18, and Thalha Jubair, 20. Image: UK National Crime Agency (NCA).</p></div>
+<p><strong>Thalha Jubair</strong>, 20, of East London and 18-year-old <strong>Owen Flowers</strong> of Walsall admitted conspiring to commit unauthorized acts against Transport for London computer systems and causing risk of serious damage to human welfare. According to <a href="https://www.bbc.com/news/articles/czx5yp9qy0do" target="_blank" rel="noopener">a report</a> from the BBC, Flowers alone admitted to being part of a conspiracy to hack into U.S. based healthcare providers SSM Health Care Corporation and Sutter Health in September 2024.</p>
+<p>Jubair is also wanted by U.S. law enforcement agencies. In September 2025, prosecutors in New Jersey unsealed <a href="https://www.justice.gov/opa/pr/united-kingdom-national-charged-connection-multiple-cyber-attacks-including-critical" target="_blank" rel="noopener">an indictment</a> alleging Jubair and other Scattered Spider members committed computer fraud, wire fraud, and money laundering in relation to 120 computer network intrusions involving 47 U.S. entities between May 2022 and September 2025, and that the group’s victims paid at least $115 million in ransom payments.</p>
+<p>In July 2025, KrebsOnSecurity <a href="https://krebsonsecurity.com/2025/07/uk-charges-four-in-scattered-spider-ransom-group/" target="_blank" rel="noopener">reported</a> that Flowers and Jubair were arrested in the United Kingdom in connection with Scattered Spider <a href="https://www.thetimes.com/uk/technology-uk/article/ransoms-hackers-cyber-crime-t5kjldwwm" target="_blank" rel="noopener">ransom attacks</a> against the retailers <strong>Marks &amp; Spencer</strong> and <strong>Harrods</strong>, and the British food retailer <strong>Co-op Group</strong>. Multiple sources familiar with those investigations said Flowers was the Scattered Spider member who anonymously <a href="https://krebsonsecurity.com/2024/09/the-dark-nexus-between-harm-groups-and-the-com/" target="_blank" rel="noopener">gave interviews to the media</a> in the days after the group’s September 2023 ransomware attacks disrupted operations at Las Vegas casinos operated by <strong>MGM Resorts</strong> and <strong>Caesars Entertainment</strong>.</p>
+<p>According to prosecutors, Jubair co-ran a bustling Telegram channel called <strong>Star Chat</strong>, the home of a <a href="https://krebsonsecurity.com/?s=SIM-swapping" target="_blank" rel="noopener">SIM-swapping</a> group that used voice- and SMS-based phishing attacks to steal credentials from employees at the major wireless providers in the U.S. and U.K. The group would then use that access to sell a service that could redirect a target’s phone number to a device the attackers controlled and intercept the victim’s calls and text messages (including one-time codes for multi-factor authentication).</p>
+<div id="attachment_72238" style="width: 819px" class="wp-caption aligncenter"><img aria-describedby="caption-attachment-72238" decoding="async" loading="lazy" class="size-full wp-image-72238" src="https://krebsonsecurity.com/wp-content/uploads/2025/09/rocketace-tmobile.png" alt="" width="809" height="915" srcset="https://krebsonsecurity.com/wp-content/uploads/2025/09/rocketace-tmobile.png 809w, https://krebsonsecurity.com/wp-content/uploads/2025/09/rocketace-tmobile-768x869.png 768w, https://krebsonsecurity.com/wp-content/uploads/2025/09/rocketace-tmobile-782x884.png 782w" sizes="(max-width: 809px) 100vw, 809px" /><p id="caption-attachment-72238" class="wp-caption-text">A receipt from Star Fraud Chat&#8217;s SIM-swapping service targeting a T-Mobile customer after the group gained access to internal T-Mobile employee tools. &#8220;Rocket Ace&#8221; was one of Jubair&#8217;s hacker handles, according to U.S. prosecutors.</p></div>
+<p>New Jersey prosecutors also allege Jubair also was involved in a <a href="https://krebsonsecurity.com/2022/08/how-1-time-passcodes-became-a-corporate-liability/" target="_blank" rel="noopener">mass SMS phishing campaign during the summer of 2022</a> that stole single sign-on credentials from employees at hundreds of companies. That weeks-long SMS phishing campaign led to intrusions and data thefts at more than 130 organizations, including <strong>LastPass</strong>, <strong>DoorDash</strong>, <strong>Mailchimp</strong>, <strong>Plex</strong> and <strong>Signal</strong>.<span id="more-73876"></span></p>
+<p>KrebsOnSecurity reported last year that one of Jubair&#8217;s alter egos at age 15 was &#8220;<strong>Everlynn</strong>,&#8221; a hacker who sold <a href="https://krebsonsecurity.com/?s=fake+edr" target="_blank" rel="noopener">fraudulent &#8220;emergency data requests&#8221;</a> that used compromised police and government email addresses to demand subscriber data (e.g. username, IP/email address) from major tech companies, claiming the requests concerned urgent matters of life and death and could not wait for a court order.</p>
+<p>In April 2026, 24-year-old British national and Scattered Spider member <strong>Tyler &#8220;Tylerb&#8221; Buchanan</strong> <a href="https://krebsonsecurity.com/2026/04/scattered-spider-member-tylerb-pleads-guilty/" target="_blank" rel="noopener">pleaded guilty</a> to wire fraud conspiracy and aggravated identity theft for participating in the group&#8217;s SMS phishing spree in the summer of 2022. The government said Buchanan, Jubair and others used the credentials harvested in that phishing campaign to steal at least $8 million in cryptocurrency from victims throughout the United States. Buchanan is currently scheduled to be sentenced on October 2.</p>
+<p>In August 2025, 20-year-old Scattered Spider member from Florida named <strong>Noah Michael Urban</strong> was <a href="https://krebsonsecurity.com/2025/08/sim-swapper-scattered-spider-hacker-gets-10-years/" target="_blank" rel="noopener">sentenced to 10 years in federal prison</a> and ordered to pay $13 million in restitution, after pleading guilty to charges of wire fraud and conspiracy.</p>
+<p>The U.S. Department of Justice says three alleged Scattered Spider defendants indicted along with Buchanan still face charges, including <strong>Ahmed Hossam Eldin Elbadawy</strong>, 24, a.k.a. “AD,” of College Station, Texas; <strong>Evans Onyeaka Osiebo</strong>, 21, of Dallas, Texas; and <strong>Joel Martin Evans</strong>, 26, a.k.a. “joeleoli,” of Jacksonville, North Carolina.</p>
+<p>Flowers and Jubair are slated to be sentenced in a London court on July 15, 2026.</p>
+]]></content:encoded>
+
+ <wfw:commentRss>https://krebsonsecurity.com/2026/06/scattered-spider-hackers-plead-guilty-on-day-1-of-trial/feed/</wfw:commentRss>
+ <slash:comments>39</slash:comments>
+
+
+ </item>
+<item>
+ <title>&#8216;Popa&#8217; Botnet Linked to Publicly-Traded Israeli Firm</title>
+ <link>https://krebsonsecurity.com/2026/06/popa-botnet-linked-to-publicly-traded-israeli-firm/</link>
+ <comments>https://krebsonsecurity.com/2026/06/popa-botnet-linked-to-publicly-traded-israeli-firm/#comments</comments>
+
+ <dc:creator><![CDATA[BrianKrebs]]></dc:creator>
+ <pubDate>Thu, 18 Jun 2026 17:37:58 +0000</pubDate>
+ <category><![CDATA[A Little Sunshine]]></category>
+ <category><![CDATA[Latest Warnings]]></category>
+ <category><![CDATA[The Coming Storm]]></category>
+ <category><![CDATA[Alarum Technologies Ltd]]></category>
+ <category><![CDATA[BadBox 2.0]]></category>
+ <category><![CDATA[Black Lotus Labs]]></category>
+ <category><![CDATA[Brendan O'Connell]]></category>
+ <category><![CDATA[Chris Formosa]]></category>
+ <category><![CDATA[Confederation of Open Access Repositories]]></category>
+ <category><![CDATA[CRICFy]]></category>
+ <category><![CDATA[CyberFlix]]></category>
+ <category><![CDATA[David Brunsdon]]></category>
+ <category><![CDATA[Directory of Open Access Journals]]></category>
+ <category><![CDATA[DooFlix]]></category>
+ <category><![CDATA[Flixoid]]></category>
+ <category><![CDATA[Include Security]]></category>
+ <category><![CDATA[Jérôme Meyer]]></category>
+ <category><![CDATA[LG]]></category>
+ <category><![CDATA[Lumen Technologies]]></category>
+ <category><![CDATA[NetNut]]></category>
+ <category><![CDATA[Nick Sundvall]]></category>
+ <category><![CDATA[Ninajtech]]></category>
+ <category><![CDATA[Nokia Deepfield]]></category>
+ <category><![CDATA[OceanStreams]]></category>
+ <category><![CDATA[Popa botnet]]></category>
+ <category><![CDATA[Qurium]]></category>
+ <category><![CDATA[Rapid Streamz]]></category>
+ <category><![CDATA[residential proxies]]></category>
+ <category><![CDATA[RoboVPN]]></category>
+ <category><![CDATA[RTS Tv]]></category>
+ <category><![CDATA[Samsung]]></category>
+ <category><![CDATA[Sean Simmons]]></category>
+ <category><![CDATA[Sprozfy]]></category>
+ <category><![CDATA[Spur]]></category>
+ <category><![CDATA[Synthient]]></category>
+ <category><![CDATA[TvMob]]></category>
+ <category><![CDATA[Vo1d botnet]]></category>
+ <guid isPermaLink="false">https://krebsonsecurity.com/?p=73832</guid>
+
+ <description><![CDATA[For the past four years, a sprawling Android-based botnet called Popa has forced millions of consumer TV boxes to relay Internet traffic linked to advertising fraud, account takeovers, and mass data-scraping efforts. This week, researchers from multiple security firms concluded that the Popa botnet is linked to NetNut, a "residential proxy" provider operated by the publicly-traded Israeli firm Alarum Technologies Ltd [NASDAQ: ALAR].]]></description>
+ <content:encoded><![CDATA[<p>For the past four years, a sprawling Android-based botnet called <strong>Popa</strong> has forced millions of consumer TV boxes to relay Internet traffic linked to advertising fraud, account takeovers, and mass data-scraping efforts. This week, researchers from multiple security firms concluded that the Popa botnet is linked to <strong>NetNut</strong>, a &#8220;residential proxy&#8221; provider operated by the publicly-traded Israeli firm <strong>Alarum Technologies Ltd </strong>[NASDAQ: ALAR].</p>
+<div id="attachment_73857" style="width: 759px" class="wp-caption aligncenter"><img aria-describedby="caption-attachment-73857" decoding="async" loading="lazy" class=" wp-image-73857" src="https://krebsonsecurity.com/wp-content/uploads/2026/06/tvboxes-proxy.png" alt="Malicious streaming devices sold online that enroll the user's home Internet address in a residential proxy service. Image: Synthient. Pictured are 8 different TV boxes, including the X96 Mini Box, stick, and other no-name brands." width="749" height="311" srcset="https://krebsonsecurity.com/wp-content/uploads/2026/06/tvboxes-proxy.png 990w, https://krebsonsecurity.com/wp-content/uploads/2026/06/tvboxes-proxy-768x319.png 768w, https://krebsonsecurity.com/wp-content/uploads/2026/06/tvboxes-proxy-782x325.png 782w" sizes="(max-width: 749px) 100vw, 749px" /><p id="caption-attachment-73857" class="wp-caption-text">Malicious streaming devices sold online that enroll the user&#8217;s home Internet address in a residential proxy service. Image: HUMAN Security.</p></div>
+<p>Popa is a massive botnet, but by all accounts it is unlike traditional botnets that enlist compromised systems in destructive activities, such as coordinating huge distributed denial-of-service attacks. Rather, Popa appears designed with a singular purpose: Implementing a persistent communications layer capable of registering a device, maintaining long-lived encrypted connections, and opening communication tunnels on demand.</p>
+<p>Experts say Popa is a plugin component associated with the <strong>Vo1d</strong> botnet, a large-scale malware campaign targeting unofficial Android-based TV boxes. These devices, which are marketed under thousands of brand names and model numbers and broadly available for purchase at top e-commerce destinations, all advertise the ability to stream hundreds of subscription video services for an up front one-time fee.</p>
+<p>But as the FBI and security industry experts have warned repeatedly, these streaming boxes typically <a href="https://krebsonsecurity.com/2025/11/is-your-android-tv-streaming-box-part-of-a-botnet/" target="_blank" rel="noopener">bundle or come pre-installed with software</a> that turns the user&#8217;s TV into a &#8220;<a href="https://synthient.com/blog/who-are-the-victims-of-residential-proxies" target="_blank" rel="noopener">residential proxy</a>&#8221; &#8212; allowing anyone to route their Internet traffic through that device for as long as it remains plugged into a wall socket and connected to a local network. More concerning, some of these proxy networks do little to stop malicious customers from communicating with and even compromising systems on the local network of the unsuspecting device owner.</p>
+<p>The first clues about Popa&#8217;s origins came in <a href="https://blog.xlab.qianxin.com/long-live-the-vo1d_botnet/" target="_blank" rel="noopener">a 2025 report</a> from the Chinese security company <strong>XLAB</strong>, which flagged at least nine domain names that were used to register and direct the activities of compromised devices. In <a href="https://www.qurium.org/forensics/finding-popa/" target="_blank" rel="noopener">a report</a> released today, the security firm <strong>Qurium</strong> described how it stumbled on some of those same domains while investigating a series of disruptive and expensive data scraping events targeting the company&#8217;s hosted organizations in May 2026, in which the scraping activity was scattered evenly across more than 1.4 million Internet addresses.</p>
+<p>Qurium said it found several dozen domains used to control Popa that were all hosted in lockstep across multiple Internet addresses over time, including <strong>gmslb[.]net</strong>, safernetwork[.]io, tera-home[.]com, and <strong>ninjatech[.]io</strong>. Digging deeper, Qurium discovered gmslb[.]net was referenced in dozens of pirated or modded video content streaming apps, such as <strong>CRICFy</strong>, <strong>DooFlix</strong>, <strong>Sprozfy</strong>, <strong>RTS Tv</strong>, <strong>Flixoid</strong>, <strong>CyberFlix</strong>, <strong>Rapid Streamz</strong>, <strong>TvMob</strong> and <strong>HD/OceanStreams</strong>.</p>
+<p>Qurium&#8217;s report notes that most of the domains long used to control the Popa botnet were <a href="https://blog.google/innovation-and-ai/technology/safety-security/google-taking-legal-action-against-the-badbox-20-botnet/" target="_blank" rel="noopener">seized or dismantled in July 2025</a>, after <strong>Google</strong>, <strong>HUMAN Security</strong> and <strong>Trend Micro</strong> teamed up to disrupt <strong>Badbox 2.0</strong>, a botnet that is closely associated with Vo1d. Qurium said that immediately after that disruption, several dozen new domains were registered to serve as controllers for the Popa botnet, but that one of those control domains was not new: <strong>ninjatech[.]io</strong>.</p>
+<p>Ninjatech is a company founded by <strong>Moishi Kramer</strong>, whose <a href="https://www.linkedin.com/in/moishikramer/" target="_blank" rel="noopener">LinkedIn profile</a> says he is vice president of research and development at NetNut. That resume credits Kramer for helping NetNut to build from the &#8220;ground up,&#8221; &#8220;designing the architecture,&#8221; and &#8220;scaling the NetNut&#8221; before the company was acquired by Alarum Technologies. A self-created listing at the job board <strong>F6S</strong> <a href="https://www.f6s.com/company/ninjatech.io" target="_blank" rel="noopener">references Kramer</a> as the sole owner of the Ninjatech domain (a screen capture of it is pictured below).</p>
+<div id="attachment_73842" style="width: 758px" class="wp-caption aligncenter"><img aria-describedby="caption-attachment-73842" decoding="async" loading="lazy" class=" wp-image-73842" src="https://krebsonsecurity.com/wp-content/uploads/2026/06/f6s-kramer-1.png" alt="" width="748" height="589" srcset="https://krebsonsecurity.com/wp-content/uploads/2026/06/f6s-kramer-1.png 1167w, https://krebsonsecurity.com/wp-content/uploads/2026/06/f6s-kramer-1-768x605.png 768w, https://krebsonsecurity.com/wp-content/uploads/2026/06/f6s-kramer-1-782x616.png 782w" sizes="(max-width: 748px) 100vw, 748px" /><p id="caption-attachment-73842" class="wp-caption-text">Image: F6S.com.</p></div>
+<p>Responding via email, Mr. Kramer said Ninjatech ceased operations approximately five years ago, when the company sold a software development kit (SDK) called Popa that was designed to use a small portion of a device&#8217;s bandwidth and to run only after the host application obtained user consent.</p>
+<p>&#8220;That code was sold and licensed to third parties including resellers years ago,&#8221; Kramer said. &#8220;Once software is distributed that way, the original developer has no control over how others later modify, rebrand, or deploy it.&#8221;</p>
+<p>Kramer said neither he nor NetNut builds, operates or maintains the infrastructure being described as Popa, nor does he control the Ninjatech domain.</p>
+<p>&#8220;I didn&#8217;t register the June 2025 domains you mention, and I don&#8217;t know who did,&#8221; he continued. &#8220;I have no control over, or visibility into, that infrastructure. I can only tell you it isn&#8217;t operated by me or by NetNut.&#8221;</p>
+<p>But in <a href="https://synthient.com/blog/popa-from-sourcing-to-distribution" target="_blank" rel="noopener">a separate Popa research report</a> released today, the proxy-tracking company <strong>Synthient</strong> said a recent analysis of the Popa SDK revealed outbound traffic clearly associated with NetNut.</p>
+<p>&#8220;The research team assesses with high confidence that devices running Popa forward traffic from Netnut clients,&#8221; Synthient wrote. &#8220;This proves without a shadow of a doubt that Popa actively continues to be used by NetNut as part of their proxy pool.&#8221;</p>
+<div id="attachment_73854" style="width: 758px" class="wp-caption aligncenter"><img aria-describedby="caption-attachment-73854" decoding="async" loading="lazy" class=" wp-image-73854" src="https://krebsonsecurity.com/wp-content/uploads/2026/06/synthient-flixview.png" alt="" width="748" height="607" /><p id="caption-attachment-73854" class="wp-caption-text">Synthient&#8217;s platform receiving outbound traffic from Popa. Image: Synthient.com.</p></div>
+<p>Alarum Technologies, NetNut&#8217;s Tel Aviv-based parent company, said the reports by Synthient and Qurium contained &#8220;demonstrably inaccurate assertions and flawed deductions rather than verified facts.&#8221; Alarum shared a statement saying they reject the basic characterization of the SDKs and technologies discussed in the reports as a &#8220;botnet.&#8221;</p>
+<p>&#8220;The SDKs at issue are designed to facilitate bandwidth-sharing functionality and do not transform user devices into malware-controlled systems or otherwise compromise the devices on which they operate,&#8221; the statement reads. &#8220;Netnut operates a commercial proxy network and maintains policies, procedures, and technological measures designed to promote lawful and responsible use of its services.&#8221;</p>
+<p>Alarum said NetNut places &#8220;significant emphasis on appropriate notice and consent mechanisms, conducts customer due diligence, monitors for potential misuse, and takes steps intended to detect and mitigate suspicious or unauthorized activity.&#8221;</p>
+<p>&#8220;This method of operation is supported both by internal procedures and policies, including performing KYC checks and additional due diligence of NetNut’s customers, as well as employing various technological measures, designed to assist in identifying and addressing suspected misuse of the network,&#8221; their statement continued.</p>
+<p>However, in a report released on June 8, the proxy tracking service <strong>Spur</strong> asserted that NetNut does not require corporate verification or meaningful &#8220;know your customer&#8221; procedures before allowing customers to purchase proxy access.</p>
+<p>&#8220;An individual can sign up, pay, and route traffic through partner address space, including space belonging to institutions whose users never opted in,&#8221; Spur <a href="https://spur.us/blog/how-proxy-providers-co-opt-entire-networks" target="_blank" rel="noopener">wrote</a>. &#8220;The &#8216;verified corporations only&#8217; claim is simply marketing for bandwidth sellers, not an access control on who actually uses the proxies.&#8221;</p>
+<p>&#8220;Nor is NetNut the only front door,&#8221; Spur continued. &#8220;A number of downstream white labelers and resellers repackage the same ISP proxy pool under their own brands. These outlets typically perform no KYC at all, less scrutiny than NetNut itself, who at the very least might assign an account manager to potential users. Anyone who knows where to look can buy access through a reseller with nothing more than a burner email address and $5 in crypto.&#8221;</p>
+<p>Synthient found that although the most recent builds of Popa (as of three months ago) have added the ability to ask the user for consent before installing proxy components, not all variants or previous versions of Popa contain this functionality.</p>
+<p>&#8220;Of the over 20 genuine Popa publishers analyzed, none of them were observed asking for user consent,&#8221; Sythient wrote.<span id="more-73832"></span></p>
+<h2>THE PREVALENCE OF POPA</h2>
+<p><strong>Chris Formosa</strong> is senior lead information security engineer for <strong>Black Lotus Labs</strong>, a division of the Internet backbone carrier <strong>Lumen Technologies</strong>.</p>
+<p>&#8220;What especially makes Popa dangerous is just how widely used NetNut is for reselling and sharing,&#8221; Formosa said, explaining that many other proxy services simply resell NetNut proxies rather than building out their own far-flung proxy networks. &#8220;So these Popa IPs appear in tons of different services all over the ecosystem, which makes it one of the most problematic and dangerous proxy botnets on the market currently.&#8221;</p>
+<p>Formosa said the Popa botnet averages between 1.5 million to 2.5 million distinct IP addresses each day, relying on between 250 and 300 Internet addresses that are used to direct its activities.</p>
+<p>&#8220;That&#8217;s why Popa is so dangerous,&#8221; Formosa said. &#8220;It may not be the largest botnet we have seen, but it is spread all over the industry, making its power very amplified.&#8221;</p>
+<p>Formosa said while that makes Popa one of the larger botnets out there today, its numbers pale in comparison to those previously boasted by <a href="https://krebsonsecurity.com/tag/ipidea/" target="_blank" rel="noopener">IPIDEA</a>, a China-based proxy provider that until recently operated a daily pool of nearly 10 million devices that they resold as proxies to anyone. In January 2026, Synthient <a href="https://krebsonsecurity.com/2026/01/the-kimwolf-botnet-is-stalking-your-local-network/" target="_blank" rel="noopener">published research</a> showing that multiple new large DDoS botnets had grown rapidly by tunneling through IPIDEA proxies into the local networks of unsuspecting TV box owners and infecting other Android-based devices behind the user&#8217;s firewall.</p>
+<p>IPIDEA is based largely on SDKs used to view pirated streaming content on a vast number of TV box devices, but the service&#8217;s numbers have dwindled since January, when Google and industry partners <a href="https://cloud.google.com/blog/topics/threat-intelligence/disrupting-largest-residential-proxy-network" target="_blank" rel="noopener">took legal action</a> to seize domain names that IPIDEA used to control devices and proxy traffic through them.</p>
+<p><strong>Jérôme Meyer</strong>, a security researcher at <strong>Nokia Deepfield</strong>, said the total population of devices participating in the Popa botnet may be far higher than Lumen&#8217;s estimates. Meyer told KrebsOnSecurity that Nokia is monitoring 26 of at least 359 known relay nodes for the botnet, and estimates that each relay node handles between 35,000 and 60,000 clients simultaneously.</p>
+<p>&#8220;On the relay node subset I am looking at (26 of them), 750,000 unique sources in 24 hours,&#8221; Meyer wrote in response to questions.</p>
+<p>Nokia Deepfield <a href="https://github.com/deepfield/public-research/blob/main/reports/2026-06-18-robovpn-neunative.md" target="_blank" rel="noopener">released its own report today</a> on <strong>RoboVPN</strong>, a VPN app tied to the Vo1d botnet&#8217;s Popa plugin that Qurium attributes to NetNut/Alarum Technologies.</p>
+<h2>THE SYMBIOSIS OF PROXIES AND DATA SCRAPING</h2>
+<p>Experts say many of the world&#8217;s largest proxy providers have updated their public-facing branding to highlight their utility for training AI platforms, implying it is a primary use case for their residential proxies. That&#8217;s because AI services tend to rely on constantly mass-scraping the Internet for new text, images and video content that can be used to train large language models (LLMs).</p>
+<div id="attachment_73850" style="width: 760px" class="wp-caption aligncenter"><a href="https://krebsonsecurity.com/wp-content/uploads/2026/06/proxy-scraping-ai.png" target="_blank" rel="noopener"><img aria-describedby="caption-attachment-73850" decoding="async" loading="lazy" class="wp-image-73850" src="https://krebsonsecurity.com/wp-content/uploads/2026/06/proxy-scraping-ai.png" alt="" width="750" height="375" srcset="https://krebsonsecurity.com/wp-content/uploads/2026/06/proxy-scraping-ai.png 1424w, https://krebsonsecurity.com/wp-content/uploads/2026/06/proxy-scraping-ai-768x384.png 768w, https://krebsonsecurity.com/wp-content/uploads/2026/06/proxy-scraping-ai-782x391.png 782w" sizes="(max-width: 750px) 100vw, 750px" /></a><p id="caption-attachment-73850" class="wp-caption-text">NetNut and other proxy services have recast themselves as critical infrastructure for the AI scraping economy. Image: Synthient.com.</p></div>
+<p>&#8220;AI companies depend on web-scraped content: for pre-training, for retrieval, for agent grounding, for search,&#8221; reads <a href="https://blog.includesecurity.com/2026/06/the-smart-tv-in-your-livingroom-is-a-node-in-the-aiscraping-economy/" target="_blank" rel="noopener">a report</a> this month from <strong>Include Security</strong> that examines the prevalence of proxy SDKs in smart TV apps. &#8220;But the modern web isn’t scrapeable from a datacenter. Cloudflare, DataDome, HUMAN, among others throttle or block requests from known cloud IPs. The workaround is residential proxies. A scraping job routed through a Comcast or T-Mobile subscriber’s connection arrives at the target site from an IP that belongs to a paying residential customer.&#8221;</p>
+<p>This non-stop content scraping has spawned <a href="https://copyrightalliance.org/ai-copyright-lawsuit-developments-2025/" target="_blank" rel="noopener">more than 70 copyright infringement lawsuits</a> against major tech companies that have acknowledged large-scale data scraping as a major source of the &#8220;brains&#8221; behind their commercial AI offerings. Ironically, much of that scraping is being aided by proxy services that are intimately tied to unofficial Android TV boxes and associated SDKs whose stated purpose is streaming pirated content.</p>
+<p>The scraping activity has become so aggressive that it often overwhelms the targeted websites, preventing them from being reachable by legitimate visitors. In many reported cases, nonprofit organizations, libraries and universities have complained of constantly battling to keep their services online in the face of relentless data-scraping firms hiding behind residential proxy services.</p>
+<p>A survey conducted last year by the <strong>Confederation of Open Access Repositories</strong> (COAR) <a href="https://www.cni.org/topics/ci/artificial-intelligence-bots-and-repositories-results-and-next-steps-from-coar-survey" target="_blank" rel="noopener">found</a> while some content scraping bots are rather innocuous, &#8220;others are sufficiently aggressive that they are increasingly causing service disruptions in repositories and other scholarly communications infrastructures.&#8221; More than 90 percent of survey respondents indicated their repository is encountering aggressive bots, usually more than once a week, and often leading to slow downs and service outages.</p>
+<p>&#8220;Automated web scraping is nothing new, and has been the key technology underlying search engines such as Google for over 30 years,&#8221; <a href="https://blog.doaj.org/2026/01/26/open-access-vs-open-excess-doaj-and-ai-scraper-bots/" target="_blank" rel="noopener">wrote</a> <strong>Brendan O&#8217;Connell</strong>, platform manager at the <strong>Directory of Open Access Journals</strong> (DOAJ), a free, community-curated index of peer-reviewed academic journals. &#8220;However, the current investor-fueled AI startup craze means there are now thousands of well-funded companies developing and deploying their own scraping tools to train AI models, alongside existing major players like OpenAI and Google.&#8221;</p>
+<h2>DON&#8217;T TOUCH THAT DIAL!</h2>
+<p>Across the United States, local communities are pushing back against the proliferation of new data centers aimed primarily at improving the capabilities of AI. But security experts say the general public remains largely unaware that using one of these unsanctioned Android TV boxes means their &#8220;smart TV&#8221; is almost certainly using a significant amount of bandwidth each month to help train modern AI models.</p>
+<p>Even households without these sketchy TV boxes can still have their smart TVs turned into residential proxy nodes, just by downloading one of thousands of apps made available on <strong>Samsung</strong> and <strong>LG</strong> smart TVs. Spur said it recently scraped the LG and Samsung app stores and found that each had approximately 3,000 apps available for download. Many of these apps are simple games or utilities that state in the fine print that the user&#8217;s Internet connection will be used to download data and that they can opt out at any time.</p>
+<p>Spur said it found that <em>more than 42 percent of apps available for download via the <strong>webOS</strong> operating system on <strong>LG</strong> smart TVs include SDKs that turn one&#8217;s television into an always-on residential proxy node. </em>More than a quarter of the apps made for Samsung&#8217;s <strong>Tizen</strong> operating system had similar residential proxy components, Spur found.</p>
+<div id="attachment_73849" style="width: 758px" class="wp-caption aligncenter"><a href="https://krebsonsecurity.com/wp-content/uploads/2026/06/spur-tvproxy.png" target="_blank" rel="noopener"><img aria-describedby="caption-attachment-73849" decoding="async" loading="lazy" class="wp-image-73849" src="https://krebsonsecurity.com/wp-content/uploads/2026/06/spur-tvproxy.png" alt="" width="748" height="304" srcset="https://krebsonsecurity.com/wp-content/uploads/2026/06/spur-tvproxy.png 1272w, https://krebsonsecurity.com/wp-content/uploads/2026/06/spur-tvproxy-768x313.png 768w, https://krebsonsecurity.com/wp-content/uploads/2026/06/spur-tvproxy-782x318.png 782w" sizes="(max-width: 748px) 100vw, 748px" /></a><p id="caption-attachment-73849" class="wp-caption-text">Image: Spur.us.</p></div>
+<p>Experts say it&#8217;s questionable whether TV apps with proxy SDKs can obtain meaningful consent from users for installing an always-on proxy connection, particularly when anyone in a household &#8212; including children &#8212; can effectively opt the family TV into a residential proxy network just by installing a simple game or app.</p>
+<p>&#8220;Privacy-policy disclosure is the wrong control surface for a TV,&#8221; Include Security wrote. &#8220;It is hard to scroll through a legal document navigated by arrow keys on a remote, and the in-app consent dialog doesn’t convey that a paying customer is about to route their scraping traffic through the user’s home internet.&#8221;</p>
+<p>Spur&#8217;s head of research <strong>Sean Simmons</strong> told KrebsOnSecurity that most people do not have a working mental model for what it means to sell access to their residential IP address, no matter what device they are using.</p>
+<p>&#8220;And on a TV, the gap is even wider,&#8221; Simmons said. &#8220;A one-time prompt navigated with a remote can disappear into the setup flow, while the app keeps monetizing the connection long after anyone remembers what they accepted.&#8221;</p>
+<p>Simmons said LG and Samsung should follow the lead of other TV platforms that have already drawn a line against residential proxy providers, pointing to policies by <strong>Amazon</strong> that prohibit apps facilitating proxy services for third parties. Likewise the TV streaming device maker <strong>Roku</strong> reportedly now bars developers from using proxy SDKs and has removed apps that bundled them.</p>
+<div id="attachment_73855" style="width: 759px" class="wp-caption aligncenter"><img aria-describedby="caption-attachment-73855" decoding="async" loading="lazy" class=" wp-image-73855" src="https://krebsonsecurity.com/wp-content/uploads/2026/06/tvstreamz.png" alt="" width="749" height="448" srcset="https://krebsonsecurity.com/wp-content/uploads/2026/06/tvstreamz.png 993w, https://krebsonsecurity.com/wp-content/uploads/2026/06/tvstreamz-768x459.png 768w, https://krebsonsecurity.com/wp-content/uploads/2026/06/tvstreamz-782x468.png 782w" sizes="(max-width: 749px) 100vw, 749px" /><p id="caption-attachment-73855" class="wp-caption-text">Piracy related apps pushing proxy SDKs onto unconsenting users. Image: Synthient.</p></div>
+<p>Apps that turn one&#8217;s device into a residential proxy node are not limited to smart TVs and no-name streaming boxes, of course. As noted by the security firm <strong>Infoblox</strong>, mobile app developers can embed SDKs provided by the residential proxy networks into their products to monetize their software, allowing them to receive a small amount of money on each installation.</p>
+<p>The result, Infoblox said, is that devices are frequently enrolled without the owner’s knowledge, typically through free applications such as VPNs, streaming apps, screensavers and &#8220;productivity&#8221; apps such as PDF viewers and break reminders.</p>
+<p>All too often, these proxy services are beaconing out from employee devices brought into the workplace, Infoblox found. In a blog post earlier this month, Infoblox said it discovered that fully 65% of its customer base was querying one or more residential proxy related domains.</p>
+<p>&#8220;We saw steady growth in these queries in 2025, with a 25% increase over the year to over 500 billion per month,&#8221; Infoblox <a href="https://www.infoblox.com/blog/threat-intelligence/residential-proxies-in-the-wild/" target="_blank" rel="noopener">wrote</a>. &#8220;Over 90% of our pharmaceutical and food &amp; beverage customers have queried residential proxy indicators. Perhaps even more concerning is that over 60% of government and banking customers have as well.&#8221;</p>
+<p>Infoblox researchers <strong>Nick Sundvall</strong> and <strong>David Brunsdon</strong> warned that with residential proxies in the corporate environment, external access is granted to an organization&#8217;s IP space.</p>
+<p>&#8220;If threat actors were to abuse the residential proxy to attack a third party, the third party’s incident response would, correctly, identify your residential proxy as the source,&#8221; they wrote. &#8220;Untangling that, by proving that you were the conduit and not the threat actor, costs time, creates legal exposure, and can damage your reputation. The stunning prevalence of these services within customer environments warrants attention from both network defenders and policy makers who should consider how the risks posed by residential proxies could be impacting their security posture.&#8221;</p>
+]]></content:encoded>
+
+ <wfw:commentRss>https://krebsonsecurity.com/2026/06/popa-botnet-linked-to-publicly-traded-israeli-firm/feed/</wfw:commentRss>
+ <slash:comments>16</slash:comments>
+
+
+ </item>
+ </channel>
+</rss>
+
+<!--
+Performance optimized by W3 Total Cache. Learn more: https://www.boldgrid.com/w3-total-cache/
+
+Object Caching 225/225 objects using memcached
+Page Caching using memcached
+Database Caching using memcached
+
+Served from: krebsonsecurity.com @ 2026-07-05 12:52:58 by W3 Total Cache
+-->
diff --git a/PROJECTS/intermediate/security-news-scraper/testdata/feeds/securityweek.xml b/PROJECTS/intermediate/security-news-scraper/testdata/feeds/securityweek.xml
new file mode 100644
index 00000000..d6580353
--- /dev/null
+++ b/PROJECTS/intermediate/security-news-scraper/testdata/feeds/securityweek.xml
@@ -0,0 +1,90 @@
+<?xml version="1.0" encoding="UTF-8"?><rss version="2.0"
+ xmlns:content="http://purl.org/rss/1.0/modules/content/"
+ xmlns:wfw="http://wellformedweb.org/CommentAPI/"
+ xmlns:dc="http://purl.org/dc/elements/1.1/"
+ xmlns:atom="http://www.w3.org/2005/Atom"
+ xmlns:sy="http://purl.org/rss/1.0/modules/syndication/"
+ xmlns:slash="http://purl.org/rss/1.0/modules/slash/"
+ >
+
+<channel>
+ <title>SecurityWeek</title>
+ <atom:link href="https://www.securityweek.com/feed/" rel="self" type="application/rss+xml" />
+ <link>https://www.securityweek.com/</link>
+ <description>Cybersecurity News, Insights &#38; Analysis</description>
+ <lastBuildDate>Fri, 03 Jul 2026 15:10:44 +0000</lastBuildDate>
+ <language>en-US</language>
+ <sy:updatePeriod>
+ hourly </sy:updatePeriod>
+ <sy:updateFrequency>
+ 1 </sy:updateFrequency>
+ <generator>https://wordpress.org/?v=7.0</generator>
+
+<image>
+ <url>https://www.securityweek.com/wp-content/uploads/2023/01/cropped-SecurityWeek-Icon-32x32.jpeg</url>
+ <title>SecurityWeek</title>
+ <link>https://www.securityweek.com/</link>
+ <width>32</width>
+ <height>32</height>
+</image>
+ <item>
+ <title>In Other News: Canadian Hacker Jailed, Open Source Zero-Days, Two Sentenced for ATM Jackpotting</title>
+ <link>https://www.securityweek.com/in-other-news-canadian-hacker-jailed-open-source-zero-days-two-sentenced-for-atm-jackpotting/</link>
+
+ <dc:creator><![CDATA[SecurityWeek News]]></dc:creator>
+ <pubDate>Fri, 03 Jul 2026 15:10:42 +0000</pubDate>
+ <category><![CDATA[Cybercrime]]></category>
+ <category><![CDATA[Management & Strategy]]></category>
+ <category><![CDATA[cybersecurity]]></category>
+ <category><![CDATA[News]]></category>
+ <guid isPermaLink="false">https://www.securityweek.com/?p=47625</guid>
+
+ <description><![CDATA[<p>Noteworthy stories that might have slipped under the radar: Anonymous-linked Canadian hacker jailed, researcher drops zero-days in open source projects, Venezuelans sentenced in the US over ATM jackpotting. </p>
+<p>The post <a href="https://www.securityweek.com/in-other-news-canadian-hacker-jailed-open-source-zero-days-two-sentenced-for-atm-jackpotting/">In Other News: Canadian Hacker Jailed, Open Source Zero-Days, Two Sentenced for ATM Jackpotting</a> appeared first on <a href="https://www.securityweek.com">SecurityWeek</a>.</p>
+]]></description>
+
+
+
+ </item>
+<item>
+ <title>Agentic AI Used to Conduct Ransomware Attack via Langflow</title>
+ <link>https://www.securityweek.com/agentic-ai-used-to-conduct-ransomware-attack-via-langflow/</link>
+
+ <dc:creator><![CDATA[Ionut Arghire]]></dc:creator>
+ <pubDate>Fri, 03 Jul 2026 11:00:00 +0000</pubDate>
+ <category><![CDATA[Artificial Intelligence]]></category>
+ <category><![CDATA[Ransomware]]></category>
+ <category><![CDATA[agentic AI]]></category>
+ <category><![CDATA[AI]]></category>
+ <category><![CDATA[Featured]]></category>
+ <guid isPermaLink="false">https://www.securityweek.com/?p=47618</guid>
+
+ <description><![CDATA[<p>Attack demonstrates how LLM agents can combine known exploitation techniques with real-time reasoning to automate complex, multi-stage intrusions.</p>
+<p>The post <a href="https://www.securityweek.com/agentic-ai-used-to-conduct-ransomware-attack-via-langflow/">Agentic AI Used to Conduct Ransomware Attack via Langflow</a> appeared first on <a href="https://www.securityweek.com">SecurityWeek</a>.</p>
+]]></description>
+
+
+
+ </item>
+<item>
+ <title>Medtronic Data Breach Impacts 3.8 Million People</title>
+ <link>https://www.securityweek.com/medtronic-data-breach-impacts-3-8-million-people/</link>
+
+ <dc:creator><![CDATA[Ionut Arghire]]></dc:creator>
+ <pubDate>Fri, 03 Jul 2026 10:00:00 +0000</pubDate>
+ <category><![CDATA[Cybercrime]]></category>
+ <category><![CDATA[Data Breaches]]></category>
+ <category><![CDATA[data breach]]></category>
+ <category><![CDATA[Medtronic]]></category>
+ <category><![CDATA[ShinyHunters]]></category>
+ <guid isPermaLink="false">https://www.securityweek.com/?p=47616</guid>
+
+ <description><![CDATA[<p>In April, ShinyHunters accessed the company’s corporate IT systems and stole patients’ personal and medical information.</p>
+<p>The post <a href="https://www.securityweek.com/medtronic-data-breach-impacts-3-8-million-people/">Medtronic Data Breach Impacts 3.8 Million People</a> appeared first on <a href="https://www.securityweek.com">SecurityWeek</a>.</p>
+]]></description>
+
+
+
+ </item>
+ </channel>
+</rss>
diff --git a/PROJECTS/intermediate/security-news-scraper/testdata/feeds/thehackernews.xml b/PROJECTS/intermediate/security-news-scraper/testdata/feeds/thehackernews.xml
new file mode 100644
index 00000000..771c4871
--- /dev/null
+++ b/PROJECTS/intermediate/security-news-scraper/testdata/feeds/thehackernews.xml
@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?><rss xmlns:atom="http://www.w3.org/2005/Atom" xmlns:content="http://purl.org/rss/1.0/modules/content/" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd" xmlns:media="http://search.yahoo.com/mrss/" xmlns:slash="http://purl.org/rss/1.0/modules/slash/" xmlns:sy="http://purl.org/rss/1.0/modules/syndication/" xmlns:wfw="http://wellformedweb.org/CommentAPI/" version="2.0"><channel><title>The Hacker News</title><link>https://thehackernews.com</link><description>Most trusted, widely-read independent cybersecurity news source for everyone; supported by hackers and IT professionals — Send TIPs to [email protected]</description><language>en-us</language><lastBuildDate>Sun, 05 Jul 2026 21:16:08 +0530</lastBuildDate><sy:updatePeriod>hourly</sy:updatePeriod><sy:updateFrequency>1</sy:updateFrequency><atom:link href="https://feeds.feedburner.com/TheHackersNews" rel="self" type="application/rss+xml"/><item><title>U.S. Government Entity Paid Kairos $1 Million in Data-Theft Extortion Case</title><description><![CDATA[A U.S. government entity paid about $1 million to keep stolen files from being leaked, according to a new&nbsp;case study by Rakesh Krishnan for Ransom-ISAC, built on a leaked negotiation chat and the blockchain trail the payment left.
+
+The odd part: the group that took the money calls itself Kairos, but it may not be a ransomware gang at all. Krishnan found no sign that it ever locked a single]]></description><link>https://thehackernews.com/2026/07/us-government-entity-paid-kairos-group.html</link><guid isPermaLink="false">https://thehackernews.com/2026/07/us-government-entity-paid-kairos-group.html</guid><pubDate>Sat, 04 Jul 2026 18:17:53 +0530</pubDate><author>[email protected] (The Hacker News)</author><enclosure length="12216320" type="image/jpeg" url="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEiEfi15-eigOUF4SV157a0LsEW4uNvhRyEIhWWYVNo8tn0tfMjuJ0Cr6zYUDeVh6VfzA-7MpWgIVX8sJgieRMD6e0KSBmB2zNmTU1ko1KPMbkBd-JHMBluVBUOjV06SebkYq4AKFssmEQ6DdGw4yy4X3qesVYJcwwgZSsQFUu4Wfy3un9alrX2Qy8IjacE/s1600/county-hacked.jpg"/></item>
+<item><title>North Korean Hackers Publish 108 Malicious Packages and Extensions in PolinRider Campaign</title><description><![CDATA[The North Korean threat actors linked to the Contagious Interview campaign have been observed publishing 108 unique packages and web browser extensions spanning npm, Packagist, Go, and Google Chrome as part of an ongoing activity referred to as PolinRider.
+
+"The campaign remains active, and new malicious packages are likely to continue appearing as threat actors compromise maintainer accounts,]]></description><link>https://thehackernews.com/2026/07/north-korean-hackers-publish-108.html</link><guid isPermaLink="false">https://thehackernews.com/2026/07/north-korean-hackers-publish-108.html</guid><pubDate>Sat, 04 Jul 2026 16:47:24 +0530</pubDate><author>[email protected] (The Hacker News)</author><enclosure length="12216320" type="image/jpeg" url="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgwdeBqtwl7nqoAp8TPhZmmr3mUTCcfxluYa7QukD2sI1AjCWoOko2YtQUrCtOLH-tJlYi2lXUA5E3RF51L26gZGyR7sT0GXu73OMB94HhINz5kajaR8-txb-tYNj2Hsm62zVwTaw7Rew6Wazf8eQgo_boWq7DjqbN1jjp5OmwSZ2yG7Y9e04hFVaroUSu6/s1600/go-code.jpg"/></item>
+<item><title>Unpatched Flaws Disclosed in Filesystem Bundled Into Millions of Embedded Devices</title><description><![CDATA[Security firm runZero has&nbsp;disclosed seven vulnerabilities&nbsp;in&nbsp;FatFs, a small filesystem library that lets a device read and write the FAT and exFAT formats used on USB drives and SD cards.
+
+The flaws matter because FatFs is nearly everywhere. It ships inside the firmware that runs security cameras, drones, industrial controllers, hardware crypto wallets, and other devices built on]]></description><link>https://thehackernews.com/2026/07/unpatched-flaws-disclosed-in-filesystem.html</link><guid isPermaLink="false">https://thehackernews.com/2026/07/unpatched-flaws-disclosed-in-filesystem.html</guid><pubDate>Sat, 04 Jul 2026 01:49:31 +0530</pubDate><author>[email protected] (The Hacker News)</author><enclosure length="12216320" type="image/jpeg" url="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEiJGo3ti2B3O-v3XpxdiFLMhvMVB_Ee5mmTlis-Qls6K8auWLQtPb4DSLL4snqvuZSxEvYXbGVBiUvRH1wBiw53ovRi-KTpdb0WqPwkZMkA_eHTSkTQVPkexu02-I2u7wc7PKESGKFZxlKRjR1SAkgOK50kIfpqZ9wcEjZCYBWIh6u_L82tXa8WfgUIhmA/s1600/fatfs.jpg"/></item></channel></rss>
diff --git a/PROJECTS/intermediate/security-news-scraper/testdata/feeds/theregister.xml b/PROJECTS/intermediate/security-news-scraper/testdata/feeds/theregister.xml
new file mode 100644
index 00000000..e76ea427
--- /dev/null
+++ b/PROJECTS/intermediate/security-news-scraper/testdata/feeds/theregister.xml
@@ -0,0 +1,48 @@
+<?xml version="1.0" encoding="utf-8"?>
+<rss version="2.0" xmlns:media="http://search.yahoo.com/mrss/" xmlns:content="http://purl.org/rss/1.0/modules/content/" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:lab="https://labradorcms.com/ns/rss">
+<channel>
+ <title>www.theregister.com - Articles</title>
+ <link>https://www.theregister.com</link>
+ <description>Articles from www.theregister.com</description>
+
+ <item>
+ <guid isPermaLink="true">https://www.theregister.com/a/5266161</guid>
+ <link>https://www.theregister.com/security/2026/07/05/mfa-optional-banks-leave-safe-doors-and-accounts-wide-open-for-thieves-to-pillage/5266161</link>
+ <pubDate>Sun, 05 Jul 2026 17:01:00 +0200</pubDate>
+ <title>MFA-optional banks leave safe doors (and accounts) wide open for thieves to pillage</title>
+ <description><![CDATA[ Financial institutions are putting their clients at risk in the name of convenience. ]]></description>
+ <category>security</category>
+ <lab:kicker><![CDATA[ Security ]]></lab:kicker>
+ <dc:modified>Fri, 03 Jul 2026 00:01:12 +0000</dc:modified>
+ <content:encoded><![CDATA[ OPINION I write a weekly column called PWNED, about how poor security practices can lead to serious damage. Usually, there’s something funny in the malfeasance, like a CEO who kept every employee’s password in an Excel file on his desktop. However, I wasn’t laughing back in May when professional thieves invaded my 84-year-old mother’s entire financial life and managed to make off with $30,000 from her bank accounts alone. And they wouldn’t have gotten in if her financial institutions required multi-factor authentication (aka MFA or 2FA), a step too many institutions won’t take. One day in May, Mom got a call from the institution that runs her retirement savings account, who had identified a suspicious transaction and asked her if it was legit. She said no and they immediately protected her account. Then she checked her bank account at a different institution to see if it was compromised and found thousands of dollars transferred out of her checking and savings accounts. The thieves knew exactly how much they could withdraw each day, and used both withdrawals and transfers to a strange account. But the financial institution hadn't flagged the fraudulent activity. The thieves were so slick that they broke into her Gmail account and created spam filters to filter any mail from her bank or retirement savings provider to the trash so she wouldn’t get alerts about the transfers or about the fake accounts they made in her name. She spent hours on the phone reporting the theft to an unhelpful and incredulous fraud department who asked “Are you sure a relative didn’t do this?” We don’t know for certain how the crims got into my mom’s accounts, but we know she used the same or similar passwords on all of her accounts, and at least one of her accounts was part of a data breach a few years ago, so that info was probably available somewhere online. The miscreants then could have used this info to get into her retirement account, her bank, and her Gmail. None of this would have been possible if she had MFA enabled on those accounts, but neither Google nor her financial institutions require it. “Many consumers assume every bank requires 2FA, but that's not the reality,” said Gregory Shein, CEO of Nomadic Soft, a SaaS company that serves fintech clients. “Some financial institutions still treat it as an optional feature because they're balancing security against friction. Every extra login step can reduce conversions, increase support tickets, and frustrate less technical customers.” Indeed, while some banks such as PNC require MFA, others such as Bank of America, Chase, Capital One, and Citibank leave it as optional. Google’s accounts are also MFA-optional. Fortunately, after they spent hours telling my mom that someone in her family could have done the deed, and repeatedly putting her on hold, then forcing her to navigate a labyrinthine phone tree, the bank eventually agreed to investigate. A few weeks later, they restored the stolen funds. A not entirely happy ending My mother was lucky, because if money is stolen from your bank account, there is no guarantee that you will get it back, at least in the US. According to the Consumer Financial Protection Bureau, you have 60 days from the date of a bank statement to dispute any transactions. The bank also has 45 days to investigate, unless your bank account was just opened in the last 30 days or the fraudulent transactions took place outside the US. But the bank could very well decide that those fraudulent transactions look legitimate and refuse to reimburse you. If the bank doesn’t agree to reimburse you, your next step is to get a lawyer and attempt to sue. A quick search revealed dozens of lawyers in my area who specialize in dealing with this problem. It would be easy to blame my mom for being robbed. Using the same password in multiple places left her wide open for exploitation. However, her bank’s lack of a required second authentication factor also contributed. The bank doesn’t let you transact without a password, and it doesn’t issue you an ATM card without a PIN, because it knows that there has to be a required minimum level of security. Banks and other financial institutions know better. Google knows better. But they’re all putting convenience ahead of security when it’s your money that’s on the line. “Different segments of the population adopt technology faster or slower. If I’m a bank, I have to consider that very closely because I don’t want to lose any banking relationships.” Andrew Shikiar, CEO of the FIDO Alliance, an industry association that advocates for stronger login security, told me in an interview. “So I think there’s some concerns around friction that have held some banks and other service providers back from really pushing this more aggressively.” How effective is MFA? According to a 2019 article from Microsoft, MFA prevents 99.9 percent of attacks on your accounts. However, other experts say this number is exaggerated, as there are many ways to get past MFA if you’re a criminal, including social engineering and interception. One of the most common types of MFA, issuing a one-time passcode via an SMS message or an email, is inherently flawed. A determined thief can use social engineering to get a SIM card with your phone number on it, then get to your texts. And if your email itself isn’t perfectly secure and it is receiving an OTP, they can get to that too. Phishers can also trick you into giving up your OTPs by creating a fake website that looks like your bank’s login page. The right way to do MFA today is with a passkey. Passkeys are cryptographic key pairs where there’s a private key on the user’s device and a public key on the server. To access the key on the device, the user must either enter a PIN, touch a physical security key like a Yubikey, or enter a biometric login such as their face or fingerprint. Passkeys cannot be phished or intercepted, which is why they are known as “phishing-resistant MFA.” Unfortunately, a lot of banks are sticking with their OTPs. For example, when I went to set up MFA for a family member’s account with US bank Chase, using its website. Chase offered the chance to receive an OTP via email, SMS, or a phone call. The bank is rolling out passkeys, according to the FIDO Alliance. So are Wells Fargo, US Bank, and Bank of America. Some banks may be using better MFA only within their mobile apps. Chase’s app, for example, asks users to use a fingerprint or facial recognition at login, even though the website does not. However, if a thief wants to log in at Chase's website, there will be no biometric challenge. And if a user doesn’t have MFA enabled at all, it’s even easier for thieves to get in. “OTP is just another password. So it’s a shorter-lived one, but it really is just another password,” Shikiar said. “And there’s also usability issues. You’re juggling between your mobile and your desktop. It’s insecure, inefficient, and a really inadequate user experience.” What banks don’t seem to understand is that you’re only as secure as your weakest entry point. If security controls only exist on mobile apps, it doesn’t help with web-based attacks. If a level of security is optional, the majority of people won’t enable it. Thieves will take the path of least resistance, so service operators need to lock down all entry paths equally by default. Unfortunately, an approach that favors convenience over security will lead to a lot more people losing their money. And, ultimately, banks will lose money when they have to reimburse people for those fraudulent transactions. “I don't expect banks to be mandating passkeys and only passkeys for some time, but the more they push them, the more comfort there is,” Shikiar told us. “The sooner we’ll get to that point where it becomes a de facto default and then becomes really something that's either required or essentially required.” That time should be now. ® ]]></content:encoded>
+ <enclosure url="https://image.theregister.com/?imageId=224126&amp;width=800" type="image/jpeg" />
+ <media:thumbnail url="https://image.theregister.com/?imageId=224126&amp;width=800" />
+ </item>
+<item>
+ <guid isPermaLink="true">https://www.theregister.com/a/5266056</guid>
+ <link>https://www.theregister.com/security/2026/07/04/confidential-computings-core-trust-mechanism-is-broken-the-fix-may-not-exist/5266056</link>
+ <pubDate>Sat, 04 Jul 2026 12:03:00 +0200</pubDate>
+ <title>Confidential computing's core trust mechanism is broken. The fix may not exist</title>
+ <description><![CDATA[ Attested TLS: the handshake that can't prove who's on the other end ]]></description>
+ <category>security</category>
+ <lab:kicker><![CDATA[ SECURITY ]]></lab:kicker>
+ <dc:modified>Fri, 03 Jul 2026 07:47:18 +0000</dc:modified>
+ <content:encoded><![CDATA[ Vendors are trying to position "confidential computing" as the technical backbone of Europe's sovereign cloud ambitions. But new research shows that a security protocol used to prove cryptographic trust in the system may have a fundamental architectural flaw. Confidential computing rests on a mechanism called remote attestation, in which a server cryptographically proves to a client that it is running inside a genuine, unmodified Trusted Execution Environment (TEE) before any sensitive data changes hands. Intel's product pages promise TDX will "add safeguards to data sovereignty and governance." Google Cloud describes its confidential computing infrastructure as offering "full, auditable control over access to customer data." In May, The Register reported that the chip beneath the chip, the management engines running below the operating system on Intel and AMD silicon, falls outside what European sovereignty frameworks like SecNumCloud actually assess. That left an open question about the layer above the silicon: the protocol meant to prove the chip itself can be trusted. New, independently verified research answers it, and the answer is not reassuring. A protocol that promises more than it proves Muhammad Usama Sardar, a researcher at TU Dresden, has spent the past two years formally verifying whether that protocol, known as attested TLS, actually does what it claims. Using ProVerif, a tool for the symbolic security analysis of protocols, he and his co-authors discovered that it largely does not. Their recent paper, Identity Crisis in Confidential Computing, published with co-authors Mariam Moustafa and Tuomas Aura and presented at the AsiaCCS 2026 conference, found diversion attacks against two state-of-the-art attested TLS protocols. A connection intended for one server can be silently redirected to a different, compromised machine running identical software, anywhere in the world, without the client ever knowing. The intended server has done nothing wrong. The attacker simply exploits the fact that the protocol checks the software's integrity, not its location. The most recent paper, Intra-handshake.fail, published with co-authors Viacheslav Dubeyko and Jean-Marie Jacquet and accepted for ESORICS 2026, goes further. It examines what the industry calls intra-handshake attestation, where evidence is generated during the TLS handshake itself, and tests seven different ways of cryptographically binding that evidence to the underlying connection. None of them prevent relay attacks, in which a client verifies the evidence of a genuine, trustworthy AI agent or server but ends up encrypting its traffic to an entirely different, malicious one. The starting assumption in all of this is that the hardware itself can be trusted. "In confidential computing, you have to trust the hardware manufacturer anyway," Sardar told The Register. "There is absolutely no way around this." With that root of trust accepted, he argues, the protocol layer was supposed to provide everything else. His research shows it provides far less than assumed. Three levels of trust The researchers formalise the problem as three increasingly strict levels of cryptographic binding between the attestation evidence and the actual TLS connection it is meant to vouch for. The weakest, level one, ties evidence only to the very first key exchange in the handshake, the Diffie-Hellman step, where client and server agree on a shared secret before either side has proven who they are. Level two ties it to the client's handshake traffic key, covering everything up to the server's identity confirmation. Level three, the strongest and the one that matters most in practice, ties evidence to the application traffic key itself, the key actually used to encrypt the sensitive data a client sends once the connection is live. Sardar's extensive analysis in ProVerif focused on intra-handshake attestation; post-handshake attestation fell outside its scope. Three of the seven binding mechanisms examined achieve level one. The rest fail even that baseline. His team's own proposed mitigation, a cryptographic binder built from the TLS handshake secret combined with the server's public key, formally achieves level two. Level three, the paper concludes, "may not be possible" within intra-handshake attestation as currently architected, without breaking properties of TLS 1.3 that the protocol was never designed to give up. In plain terms: the best fix available today proves a client is talking to the right machine at the start of a handshake. It cannot prove that the data sent minutes later is still going to that same machine. Production systems, not laboratory proofs of concept The vulnerability is not confined to academic models. Sardar's team formally analysed four real-world implementations of intra-handshake attestation: Meta's Private Processing system for WhatsApp, Edgeless Systems' Contrast, the open-source Cocos AI platform, and a proof-of-concept maintained by the Confidential Computing Consortium's (CCC) Attestation Special Interest Group. The first three of the four are running in production today. The attacks apply to every version of Cocos AI between 0.4.0 and 0.8.2. The class of flaw itself is not new. Sardar's team notes the attacks are subtle enough to have gone undiscovered for years before formal analysis caught them. The responsible disclosure resulted in CVE-2026-33697, rated 7.5 on the Common Vulnerability Scoring System, high severity. For comparison, the researchers note in their paper that BadRAM, the 2024 memory aliasing attack against AMD's SEV-SNP that made headlines in its own right, scored 5.3. The CCC Attestation SIG's repository lists CVE-2026-33697 as the highest-scoring vulnerability among a cluster of recent confidential computing flaws, ahead of Fabricked (5.9), BreakFAST (5.9) and Staleus (4.0). The working group and the IETF's TLS working group have both formally acknowledged the relay attacks. "As implemented today, attested TLS is not mature yet," Sardar told The Register. "We are investigating further, and we are confident there are more issues yet to be discovered." What makes the finding more pointed is who missed it first. Meta commissioned an extensive security review of its WhatsApp implementation from Trail of Bits, a well-regarded security firm, before Sardar's team examined it. That review did not detect the relay attack. It is methodology, not incompetence, that explains the gap. The ESORICS paper records that Sardar's team contacted Trail of Bits directly, who confirmed no formal methods were used in their review process. Formal verification tools like ProVerif check a protocol exhaustively against every scenario a defined threat model allows. A manual audit, however thorough, samples. A subtle flaw in how evidence is bound to a connection can slip past a sampled review and still be provably broken under exhaustive formal analysis. The Attestation Special Interest Group of the CCC, which governs the adopted proof-of-concept project Sardar tested, found its own system vulnerable to the same relay attacks. A repository nobody would create The vulnerability itself had already been through a lengthy, orderly disclosure process. Sardar's team flagged it to Cocos AI in October 2025, the vendor acknowledged it two months later, and the CVE was published in March 2026. What happened next was different. On 14 June, Sardar wrote to the chairs of the CCC's Attestation Special Interest Group requesting a new public GitHub repository, named relay-attacks-in-intra-handshake, so his formal analysis artefacts for the relay attacks could be released under an Apache 2.0 licence, for use by researchers and the standardization community. He referenced an existing, adopted project under the same group's governance, the kind of administrative step that, on paper, should take minutes. Three days later, on 17 June, he sent a reminder. The following day, a second, noting the artefact link was needed for the paper's final version. On 24 June, ten days after the original request, he wrote again, this time without the diplomatic padding: "I do not see a good reason for such a delay, since the requested repo is part of an adopted project and creation of a new repo is not such a time-consuming task." The new repository still did not exist. The CCC's Attestation Special Interest Group is made up of representatives from the hardware and cloud vendors whose products the research concerns. That fact requires no embellishment. A working group populated by the companies whose attestation implementations were just shown to be vulnerable to relay attacks did not act, for over a week and across three written reminders, on a request to publish proof of that vulnerability. Since no repository had been created before the paper's final version went to the publisher, Sardar published the artefacts anyway, but inside an existing CCC-affiliated repository rather than the dedicated one he had asked for. He told The Register the repository had originally been built for an unrelated project: "Since the monopoly [of vendor-dominated working groups over this infrastructure] continues, we have released the artifacts to inform the community and for researchers to analyse it independently." The CVE stands regardless, credited and public. The delay changes nothing about the underlying mathematics. BSI reaches the same verdict None of this requires taking Sardar's interpretation on faith. A world away from the IETF mailing lists, Germany's Federal Office for Information Security (BSI) arrived at a closely related conclusion through its own, entirely separate channel. Carina Hilt, deputy press spokesperson at BSI, was asked directly about confidential computing's role in digital sovereignty. She told The Register the technology functions as "a defense-in-depth component," strengthening tenant isolation and protecting confidentiality and integrity, but not availability. Crucially, she added that "dependencies on other services, such as identity and key management etc., are also not mitigated by CC." That is, in other words, an institutional echo of exactly the gap Sardar's protocol analysis exposes: confidential computing's guarantees stop well short of guaranteeing who actually controls the keys and the identity infrastructure a deployment depends on. Pressed further on vendor marketing claims, BSI did not soften its position. "The vendors' positioning on CC might give too much weight to its technical capabilities," the spokesperson told The Register. "CC alone cannot satisfy the requirements for digital sovereignty." What the chipmakers say Mikael Moreau, Intel's France Communication Manager, was asked specifically about the attestation infrastructure underpinning its TDX confidential computing technology, and whether Intel's own role in that infrastructure constitutes a dependency. He said the company does "not consider its attestation infrastructure to be a limitation to sovereignty guarantees," arguing that any reliance on Intel's silicon and certificate root of trust is "bounded." Intel is not in the customer's workload data path, does not receive customer plaintext through attestation, and the operational trust decision can be delegated to an independent verifier or retained by the customer. That is a carefully constructed, technically defensible answer. It explains the architecture, not the law. Intel was asked whether its attestation infrastructure poses a sovereignty risk under RISAA, the 2024 US law that can compel hardware manufacturers to cooperate with secret intelligence orders. That question went unanswered. Google did not respond to a request for comment for this article. Acknowledged everywhere except the sales pitch Sardar's findings prompted four different institutional responses. The IETF's Secure Evidence and Attestation Transport (SEAT) working group, formed after a group including Sardar successfully argued for it at a Birds of a Feather session at IETF 123 in Madrid in July 2025, wrote his correlation properties directly into its charter as an explicit, mandatory requirement for any new specification work. That is a standards body doing exactly what it should, building formal verification into the process rather than bolting it on afterwards. The IETF's TLS working group formally acknowledged the same attacks, without adopting a binding requirement of its own. The CCC's inaction over ten days meant Sardar published the evidence himself, without the working group's help. None of that reached the sales conversation. Intel and Google continue to market confidential computing as proof of sovereign, verified protection. Asked directly about the infrastructure underpinning that claim, Intel's answer stopped short of the legal question at its centre. Google did not answer at all. For European CIOs and procurement officers, this raises a question beyond the one usually asked. It is no longer only which company owns the cloud or which government can compel which hardware manufacturer. It is whether the cryptographic handshake meant to prove a workload is running where it claims to be running can be trusted at all. The level that timing rules out Sardar's own mitigation reaches level two. Level three, the one that actually matters to a customer trying to verify their workload is still protected once data starts flowing, may not be achievable at all within the current architecture of intra-handshake attestation, where evidence is generated during the handshake itself. The timing is the problem. Level three requires binding the evidence to the key that encrypts the actual application data, but by the time that key exists, the evidence has already been sent, unless the TLS protocol itself is significantly changed. Post-handshake attestation waits until after that point, when the key is already there to bind against. "We believe post-handshake attestation alone can achieve level three binding," Sardar told The Register, warning that newer proposals combining both approaches add unnecessary complexity without adding security. His recommendation to the IETF's TLS working group is blunt: developers should abandon intra-handshake attestation altogether. ® ]]></content:encoded>
+ <enclosure url="https://image.theregister.com/?imageId=5266120&amp;width=800" type="image/jpeg" />
+ <media:thumbnail url="https://image.theregister.com/?imageId=5266120&amp;width=800" />
+ </item>
+<item>
+ <guid isPermaLink="true">https://www.theregister.com/a/5266512</guid>
+ <link>https://www.theregister.com/security/2026/07/03/adapthealth-crooks-stole-our-passwords-patient-health-data/5266512</link>
+ <pubDate>Fri, 03 Jul 2026 16:29:00 +0200</pubDate>
+ <title>AdaptHealth says attackers sweet-talked their way into cloud systems and stole patient data</title>
+ <description><![CDATA[ Third-party contractor compromise exposed health information and insurance billing passwords ]]></description>
+ <category>security</category>
+ <lab:kicker><![CDATA[ Security ]]></lab:kicker>
+ <dc:modified>Fri, 03 Jul 2026 13:50:25 +0000</dc:modified>
+ <content:encoded><![CDATA[ AdaptHealth says attackers used social engineering to breach its systems and steal sensitive patient data, including passwords associated with insurance billing. The medical equipment company disclosed the attack to the Securities and Exchange Commission (SEC) on Thursday, noting that attackers accessed internal patient management systems, document storage platforms, and external electronic health record system portals. The attack targeted an unwitting third-party contractor, through which the cybercriminals gained entry to the company's cloud environment, where they accessed business applications holding sensitive data. AdaptHealth activated its incident response protocols soon after the attacker contacted the company on June 15 and disclosed the theft. It did not specify whether an extortion demand was made, nor whether one was paid, and no cybercrime group had claimed responsibility at the time of writing. The company's response included disabling the contractor's user account, resetting credentials, and implementing additional access controls. It believes the attack is now contained. In addition to the "password file associated with insurance billing," AdaptHealth confirmed that personally identifiable information (PII) and protected health information of certain patients were also stolen. Social Security numbers and payment details are not thought to be affected. On June 27, AdaptHealth determined that "due to the nature and potential volume of the data that is at risk," the attack can be considered material, requiring disclosure to the SEC. The company did not comment on the exact scale of the attack or the related data theft, but said investigations continue to determine the scope of the breach. It also said it "has since taken steps intended to mitigate the risk of dissemination of the exfiltrated data." The Register asked AdaptHealth for more information, including whether it received any extortion demands and what steps it took to reduce the risk of the stolen data being distributed or misused. Pennsylvania-based AdaptHealth provides home medical equipment and related services for patients with chronic and serious conditions. Founded in 2012, it specializes in respiratory, sleep, and diabetes therapies. According to a 2024 annual report, it serves more than 4.2 million patients across all 50 US states. ® ]]></content:encoded>
+ <enclosure url="https://image.theregister.com/?imageId=5266544&amp;width=800" type="image/jpeg" />
+ <media:thumbnail url="https://image.theregister.com/?imageId=5266544&amp;width=800" />
+ </item>
+</channel>
+</rss>