Skip to content

Building 70 Projects ranging from beginner to advanced so anyone can — learn from, build upon, use as a reference, or even copy directly. Owned by my cybersecurity company đŸ‘‡

feat(nadezhda): M4 ranking + digest + Markdown/JSON export

CarterPerez-dev8713712bf9d6
diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/digest.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/digest.go
new file mode 100644
index 00000000..b4aed201
--- /dev/null
+++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/digest.go
@@ -0,0 +1,101 @@
+// ©AngelaMos | 2026
+// digest.go
+
+package main
+
+import (
+ "fmt"
+ "os"
+ "time"
+
+ "github.com/spf13/cobra"
+
+ "github.com/CarterPerez-dev/nadezhda/internal/export"
+ "github.com/CarterPerez-dev/nadezhda/internal/rank"
+ "github.com/CarterPerez-dev/nadezhda/internal/store"
+)
+
+const (
+ defaultDigestTop = 20
+ formatMarkdown = "md"
+ formatJSON = "json"
+ outFilePerm = 0o644
+)
+
+var (
+ digestTop int
+ digestSince string
+ digestFormat string
+ digestOut string
+)
+
+var digestCmd = &cobra.Command{
+ Use: "digest",
+ Short: "Render a ranked digest of story clusters to Markdown or JSON",
+ RunE: runDigest,
+}
+
+func init() {
+ digestCmd.Flags().IntVar(&digestTop, "top", defaultDigestTop, "show the top N ranked clusters")
+ digestCmd.Flags().StringVar(&digestSince, "since", "", "only clusters active within this window (e.g. 24h, 168h)")
+ digestCmd.Flags().StringVar(&digestFormat, "format", formatMarkdown, "output format: md or json")
+ digestCmd.Flags().StringVar(&digestOut, "out", "", "write to this file instead of stdout")
+ rootCmd.AddCommand(digestCmd)
+}
+
+func runDigest(cmd *cobra.Command, args []string) error {
+ cfg, err := loadConfig()
+ if err != nil {
+ return err
+ }
+ if digestFormat != formatMarkdown && digestFormat != formatJSON {
+ return fmt.Errorf("invalid --format %q: want md or json", digestFormat)
+ }
+
+ var since int64
+ now := time.Now()
+ if digestSince != "" {
+ d, err := time.ParseDuration(digestSince)
+ if err != nil {
+ return fmt.Errorf("invalid --since %q: %w", digestSince, err)
+ }
+ since = now.Add(-d).Unix()
+ }
+
+ st, err := store.Open(cfg.DBPath)
+ if err != nil {
+ return err
+ }
+ defer st.Close()
+
+ clusters, err := st.DigestClusters(since)
+ if err != nil {
+ return err
+ }
+ scored := rank.Rank(clusters, cfg.Rank, cfg.Watchlist, now)
+
+ shown := len(scored)
+ if digestTop > 0 && digestTop < shown {
+ shown = digestTop
+ }
+
+ var rendered string
+ if digestFormat == formatJSON {
+ rendered, err = export.JSON(scored, digestTop)
+ if err != nil {
+ return err
+ }
+ } else {
+ rendered = export.Markdown(scored, digestTop)
+ }
+
+ if digestOut != "" {
+ if err := os.WriteFile(digestOut, []byte(rendered), outFilePerm); err != nil {
+ return fmt.Errorf("write digest to %s: %w", digestOut, err)
+ }
+ fmt.Fprintf(cmd.OutOrStdout(), "wrote %d clusters to %s\n", shown, digestOut)
+ return nil
+ }
+ fmt.Fprint(cmd.OutOrStdout(), rendered)
+ return nil
+}
diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go
index c1a4b829..e12b52fe 100644
--- a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go
+++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go
@@ -21,7 +21,6 @@ func init() {
short string
milestone string
}{
- {"digest", "Render a ranked digest to Markdown or JSON", "milestone M4"},
{"tui", "Browse aggregated news in an interactive terminal UI", "milestone M5"},
{"ideate", "Generate content angles from ranked clusters via an AI provider", "milestone M6"},
{"watch", "Run as a daemon, re-ingesting on an interval", "milestone M7"},
diff --git a/PROJECTS/intermediate/security-news-scraper/internal/export/export.go b/PROJECTS/intermediate/security-news-scraper/internal/export/export.go
new file mode 100644
index 00000000..38e0a148
--- /dev/null
+++ b/PROJECTS/intermediate/security-news-scraper/internal/export/export.go
@@ -0,0 +1,172 @@
+// ©AngelaMos | 2026
+// export.go
+
+package export
+
+import (
+ "encoding/json"
+ "fmt"
+ "sort"
+ "strings"
+ "time"
+
+ "github.com/CarterPerez-dev/nadezhda/internal/rank"
+ "github.com/CarterPerez-dev/nadezhda/internal/store"
+)
+
+const (
+ dateLayout = "2006-01-02"
+ noDate = "----------"
+ scoreRounder = 10000.0
+ digestTitle = "# Nadezhda Digest"
+)
+
+type CVEEntry struct {
+ ID string `json:"id"`
+ CVSS *float64 `json:"cvss,omitempty"`
+ EPSS *float64 `json:"epss,omitempty"`
+ KEV bool `json:"kev"`
+}
+
+type Entry struct {
+ Rank int `json:"rank"`
+ Score float64 `json:"score"`
+ Headline string `json:"headline"`
+ URL string `json:"url"`
+ Outlets []string `json:"outlets"`
+ FirstSeen string `json:"first_seen"`
+ LastSeen string `json:"last_seen"`
+ CVEs []CVEEntry `json:"cves,omitempty"`
+}
+
+type Digest struct {
+ Count int `json:"count"`
+ Entries []Entry `json:"entries"`
+}
+
+func Build(scored []rank.Scored, top int) Digest {
+ limit := len(scored)
+ if top > 0 && top < limit {
+ limit = top
+ }
+ entries := make([]Entry, 0, limit)
+ for i := 0; i < limit; i++ {
+ s := scored[i]
+ h := headline(s.Cluster.Articles)
+ entries = append(entries, Entry{
+ Rank: i + 1,
+ Score: roundScore(s.Score),
+ Headline: h.Title,
+ URL: h.CanonicalURL,
+ Outlets: outlets(s.Cluster.Articles),
+ FirstSeen: formatDate(s.Cluster.FirstSeen),
+ LastSeen: formatDate(s.Cluster.LastSeen),
+ CVEs: cveEntries(s.Cluster.CVEs),
+ })
+ }
+ return Digest{Count: len(entries), Entries: entries}
+}
+
+func JSON(scored []rank.Scored, top int) (string, error) {
+ b, err := json.MarshalIndent(Build(scored, top), "", " ")
+ if err != nil {
+ return "", fmt.Errorf("export json: %w", err)
+ }
+ return string(b), nil
+}
+
+func Markdown(scored []rank.Scored, top int) string {
+ d := Build(scored, top)
+ var b strings.Builder
+ b.WriteString(digestTitle + "\n\n")
+ for _, e := range d.Entries {
+ fmt.Fprintf(&b, "## %d. %s\n", e.Rank, oneLine(e.Headline))
+ fmt.Fprintf(&b, "score %.2f | %d outlet(s): %s\n", e.Score, len(e.Outlets), strings.Join(e.Outlets, ", "))
+ for _, c := range e.CVEs {
+ fmt.Fprintf(&b, "- %s\n", cveLine(c))
+ }
+ fmt.Fprintf(&b, "%s\n\n", e.URL)
+ }
+ return b.String()
+}
+
+func headline(articles []store.DigestArticle) store.DigestArticle {
+ var best store.DigestArticle
+ for i, a := range articles {
+ if i == 0 || a.PublishedAt > best.PublishedAt ||
+ (a.PublishedAt == best.PublishedAt && a.ID < best.ID) {
+ best = a
+ }
+ }
+ return best
+}
+
+func outlets(articles []store.DigestArticle) []string {
+ seen := make(map[string]struct{}, len(articles))
+ var out []string
+ for _, a := range articles {
+ if _, ok := seen[a.SourceName]; ok {
+ continue
+ }
+ seen[a.SourceName] = struct{}{}
+ out = append(out, a.SourceName)
+ }
+ sort.Strings(out)
+ return out
+}
+
+func cveEntries(cves []store.DigestCVE) []CVEEntry {
+ out := make([]CVEEntry, 0, len(cves))
+ for _, v := range cves {
+ out = append(out, CVEEntry{ID: v.ID, CVSS: v.CVSSScore, EPSS: v.EPSS, KEV: v.IsKEV})
+ }
+ sort.SliceStable(out, func(a, b int) bool {
+ if out[a].KEV != out[b].KEV {
+ return out[a].KEV
+ }
+ if cvssOf(out[a]) != cvssOf(out[b]) {
+ return cvssOf(out[a]) > cvssOf(out[b])
+ }
+ return out[a].ID < out[b].ID
+ })
+ return out
+}
+
+func cvssOf(c CVEEntry) float64 {
+ if c.CVSS == nil {
+ return -1
+ }
+ return *c.CVSS
+}
+
+func cveLine(c CVEEntry) string {
+ var parts []string
+ if c.KEV {
+ parts = append(parts, "KEV")
+ }
+ if c.CVSS != nil {
+ parts = append(parts, fmt.Sprintf("CVSS %.1f", *c.CVSS))
+ }
+ if c.EPSS != nil {
+ parts = append(parts, fmt.Sprintf("EPSS %.2f", *c.EPSS))
+ }
+ if len(parts) == 0 {
+ return c.ID
+ }
+ return fmt.Sprintf("%s (%s)", c.ID, strings.Join(parts, ", "))
+}
+
+func oneLine(s string) string {
+ return strings.Join(strings.Fields(s), " ")
+}
+
+func roundScore(v float64) float64 {
+ return float64(int64(v*scoreRounder+0.5)) / scoreRounder
+}
+
+func formatDate(unix int64) string {
+ if unix == 0 {
+ return noDate
+ }
+ return time.Unix(unix, 0).UTC().Format(dateLayout)
+}
diff --git a/PROJECTS/intermediate/security-news-scraper/internal/export/export_test.go b/PROJECTS/intermediate/security-news-scraper/internal/export/export_test.go
new file mode 100644
index 00000000..88c499fc
--- /dev/null
+++ b/PROJECTS/intermediate/security-news-scraper/internal/export/export_test.go
@@ -0,0 +1,104 @@
+// ©AngelaMos | 2026
+// export_test.go
+
+package export
+
+import (
+ "encoding/json"
+ "strings"
+ "testing"
+
+ "github.com/CarterPerez-dev/nadezhda/internal/rank"
+ "github.com/CarterPerez-dev/nadezhda/internal/store"
+)
+
+func f(v float64) *float64 { return &v }
+
+func sample() []rank.Scored {
+ return []rank.Scored{
+ {
+ Score: 0.9412,
+ Cluster: store.DigestCluster{
+ Size: 3, FirstSeen: 1751600000, LastSeen: 1751700000,
+ Articles: []store.DigestArticle{
+ {ID: 1, SourceName: "thehackernews", Title: "Old framing", CanonicalURL: "https://thn/1", PublishedAt: 100},
+ {ID: 2, SourceName: "bleepingcomputer", Title: "Freshest framing", CanonicalURL: "https://bc/2", PublishedAt: 300},
+ {ID: 3, SourceName: "bleepingcomputer", Title: "Mid framing", CanonicalURL: "https://bc/3", PublishedAt: 200},
+ },
+ CVEs: []store.DigestCVE{
+ {ID: "CVE-2026-2", CVSSScore: f(7.5), IsKEV: false},
+ {ID: "CVE-2026-1", CVSSScore: f(9.8), EPSS: f(0.91), IsKEV: true},
+ },
+ },
+ },
+ {
+ Score: 0.10,
+ Cluster: store.DigestCluster{
+ Size: 1, FirstSeen: 1751000000, LastSeen: 1751000000,
+ Articles: []store.DigestArticle{{ID: 9, SourceName: "krebs", Title: "Quiet story", CanonicalURL: "https://k/9", PublishedAt: 50}},
+ },
+ },
+ }
+}
+
+func TestBuildHeadlineAndOutlets(t *testing.T) {
+ d := Build(sample(), 0)
+ if d.Count != 2 {
+ t.Fatalf("count = %d, want 2", d.Count)
+ }
+ e := d.Entries[0]
+ if e.Rank != 1 || e.Score != 0.9412 {
+ t.Errorf("rank/score = %d/%v", e.Rank, e.Score)
+ }
+ if e.Headline != "Freshest framing" {
+ t.Errorf("headline = %q, want the freshest member", e.Headline)
+ }
+ if e.URL != "https://bc/2" {
+ t.Errorf("url = %q, want the freshest member's", e.URL)
+ }
+ if len(e.Outlets) != 2 || e.Outlets[0] != "bleepingcomputer" || e.Outlets[1] != "thehackernews" {
+ t.Errorf("outlets = %v, want distinct + sorted", e.Outlets)
+ }
+ if e.CVEs[0].ID != "CVE-2026-1" {
+ t.Errorf("first cve = %q, want the KEV one first", e.CVEs[0].ID)
+ }
+}
+
+func TestBuildTopLimit(t *testing.T) {
+ d := Build(sample(), 1)
+ if d.Count != 1 {
+ t.Errorf("top=1 should yield 1 entry, got %d", d.Count)
+ }
+}
+
+func TestJSONValidAndStable(t *testing.T) {
+ out, err := JSON(sample(), 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var back Digest
+ if err := json.Unmarshal([]byte(out), &back); err != nil {
+ t.Fatalf("output is not valid JSON: %v", err)
+ }
+ if back.Entries[0].Headline != "Freshest framing" {
+ t.Errorf("round-trip headline = %q", back.Entries[0].Headline)
+ }
+ if !back.Entries[0].CVEs[0].KEV {
+ t.Error("first cve should be the KEV one")
+ }
+}
+
+func TestMarkdownContainsSignals(t *testing.T) {
+ md := Markdown(sample(), 0)
+ for _, want := range []string{
+ "# Nadezhda Digest",
+ "## 1. Freshest framing",
+ "2 outlet(s): bleepingcomputer, thehackernews",
+ "CVE-2026-1 (KEV, CVSS 9.8, EPSS 0.91)",
+ "https://bc/2",
+ } {
+ if !strings.Contains(md, want) {
+ t.Errorf("markdown missing %q\n---\n%s", want, md)
+ }
+ }
+}
diff --git a/PROJECTS/intermediate/security-news-scraper/internal/rank/rank.go b/PROJECTS/intermediate/security-news-scraper/internal/rank/rank.go
new file mode 100644
index 00000000..04f2bf33
--- /dev/null
+++ b/PROJECTS/intermediate/security-news-scraper/internal/rank/rank.go
@@ -0,0 +1,159 @@
+// ©AngelaMos | 2026
+// rank.go
+
+package rank
+
+import (
+ "math"
+ "sort"
+ "strings"
+ "time"
+
+ "github.com/CarterPerez-dev/nadezhda/internal/config"
+ "github.com/CarterPerez-dev/nadezhda/internal/store"
+)
+
+const (
+ cvssMax = 10.0
+ minVelocityWindowHours = 1.0
+ secondsPerHour = 3600.0
+)
+
+type Signals struct {
+ AgeHours float64
+ MaxCVSS float64
+ KEV bool
+ MaxEPSS float64
+ ClusterSize int
+ ClusterAgeHours float64
+ SourceWeight float64
+ KeywordMatch bool
+}
+
+func Score(s Signals, cfg config.Rank) float64 {
+ w := cfg.Weights
+ return w.Recency*recency(s.AgeHours, cfg.HalfLifeHours) +
+ w.CVSS*clamp01(s.MaxCVSS/cvssMax) +
+ w.KEV*boolScore(s.KEV) +
+ w.EPSS*clamp01(s.MaxEPSS) +
+ w.Velocity*velocity(s.ClusterSize, s.ClusterAgeHours, cfg.VelocityNorm) +
+ w.Source*clamp01(s.SourceWeight) +
+ w.Keyword*boolScore(s.KeywordMatch)
+}
+
+func recency(ageHours float64, halfLifeHours int) float64 {
+ if ageHours < 0 {
+ ageHours = 0
+ }
+ if halfLifeHours < 1 {
+ halfLifeHours = 1
+ }
+ return math.Exp(-math.Ln2 * ageHours / float64(halfLifeHours))
+}
+
+func velocity(size int, ageHours, norm float64) float64 {
+ if size <= 1 || norm <= 0 {
+ return 0
+ }
+ if ageHours < minVelocityWindowHours {
+ ageHours = minVelocityWindowHours
+ }
+ return clamp01((float64(size) / ageHours) / norm)
+}
+
+func boolScore(b bool) float64 {
+ if b {
+ return 1
+ }
+ return 0
+}
+
+func clamp01(v float64) float64 {
+ if v < 0 {
+ return 0
+ }
+ if v > 1 {
+ return 1
+ }
+ return v
+}
+
+type Scored struct {
+ Cluster store.DigestCluster
+ Score float64
+}
+
+func Rank(clusters []store.DigestCluster, cfg config.Rank, watchlist []string, now time.Time) []Scored {
+ out := make([]Scored, len(clusters))
+ for i, c := range clusters {
+ out[i] = Scored{Cluster: c, Score: Score(signalsFor(c, watchlist, now), cfg)}
+ }
+ sort.SliceStable(out, func(a, b int) bool {
+ if out[a].Score != out[b].Score {
+ return out[a].Score > out[b].Score
+ }
+ return out[a].Cluster.LastSeen > out[b].Cluster.LastSeen
+ })
+ return out
+}
+
+func signalsFor(c store.DigestCluster, watchlist []string, now time.Time) Signals {
+ s := Signals{
+ AgeHours: hoursSince(c.LastSeen, now),
+ ClusterSize: c.Size,
+ ClusterAgeHours: float64(c.LastSeen-c.FirstSeen) / secondsPerHour,
+ SourceWeight: maxSourceWeight(c.Articles),
+ KeywordMatch: matchesWatchlist(c, watchlist),
+ }
+ for _, v := range c.CVEs {
+ if v.CVSSScore != nil && *v.CVSSScore > s.MaxCVSS {
+ s.MaxCVSS = *v.CVSSScore
+ }
+ if v.EPSS != nil && *v.EPSS > s.MaxEPSS {
+ s.MaxEPSS = *v.EPSS
+ }
+ if v.IsKEV {
+ s.KEV = true
+ }
+ }
+ return s
+}
+
+func hoursSince(unix int64, now time.Time) float64 {
+ return float64(now.Unix()-unix) / secondsPerHour
+}
+
+func maxSourceWeight(articles []store.DigestArticle) float64 {
+ var max float64
+ for _, a := range articles {
+ if a.SourceWeight > max {
+ max = a.SourceWeight
+ }
+ }
+ return max
+}
+
+func matchesWatchlist(c store.DigestCluster, watchlist []string) bool {
+ if len(watchlist) == 0 {
+ return false
+ }
+ var sb strings.Builder
+ for _, a := range c.Articles {
+ sb.WriteString(strings.ToLower(a.Title))
+ sb.WriteByte(' ')
+ }
+ for _, v := range c.CVEs {
+ sb.WriteString(strings.ToLower(v.ID))
+ sb.WriteByte(' ')
+ }
+ hay := sb.String()
+ for _, term := range watchlist {
+ if term == "" {
+ continue
+ }
+ if strings.Contains(hay, strings.ToLower(term)) {
+ return true
+ }
+ }
+ return false
+}
diff --git a/PROJECTS/intermediate/security-news-scraper/internal/rank/rank_test.go b/PROJECTS/intermediate/security-news-scraper/internal/rank/rank_test.go
new file mode 100644
index 00000000..1748bc13
--- /dev/null
+++ b/PROJECTS/intermediate/security-news-scraper/internal/rank/rank_test.go
@@ -0,0 +1,106 @@
+// ©AngelaMos | 2026
+// rank_test.go
+
+package rank
+
+import (
+ "math"
+ "testing"
+ "time"
+
+ "github.com/CarterPerez-dev/nadezhda/internal/config"
+ "github.com/CarterPerez-dev/nadezhda/internal/store"
+)
+
+func f(v float64) *float64 { return &v }
+
+func TestScoreGoldenOrder(t *testing.T) {
+ cfg := config.Default().Rank
+
+ a := Signals{
+ AgeHours: 2, MaxCVSS: 9.8, KEV: true, MaxEPSS: 0.97,
+ ClusterSize: 5, ClusterAgeHours: 6, SourceWeight: 1.0, KeywordMatch: true,
+ }
+ b := Signals{
+ AgeHours: 120, MaxCVSS: 4.3, KEV: false, MaxEPSS: 0.02,
+ ClusterSize: 1, ClusterAgeHours: 0, SourceWeight: 0.6, KeywordMatch: false,
+ }
+
+ sa, sb := Score(a, cfg), Score(b, cfg)
+ if !(sa > sb) {
+ t.Fatalf("expected A (%.4f) to rank strictly above B (%.4f)", sa, sb)
+ }
+ if sa < 0.95 {
+ t.Errorf("A score = %.4f, want ~0.98", sa)
+ }
+ if sb > 0.20 {
+ t.Errorf("B score = %.4f, want ~0.14", sb)
+ }
+}
+
+func TestRecencyDecay(t *testing.T) {
+ if got := recency(0, 48); math.Abs(got-1.0) > 1e-9 {
+ t.Errorf("recency(0) = %v, want 1.0", got)
+ }
+ if got := recency(48, 48); math.Abs(got-0.5) > 1e-9 {
+ t.Errorf("recency(one half-life) = %v, want 0.5", got)
+ }
+ if got := recency(-5, 48); got != 1.0 {
+ t.Errorf("negative age should clamp to 1.0, got %v", got)
+ }
+}
+
+func TestVelocity(t *testing.T) {
+ if v := velocity(1, 0, 0.5); v != 0 {
+ t.Errorf("size-1 velocity = %v, want 0", v)
+ }
+ if v := velocity(6, 12, 0.5); v != 1.0 {
+ t.Errorf("size-6 in 12h should saturate, got %v", v)
+ }
+ if v := velocity(3, 0, 0.5); v != 1.0 {
+ t.Errorf("burst (age<floor) should saturate, got %v", v)
+ }
+ if v := velocity(2, 100, 0.5); math.Abs(v-0.04) > 1e-9 {
+ t.Errorf("slow cluster velocity = %v, want 0.04", v)
+ }
+}
+
+func TestRankOrdersByScore(t *testing.T) {
+ now := time.Unix(1_000_000, 0)
+ hot := store.DigestCluster{
+ ClusterID: 1, Size: 4, FirstSeen: now.Unix() - 3600, LastSeen: now.Unix() - 60,
+ Articles: []store.DigestArticle{{SourceName: "a", SourceWeight: 1.0, PublishedAt: now.Unix() - 60}},
+ CVEs: []store.DigestCVE{{ID: "CVE-2026-1", CVSSScore: f(9.9), EPSS: f(0.95), IsKEV: true}},
+ }
+ cold := store.DigestCluster{
+ ClusterID: 2, Size: 1, FirstSeen: now.Unix() - 500*3600, LastSeen: now.Unix() - 500*3600,
+ Articles: []store.DigestArticle{{SourceName: "b", SourceWeight: 0.5, PublishedAt: now.Unix() - 500*3600}},
+ }
+
+ scored := Rank([]store.DigestCluster{cold, hot}, config.Default().Rank, nil, now)
+ if scored[0].Cluster.ClusterID != 1 {
+ t.Errorf("hot KEV cluster should rank first, got cluster %d", scored[0].Cluster.ClusterID)
+ }
+ if !(scored[0].Score > scored[1].Score) {
+ t.Errorf("scores not descending: %.4f then %.4f", scored[0].Score, scored[1].Score)
+ }
+}
+
+func TestKeywordMatch(t *testing.T) {
+ c := store.DigestCluster{
+ Articles: []store.DigestArticle{{Title: "Fortinet FortiOS flaw exploited"}},
+ CVEs: []store.DigestCVE{{ID: "CVE-2026-1"}},
+ }
+ if !matchesWatchlist(c, []string{"fortinet"}) {
+ t.Error("case-insensitive watchlist term should match title")
+ }
+ if !matchesWatchlist(c, []string{"CVE-2026-1"}) {
+ t.Error("watchlist should match against CVE ids")
+ }
+ if matchesWatchlist(c, []string{"cisco"}) {
+ t.Error("non-matching term should not match")
+ }
+ if matchesWatchlist(c, nil) {
+ t.Error("empty watchlist should not match")
+ }
+}
diff --git a/PROJECTS/intermediate/security-news-scraper/internal/store/digest_test.go b/PROJECTS/intermediate/security-news-scraper/internal/store/digest_test.go
new file mode 100644
index 00000000..b6230eba
--- /dev/null
+++ b/PROJECTS/intermediate/security-news-scraper/internal/store/digest_test.go
@@ -0,0 +1,110 @@
+// ©AngelaMos | 2026
+// digest_test.go
+
+package store
+
+import "testing"
+
+func TestDigestClustersAggregates(t *testing.T) {
+ s := openTemp(t)
+
+ src1, _ := s.UpsertSource(SourceInput{Name: "a", URL: "https://a/f", Type: "rss", Weight: 1.0, Enabled: true})
+ src2, _ := s.UpsertSource(SourceInput{Name: "b", URL: "https://b/f", Type: "rss", Weight: 0.8, Enabled: true})
+
+ a1, _ := s.InsertArticle(Article{SourceID: src1, CanonicalURL: "https://a/1", ContentHash: "c1", TitleHash: "t1", Title: "Story one", PublishedAt: 1000})
+ a2, _ := s.InsertArticle(Article{SourceID: src2, CanonicalURL: "https://b/1", ContentHash: "c2", TitleHash: "t2", Title: "Story two", PublishedAt: 2000})
+
+ if err := s.UpsertCVEStub("CVE-2026-1"); err != nil {
+ t.Fatal(err)
+ }
+ score := 9.8
+ if err := s.UpdateCVEEnrichment(CVE{ID: "CVE-2026-1", CVSSScore: &score, IsKEV: true, EnrichedAt: 1, EnrichStatus: EnrichStatusOK}); err != nil {
+ t.Fatal(err)
+ }
+ if err := s.LinkArticleCVE(a1, "CVE-2026-1"); err != nil {
+ t.Fatal(err)
+ }
+ if err := s.LinkArticleCVE(a2, "CVE-2026-1"); err != nil {
+ t.Fatal(err)
+ }
+
+ if err := s.ReplaceClusters([]ClusterRow{
+ {Key: "1", Members: []int64{a1, a2}, FirstSeen: 1000, LastSeen: 2000},
+ }); err != nil {
+ t.Fatal(err)
+ }
+
+ clusters, err := s.DigestClusters(0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(clusters) != 1 {
+ t.Fatalf("clusters = %d, want 1", len(clusters))
+ }
+ c := clusters[0]
+ if len(c.Articles) != 2 {
+ t.Errorf("articles = %d, want 2", len(c.Articles))
+ }
+ if len(c.CVEs) != 1 {
+ t.Fatalf("cves = %d, want 1 (deduped across both articles)", len(c.CVEs))
+ }
+ if !c.CVEs[0].IsKEV || c.CVEs[0].CVSSScore == nil || *c.CVEs[0].CVSSScore != 9.8 {
+ t.Errorf("cve signals not aggregated: %+v", c.CVEs[0])
+ }
+}
+
+func TestDigestClustersAttachToCorrectCluster(t *testing.T) {
+ s := openTemp(t)
+ src, _ := s.UpsertSource(SourceInput{Name: "a", URL: "https://a/f", Type: "rss", Weight: 1, Enabled: true})
+
+ a1, _ := s.InsertArticle(Article{SourceID: src, CanonicalURL: "https://a/1", ContentHash: "c1", TitleHash: "t1", Title: "one"})
+ a2, _ := s.InsertArticle(Article{SourceID: src, CanonicalURL: "https://a/2", ContentHash: "c2", TitleHash: "t2", Title: "two"})
+
+ for _, id := range []string{"CVE-2026-1", "CVE-2026-2"} {
+ if err := s.UpsertCVEStub(id); err != nil {
+ t.Fatal(err)
+ }
+ }
+ _ = s.LinkArticleCVE(a1, "CVE-2026-1")
+ _ = s.LinkArticleCVE(a2, "CVE-2026-2")
+
+ if err := s.ReplaceClusters([]ClusterRow{
+ {Key: "1", Members: []int64{a1}, FirstSeen: 1000, LastSeen: 1000},
+ {Key: "2", Members: []int64{a2}, FirstSeen: 2000, LastSeen: 2000},
+ }); err != nil {
+ t.Fatal(err)
+ }
+
+ clusters, err := s.DigestClusters(0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(clusters) != 2 {
+ t.Fatalf("clusters = %d, want 2", len(clusters))
+ }
+ byArticle := map[int64]DigestCluster{}
+ for _, c := range clusters {
+ if len(c.Articles) != 1 || len(c.CVEs) != 1 {
+ t.Fatalf("cluster %d: articles=%d cves=%d, want 1/1", c.ClusterID, len(c.Articles), len(c.CVEs))
+ }
+ byArticle[c.Articles[0].ID] = c
+ }
+ if byArticle[a1].CVEs[0].ID != "CVE-2026-1" || byArticle[a2].CVEs[0].ID != "CVE-2026-2" {
+ t.Error("CVEs attached to the wrong cluster")
+ }
+}
+
+func TestDigestClustersSinceFilter(t *testing.T) {
+ s := openTemp(t)
+ src, _ := s.UpsertSource(SourceInput{Name: "a", URL: "https://a/f", Type: "rss", Weight: 1, Enabled: true})
+ a1, _ := s.InsertArticle(Article{SourceID: src, CanonicalURL: "https://a/1", ContentHash: "c1", TitleHash: "t1", Title: "old"})
+ if err := s.ReplaceClusters([]ClusterRow{{Key: "1", Members: []int64{a1}, FirstSeen: 500, LastSeen: 500}}); err != nil {
+ t.Fatal(err)
+ }
+ if got, _ := s.DigestClusters(1000); len(got) != 0 {
+ t.Errorf("cluster with last_seen 500 should be excluded by since=1000, got %d", len(got))
+ }
+ if got, _ := s.DigestClusters(0); len(got) != 1 {
+ t.Errorf("since=0 should include it, got %d", len(got))
+ }
+}
diff --git a/PROJECTS/intermediate/security-news-scraper/internal/store/store.go b/PROJECTS/intermediate/security-news-scraper/internal/store/store.go
index b65e681e..e28bb698 100644
--- a/PROJECTS/intermediate/security-news-scraper/internal/store/store.go
+++ b/PROJECTS/intermediate/security-news-scraper/internal/store/store.go
@@ -295,6 +295,122 @@ func (s *Store) ArticlesForCVE(id string) ([]ArticleSummary, error) {
return scanArticleSummaries(rows)
}
+type DigestArticle struct {
+ ID int64
+ SourceName string
+ SourceWeight float64
+ Title string
+ CanonicalURL string
+ PublishedAt int64
+}
+
+type DigestCVE struct {
+ ID string
+ CVSSScore *float64
+ EPSS *float64
+ IsKEV bool
+}
+
+type DigestCluster struct {
+ ClusterID int64
+ Key string
+ Size int
+ FirstSeen int64
+ LastSeen int64
+ Articles []DigestArticle
+ CVEs []DigestCVE
+}
+
+func (s *Store) DigestClusters(since int64) ([]DigestCluster, error) {
+ byID, order, err := s.digestClusterRows(since)
+ if err != nil {
+ return nil, err
+ }
+ if err := s.digestAttachArticles(byID); err != nil {
+ return nil, err
+ }
+ if err := s.digestAttachCVEs(byID); err != nil {
+ return nil, err
+ }
+ out := make([]DigestCluster, 0, len(order))
+ for _, id := range order {
+ out = append(out, *byID[id])
+ }
+ return out, nil
+}
+
+func (s *Store) digestClusterRows(since int64) (map[int64]*DigestCluster, []int64, error) {
+ rows, err := s.db.Query(`
+ SELECT id, cluster_key, size, first_seen, last_seen
+ FROM clusters WHERE last_seen >= ? ORDER BY id`, since)
+ if err != nil {
+ return nil, nil, fmt.Errorf("digest clusters: %w", err)
+ }
+ defer rows.Close()
+ byID := make(map[int64]*DigestCluster)
+ var order []int64
+ for rows.Next() {
+ var dc DigestCluster
+ if err := rows.Scan(&dc.ClusterID, &dc.Key, &dc.Size, &dc.FirstSeen, &dc.LastSeen); err != nil {
+ return nil, nil, fmt.Errorf("digest clusters: scan: %w", err)
+ }
+ clone := dc
+ byID[dc.ClusterID] = &clone
+ order = append(order, dc.ClusterID)
+ }
+ return byID, order, rows.Err()
+}
+
+func (s *Store) digestAttachArticles(byID map[int64]*DigestCluster) error {
+ rows, err := s.db.Query(`
+ SELECT cm.cluster_id, a.id, s.name, s.weight, a.title, a.canonical_url, a.published_at
+ FROM cluster_members cm
+ JOIN articles a ON a.id = cm.article_id
+ JOIN sources s ON s.id = a.source_id
+ ORDER BY cm.cluster_id, a.id`)
+ if err != nil {
+ return fmt.Errorf("digest articles: %w", err)
+ }
+ defer rows.Close()
+ for rows.Next() {
+ var clusterID int64
+ var a DigestArticle
+ if err := rows.Scan(&clusterID, &a.ID, &a.SourceName, &a.SourceWeight, &a.Title, &a.CanonicalURL, &a.PublishedAt); err != nil {
+ return fmt.Errorf("digest articles: scan: %w", err)
+ }
+ if dc, ok := byID[clusterID]; ok {
+ dc.Articles = append(dc.Articles, a)
+ }
+ }
+ return rows.Err()
+}
+
+func (s *Store) digestAttachCVEs(byID map[int64]*DigestCluster) error {
+ rows, err := s.db.Query(`
+ SELECT DISTINCT cm.cluster_id, c.id, c.cvss_score, c.epss, c.is_kev
+ FROM cluster_members cm
+ JOIN article_cves ac ON ac.article_id = cm.article_id
+ JOIN cves c ON c.id = ac.cve_id
+ ORDER BY cm.cluster_id, c.id`)
+ if err != nil {
+ return fmt.Errorf("digest cves: %w", err)
+ }
+ defer rows.Close()
+ for rows.Next() {
+ var clusterID int64
+ var v DigestCVE
+ var isKEV int
+ if err := rows.Scan(&clusterID, &v.ID, &v.CVSSScore, &v.EPSS, &isKEV); err != nil {
+ return fmt.Errorf("digest cves: scan: %w", err)
+ }
+ v.IsKEV = isKEV != 0
+ if dc, ok := byID[clusterID]; ok {
+ dc.CVEs = append(dc.CVEs, v)
+ }
+ }
+ return rows.Err()
+}
+
func (s *Store) UpsertCVEStub(id string) error {
_, err := s.db.Exec(`INSERT INTO cves (id) VALUES (?) ON CONFLICT(id) DO NOTHING`, id)
if err != nil {