Building 70 Projects ranging from beginner to advanced so anyone can — learn from, build upon, use as a reference, or even copy directly. Owned by my cybersecurity company đŸ‘‡
Go aicertificationcloud-engineeringcybercybersecuritycybersecurity-analystcybersecurity-certificationcybersecurity-certificationscybersecurity-educationcybersecurity-jobscybersecurity-portfoliocybersecurity-projectscybersecurity-toolsgrchackinglearningpentestingproject-repositoryprojetcspython
feat(nadezhda): M4 ranking + digest + Markdown/JSON export
CarterPerez-dev8713712bf9d6
diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/digest.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/digest.gonew file mode 100644index 00000000..b4aed201--- /dev/null+++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/digest.go@@ -0,0 +1,101 @@+// ©AngelaMos | 2026+// digest.go++package main++import (+ "fmt"+ "os"+ "time"++ "github.com/spf13/cobra"++ "github.com/CarterPerez-dev/nadezhda/internal/export"+ "github.com/CarterPerez-dev/nadezhda/internal/rank"+ "github.com/CarterPerez-dev/nadezhda/internal/store"+)++const (+ defaultDigestTop = 20+ formatMarkdown = "md"+ formatJSON = "json"+ outFilePerm = 0o644+)++var (+ digestTop int+ digestSince string+ digestFormat string+ digestOut string+)++var digestCmd = &cobra.Command{+ Use: "digest",+ Short: "Render a ranked digest of story clusters to Markdown or JSON",+ RunE: runDigest,+}++func init() {+ digestCmd.Flags().IntVar(&digestTop, "top", defaultDigestTop, "show the top N ranked clusters")+ digestCmd.Flags().StringVar(&digestSince, "since", "", "only clusters active within this window (e.g. 24h, 168h)")+ digestCmd.Flags().StringVar(&digestFormat, "format", formatMarkdown, "output format: md or json")+ digestCmd.Flags().StringVar(&digestOut, "out", "", "write to this file instead of stdout")+ rootCmd.AddCommand(digestCmd)+}++func runDigest(cmd *cobra.Command, args []string) error {+ cfg, err := loadConfig()+ if err != nil {+ return err+ }+ if digestFormat != formatMarkdown && digestFormat != formatJSON {+ return fmt.Errorf("invalid --format %q: want md or json", digestFormat)+ }++ var since int64+ now := time.Now()+ if digestSince != "" {+ d, err := time.ParseDuration(digestSince)+ if err != nil {+ return fmt.Errorf("invalid --since %q: %w", digestSince, err)+ }+ since = now.Add(-d).Unix()+ }++ st, err := store.Open(cfg.DBPath)+ if err != nil {+ return err+ }+ defer st.Close()++ clusters, err := st.DigestClusters(since)+ if err != nil {+ return err+ }+ scored := rank.Rank(clusters, cfg.Rank, cfg.Watchlist, now)++ shown := len(scored)+ if digestTop > 0 && digestTop < shown {+ shown = digestTop+ }++ var rendered string+ if digestFormat == formatJSON {+ rendered, err = export.JSON(scored, digestTop)+ if err != nil {+ return err+ }+ } else {+ rendered = export.Markdown(scored, digestTop)+ }++ if digestOut != "" {+ if err := os.WriteFile(digestOut, []byte(rendered), outFilePerm); err != nil {+ return fmt.Errorf("write digest to %s: %w", digestOut, err)+ }+ fmt.Fprintf(cmd.OutOrStdout(), "wrote %d clusters to %s\n", shown, digestOut)+ return nil+ }+ fmt.Fprint(cmd.OutOrStdout(), rendered)+ return nil+}diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.goindex c1a4b829..e12b52fe 100644--- a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go+++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go@@ -21,7 +21,6 @@ func init() {short stringmilestone string}{- {"digest", "Render a ranked digest to Markdown or JSON", "milestone M4"},{"tui", "Browse aggregated news in an interactive terminal UI", "milestone M5"},{"ideate", "Generate content angles from ranked clusters via an AI provider", "milestone M6"},{"watch", "Run as a daemon, re-ingesting on an interval", "milestone M7"},diff --git a/PROJECTS/intermediate/security-news-scraper/internal/export/export.go b/PROJECTS/intermediate/security-news-scraper/internal/export/export.gonew file mode 100644index 00000000..38e0a148--- /dev/null+++ b/PROJECTS/intermediate/security-news-scraper/internal/export/export.go@@ -0,0 +1,172 @@+// ©AngelaMos | 2026+// export.go++package export++import (+ "encoding/json"+ "fmt"+ "sort"+ "strings"+ "time"++ "github.com/CarterPerez-dev/nadezhda/internal/rank"+ "github.com/CarterPerez-dev/nadezhda/internal/store"+)++const (+ dateLayout = "2006-01-02"+ noDate = "----------"+ scoreRounder = 10000.0+ digestTitle = "# Nadezhda Digest"+)++type CVEEntry struct {+ ID string `json:"id"`+ CVSS *float64 `json:"cvss,omitempty"`+ EPSS *float64 `json:"epss,omitempty"`+ KEV bool `json:"kev"`+}++type Entry struct {+ Rank int `json:"rank"`+ Score float64 `json:"score"`+ Headline string `json:"headline"`+ URL string `json:"url"`+ Outlets []string `json:"outlets"`+ FirstSeen string `json:"first_seen"`+ LastSeen string `json:"last_seen"`+ CVEs []CVEEntry `json:"cves,omitempty"`+}++type Digest struct {+ Count int `json:"count"`+ Entries []Entry `json:"entries"`+}++func Build(scored []rank.Scored, top int) Digest {+ limit := len(scored)+ if top > 0 && top < limit {+ limit = top+ }+ entries := make([]Entry, 0, limit)+ for i := 0; i < limit; i++ {+ s := scored[i]+ h := headline(s.Cluster.Articles)+ entries = append(entries, Entry{+ Rank: i + 1,+ Score: roundScore(s.Score),+ Headline: h.Title,+ URL: h.CanonicalURL,+ Outlets: outlets(s.Cluster.Articles),+ FirstSeen: formatDate(s.Cluster.FirstSeen),+ LastSeen: formatDate(s.Cluster.LastSeen),+ CVEs: cveEntries(s.Cluster.CVEs),+ })+ }+ return Digest{Count: len(entries), Entries: entries}+}++func JSON(scored []rank.Scored, top int) (string, error) {+ b, err := json.MarshalIndent(Build(scored, top), "", " ")+ if err != nil {+ return "", fmt.Errorf("export json: %w", err)+ }+ return string(b), nil+}++func Markdown(scored []rank.Scored, top int) string {+ d := Build(scored, top)+ var b strings.Builder+ b.WriteString(digestTitle + "\n\n")+ for _, e := range d.Entries {+ fmt.Fprintf(&b, "## %d. %s\n", e.Rank, oneLine(e.Headline))+ fmt.Fprintf(&b, "score %.2f | %d outlet(s): %s\n", e.Score, len(e.Outlets), strings.Join(e.Outlets, ", "))+ for _, c := range e.CVEs {+ fmt.Fprintf(&b, "- %s\n", cveLine(c))+ }+ fmt.Fprintf(&b, "%s\n\n", e.URL)+ }+ return b.String()+}++func headline(articles []store.DigestArticle) store.DigestArticle {+ var best store.DigestArticle+ for i, a := range articles {+ if i == 0 || a.PublishedAt > best.PublishedAt ||+ (a.PublishedAt == best.PublishedAt && a.ID < best.ID) {+ best = a+ }+ }+ return best+}++func outlets(articles []store.DigestArticle) []string {+ seen := make(map[string]struct{}, len(articles))+ var out []string+ for _, a := range articles {+ if _, ok := seen[a.SourceName]; ok {+ continue+ }+ seen[a.SourceName] = struct{}{}+ out = append(out, a.SourceName)+ }+ sort.Strings(out)+ return out+}++func cveEntries(cves []store.DigestCVE) []CVEEntry {+ out := make([]CVEEntry, 0, len(cves))+ for _, v := range cves {+ out = append(out, CVEEntry{ID: v.ID, CVSS: v.CVSSScore, EPSS: v.EPSS, KEV: v.IsKEV})+ }+ sort.SliceStable(out, func(a, b int) bool {+ if out[a].KEV != out[b].KEV {+ return out[a].KEV+ }+ if cvssOf(out[a]) != cvssOf(out[b]) {+ return cvssOf(out[a]) > cvssOf(out[b])+ }+ return out[a].ID < out[b].ID+ })+ return out+}++func cvssOf(c CVEEntry) float64 {+ if c.CVSS == nil {+ return -1+ }+ return *c.CVSS+}++func cveLine(c CVEEntry) string {+ var parts []string+ if c.KEV {+ parts = append(parts, "KEV")+ }+ if c.CVSS != nil {+ parts = append(parts, fmt.Sprintf("CVSS %.1f", *c.CVSS))+ }+ if c.EPSS != nil {+ parts = append(parts, fmt.Sprintf("EPSS %.2f", *c.EPSS))+ }+ if len(parts) == 0 {+ return c.ID+ }+ return fmt.Sprintf("%s (%s)", c.ID, strings.Join(parts, ", "))+}++func oneLine(s string) string {+ return strings.Join(strings.Fields(s), " ")+}++func roundScore(v float64) float64 {+ return float64(int64(v*scoreRounder+0.5)) / scoreRounder+}++func formatDate(unix int64) string {+ if unix == 0 {+ return noDate+ }+ return time.Unix(unix, 0).UTC().Format(dateLayout)+}diff --git a/PROJECTS/intermediate/security-news-scraper/internal/export/export_test.go b/PROJECTS/intermediate/security-news-scraper/internal/export/export_test.gonew file mode 100644index 00000000..88c499fc--- /dev/null+++ b/PROJECTS/intermediate/security-news-scraper/internal/export/export_test.go@@ -0,0 +1,104 @@+// ©AngelaMos | 2026+// export_test.go++package export++import (+ "encoding/json"+ "strings"+ "testing"++ "github.com/CarterPerez-dev/nadezhda/internal/rank"+ "github.com/CarterPerez-dev/nadezhda/internal/store"+)++func f(v float64) *float64 { return &v }++func sample() []rank.Scored {+ return []rank.Scored{+ {+ Score: 0.9412,+ Cluster: store.DigestCluster{+ Size: 3, FirstSeen: 1751600000, LastSeen: 1751700000,+ Articles: []store.DigestArticle{+ {ID: 1, SourceName: "thehackernews", Title: "Old framing", CanonicalURL: "https://thn/1", PublishedAt: 100},+ {ID: 2, SourceName: "bleepingcomputer", Title: "Freshest framing", CanonicalURL: "https://bc/2", PublishedAt: 300},+ {ID: 3, SourceName: "bleepingcomputer", Title: "Mid framing", CanonicalURL: "https://bc/3", PublishedAt: 200},+ },+ CVEs: []store.DigestCVE{+ {ID: "CVE-2026-2", CVSSScore: f(7.5), IsKEV: false},+ {ID: "CVE-2026-1", CVSSScore: f(9.8), EPSS: f(0.91), IsKEV: true},+ },+ },+ },+ {+ Score: 0.10,+ Cluster: store.DigestCluster{+ Size: 1, FirstSeen: 1751000000, LastSeen: 1751000000,+ Articles: []store.DigestArticle{{ID: 9, SourceName: "krebs", Title: "Quiet story", CanonicalURL: "https://k/9", PublishedAt: 50}},+ },+ },+ }+}++func TestBuildHeadlineAndOutlets(t *testing.T) {+ d := Build(sample(), 0)+ if d.Count != 2 {+ t.Fatalf("count = %d, want 2", d.Count)+ }+ e := d.Entries[0]+ if e.Rank != 1 || e.Score != 0.9412 {+ t.Errorf("rank/score = %d/%v", e.Rank, e.Score)+ }+ if e.Headline != "Freshest framing" {+ t.Errorf("headline = %q, want the freshest member", e.Headline)+ }+ if e.URL != "https://bc/2" {+ t.Errorf("url = %q, want the freshest member's", e.URL)+ }+ if len(e.Outlets) != 2 || e.Outlets[0] != "bleepingcomputer" || e.Outlets[1] != "thehackernews" {+ t.Errorf("outlets = %v, want distinct + sorted", e.Outlets)+ }+ if e.CVEs[0].ID != "CVE-2026-1" {+ t.Errorf("first cve = %q, want the KEV one first", e.CVEs[0].ID)+ }+}++func TestBuildTopLimit(t *testing.T) {+ d := Build(sample(), 1)+ if d.Count != 1 {+ t.Errorf("top=1 should yield 1 entry, got %d", d.Count)+ }+}++func TestJSONValidAndStable(t *testing.T) {+ out, err := JSON(sample(), 0)+ if err != nil {+ t.Fatal(err)+ }+ var back Digest+ if err := json.Unmarshal([]byte(out), &back); err != nil {+ t.Fatalf("output is not valid JSON: %v", err)+ }+ if back.Entries[0].Headline != "Freshest framing" {+ t.Errorf("round-trip headline = %q", back.Entries[0].Headline)+ }+ if !back.Entries[0].CVEs[0].KEV {+ t.Error("first cve should be the KEV one")+ }+}++func TestMarkdownContainsSignals(t *testing.T) {+ md := Markdown(sample(), 0)+ for _, want := range []string{+ "# Nadezhda Digest",+ "## 1. Freshest framing",+ "2 outlet(s): bleepingcomputer, thehackernews",+ "CVE-2026-1 (KEV, CVSS 9.8, EPSS 0.91)",+ "https://bc/2",+ } {+ if !strings.Contains(md, want) {+ t.Errorf("markdown missing %q\n---\n%s", want, md)+ }+ }+}diff --git a/PROJECTS/intermediate/security-news-scraper/internal/rank/rank.go b/PROJECTS/intermediate/security-news-scraper/internal/rank/rank.gonew file mode 100644index 00000000..04f2bf33--- /dev/null+++ b/PROJECTS/intermediate/security-news-scraper/internal/rank/rank.go@@ -0,0 +1,159 @@+// ©AngelaMos | 2026+// rank.go++package rank++import (+ "math"+ "sort"+ "strings"+ "time"++ "github.com/CarterPerez-dev/nadezhda/internal/config"+ "github.com/CarterPerez-dev/nadezhda/internal/store"+)++const (+ cvssMax = 10.0+ minVelocityWindowHours = 1.0+ secondsPerHour = 3600.0+)++type Signals struct {+ AgeHours float64+ MaxCVSS float64+ KEV bool+ MaxEPSS float64+ ClusterSize int+ ClusterAgeHours float64+ SourceWeight float64+ KeywordMatch bool+}++func Score(s Signals, cfg config.Rank) float64 {+ w := cfg.Weights+ return w.Recency*recency(s.AgeHours, cfg.HalfLifeHours) ++ w.CVSS*clamp01(s.MaxCVSS/cvssMax) ++ w.KEV*boolScore(s.KEV) ++ w.EPSS*clamp01(s.MaxEPSS) ++ w.Velocity*velocity(s.ClusterSize, s.ClusterAgeHours, cfg.VelocityNorm) ++ w.Source*clamp01(s.SourceWeight) ++ w.Keyword*boolScore(s.KeywordMatch)+}++func recency(ageHours float64, halfLifeHours int) float64 {+ if ageHours < 0 {+ ageHours = 0+ }+ if halfLifeHours < 1 {+ halfLifeHours = 1+ }+ return math.Exp(-math.Ln2 * ageHours / float64(halfLifeHours))+}++func velocity(size int, ageHours, norm float64) float64 {+ if size <= 1 || norm <= 0 {+ return 0+ }+ if ageHours < minVelocityWindowHours {+ ageHours = minVelocityWindowHours+ }+ return clamp01((float64(size) / ageHours) / norm)+}++func boolScore(b bool) float64 {+ if b {+ return 1+ }+ return 0+}++func clamp01(v float64) float64 {+ if v < 0 {+ return 0+ }+ if v > 1 {+ return 1+ }+ return v+}++type Scored struct {+ Cluster store.DigestCluster+ Score float64+}++func Rank(clusters []store.DigestCluster, cfg config.Rank, watchlist []string, now time.Time) []Scored {+ out := make([]Scored, len(clusters))+ for i, c := range clusters {+ out[i] = Scored{Cluster: c, Score: Score(signalsFor(c, watchlist, now), cfg)}+ }+ sort.SliceStable(out, func(a, b int) bool {+ if out[a].Score != out[b].Score {+ return out[a].Score > out[b].Score+ }+ return out[a].Cluster.LastSeen > out[b].Cluster.LastSeen+ })+ return out+}++func signalsFor(c store.DigestCluster, watchlist []string, now time.Time) Signals {+ s := Signals{+ AgeHours: hoursSince(c.LastSeen, now),+ ClusterSize: c.Size,+ ClusterAgeHours: float64(c.LastSeen-c.FirstSeen) / secondsPerHour,+ SourceWeight: maxSourceWeight(c.Articles),+ KeywordMatch: matchesWatchlist(c, watchlist),+ }+ for _, v := range c.CVEs {+ if v.CVSSScore != nil && *v.CVSSScore > s.MaxCVSS {+ s.MaxCVSS = *v.CVSSScore+ }+ if v.EPSS != nil && *v.EPSS > s.MaxEPSS {+ s.MaxEPSS = *v.EPSS+ }+ if v.IsKEV {+ s.KEV = true+ }+ }+ return s+}++func hoursSince(unix int64, now time.Time) float64 {+ return float64(now.Unix()-unix) / secondsPerHour+}++func maxSourceWeight(articles []store.DigestArticle) float64 {+ var max float64+ for _, a := range articles {+ if a.SourceWeight > max {+ max = a.SourceWeight+ }+ }+ return max+}++func matchesWatchlist(c store.DigestCluster, watchlist []string) bool {+ if len(watchlist) == 0 {+ return false+ }+ var sb strings.Builder+ for _, a := range c.Articles {+ sb.WriteString(strings.ToLower(a.Title))+ sb.WriteByte(' ')+ }+ for _, v := range c.CVEs {+ sb.WriteString(strings.ToLower(v.ID))+ sb.WriteByte(' ')+ }+ hay := sb.String()+ for _, term := range watchlist {+ if term == "" {+ continue+ }+ if strings.Contains(hay, strings.ToLower(term)) {+ return true+ }+ }+ return false+}diff --git a/PROJECTS/intermediate/security-news-scraper/internal/rank/rank_test.go b/PROJECTS/intermediate/security-news-scraper/internal/rank/rank_test.gonew file mode 100644index 00000000..1748bc13--- /dev/null+++ b/PROJECTS/intermediate/security-news-scraper/internal/rank/rank_test.go@@ -0,0 +1,106 @@+// ©AngelaMos | 2026+// rank_test.go++package rank++import (+ "math"+ "testing"+ "time"++ "github.com/CarterPerez-dev/nadezhda/internal/config"+ "github.com/CarterPerez-dev/nadezhda/internal/store"+)++func f(v float64) *float64 { return &v }++func TestScoreGoldenOrder(t *testing.T) {+ cfg := config.Default().Rank++ a := Signals{+ AgeHours: 2, MaxCVSS: 9.8, KEV: true, MaxEPSS: 0.97,+ ClusterSize: 5, ClusterAgeHours: 6, SourceWeight: 1.0, KeywordMatch: true,+ }+ b := Signals{+ AgeHours: 120, MaxCVSS: 4.3, KEV: false, MaxEPSS: 0.02,+ ClusterSize: 1, ClusterAgeHours: 0, SourceWeight: 0.6, KeywordMatch: false,+ }++ sa, sb := Score(a, cfg), Score(b, cfg)+ if !(sa > sb) {+ t.Fatalf("expected A (%.4f) to rank strictly above B (%.4f)", sa, sb)+ }+ if sa < 0.95 {+ t.Errorf("A score = %.4f, want ~0.98", sa)+ }+ if sb > 0.20 {+ t.Errorf("B score = %.4f, want ~0.14", sb)+ }+}++func TestRecencyDecay(t *testing.T) {+ if got := recency(0, 48); math.Abs(got-1.0) > 1e-9 {+ t.Errorf("recency(0) = %v, want 1.0", got)+ }+ if got := recency(48, 48); math.Abs(got-0.5) > 1e-9 {+ t.Errorf("recency(one half-life) = %v, want 0.5", got)+ }+ if got := recency(-5, 48); got != 1.0 {+ t.Errorf("negative age should clamp to 1.0, got %v", got)+ }+}++func TestVelocity(t *testing.T) {+ if v := velocity(1, 0, 0.5); v != 0 {+ t.Errorf("size-1 velocity = %v, want 0", v)+ }+ if v := velocity(6, 12, 0.5); v != 1.0 {+ t.Errorf("size-6 in 12h should saturate, got %v", v)+ }+ if v := velocity(3, 0, 0.5); v != 1.0 {+ t.Errorf("burst (age<floor) should saturate, got %v", v)+ }+ if v := velocity(2, 100, 0.5); math.Abs(v-0.04) > 1e-9 {+ t.Errorf("slow cluster velocity = %v, want 0.04", v)+ }+}++func TestRankOrdersByScore(t *testing.T) {+ now := time.Unix(1_000_000, 0)+ hot := store.DigestCluster{+ ClusterID: 1, Size: 4, FirstSeen: now.Unix() - 3600, LastSeen: now.Unix() - 60,+ Articles: []store.DigestArticle{{SourceName: "a", SourceWeight: 1.0, PublishedAt: now.Unix() - 60}},+ CVEs: []store.DigestCVE{{ID: "CVE-2026-1", CVSSScore: f(9.9), EPSS: f(0.95), IsKEV: true}},+ }+ cold := store.DigestCluster{+ ClusterID: 2, Size: 1, FirstSeen: now.Unix() - 500*3600, LastSeen: now.Unix() - 500*3600,+ Articles: []store.DigestArticle{{SourceName: "b", SourceWeight: 0.5, PublishedAt: now.Unix() - 500*3600}},+ }++ scored := Rank([]store.DigestCluster{cold, hot}, config.Default().Rank, nil, now)+ if scored[0].Cluster.ClusterID != 1 {+ t.Errorf("hot KEV cluster should rank first, got cluster %d", scored[0].Cluster.ClusterID)+ }+ if !(scored[0].Score > scored[1].Score) {+ t.Errorf("scores not descending: %.4f then %.4f", scored[0].Score, scored[1].Score)+ }+}++func TestKeywordMatch(t *testing.T) {+ c := store.DigestCluster{+ Articles: []store.DigestArticle{{Title: "Fortinet FortiOS flaw exploited"}},+ CVEs: []store.DigestCVE{{ID: "CVE-2026-1"}},+ }+ if !matchesWatchlist(c, []string{"fortinet"}) {+ t.Error("case-insensitive watchlist term should match title")+ }+ if !matchesWatchlist(c, []string{"CVE-2026-1"}) {+ t.Error("watchlist should match against CVE ids")+ }+ if matchesWatchlist(c, []string{"cisco"}) {+ t.Error("non-matching term should not match")+ }+ if matchesWatchlist(c, nil) {+ t.Error("empty watchlist should not match")+ }+}diff --git a/PROJECTS/intermediate/security-news-scraper/internal/store/digest_test.go b/PROJECTS/intermediate/security-news-scraper/internal/store/digest_test.gonew file mode 100644index 00000000..b6230eba--- /dev/null+++ b/PROJECTS/intermediate/security-news-scraper/internal/store/digest_test.go@@ -0,0 +1,110 @@+// ©AngelaMos | 2026+// digest_test.go++package store++import "testing"++func TestDigestClustersAggregates(t *testing.T) {+ s := openTemp(t)++ src1, _ := s.UpsertSource(SourceInput{Name: "a", URL: "https://a/f", Type: "rss", Weight: 1.0, Enabled: true})+ src2, _ := s.UpsertSource(SourceInput{Name: "b", URL: "https://b/f", Type: "rss", Weight: 0.8, Enabled: true})++ a1, _ := s.InsertArticle(Article{SourceID: src1, CanonicalURL: "https://a/1", ContentHash: "c1", TitleHash: "t1", Title: "Story one", PublishedAt: 1000})+ a2, _ := s.InsertArticle(Article{SourceID: src2, CanonicalURL: "https://b/1", ContentHash: "c2", TitleHash: "t2", Title: "Story two", PublishedAt: 2000})++ if err := s.UpsertCVEStub("CVE-2026-1"); err != nil {+ t.Fatal(err)+ }+ score := 9.8+ if err := s.UpdateCVEEnrichment(CVE{ID: "CVE-2026-1", CVSSScore: &score, IsKEV: true, EnrichedAt: 1, EnrichStatus: EnrichStatusOK}); err != nil {+ t.Fatal(err)+ }+ if err := s.LinkArticleCVE(a1, "CVE-2026-1"); err != nil {+ t.Fatal(err)+ }+ if err := s.LinkArticleCVE(a2, "CVE-2026-1"); err != nil {+ t.Fatal(err)+ }++ if err := s.ReplaceClusters([]ClusterRow{+ {Key: "1", Members: []int64{a1, a2}, FirstSeen: 1000, LastSeen: 2000},+ }); err != nil {+ t.Fatal(err)+ }++ clusters, err := s.DigestClusters(0)+ if err != nil {+ t.Fatal(err)+ }+ if len(clusters) != 1 {+ t.Fatalf("clusters = %d, want 1", len(clusters))+ }+ c := clusters[0]+ if len(c.Articles) != 2 {+ t.Errorf("articles = %d, want 2", len(c.Articles))+ }+ if len(c.CVEs) != 1 {+ t.Fatalf("cves = %d, want 1 (deduped across both articles)", len(c.CVEs))+ }+ if !c.CVEs[0].IsKEV || c.CVEs[0].CVSSScore == nil || *c.CVEs[0].CVSSScore != 9.8 {+ t.Errorf("cve signals not aggregated: %+v", c.CVEs[0])+ }+}++func TestDigestClustersAttachToCorrectCluster(t *testing.T) {+ s := openTemp(t)+ src, _ := s.UpsertSource(SourceInput{Name: "a", URL: "https://a/f", Type: "rss", Weight: 1, Enabled: true})++ a1, _ := s.InsertArticle(Article{SourceID: src, CanonicalURL: "https://a/1", ContentHash: "c1", TitleHash: "t1", Title: "one"})+ a2, _ := s.InsertArticle(Article{SourceID: src, CanonicalURL: "https://a/2", ContentHash: "c2", TitleHash: "t2", Title: "two"})++ for _, id := range []string{"CVE-2026-1", "CVE-2026-2"} {+ if err := s.UpsertCVEStub(id); err != nil {+ t.Fatal(err)+ }+ }+ _ = s.LinkArticleCVE(a1, "CVE-2026-1")+ _ = s.LinkArticleCVE(a2, "CVE-2026-2")++ if err := s.ReplaceClusters([]ClusterRow{+ {Key: "1", Members: []int64{a1}, FirstSeen: 1000, LastSeen: 1000},+ {Key: "2", Members: []int64{a2}, FirstSeen: 2000, LastSeen: 2000},+ }); err != nil {+ t.Fatal(err)+ }++ clusters, err := s.DigestClusters(0)+ if err != nil {+ t.Fatal(err)+ }+ if len(clusters) != 2 {+ t.Fatalf("clusters = %d, want 2", len(clusters))+ }+ byArticle := map[int64]DigestCluster{}+ for _, c := range clusters {+ if len(c.Articles) != 1 || len(c.CVEs) != 1 {+ t.Fatalf("cluster %d: articles=%d cves=%d, want 1/1", c.ClusterID, len(c.Articles), len(c.CVEs))+ }+ byArticle[c.Articles[0].ID] = c+ }+ if byArticle[a1].CVEs[0].ID != "CVE-2026-1" || byArticle[a2].CVEs[0].ID != "CVE-2026-2" {+ t.Error("CVEs attached to the wrong cluster")+ }+}++func TestDigestClustersSinceFilter(t *testing.T) {+ s := openTemp(t)+ src, _ := s.UpsertSource(SourceInput{Name: "a", URL: "https://a/f", Type: "rss", Weight: 1, Enabled: true})+ a1, _ := s.InsertArticle(Article{SourceID: src, CanonicalURL: "https://a/1", ContentHash: "c1", TitleHash: "t1", Title: "old"})+ if err := s.ReplaceClusters([]ClusterRow{{Key: "1", Members: []int64{a1}, FirstSeen: 500, LastSeen: 500}}); err != nil {+ t.Fatal(err)+ }+ if got, _ := s.DigestClusters(1000); len(got) != 0 {+ t.Errorf("cluster with last_seen 500 should be excluded by since=1000, got %d", len(got))+ }+ if got, _ := s.DigestClusters(0); len(got) != 1 {+ t.Errorf("since=0 should include it, got %d", len(got))+ }+}diff --git a/PROJECTS/intermediate/security-news-scraper/internal/store/store.go b/PROJECTS/intermediate/security-news-scraper/internal/store/store.goindex b65e681e..e28bb698 100644--- a/PROJECTS/intermediate/security-news-scraper/internal/store/store.go+++ b/PROJECTS/intermediate/security-news-scraper/internal/store/store.go@@ -295,6 +295,122 @@ func (s *Store) ArticlesForCVE(id string) ([]ArticleSummary, error) {return scanArticleSummaries(rows)}+type DigestArticle struct {+ ID int64+ SourceName string+ SourceWeight float64+ Title string+ CanonicalURL string+ PublishedAt int64+}++type DigestCVE struct {+ ID string+ CVSSScore *float64+ EPSS *float64+ IsKEV bool+}++type DigestCluster struct {+ ClusterID int64+ Key string+ Size int+ FirstSeen int64+ LastSeen int64+ Articles []DigestArticle+ CVEs []DigestCVE+}++func (s *Store) DigestClusters(since int64) ([]DigestCluster, error) {+ byID, order, err := s.digestClusterRows(since)+ if err != nil {+ return nil, err+ }+ if err := s.digestAttachArticles(byID); err != nil {+ return nil, err+ }+ if err := s.digestAttachCVEs(byID); err != nil {+ return nil, err+ }+ out := make([]DigestCluster, 0, len(order))+ for _, id := range order {+ out = append(out, *byID[id])+ }+ return out, nil+}++func (s *Store) digestClusterRows(since int64) (map[int64]*DigestCluster, []int64, error) {+ rows, err := s.db.Query(`+ SELECT id, cluster_key, size, first_seen, last_seen+ FROM clusters WHERE last_seen >= ? ORDER BY id`, since)+ if err != nil {+ return nil, nil, fmt.Errorf("digest clusters: %w", err)+ }+ defer rows.Close()+ byID := make(map[int64]*DigestCluster)+ var order []int64+ for rows.Next() {+ var dc DigestCluster+ if err := rows.Scan(&dc.ClusterID, &dc.Key, &dc.Size, &dc.FirstSeen, &dc.LastSeen); err != nil {+ return nil, nil, fmt.Errorf("digest clusters: scan: %w", err)+ }+ clone := dc+ byID[dc.ClusterID] = &clone+ order = append(order, dc.ClusterID)+ }+ return byID, order, rows.Err()+}++func (s *Store) digestAttachArticles(byID map[int64]*DigestCluster) error {+ rows, err := s.db.Query(`+ SELECT cm.cluster_id, a.id, s.name, s.weight, a.title, a.canonical_url, a.published_at+ FROM cluster_members cm+ JOIN articles a ON a.id = cm.article_id+ JOIN sources s ON s.id = a.source_id+ ORDER BY cm.cluster_id, a.id`)+ if err != nil {+ return fmt.Errorf("digest articles: %w", err)+ }+ defer rows.Close()+ for rows.Next() {+ var clusterID int64+ var a DigestArticle+ if err := rows.Scan(&clusterID, &a.ID, &a.SourceName, &a.SourceWeight, &a.Title, &a.CanonicalURL, &a.PublishedAt); err != nil {+ return fmt.Errorf("digest articles: scan: %w", err)+ }+ if dc, ok := byID[clusterID]; ok {+ dc.Articles = append(dc.Articles, a)+ }+ }+ return rows.Err()+}++func (s *Store) digestAttachCVEs(byID map[int64]*DigestCluster) error {+ rows, err := s.db.Query(`+ SELECT DISTINCT cm.cluster_id, c.id, c.cvss_score, c.epss, c.is_kev+ FROM cluster_members cm+ JOIN article_cves ac ON ac.article_id = cm.article_id+ JOIN cves c ON c.id = ac.cve_id+ ORDER BY cm.cluster_id, c.id`)+ if err != nil {+ return fmt.Errorf("digest cves: %w", err)+ }+ defer rows.Close()+ for rows.Next() {+ var clusterID int64+ var v DigestCVE+ var isKEV int+ if err := rows.Scan(&clusterID, &v.ID, &v.CVSSScore, &v.EPSS, &isKEV); err != nil {+ return fmt.Errorf("digest cves: scan: %w", err)+ }+ v.IsKEV = isKEV != 0+ if dc, ok := byID[clusterID]; ok {+ dc.CVEs = append(dc.CVEs, v)+ }+ }+ return rows.Err()+}+func (s *Store) UpsertCVEStub(id string) error {_, err := s.db.Exec(`INSERT INTO cves (id) VALUES (?) ON CONFLICT(id) DO NOTHING`, id)if err != nil {