Building 70 Projects ranging from beginner to advanced so anyone can — learn from, build upon, use as a reference, or even copy directly. Owned by my cybersecurity company đŸ‘‡
Go aicertificationcloud-engineeringcybercybersecuritycybersecurity-analystcybersecurity-certificationcybersecurity-certificationscybersecurity-educationcybersecurity-jobscybersecurity-portfoliocybersecurity-projectscybersecurity-toolsgrchackinglearningpentestingproject-repositoryprojetcspython
feat(nadezhda): M2 cross-source dedup + clustering
CarterPerez-dev312b13b3488f
diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/scrape.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/scrape.goindex 6d3c5587..0b49e69d 100644--- a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/scrape.go+++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/scrape.go@@ -12,12 +12,15 @@ import ("github.com/spf13/cobra"+ "github.com/CarterPerez-dev/nadezhda/internal/cluster""github.com/CarterPerez-dev/nadezhda/internal/fetch""github.com/CarterPerez-dev/nadezhda/internal/ingest""github.com/CarterPerez-dev/nadezhda/internal/source""github.com/CarterPerez-dev/nadezhda/internal/store")+const secondsPerHour = 3600+const (statusNotModified = "304"statusError = "error"@@ -70,12 +73,21 @@ func runScrape(cmd *cobra.Command, args []string) error {ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt)defer stop()- summary, err := ingest.Run(ctx, fc, st, cfg, targets, time.Now())+ now := time.Now()+ summary, err := ingest.Run(ctx, fc, st, cfg, targets, now)if err != nil {return err}-printSummary(cmd, summary)++ sinceUnix := now.Unix() - int64(cfg.Cluster.LookbackHours)*secondsPerHour+ windowSeconds := int64(cfg.Cluster.WindowHours) * secondsPerHour+ stats, err := cluster.Rebuild(st, cfg.Cluster.TitleJaccard, windowSeconds, sinceUnix)+ if err != nil {+ return err+ }+ fmt.Fprintf(cmd.OutOrStdout(), "%d clusters (%d multi-source, largest %d)\n",+ stats.Total, stats.MultiSource, stats.LargestSize)return nil}diff --git a/PROJECTS/intermediate/security-news-scraper/internal/cluster/cluster.go b/PROJECTS/intermediate/security-news-scraper/internal/cluster/cluster.gonew file mode 100644index 00000000..2176e2df--- /dev/null+++ b/PROJECTS/intermediate/security-news-scraper/internal/cluster/cluster.go@@ -0,0 +1,208 @@+// ©AngelaMos | 2026+// cluster.go++package cluster++import (+ "sort"+ "strconv"+ "strings"++ "github.com/CarterPerez-dev/nadezhda/internal/normalize"+)++type Item struct {+ ID int64+ SourceID int64+ Title string+ CVEs []string+ Time int64+}++type Cluster struct {+ Key string+ Members []int64+ SourceCount int+ FirstSeen int64+ LastSeen int64+}++type tokenized struct {+ item Item+ tokens map[string]struct{}+ cves map[string]struct{}+}++func Compute(items []Item, jaccardThreshold float64, windowSeconds int64) []Cluster {+ n := len(items)+ prepared := make([]tokenized, n)+ for i, it := range items {+ prepared[i] = tokenized{+ item: it,+ tokens: tokenSet(it.Title),+ cves: stringSet(it.CVEs),+ }+ }++ uf := newUnionFind(n)+ for i := 0; i < n; i++ {+ for j := i + 1; j < n; j++ {+ if abs64(prepared[i].item.Time-prepared[j].item.Time) > windowSeconds {+ continue+ }+ sharedCVE := shareAny(prepared[i].cves, prepared[j].cves)+ crossSource := prepared[i].item.SourceID != prepared[j].item.SourceID+ titleMatch := crossSource &&+ jaccard(prepared[i].tokens, prepared[j].tokens) >= jaccardThreshold+ if sharedCVE || titleMatch {+ uf.union(i, j)+ }+ }+ }++ groups := make(map[int][]int)+ for i := 0; i < n; i++ {+ root := uf.find(i)+ groups[root] = append(groups[root], i)+ }++ clusters := make([]Cluster, 0, len(groups))+ for _, members := range groups {+ clusters = append(clusters, buildCluster(prepared, members))+ }+ sort.Slice(clusters, func(a, b int) bool {+ if clusters[a].FirstSeen != clusters[b].FirstSeen {+ return clusters[a].FirstSeen < clusters[b].FirstSeen+ }+ return clusters[a].Members[0] < clusters[b].Members[0]+ })+ return clusters+}++func buildCluster(prepared []tokenized, members []int) Cluster {+ earliest := prepared[members[0]].item+ ids := make([]int64, 0, len(members))+ sources := make(map[int64]struct{}, len(members))+ first := earliest.Time+ last := earliest.Time+ for _, m := range members {+ it := prepared[m].item+ ids = append(ids, it.ID)+ sources[it.SourceID] = struct{}{}+ if it.Time < first {+ first = it.Time+ }+ if it.Time > last {+ last = it.Time+ }+ if it.Time < earliest.Time || (it.Time == earliest.Time && it.ID < earliest.ID) {+ earliest = it+ }+ }+ sort.Slice(ids, func(a, b int) bool { return ids[a] < ids[b] })+ return Cluster{+ Key: strconv.FormatInt(earliest.ID, 10),+ Members: ids,+ SourceCount: len(sources),+ FirstSeen: first,+ LastSeen: last,+ }+}++func tokenSet(title string) map[string]struct{} {+ fields := strings.Fields(normalize.NormalizeTitle(title))+ set := make(map[string]struct{}, len(fields))+ for _, f := range fields {+ set[f] = struct{}{}+ }+ return set+}++func stringSet(values []string) map[string]struct{} {+ set := make(map[string]struct{}, len(values))+ for _, v := range values {+ if v != "" {+ set[v] = struct{}{}+ }+ }+ return set+}++func jaccard(a, b map[string]struct{}) float64 {+ if len(a) == 0 || len(b) == 0 {+ return 0+ }+ small, large := a, b+ if len(small) > len(large) {+ small, large = large, small+ }+ intersection := 0+ for k := range small {+ if _, ok := large[k]; ok {+ intersection+++ }+ }+ union := len(a) + len(b) - intersection+ if union == 0 {+ return 0+ }+ return float64(intersection) / float64(union)+}++func shareAny(a, b map[string]struct{}) bool {+ if len(a) == 0 || len(b) == 0 {+ return false+ }+ small, large := a, b+ if len(small) > len(large) {+ small, large = large, small+ }+ for k := range small {+ if _, ok := large[k]; ok {+ return true+ }+ }+ return false+}++func abs64(v int64) int64 {+ if v < 0 {+ return -v+ }+ return v+}++type unionFind struct {+ parent []int+ rank []int+}++func newUnionFind(n int) *unionFind {+ uf := &unionFind{parent: make([]int, n), rank: make([]int, n)}+ for i := range uf.parent {+ uf.parent[i] = i+ }+ return uf+}++func (uf *unionFind) find(x int) int {+ for uf.parent[x] != x {+ uf.parent[x] = uf.parent[uf.parent[x]]+ x = uf.parent[x]+ }+ return x+}++func (uf *unionFind) union(a, b int) {+ ra, rb := uf.find(a), uf.find(b)+ if ra == rb {+ return+ }+ if uf.rank[ra] < uf.rank[rb] {+ ra, rb = rb, ra+ }+ uf.parent[rb] = ra+ if uf.rank[ra] == uf.rank[rb] {+ uf.rank[ra]+++ }+}diff --git a/PROJECTS/intermediate/security-news-scraper/internal/cluster/cluster_test.go b/PROJECTS/intermediate/security-news-scraper/internal/cluster/cluster_test.gonew file mode 100644index 00000000..4c93afb9--- /dev/null+++ b/PROJECTS/intermediate/security-news-scraper/internal/cluster/cluster_test.go@@ -0,0 +1,148 @@+// ©AngelaMos | 2026+// cluster_test.go++package cluster++import (+ "sort"+ "testing"+)++const bigWindow = 1 << 40++func sizes(clusters []Cluster) []int {+ out := make([]int, len(clusters))+ for i, c := range clusters {+ out[i] = len(c.Members)+ }+ sort.Ints(out)+ return out+}++func findCluster(clusters []Cluster, id int64) Cluster {+ for _, c := range clusters {+ for _, m := range c.Members {+ if m == id {+ return c+ }+ }+ }+ return Cluster{}+}++func TestComputeTitleSimilarityGroups(t *testing.T) {+ items := []Item{+ {ID: 1, SourceID: 1, Title: "Acme Corp data breach exposes millions", Time: 100},+ {ID: 2, SourceID: 2, Title: "Acme Corp data breach exposed millions", Time: 200},+ {ID: 3, SourceID: 3, Title: "Acme Corp data breach hits millions", Time: 300},+ {ID: 4, SourceID: 4, Title: "Linux kernel 7 released with new scheduler", Time: 400},+ }+ clusters := Compute(items, 0.6, bigWindow)++ if got := sizes(clusters); len(got) != 2 || got[0] != 1 || got[1] != 3 {+ t.Fatalf("cluster sizes = %v, want [1 3]", got)+ }+ c := findCluster(clusters, 1)+ if len(c.Members) != 3 {+ t.Errorf("article 1 cluster size = %d, want 3", len(c.Members))+ }+ if c.SourceCount != 3 {+ t.Errorf("SourceCount = %d, want 3 (three distinct outlets)", c.SourceCount)+ }+ if c.Key != "1" {+ t.Errorf("cluster key = %q, want 1 (earliest article)", c.Key)+ }+ if c.FirstSeen != 100 || c.LastSeen != 300 {+ t.Errorf("span = [%d,%d], want [100,300]", c.FirstSeen, c.LastSeen)+ }+}++func TestComputeSameSourceTitlesDoNotMerge(t *testing.T) {+ items := []Item{+ {ID: 1, SourceID: 1, Title: "CISA Adds One Known Exploited Vulnerability to Catalog", Time: 100},+ {ID: 2, SourceID: 1, Title: "CISA Adds One Known Exploited Vulnerability to Catalog", Time: 200},+ }+ if got := sizes(Compute(items, 0.6, bigWindow)); len(got) != 2 {+ t.Errorf("same-source identical titles sizes = %v, want [1 1] (clustering is cross-outlet)", got)+ }+}++func TestComputeSharedCVEJoinsRegardlessOfSource(t *testing.T) {+ items := []Item{+ {ID: 1, SourceID: 1, Title: "Router vendor patches critical flaw", Time: 100, CVEs: []string{"CVE-2026-1111"}},+ {ID: 2, SourceID: 1, Title: "Enterprise gear gets emergency update", Time: 200, CVEs: []string{"CVE-2026-1111"}},+ {ID: 3, SourceID: 2, Title: "Totally unrelated privacy story", Time: 300, CVEs: []string{"CVE-2026-9999"}},+ }+ clusters := Compute(items, 0.6, bigWindow)+ if got := sizes(clusters); len(got) != 2 || got[0] != 1 || got[1] != 2 {+ t.Fatalf("cluster sizes = %v, want [1 2]", got)+ }+ joined := findCluster(clusters, 1)+ if len(joined.Members) != 2 {+ t.Error("shared-CVE articles should join even from the same source")+ }+ if joined.SourceCount != 1 {+ t.Errorf("SourceCount = %d, want 1 (a same-source cluster is NOT multi-source)", joined.SourceCount)+ }+}++func TestComputeThresholdBoundary(t *testing.T) {+ items := []Item{+ {ID: 1, SourceID: 1, Title: "alpha bravo charlie delta", Time: 100},+ {ID: 2, SourceID: 2, Title: "alpha bravo charlie echo", Time: 200},+ }+ if got := sizes(Compute(items, 0.6, bigWindow)); len(got) != 1 || got[0] != 2 {+ t.Errorf("at threshold 0.6 (jaccard==0.6) sizes = %v, want [2] (>= joins)", got)+ }+ if got := sizes(Compute(items, 0.61, bigWindow)); len(got) != 2 {+ t.Errorf("at threshold 0.61 sizes = %v, want [1 1] (below threshold)", got)+ }+}++func TestComputeWindowSeparates(t *testing.T) {+ items := []Item{+ {ID: 1, SourceID: 1, Title: "identical breaking headline text", Time: 0},+ {ID: 2, SourceID: 2, Title: "identical breaking headline text", Time: 200},+ }+ if got := sizes(Compute(items, 0.6, 100)); len(got) != 2 {+ t.Errorf("outside window sizes = %v, want [1 1] (time gap > window)", got)+ }+ if got := sizes(Compute(items, 0.6, 300)); len(got) != 1 || got[0] != 2 {+ t.Errorf("inside window sizes = %v, want [2]", got)+ }+}++func TestComputeKeyIsEarliestByTime(t *testing.T) {+ items := []Item{+ {ID: 10, SourceID: 1, Title: "shared story about the incident", Time: 500},+ {ID: 20, SourceID: 2, Title: "shared story about the incident", Time: 100},+ }+ clusters := Compute(items, 0.6, bigWindow)+ if len(clusters) != 1 {+ t.Fatalf("clusters = %d, want 1", len(clusters))+ }+ if clusters[0].Key != "20" {+ t.Errorf("key = %q, want 20 (earliest by time, not lowest id)", clusters[0].Key)+ }+ if clusters[0].FirstSeen != 100 || clusters[0].LastSeen != 500 {+ t.Errorf("span = [%d,%d], want [100,500]", clusters[0].FirstSeen, clusters[0].LastSeen)+ }+}++func TestComputeTransitiveChain(t *testing.T) {+ items := []Item{+ {ID: 1, SourceID: 1, Title: "one two three four five", Time: 100},+ {ID: 2, SourceID: 2, Title: "one two three four six", Time: 100},+ {ID: 3, SourceID: 3, Title: "one two three six five", Time: 100},+ }+ clusters := Compute(items, 0.6, bigWindow)+ if len(clusters) != 1 || len(clusters[0].Members) != 3 {+ t.Errorf("transitive similarity should merge all three, got %d clusters", len(clusters))+ }+}++func TestComputeEmpty(t *testing.T) {+ if c := Compute(nil, 0.6, bigWindow); len(c) != 0 {+ t.Errorf("empty input -> %d clusters, want 0", len(c))+ }+}diff --git a/PROJECTS/intermediate/security-news-scraper/internal/cluster/rebuild.go b/PROJECTS/intermediate/security-news-scraper/internal/cluster/rebuild.gonew file mode 100644index 00000000..b1103f43--- /dev/null+++ b/PROJECTS/intermediate/security-news-scraper/internal/cluster/rebuild.go@@ -0,0 +1,51 @@+// ©AngelaMos | 2026+// rebuild.go++package cluster++import (+ "github.com/CarterPerez-dev/nadezhda/internal/store"+)++type Stats struct {+ Total int+ MultiSource int+ LargestSize int+}++func Rebuild(st *store.Store, jaccardThreshold float64, windowSeconds, sinceUnix int64) (Stats, error) {+ candidates, err := st.ClusterCandidates(sinceUnix)+ if err != nil {+ return Stats{}, err+ }+ cveMap, err := st.ArticleCVEMap()+ if err != nil {+ return Stats{}, err+ }++ items := make([]Item, len(candidates))+ for i, c := range candidates {+ items[i] = Item{ID: c.ID, SourceID: c.SourceID, Title: c.Title, Time: c.Time, CVEs: cveMap[c.ID]}+ }++ clusters := Compute(items, jaccardThreshold, windowSeconds)++ rows := make([]store.ClusterRow, len(clusters))+ stats := Stats{Total: len(clusters)}+ for i, c := range clusters {+ rows[i] = store.ClusterRow{+ Key: c.Key, Members: c.Members, FirstSeen: c.FirstSeen, LastSeen: c.LastSeen,+ }+ if c.SourceCount > 1 {+ stats.MultiSource+++ }+ if size := len(c.Members); size > stats.LargestSize {+ stats.LargestSize = size+ }+ }++ if err := st.ReplaceClusters(rows); err != nil {+ return Stats{}, err+ }+ return stats, nil+}diff --git a/PROJECTS/intermediate/security-news-scraper/internal/cluster/rebuild_test.go b/PROJECTS/intermediate/security-news-scraper/internal/cluster/rebuild_test.gonew file mode 100644index 00000000..6204d029--- /dev/null+++ b/PROJECTS/intermediate/security-news-scraper/internal/cluster/rebuild_test.go@@ -0,0 +1,133 @@+// ©AngelaMos | 2026+// rebuild_test.go++package cluster++import (+ "fmt"+ "path/filepath"+ "testing"++ "github.com/CarterPerez-dev/nadezhda/internal/store"+)++func openStore(t *testing.T) *store.Store {+ t.Helper()+ st, err := store.Open(filepath.Join(t.TempDir(), "cluster.db"))+ if err != nil {+ t.Fatalf("open store: %v", err)+ }+ t.Cleanup(func() { _ = st.Close() })+ return st+}++func seedArticles(t *testing.T, st *store.Store) {+ t.Helper()+ titles := []string{+ "Acme Corp data breach exposes millions",+ "Acme Corp data breach exposed millions",+ "Acme Corp data breach hits millions",+ "Linux kernel 7 released with new scheduler",+ }+ for i, title := range titles {+ sourceID, err := st.UpsertSource(store.SourceInput{+ Name: fmt.Sprintf("src%d", i), URL: fmt.Sprintf("https://outlet%d.example/feed", i),+ Type: "rss", Weight: 1, Enabled: true,+ })+ if err != nil {+ t.Fatalf("upsert source %d: %v", i, err)+ }+ if _, err := st.InsertArticle(store.Article{+ SourceID: sourceID,+ CanonicalURL: fmt.Sprintf("https://example.com/a/%d", i),+ ContentHash: fmt.Sprintf("chash-%d", i),+ TitleHash: fmt.Sprintf("thash-%d", i),+ Title: title,+ PublishedAt: int64(100 * (i + 1)),+ FetchedAt: int64(100 * (i + 1)),+ }); err != nil {+ t.Fatalf("insert article %d: %v", i, err)+ }+ }+}++func clusterSizes(t *testing.T, st *store.Store) []int {+ t.Helper()+ rows, err := st.DB().Query(`SELECT size FROM clusters ORDER BY size`)+ if err != nil {+ t.Fatal(err)+ }+ defer rows.Close()+ var out []int+ for rows.Next() {+ var n int+ if err := rows.Scan(&n); err != nil {+ t.Fatal(err)+ }+ out = append(out, n)+ }+ return out+}++func countMembers(t *testing.T, st *store.Store) int {+ t.Helper()+ var n int+ if err := st.DB().QueryRow(`SELECT COUNT(*) FROM cluster_members`).Scan(&n); err != nil {+ t.Fatal(err)+ }+ return n+}++func TestRebuildClustersEndToEnd(t *testing.T) {+ st := openStore(t)+ seedArticles(t, st)++ stats, err := Rebuild(st, 0.6, bigWindow, 0)+ if err != nil {+ t.Fatalf("Rebuild: %v", err)+ }+ if stats.Total != 2 || stats.MultiSource != 1 || stats.LargestSize != 3 {+ t.Fatalf("stats = %+v, want {Total:2 MultiSource:1 LargestSize:3}", stats)+ }+ if got := clusterSizes(t, st); len(got) != 2 || got[0] != 1 || got[1] != 3 {+ t.Errorf("cluster sizes = %v, want [1 3]", got)+ }+ if m := countMembers(t, st); m != 4 {+ t.Errorf("cluster_members = %d, want 4 (every article assigned once)", m)+ }+}++func TestRebuildIsIdempotent(t *testing.T) {+ st := openStore(t)+ seedArticles(t, st)++ if _, err := Rebuild(st, 0.6, bigWindow, 0); err != nil {+ t.Fatalf("first Rebuild: %v", err)+ }+ stats, err := Rebuild(st, 0.6, bigWindow, 0)+ if err != nil {+ t.Fatalf("second Rebuild: %v", err)+ }+ if stats.Total != 2 {+ t.Errorf("second run clusters = %d, want 2 (rebuild replaces, no accumulation)", stats.Total)+ }+ if m := countMembers(t, st); m != 4 {+ t.Errorf("cluster_members = %d, want 4 after re-run", m)+ }+}++func TestRebuildLookbackExcludesOld(t *testing.T) {+ st := openStore(t)+ seedArticles(t, st)++ stats, err := Rebuild(st, 0.6, bigWindow, 250)+ if err != nil {+ t.Fatalf("Rebuild: %v", err)+ }+ if stats.Total != 2 {+ t.Errorf("with since=250 only articles at t>=250 cluster, total = %d, want 2", stats.Total)+ }+ if m := countMembers(t, st); m != 2 {+ t.Errorf("members = %d, want 2 (two articles below lookback excluded)", m)+ }+}diff --git a/PROJECTS/intermediate/security-news-scraper/internal/config/config.go b/PROJECTS/intermediate/security-news-scraper/internal/config/config.goindex a8692f5f..1f2677f2 100644--- a/PROJECTS/intermediate/security-news-scraper/internal/config/config.go+++ b/PROJECTS/intermediate/security-news-scraper/internal/config/config.go@@ -24,8 +24,9 @@ const (defaultCacheTTLHours = 24defaultNegativeTTLHours = 3- defaultTitleJaccard = 0.6- defaultWindowHours = 72+ defaultTitleJaccard = 0.6+ defaultWindowHours = 72+ defaultLookbackHours = 168trackingUTMPrefix = "utm_*"@@ -73,6 +74,7 @@ type Enrich struct {type Cluster struct {TitleJaccard float64 `yaml:"title_jaccard_threshold"`WindowHours int `yaml:"window_hours"`+ LookbackHours int `yaml:"lookback_hours"`TrackingParams []string `yaml:"tracking_params"`}@@ -136,6 +138,7 @@ func Default() Config {Cluster: Cluster{TitleJaccard: defaultTitleJaccard,WindowHours: defaultWindowHours,+ LookbackHours: defaultLookbackHours,TrackingParams: defaultTrackingParams,},Rank: Rank{@@ -208,6 +211,15 @@ func (c Config) validate() error {if c.Cluster.TitleJaccard < 0 || c.Cluster.TitleJaccard > 1 {return fmt.Errorf("config: cluster.title_jaccard_threshold must be in [0,1], got %v", c.Cluster.TitleJaccard)}+ if c.Cluster.WindowHours < 1 {+ return fmt.Errorf("config: cluster.window_hours must be >= 1, got %d", c.Cluster.WindowHours)+ }+ if c.Cluster.LookbackHours < 1 {+ return fmt.Errorf("config: cluster.lookback_hours must be >= 1, got %d", c.Cluster.LookbackHours)+ }+ if c.Cluster.LookbackHours < c.Cluster.WindowHours {+ return fmt.Errorf("config: cluster.lookback_hours (%d) must be >= cluster.window_hours (%d) or window edges near the corpus boundary are silently dropped", c.Cluster.LookbackHours, c.Cluster.WindowHours)+ }if c.Rank.HalfLifeHours < 1 {return fmt.Errorf("config: rank.half_life_hours must be >= 1, got %d", c.Rank.HalfLifeHours)}diff --git a/PROJECTS/intermediate/security-news-scraper/internal/config/config_test.go b/PROJECTS/intermediate/security-news-scraper/internal/config/config_test.goindex 9a490a58..2eaaf370 100644--- a/PROJECTS/intermediate/security-news-scraper/internal/config/config_test.go+++ b/PROJECTS/intermediate/security-news-scraper/internal/config/config_test.go@@ -92,3 +92,15 @@ func TestValidateRejectsBadProvider(t *testing.T) {t.Error("expected error for invalid ai.provider")}}++func TestValidateRejectsLookbackBelowWindow(t *testing.T) {+ dir := t.TempDir()+ path := filepath.Join(dir, "lb.yaml")+ body := "cluster:\n window_hours: 72\n lookback_hours: 24\n"+ if err := os.WriteFile(path, []byte(body), 0o600); err != nil {+ t.Fatal(err)+ }+ if _, err := Load(path); err == nil {+ t.Error("expected error when lookback_hours < window_hours")+ }+}diff --git a/PROJECTS/intermediate/security-news-scraper/internal/store/store.go b/PROJECTS/intermediate/security-news-scraper/internal/store/store.goindex a19dddc7..b56b7588 100644--- a/PROJECTS/intermediate/security-news-scraper/internal/store/store.go+++ b/PROJECTS/intermediate/security-news-scraper/internal/store/store.go@@ -179,6 +179,100 @@ func (s *Store) UpsertFetchState(sourceID int64, fs FetchState) error {return nil}+type CandidateArticle struct {+ ID int64+ SourceID int64+ Title string+ Time int64+}++type ClusterRow struct {+ Key string+ Members []int64+ FirstSeen int64+ LastSeen int64+}++func (s *Store) ClusterCandidates(since int64) ([]CandidateArticle, error) {+ rows, err := s.db.Query(`+ SELECT id, source_id, title, COALESCE(NULLIF(published_at, 0), fetched_at) AS t+ FROM articles+ WHERE COALESCE(NULLIF(published_at, 0), fetched_at) >= ?+ ORDER BY id`, since)+ if err != nil {+ return nil, fmt.Errorf("cluster candidates: %w", err)+ }+ defer rows.Close()++ var out []CandidateArticle+ for rows.Next() {+ var c CandidateArticle+ if err := rows.Scan(&c.ID, &c.SourceID, &c.Title, &c.Time); err != nil {+ return nil, fmt.Errorf("cluster candidates: scan: %w", err)+ }+ out = append(out, c)+ }+ return out, rows.Err()+}++func (s *Store) ArticleCVEMap() (map[int64][]string, error) {+ rows, err := s.db.Query(`SELECT article_id, cve_id FROM article_cves`)+ if err != nil {+ return nil, fmt.Errorf("article cve map: %w", err)+ }+ defer rows.Close()++ out := make(map[int64][]string)+ for rows.Next() {+ var articleID int64+ var cveID string+ if err := rows.Scan(&articleID, &cveID); err != nil {+ return nil, fmt.Errorf("article cve map: scan: %w", err)+ }+ out[articleID] = append(out[articleID], cveID)+ }+ return out, rows.Err()+}++func (s *Store) ReplaceClusters(rows []ClusterRow) error {+ tx, err := s.db.Begin()+ if err != nil {+ return fmt.Errorf("replace clusters: begin: %w", err)+ }+ defer func() { _ = tx.Rollback() }()++ if _, err := tx.Exec(`DELETE FROM cluster_members`); err != nil {+ return fmt.Errorf("replace clusters: clear members: %w", err)+ }+ if _, err := tx.Exec(`DELETE FROM clusters`); err != nil {+ return fmt.Errorf("replace clusters: clear clusters: %w", err)+ }++ for _, r := range rows {+ var clusterID int64+ if err := tx.QueryRow(`+ INSERT INTO clusters (cluster_key, first_seen, last_seen, size)+ VALUES (?, ?, ?, ?) RETURNING id`,+ r.Key, r.FirstSeen, r.LastSeen, len(r.Members),+ ).Scan(&clusterID); err != nil {+ return fmt.Errorf("replace clusters: insert cluster %q: %w", r.Key, err)+ }+ for _, articleID := range r.Members {+ if _, err := tx.Exec(`+ INSERT INTO cluster_members (cluster_id, article_id) VALUES (?, ?)`,+ clusterID, articleID,+ ); err != nil {+ return fmt.Errorf("replace clusters: insert member %d: %w", articleID, err)+ }+ }+ }++ if err := tx.Commit(); err != nil {+ return fmt.Errorf("replace clusters: commit: %w", err)+ }+ return nil+}+func boolToInt(b bool) int {if b {return 1