ruvnet.
Python
fix: separate registry and GitHub provenance
rUv68750e999c8b
diff --git a/data/metrics.json b/data/metrics.jsonindex 41ec9a7..46c1f7c 100644--- a/data/metrics.json+++ b/data/metrics.json@@ -65,7 +65,9 @@],"npm_14_day_complete_utc_days": 14,"npm_14_day_confirmed_days": 9,- "npm_14_day_is_rolling_24h": false+ "npm_14_day_is_rolling_24h": false,+ "npm_14_day_start": "2026-10-18",+ "npm_14_day_end": "2026-10-31"},"verification_dates": {"github": "2026-11-01T13:57:51+00:00",diff --git a/scripts/render-registry-metrics.mjs b/scripts/render-registry-metrics.mjsindex 6e1a560..915b543 100644--- a/scripts/render-registry-metrics.mjs+++ b/scripts/render-registry-metrics.mjs@@ -7,6 +7,7 @@ const read = p => fs.readFileSync(path.join(root, p), 'utf8');const save = (p, s) => fs.writeFileSync(path.join(root, p), s);const data = JSON.parse(read('data/registry-stats.json'));const metrics = JSON.parse(read('data/metrics.json'));+const runningTotals = JSON.parse(read('data/ecosystem-running-totals.json'));const npm = data.npm, crates = data.crates_io;const format = n => new Intl.NumberFormat('en-US').format(n);const date = value => value.slice(0, 10);@@ -108,7 +109,20 @@ if (latestDay) Object.assign(metrics.current_windows, {npm_latest_confirmed_daily_package_count: cohort,npm_latest_daily_is_rolling_24h: false,});-metrics.notes = metrics.notes.filter(note => !note.startsWith('The npm download cohort is'));+if (runningTotals?.npm) {+ const npmWindowDays = (Date.parse(runningTotals.npm.period_end) - Date.parse(runningTotals.npm.period_start)) / 86400000 + 1;+ Object.assign(metrics.current_windows, {+ npm_14_day_start: runningTotals.npm.period_start,+ npm_14_day_end: runningTotals.npm.period_end,+ npm_14_day_confirmed_download_events: runningTotals.npm.observed_downloads,+ npm_14_day_unknown_days: runningTotals.npm.unknown_days,+ npm_14_day_complete_utc_days: npmWindowDays,+ npm_14_day_confirmed_days: runningTotals.npm.confirmed_days,+ npm_14_day_is_rolling_24h: false,+ });+}+metrics.notes = metrics.notes.filter(note => !note.startsWith('The npm download cohort is') && !note.includes('all-package-zero npm dates'));+if (runningTotals?.npm) metrics.notes.push(`${runningTotals.npm.unknown_days.length} all-package-zero npm dates in ${runningTotals.npm.period_start} through ${runningTotals.npm.period_end} UTC are unknown (null), never zero; ${date(data.generated_at)} is incomplete and excluded.`);metrics.notes.push(`The npm download cohort is ${cohort} packages; the current inventory contains ${npm.package_count}.`);save('data/metrics.json', JSON.stringify(metrics, null, 2) + '\n');console.log(`Rendered ${npm.package_count} npm packages, ${crates.crate_count} crates; downloads dated ${downloadDate}.`);diff --git a/scripts/tests/test_registry_refresh.py b/scripts/tests/test_registry_refresh.pyindex d29a8b5..319f56f 100644--- a/scripts/tests/test_registry_refresh.py+++ b/scripts/tests/test_registry_refresh.py@@ -61,6 +61,30 @@ class RegistryRefreshTest(unittest.TestCase):shutil.copytree(entry,root/entry.name,ignore=shutil.ignore_patterns('__pycache__'))else:shutil.copyfile(entry,root/entry.name)+ registry_path=root/'data/registry-stats.json'+ registry=json.loads(registry_path.read_text())+ refreshed={+ 'npm_inventory':'2026-11-02T05:19:29.846430Z',+ 'npm_downloads':'2026-11-02T05:40:00.000000Z',+ 'crates_io':'2026-11-02T05:40:00.245693Z',+ }+ registry['generated_at']=refreshed['crates_io']+ registry['npm']['inventory_verified_at']=refreshed['npm_inventory']+ registry['npm']['downloads_verified_at']=refreshed['npm_downloads']+ registry['crates_io']['verified_at']=refreshed['crates_io']+ for section,path_key,date_key in [+ (registry['npm'],'inventory_path','npm_inventory'),+ (registry['npm'],'downloads_evidence_path','npm_downloads'),+ (registry['crates_io'],'inventory_path','crates_io')]:+ old_path=root/section[path_key]+ new_rel=f'data/snapshots/2026-11-02/{old_path.name}'+ receipt_path=root/new_rel+ receipt_path.parent.mkdir(parents=True,exist_ok=True)+ receipt=json.loads(old_path.read_text())+ receipt['retrieved_at']=refreshed[date_key]+ receipt_path.write_text(json.dumps(receipt))+ section[path_key]=new_rel+ registry_path.write_text(json.dumps(registry))(root/'input.json').write_text(json.dumps(payload))run=subprocess.run([sys.executable,'scripts/update-ecosystem-snapshot.py','--date',snapshot_date,'--input','input.json'],cwd=root,@@ -85,6 +109,27 @@ class RegistryRefreshTest(unittest.TestCase):running['npm']['unknown_days'])self.assertEqual(metrics['current_windows']['npm_14_day_confirmed_days'],running['npm']['confirmed_days'])+ self.assertEqual(metrics['verification_dates']['npm_inventory'],+ refreshed['npm_inventory'])+ self.assertEqual(metrics['verification_dates']['npm_downloads'],+ refreshed['npm_downloads'])+ self.assertEqual(metrics['verification_dates']['crates_io'],+ refreshed['crates_io'])++ running['npm'].update(+ period_start='2026-10-19',period_end='2026-11-01',+ observed_downloads=123456,unknown_days=['2026-11-01'],+ confirmed_days=13)+ (root/'data/ecosystem-running-totals.json').write_text(json.dumps(running))+ render=subprocess.run(['node','scripts/render-registry-metrics.mjs'],cwd=root,+ capture_output=True,text=True)+ self.assertEqual(render.returncode,0,render.stderr)+ metrics=json.loads((root/'data/metrics.json').read_text())+ self.assertEqual(metrics['current_windows']['npm_14_day_start'],'2026-10-19')+ self.assertEqual(metrics['current_windows']['npm_14_day_end'],'2026-11-01')+ self.assertEqual(metrics['current_windows']['npm_14_day_confirmed_download_events'],123456)+ self.assertEqual(metrics['current_windows']['npm_14_day_unknown_days'],['2026-11-01'])+ self.assertEqual(metrics['current_windows']['npm_14_day_confirmed_days'],13)def test_new_npm_package_404_is_retried_without_inventing_zero(self):url='https://api.npmjs.org/downloads/range/2025-10-01:2026-10-11/new-package'diff --git a/scripts/update-ecosystem-snapshot.py b/scripts/update-ecosystem-snapshot.pyindex 2dde0b2..180e42f 100644--- a/scripts/update-ecosystem-snapshot.py+++ b/scripts/update-ecosystem-snapshot.py@@ -304,6 +304,8 @@ def main():'npm_download_package_count': registry['npm']['download_package_count'],'github_activity_start': window_start,'github_activity_end': window_end,+ 'npm_14_day_start': window_start,+ 'npm_14_day_end': window_end,'npm_14_day_confirmed_download_events': running['npm']['observed_downloads'],'npm_14_day_unknown_days': running['npm']['unknown_days'],'npm_14_day_complete_utc_days': len(expected),diff --git a/scripts/verify-profile.mjs b/scripts/verify-profile.mjsindex 5dbba8a..581db41 100644--- a/scripts/verify-profile.mjs+++ b/scripts/verify-profile.mjs@@ -43,12 +43,11 @@ const watermark = json(expectedWatermarkPath);assert.equal(watermark.repository, 'ruvnet/ruvnet');assert.equal(watermark.snapshot_date, path.posix.basename(path.posix.dirname(expectedWatermarkPath)));assert.equal(watermark.github_verified_at, github.verified_at);-assert.equal(watermark.npm_inventory_verified_at, metrics.verification_dates.npm_inventory);-assert.equal(watermark.npm_downloads_verified_at, metrics.verification_dates.npm_downloads);-assert.equal(watermark.crates_io_verified_at, metrics.verification_dates.crates_io);assert.equal(watermark.public_repository_ids, expected.public_repositories);-assert.equal(watermark.npm_coverage.packages, npm.package_count);-assert.equal(watermark.crates_io_coverage.crates, crates.crate_count);+assert.equal(metrics.verification_dates.github, github.verified_at);+assert.equal(metrics.verification_dates.npm_inventory, npm.inventory_verified_at);+assert.equal(metrics.verification_dates.npm_downloads, npm.downloads_verified_at);+assert.equal(metrics.verification_dates.crates_io, crates.verified_at);assert.equal(metrics.counts.github_public_repositories, expected.public_repositories);assert.equal(metrics.counts.github_stars_owned_nonfork, expected.stars_across_owned_nonfork);assert.equal(metrics.counts.npm_packages, npm.package_count);@@ -63,7 +62,11 @@ const monthIndex = value => Number(value.slice(0, 4)) * 12 + Number(value.slice(assert.equal(npm.monthly_downloads.length, monthIndex(npm.monthly_period_end) - monthIndex(npm.monthly_period_start) + 1);assert.equal(npm.monthly_downloads[0].month, npm.monthly_period_start.slice(0, 7));assert.equal(npm.monthly_downloads.at(-1).month, npm.monthly_period_end.slice(0, 7));-if (npm.inventory_path) assert.equal(json(npm.inventory_path).packages.length, npm.package_count);+if (npm.inventory_path) {+ const npmInventory = json(npm.inventory_path);+ assert.equal(npmInventory.retrieved_at, npm.inventory_verified_at);+ assert.equal(npmInventory.packages.length, npm.package_count);+}if (npm.current_month) {const partial = npm.current_month;assert.ok(partial.month > npm.monthly_downloads.at(-1).month);@@ -73,6 +76,7 @@ if (npm.current_month) {}if (npm.downloads_evidence_path) {const receipt = json(npm.downloads_evidence_path);+ assert.equal(receipt.retrieved_at, npm.downloads_verified_at);assert.equal(receipt.packages.length, npm.download_package_count);assert.equal(new Set(receipt.packages.map(p => p.package)).size, npm.download_package_count);assert.equal(receipt.rolling_start, npm.period_start);@@ -108,7 +112,9 @@ if (npm.downloads_evidence_path) {}}if (crates.inventory_path) {- const list = json(crates.inventory_path).crates;+ const crateInventory = json(crates.inventory_path);+ assert.equal(crateInventory.retrieved_at, crates.verified_at);+ const list = crateInventory.crates;assert.equal(list.length, crates.crate_count);assert.equal(list.reduce((n, c) => n + c.downloads, 0), crates.cumulative_downloads);}@@ -123,10 +129,15 @@ const running = json('data/ecosystem-running-totals.json');assert.deepEqual(running.source_watermark, watermark, 'Embedded source watermark must match its dated file');assert.equal(metrics.current_windows.github_activity_start, watermark.activity_window_utc.start);assert.equal(metrics.current_windows.github_activity_end, watermark.activity_window_utc.end);+assert.equal(metrics.current_windows.npm_14_day_start, running.npm.period_start);+assert.equal(metrics.current_windows.npm_14_day_end, running.npm.period_end);assert.equal(metrics.current_windows.npm_14_day_confirmed_download_events, running.npm.observed_downloads);assert.deepEqual(metrics.current_windows.npm_14_day_unknown_days, running.npm.unknown_days);-assert.equal(metrics.current_windows.npm_14_day_complete_utc_days, watermark.activity_window_utc.complete_days);+assert.equal(metrics.current_windows.npm_14_day_complete_utc_days,+ (Date.parse(running.npm.period_end) - Date.parse(running.npm.period_start)) / 86400000 + 1);assert.equal(metrics.current_windows.npm_14_day_confirmed_days, running.npm.confirmed_days);+assert.equal(metrics.current_windows.npm_14_day_confirmed_days + metrics.current_windows.npm_14_day_unknown_days.length,+ metrics.current_windows.npm_14_day_complete_utc_days);assert.equal(metrics.current_windows.npm_14_day_is_rolling_24h, false);const milestone = running.download_milestone;const periodTotal = npm.monthly_downloads.reduce((sum,row)=>sum+row.downloads,0)+(npm.current_month?.downloads || 0);