Skip to content

ruvnet.

Python

fix: separate registry and GitHub provenance

rUv68750e999c8b
diff --git a/data/metrics.json b/data/metrics.json
index 41ec9a7..46c1f7c 100644
--- a/data/metrics.json
+++ b/data/metrics.json
@@ -65,7 +65,9 @@
],
"npm_14_day_complete_utc_days": 14,
"npm_14_day_confirmed_days": 9,
- "npm_14_day_is_rolling_24h": false
+ "npm_14_day_is_rolling_24h": false,
+ "npm_14_day_start": "2026-10-18",
+ "npm_14_day_end": "2026-10-31"
},
"verification_dates": {
"github": "2026-11-01T13:57:51+00:00",
diff --git a/scripts/render-registry-metrics.mjs b/scripts/render-registry-metrics.mjs
index 6e1a560..915b543 100644
--- a/scripts/render-registry-metrics.mjs
+++ b/scripts/render-registry-metrics.mjs
@@ -7,6 +7,7 @@ const read = p => fs.readFileSync(path.join(root, p), 'utf8');
const save = (p, s) => fs.writeFileSync(path.join(root, p), s);
const data = JSON.parse(read('data/registry-stats.json'));
const metrics = JSON.parse(read('data/metrics.json'));
+const runningTotals = JSON.parse(read('data/ecosystem-running-totals.json'));
const npm = data.npm, crates = data.crates_io;
const format = n => new Intl.NumberFormat('en-US').format(n);
const date = value => value.slice(0, 10);
@@ -108,7 +109,20 @@ if (latestDay) Object.assign(metrics.current_windows, {
npm_latest_confirmed_daily_package_count: cohort,
npm_latest_daily_is_rolling_24h: false,
});
-metrics.notes = metrics.notes.filter(note => !note.startsWith('The npm download cohort is'));
+if (runningTotals?.npm) {
+ const npmWindowDays = (Date.parse(runningTotals.npm.period_end) - Date.parse(runningTotals.npm.period_start)) / 86400000 + 1;
+ Object.assign(metrics.current_windows, {
+ npm_14_day_start: runningTotals.npm.period_start,
+ npm_14_day_end: runningTotals.npm.period_end,
+ npm_14_day_confirmed_download_events: runningTotals.npm.observed_downloads,
+ npm_14_day_unknown_days: runningTotals.npm.unknown_days,
+ npm_14_day_complete_utc_days: npmWindowDays,
+ npm_14_day_confirmed_days: runningTotals.npm.confirmed_days,
+ npm_14_day_is_rolling_24h: false,
+ });
+}
+metrics.notes = metrics.notes.filter(note => !note.startsWith('The npm download cohort is') && !note.includes('all-package-zero npm dates'));
+if (runningTotals?.npm) metrics.notes.push(`${runningTotals.npm.unknown_days.length} all-package-zero npm dates in ${runningTotals.npm.period_start} through ${runningTotals.npm.period_end} UTC are unknown (null), never zero; ${date(data.generated_at)} is incomplete and excluded.`);
metrics.notes.push(`The npm download cohort is ${cohort} packages; the current inventory contains ${npm.package_count}.`);
save('data/metrics.json', JSON.stringify(metrics, null, 2) + '\n');
console.log(`Rendered ${npm.package_count} npm packages, ${crates.crate_count} crates; downloads dated ${downloadDate}.`);
diff --git a/scripts/tests/test_registry_refresh.py b/scripts/tests/test_registry_refresh.py
index d29a8b5..319f56f 100644
--- a/scripts/tests/test_registry_refresh.py
+++ b/scripts/tests/test_registry_refresh.py
@@ -61,6 +61,30 @@ class RegistryRefreshTest(unittest.TestCase):
shutil.copytree(entry,root/entry.name,ignore=shutil.ignore_patterns('__pycache__'))
else:
shutil.copyfile(entry,root/entry.name)
+ registry_path=root/'data/registry-stats.json'
+ registry=json.loads(registry_path.read_text())
+ refreshed={
+ 'npm_inventory':'2026-11-02T05:19:29.846430Z',
+ 'npm_downloads':'2026-11-02T05:40:00.000000Z',
+ 'crates_io':'2026-11-02T05:40:00.245693Z',
+ }
+ registry['generated_at']=refreshed['crates_io']
+ registry['npm']['inventory_verified_at']=refreshed['npm_inventory']
+ registry['npm']['downloads_verified_at']=refreshed['npm_downloads']
+ registry['crates_io']['verified_at']=refreshed['crates_io']
+ for section,path_key,date_key in [
+ (registry['npm'],'inventory_path','npm_inventory'),
+ (registry['npm'],'downloads_evidence_path','npm_downloads'),
+ (registry['crates_io'],'inventory_path','crates_io')]:
+ old_path=root/section[path_key]
+ new_rel=f'data/snapshots/2026-11-02/{old_path.name}'
+ receipt_path=root/new_rel
+ receipt_path.parent.mkdir(parents=True,exist_ok=True)
+ receipt=json.loads(old_path.read_text())
+ receipt['retrieved_at']=refreshed[date_key]
+ receipt_path.write_text(json.dumps(receipt))
+ section[path_key]=new_rel
+ registry_path.write_text(json.dumps(registry))
(root/'input.json').write_text(json.dumps(payload))
run=subprocess.run([sys.executable,'scripts/update-ecosystem-snapshot.py',
'--date',snapshot_date,'--input','input.json'],cwd=root,
@@ -85,6 +109,27 @@ class RegistryRefreshTest(unittest.TestCase):
running['npm']['unknown_days'])
self.assertEqual(metrics['current_windows']['npm_14_day_confirmed_days'],
running['npm']['confirmed_days'])
+ self.assertEqual(metrics['verification_dates']['npm_inventory'],
+ refreshed['npm_inventory'])
+ self.assertEqual(metrics['verification_dates']['npm_downloads'],
+ refreshed['npm_downloads'])
+ self.assertEqual(metrics['verification_dates']['crates_io'],
+ refreshed['crates_io'])
+
+ running['npm'].update(
+ period_start='2026-10-19',period_end='2026-11-01',
+ observed_downloads=123456,unknown_days=['2026-11-01'],
+ confirmed_days=13)
+ (root/'data/ecosystem-running-totals.json').write_text(json.dumps(running))
+ render=subprocess.run(['node','scripts/render-registry-metrics.mjs'],cwd=root,
+ capture_output=True,text=True)
+ self.assertEqual(render.returncode,0,render.stderr)
+ metrics=json.loads((root/'data/metrics.json').read_text())
+ self.assertEqual(metrics['current_windows']['npm_14_day_start'],'2026-10-19')
+ self.assertEqual(metrics['current_windows']['npm_14_day_end'],'2026-11-01')
+ self.assertEqual(metrics['current_windows']['npm_14_day_confirmed_download_events'],123456)
+ self.assertEqual(metrics['current_windows']['npm_14_day_unknown_days'],['2026-11-01'])
+ self.assertEqual(metrics['current_windows']['npm_14_day_confirmed_days'],13)
def test_new_npm_package_404_is_retried_without_inventing_zero(self):
url='https://api.npmjs.org/downloads/range/2025-10-01:2026-10-11/new-package'
diff --git a/scripts/update-ecosystem-snapshot.py b/scripts/update-ecosystem-snapshot.py
index 2dde0b2..180e42f 100644
--- a/scripts/update-ecosystem-snapshot.py
+++ b/scripts/update-ecosystem-snapshot.py
@@ -304,6 +304,8 @@ def main():
'npm_download_package_count': registry['npm']['download_package_count'],
'github_activity_start': window_start,
'github_activity_end': window_end,
+ 'npm_14_day_start': window_start,
+ 'npm_14_day_end': window_end,
'npm_14_day_confirmed_download_events': running['npm']['observed_downloads'],
'npm_14_day_unknown_days': running['npm']['unknown_days'],
'npm_14_day_complete_utc_days': len(expected),
diff --git a/scripts/verify-profile.mjs b/scripts/verify-profile.mjs
index 5dbba8a..581db41 100644
--- a/scripts/verify-profile.mjs
+++ b/scripts/verify-profile.mjs
@@ -43,12 +43,11 @@ const watermark = json(expectedWatermarkPath);
assert.equal(watermark.repository, 'ruvnet/ruvnet');
assert.equal(watermark.snapshot_date, path.posix.basename(path.posix.dirname(expectedWatermarkPath)));
assert.equal(watermark.github_verified_at, github.verified_at);
-assert.equal(watermark.npm_inventory_verified_at, metrics.verification_dates.npm_inventory);
-assert.equal(watermark.npm_downloads_verified_at, metrics.verification_dates.npm_downloads);
-assert.equal(watermark.crates_io_verified_at, metrics.verification_dates.crates_io);
assert.equal(watermark.public_repository_ids, expected.public_repositories);
-assert.equal(watermark.npm_coverage.packages, npm.package_count);
-assert.equal(watermark.crates_io_coverage.crates, crates.crate_count);
+assert.equal(metrics.verification_dates.github, github.verified_at);
+assert.equal(metrics.verification_dates.npm_inventory, npm.inventory_verified_at);
+assert.equal(metrics.verification_dates.npm_downloads, npm.downloads_verified_at);
+assert.equal(metrics.verification_dates.crates_io, crates.verified_at);
assert.equal(metrics.counts.github_public_repositories, expected.public_repositories);
assert.equal(metrics.counts.github_stars_owned_nonfork, expected.stars_across_owned_nonfork);
assert.equal(metrics.counts.npm_packages, npm.package_count);
@@ -63,7 +62,11 @@ const monthIndex = value => Number(value.slice(0, 4)) * 12 + Number(value.slice(
assert.equal(npm.monthly_downloads.length, monthIndex(npm.monthly_period_end) - monthIndex(npm.monthly_period_start) + 1);
assert.equal(npm.monthly_downloads[0].month, npm.monthly_period_start.slice(0, 7));
assert.equal(npm.monthly_downloads.at(-1).month, npm.monthly_period_end.slice(0, 7));
-if (npm.inventory_path) assert.equal(json(npm.inventory_path).packages.length, npm.package_count);
+if (npm.inventory_path) {
+ const npmInventory = json(npm.inventory_path);
+ assert.equal(npmInventory.retrieved_at, npm.inventory_verified_at);
+ assert.equal(npmInventory.packages.length, npm.package_count);
+}
if (npm.current_month) {
const partial = npm.current_month;
assert.ok(partial.month > npm.monthly_downloads.at(-1).month);
@@ -73,6 +76,7 @@ if (npm.current_month) {
}
if (npm.downloads_evidence_path) {
const receipt = json(npm.downloads_evidence_path);
+ assert.equal(receipt.retrieved_at, npm.downloads_verified_at);
assert.equal(receipt.packages.length, npm.download_package_count);
assert.equal(new Set(receipt.packages.map(p => p.package)).size, npm.download_package_count);
assert.equal(receipt.rolling_start, npm.period_start);
@@ -108,7 +112,9 @@ if (npm.downloads_evidence_path) {
}
}
if (crates.inventory_path) {
- const list = json(crates.inventory_path).crates;
+ const crateInventory = json(crates.inventory_path);
+ assert.equal(crateInventory.retrieved_at, crates.verified_at);
+ const list = crateInventory.crates;
assert.equal(list.length, crates.crate_count);
assert.equal(list.reduce((n, c) => n + c.downloads, 0), crates.cumulative_downloads);
}
@@ -123,10 +129,15 @@ const running = json('data/ecosystem-running-totals.json');
assert.deepEqual(running.source_watermark, watermark, 'Embedded source watermark must match its dated file');
assert.equal(metrics.current_windows.github_activity_start, watermark.activity_window_utc.start);
assert.equal(metrics.current_windows.github_activity_end, watermark.activity_window_utc.end);
+assert.equal(metrics.current_windows.npm_14_day_start, running.npm.period_start);
+assert.equal(metrics.current_windows.npm_14_day_end, running.npm.period_end);
assert.equal(metrics.current_windows.npm_14_day_confirmed_download_events, running.npm.observed_downloads);
assert.deepEqual(metrics.current_windows.npm_14_day_unknown_days, running.npm.unknown_days);
-assert.equal(metrics.current_windows.npm_14_day_complete_utc_days, watermark.activity_window_utc.complete_days);
+assert.equal(metrics.current_windows.npm_14_day_complete_utc_days,
+ (Date.parse(running.npm.period_end) - Date.parse(running.npm.period_start)) / 86400000 + 1);
assert.equal(metrics.current_windows.npm_14_day_confirmed_days, running.npm.confirmed_days);
+assert.equal(metrics.current_windows.npm_14_day_confirmed_days + metrics.current_windows.npm_14_day_unknown_days.length,
+ metrics.current_windows.npm_14_day_complete_utc_days);
assert.equal(metrics.current_windows.npm_14_day_is_rolling_24h, false);
const milestone = running.download_milestone;
const periodTotal = npm.monthly_downloads.reduce((sum,row)=>sum+row.downloads,0)+(npm.current_month?.downloads || 0);