Skip to content

ruvnet.

Python
21.1 KB
Raw Blame
"""The milestone period and reporting-lag rules must survive daily refreshes."""
import datetime as dt
import importlib.util
import json
from pathlib import Path
import tempfile
import unittest
import io
import shutil
import subprocess
import sys
import urllib.error
from unittest.mock import patch

ROOT = Path(__file__).resolve().parents[2]
spec = importlib.util.spec_from_file_location('registry_refresh', ROOT / 'scripts/refresh-registry-metrics.py')
refresh = importlib.util.module_from_spec(spec)
spec.loader.exec_module(refresh)


def result(package='example', positive_through='2026-10-05', requested_end='2026-10-07', count=1):
    start = dt.date.fromisoformat(refresh.PERIOD_START)
    end = dt.date.fromisoformat(requested_end)
    points = []
    for offset in range((end - start).days + 1):
        day = (start + dt.timedelta(days=offset)).isoformat()
        points.append({'day': day, 'downloads': count if day <= positive_through else 0})
    return {
        'package': package,
        'body': {'package': package, 'start': start.isoformat(), 'end': requested_end, 'downloads': points},
        'receipt': {'response_sha256': '0' * 64, 'retrieved_at': '2026-10-08T12:00:00Z'},
    }


class RegistryRefreshTest(unittest.TestCase):
    def test_ecosystem_compositor_preserves_canonical_latest_day(self):
        """The GitHub compositor must not drop fields needed by profile verification."""
        inventory=json.loads((ROOT/'data/github-stats.json').read_text())['inventory']
        snapshot=ROOT/Path(inventory).parent
        snapshot_date=snapshot.name
        old=json.loads((snapshot/'ecosystem-summary.json').read_text())
        evidence=json.loads((snapshot/'ecosystem-evidence.json').read_text())
        payload={
            'verified_at':old['github']['verified_at'],
            'window_start':evidence['window']['start'],'window_end':evidence['window']['end'],
            'followers':old['github']['followers'],
            'github':{**old['github'], 'public_forks':old['github']['all_owned']['repositories']-old['github']['nonfork']['repositories'],
                      'releases':old['release_assets']['releases'],
                      'published_in_14_day_window':old['release_assets']['published_in_14_day_window']},
            'activity':{**old['activity'],**old['open_items']},
            'actions':old['actions'],'traffic':evidence['traffic'],
        }
        with tempfile.TemporaryDirectory() as folder:
            root=Path(folder)
            for name in ['data','docs','scripts']:
                shutil.copytree(ROOT/name,root/name,ignore=shutil.ignore_patterns('__pycache__'))
            shutil.copyfile(ROOT/'README.md',root/'README.md')
            for entry in ROOT.iterdir():
                if entry.name != '.git' and not (root/entry.name).exists():
                    if entry.is_dir():
                        shutil.copytree(entry,root/entry.name,ignore=shutil.ignore_patterns('__pycache__'))
                    else:
                        shutil.copyfile(entry,root/entry.name)
            registry_path=root/'data/registry-stats.json'
            registry=json.loads(registry_path.read_text())
            refreshed={
                'npm_inventory':'2026-11-02T05:19:29.846430Z',
                'npm_downloads':'2026-11-02T05:40:00.000000Z',
                'crates_io':'2026-11-02T05:40:00.245693Z',
            }
            registry['generated_at']=refreshed['crates_io']
            registry['npm']['inventory_verified_at']=refreshed['npm_inventory']
            registry['npm']['downloads_verified_at']=refreshed['npm_downloads']
            registry['crates_io']['verified_at']=refreshed['crates_io']
            for section,path_key,date_key in [
                    (registry['npm'],'inventory_path','npm_inventory'),
                    (registry['npm'],'downloads_evidence_path','npm_downloads'),
                    (registry['crates_io'],'inventory_path','crates_io')]:
                old_path=root/section[path_key]
                new_rel=f'data/snapshots/2026-11-02/{old_path.name}'
                receipt_path=root/new_rel
                receipt_path.parent.mkdir(parents=True,exist_ok=True)
                receipt=json.loads(old_path.read_text())
                receipt['retrieved_at']=refreshed[date_key]
                receipt_path.write_text(json.dumps(receipt))
                section[path_key]=new_rel
            registry_path.write_text(json.dumps(registry))
            (root/'input.json').write_text(json.dumps(payload))
            run=subprocess.run([sys.executable,'scripts/update-ecosystem-snapshot.py',
                                '--date',snapshot_date,'--input','input.json'],cwd=root,
                               capture_output=True,text=True)
            self.assertEqual(run.returncode,0,run.stderr)
            check=subprocess.run(['node','scripts/verify-profile.mjs'],cwd=root,
                                 capture_output=True,text=True)
            self.assertEqual(check.returncode,0,check.stderr)
            current=json.loads((root/'data/registry-stats.json').read_text())
            running=json.loads((root/'data/ecosystem-running-totals.json').read_text())
            metrics=json.loads((root/'data/metrics.json').read_text())
            self.assertEqual(running['npm']['latest_confirmed_daily']['day'],current['npm']['period_end'])
            self.assertEqual(metrics['source_watermark'],
                             f'data/snapshots/{snapshot_date}/collection-watermark.json')
            self.assertEqual(metrics['current_windows']['github_activity_start'],
                             evidence['window']['start'])
            self.assertEqual(metrics['current_windows']['github_activity_end'],
                             evidence['window']['end'])
            self.assertEqual(metrics['current_windows']['npm_14_day_confirmed_download_events'],
                             running['npm']['observed_downloads'])
            self.assertEqual(metrics['current_windows']['npm_14_day_unknown_days'],
                             running['npm']['unknown_days'])
            self.assertEqual(metrics['current_windows']['npm_14_day_confirmed_days'],
                             running['npm']['confirmed_days'])
            self.assertEqual(metrics['verification_dates']['npm_inventory'],
                             refreshed['npm_inventory'])
            self.assertEqual(metrics['verification_dates']['npm_downloads'],
                             refreshed['npm_downloads'])
            self.assertEqual(metrics['verification_dates']['crates_io'],
                             refreshed['crates_io'])

            running['npm'].update(
                period_start='2026-10-19',period_end='2026-11-01',
                observed_downloads=123456,unknown_days=['2026-11-01'],
                confirmed_days=13)
            (root/'data/ecosystem-running-totals.json').write_text(json.dumps(running))
            render=subprocess.run(['node','scripts/render-registry-metrics.mjs'],cwd=root,
                                  capture_output=True,text=True)
            self.assertEqual(render.returncode,0,render.stderr)
            metrics=json.loads((root/'data/metrics.json').read_text())
            self.assertEqual(metrics['current_windows']['npm_14_day_start'],'2026-10-19')
            self.assertEqual(metrics['current_windows']['npm_14_day_end'],'2026-11-01')
            self.assertEqual(metrics['current_windows']['npm_14_day_confirmed_download_events'],123456)
            self.assertEqual(metrics['current_windows']['npm_14_day_unknown_days'],['2026-11-01'])
            self.assertEqual(metrics['current_windows']['npm_14_day_confirmed_days'],13)

    def test_new_npm_package_404_is_retried_without_inventing_zero(self):
        url='https://api.npmjs.org/downloads/range/2025-10-01:2026-10-11/new-package'
        error=urllib.error.HTTPError(url,404,'Not Found',{},None)
        with patch.object(refresh.urllib.request,'urlopen',side_effect=[error,io.BytesIO(b'{"downloads": 17}')]), patch.object(refresh.time,'sleep') as sleep:
            body,receipt=refresh.fetch_json(url)
        self.assertEqual(body['downloads'],17)
        self.assertEqual(receipt['source'],url)
        sleep.assert_called_once()
        with patch.object(refresh.urllib.request,'urlopen',side_effect=error) as fetch, patch.object(refresh.time,'sleep'):
            with self.assertRaisesRegex(refresh.NpmRangeNotFound,'previous verified data retained'):
                refresh.fetch_json(url)
        self.assertEqual(fetch.call_count,8)

    def test_registry_milestone_refreshes_from_evidence_idempotently(self):
        registry=json.loads((ROOT/'data/registry-stats.json').read_text())
        evidence=json.loads((ROOT/registry['npm']['downloads_evidence_path']).read_text())
        registry['crates_io']['cumulative_downloads'] += 123
        with tempfile.TemporaryDirectory() as folder:
            root=Path(folder);(root/'data').mkdir()
            for name in ['ecosystem-running-totals.json','ecosystem-daily-ledger.json']:
                shutil.copyfile(ROOT/'data'/name,root/'data'/name)
            stale=json.loads((root/'data/ecosystem-running-totals.json').read_text())
            stale['npm'].update(errors=[{'package':'resolved-package','error':'HTTP 404'}],
                                status='stale coverage',latest_confirmed_daily={
                                    'day':'2020-01-01','downloads':999,'collected_packages':1})
            stale['download_milestone']['status']='stale coverage'
            stale['validation'].update(npm_unavailable_packages=99,
                                       npm_download_package_coverage_complete=False)
            (root/'data/ecosystem-running-totals.json').write_text(json.dumps(stale))
            with patch.object(refresh,'ROOT',root):
                refresh.update_running_totals(registry,evidence)
                first=(root/'data/ecosystem-running-totals.json').read_bytes()
                refresh.update_running_totals(registry,evidence)
                self.assertEqual(first,(root/'data/ecosystem-running-totals.json').read_bytes())
            m=json.loads(first)['download_milestone']
            expected=sum((row.get('downloads') or 0) for row in evidence['daily_totals'] if row['day']<=evidence['reported_through'])
            self.assertEqual(m['combined_observed'],expected+registry['crates_io']['cumulative_downloads'])
            self.assertEqual(m['npm_verified_at'],registry['npm']['downloads_verified_at'])
            self.assertEqual(m['remaining'],max(0,m['target']-m['combined_observed']))
            running=json.loads(first)
            latest=running['npm']['latest_confirmed_daily']
            last=next(row for row in evidence['daily_totals']
                      if row['day']==evidence['reported_through'])
            self.assertEqual(latest['day'],last['day'])
            self.assertEqual(latest['downloads'],last['downloads'])
            self.assertEqual(latest['collected_packages'],len(evidence['packages']))
            self.assertEqual(latest['verified_at'],evidence['retrieved_at'])
            self.assertEqual(latest['source'],evidence['source'])
            self.assertEqual(running['npm']['errors'],evidence.get('unavailable_packages',[]))
            self.assertEqual(running['validation']['npm_unavailable_packages'],len(running['npm']['errors']))
            self.assertEqual(running['validation']['npm_download_package_coverage_complete'],
                             len(evidence['packages'])==registry['npm']['package_count'])
            coverage=f"{len(evidence['packages'])} of {registry['npm']['package_count']}"
            self.assertIn(coverage,running['npm']['status'])
            self.assertIn(coverage,m['status'])

    def test_provisional_zero_tail_is_unknown_in_daily_ledger(self):
        row=result(positive_through='2026-10-05',requested_end='2026-10-07')
        row['receipt']['source']='https://api.npmjs.org/downloads/range/example'
        with tempfile.TemporaryDirectory() as folder:
            root=Path(folder);(root/'data').mkdir()
            (root/'data/ecosystem-daily-ledger.json').write_text(json.dumps({'values':{}}))
            with patch.object(refresh,'ROOT',root):
                refresh.update_daily_ledger([row],dt.date(2026,10,8),'2026-10-08T12:00:00Z')
            values=json.loads((root/'data/ecosystem-daily-ledger.json').read_text())['values']
            self.assertEqual(values['npm|example|downloads|2026-10-05'],1)
            self.assertIsNone(values['npm|example|downloads|2026-10-06'])
            self.assertIsNone(values['npm|example|downloads|2026-10-07'])

    def test_missing_inventory_package_keeps_window_incomplete(self):
        registry=json.loads((ROOT/'data/registry-stats.json').read_text())
        evidence=json.loads((ROOT/registry['npm']['downloads_evidence_path']).read_text())
        registry['npm']['package_count'] += 1
        evidence['unavailable_packages']=[{'package':'pending-package','error':'HTTP 404'}]
        with tempfile.TemporaryDirectory() as folder:
            root=Path(folder);(root/'data').mkdir()
            for name in ['ecosystem-running-totals.json','ecosystem-daily-ledger.json']:
                shutil.copyfile(ROOT/'data'/name,root/'data'/name)
            with patch.object(refresh,'ROOT',root):
                refresh.update_running_totals(registry,evidence)
            running=json.loads((root/'data/ecosystem-running-totals.json').read_text())
            self.assertEqual(len(running['npm']['missing_days']),14)
            self.assertFalse(running['validation']['npm_download_package_coverage_complete'])
            self.assertEqual(running['validation']['npm_unavailable_packages'],1)

    def test_daily_ledger_upserts_complete_window_idempotently(self):
        today = dt.date(2026, 10, 8)
        row = result(positive_through='2026-10-07', requested_end='2026-10-07')
        row['receipt']['source'] = 'https://api.npmjs.org/downloads/range/example'
        with tempfile.TemporaryDirectory() as folder:
            root = Path(folder)
            data = root / 'data'
            data.mkdir()
            ledger = {
                'schema_version': '1.0', 'observed_from': '2026-09-01',
                'values': {'npm|example|downloads|2026-10-07': 99},
                'package_provenance': {}, 'coverage': {}}
            (data / 'ecosystem-daily-ledger.json').write_text(json.dumps(ledger))
            with patch.object(refresh, 'ROOT', root):
                refresh.update_daily_ledger([row], today, '2026-10-08T12:00:00Z')
                first = (data / 'ecosystem-daily-ledger.json').read_bytes()
                refresh.update_daily_ledger([row], today, '2026-10-08T12:00:00Z')
                second = (data / 'ecosystem-daily-ledger.json').read_bytes()
            updated = json.loads(first)
            self.assertEqual(first, second)
            self.assertEqual(updated['observed_from'], '2026-09-01')
            self.assertEqual(updated['observed_through'], '2026-10-07')
            self.assertEqual(updated['values']['npm|example|downloads|2026-10-07'], 1)
            self.assertEqual(updated['coverage']['collected_packages'], 1)

    def test_daily_ledger_revises_existing_historical_cells(self):
        today = dt.date(2026, 10, 8)
        rows = [result('example', '2026-10-07', '2026-10-07'),
                result('companion', '2026-10-07', '2026-10-07')]
        for row in rows:
            row['receipt']['source'] = 'https://api.npmjs.org/downloads/range/' + row['package']
            for point in row['body']['downloads']:
                if point['day'] == '2026-09-03':
                    point['downloads'] = 0
                if row['package'] == 'example':
                    if point['day'] == '2026-09-01':
                        point['downloads'] = 0
                    elif point['day'] == '2026-09-02':
                        point['downloads'] = 7
        prefix = 'npm|example|downloads|'
        with tempfile.TemporaryDirectory() as folder:
            root = Path(folder); (root / 'data').mkdir()
            path = root / 'data/ecosystem-daily-ledger.json'
            path.write_text(json.dumps({'values': {
                prefix + '2026-09-01': None, prefix + '2026-09-02': 99,
                prefix + '2026-09-03': 9,
                'npm|unavailable|downloads|2026-09-01': None}}))
            with patch.object(refresh, 'ROOT', root):
                refresh.update_daily_ledger(rows, today, '2026-10-08T12:00:00Z')
                first = path.read_bytes()
                refresh.update_daily_ledger(rows, today, '2026-10-08T12:00:00Z')
                self.assertEqual(first, path.read_bytes())
            values = json.loads(first)['values']
            self.assertEqual(values[prefix + '2026-09-01'], 0)
            self.assertEqual(values[prefix + '2026-09-02'], 7)
            self.assertIsNone(values[prefix + '2026-09-03'])
            self.assertIsNone(values['npm|unavailable|downloads|2026-09-01'])
            self.assertNotIn(prefix + '2026-08-01', values)
            self.assertNotIn(prefix + '2026-10-08', values)

    def test_daily_ledger_records_missing_package_without_inventing_zero(self):
        today = dt.date(2026, 10, 8)
        row = result(positive_through='2026-10-07', requested_end='2026-10-07')
        row['receipt']['source'] = 'https://api.npmjs.org/downloads/range/example'
        error = {'package': 'new-package', 'error': 'HTTP 404', 'checked_at': '2026-10-08T12:00:00Z'}
        with tempfile.TemporaryDirectory() as folder:
            root = Path(folder); data = root / 'data'; data.mkdir()
            (data / 'ecosystem-daily-ledger.json').write_text(json.dumps({
                'schema_version': '1.0', 'observed_from': '2026-09-01',
                'values': {}, 'package_provenance': {}, 'coverage': {}}))
            with patch.object(refresh, 'ROOT', root):
                refresh.update_daily_ledger([row], today, '2026-10-08T12:00:00Z',
                                            expected_packages=2, errors=[error])
            updated = json.loads((data / 'ecosystem-daily-ledger.json').read_text())
            self.assertEqual(updated['coverage']['expected_packages'], 2)
            self.assertEqual(updated['coverage']['collected_packages'], 1)
            self.assertEqual(updated['coverage']['errors'], [error])
            self.assertFalse(any('|new-package|' in key for key in updated['values']))

    def test_zero_tail_does_not_advance_or_shrink_rolling_window(self):
        rows = [result('one'), result('two', count=2)]
        evidence, months = refresh.summarize_ranges(rows, refresh.PERIOD_START, '2026-10-07')
        self.assertEqual(evidence['reported_through'], '2026-10-05')
        self.assertEqual(evidence['trailing_all_zero_days_excluded'], ['2026-10-06', '2026-10-07'])
        self.assertEqual(evidence['rolling_start'], '2025-10-06')
        self.assertEqual(sum(row['rolling_365_days'] for row in evidence['packages']), 365 * 3)
        self.assertEqual(sum(months.values()), 370 * 3)
        self.assertEqual(months['2026-10'], 5 * 3)

    def test_fixed_period_is_preserved_when_reporting_advances(self):
        earlier, old_months = refresh.summarize_ranges([result()], refresh.PERIOD_START, '2026-10-07')
        later, new_months = refresh.summarize_ranges(
            [result(positive_through='2026-10-08', requested_end='2026-10-09')],
            refresh.PERIOD_START, '2026-10-09')
        self.assertEqual(earlier['requested_start'], later['requested_start'])
        self.assertEqual(sum(new_months.values()) - sum(old_months.values()), 3)
        self.assertEqual(later['rolling_start'], '2025-10-09')
        self.assertEqual(later['packages'][0]['rolling_365_days'], 365)
        self.assertGreater(sum(new_months.values()), later['packages'][0]['rolling_365_days'])

    def test_invalid_or_incomplete_data_is_rejected(self):
        cases = []
        missing = result(); missing['body']['downloads'].pop(); cases.append(missing)
        negative = result(); negative['body']['downloads'][0]['downloads'] = -1; cases.append(negative)
        boolean = result(); boolean['body']['downloads'][0]['downloads'] = True; cases.append(boolean)
        wrong_day = result(); wrong_day['body']['downloads'][0]['day'] = '2025-10-02'; cases.append(wrong_day)
        wrong_package = result(); wrong_package['body']['package'] = 'other'; cases.append(wrong_package)
        all_zero = result(count=0); cases.append(all_zero)
        for row in cases:
            with self.subTest(row=row['body']['package']):
                with self.assertRaises(ValueError):
                    refresh.summarize_ranges([row], refresh.PERIOD_START, '2026-10-07')
        with self.assertRaises(ValueError):
            refresh.summarize_ranges([result(), result()], refresh.PERIOD_START, '2026-10-07')

    def test_source_failure_does_not_write_or_render(self):
        receipt = {'retrieved_at': '2026-10-08T12:00:00Z'}
        with patch.object(refresh, 'npm_inventory', return_value=(['example'], [receipt])), \
                patch.object(refresh, 'fetch_json', side_effect=RuntimeError('API unavailable')), \
                patch.object(refresh, 'save') as save, \
                patch.object(refresh.subprocess, 'run') as run:
            with self.assertRaisesRegex(RuntimeError, 'API unavailable'):
                refresh.main()
            save.assert_not_called()
            run.assert_not_called()


if __name__ == '__main__':
    unittest.main()