The definitive list of the best of (consumer facing) open source.
Added fallback in case of file corruption, api fail, etc
Lucas08d6a09d0e91
diff --git a/core/source/maintenance/backlog_updator.py b/core/source/maintenance/backlog_updator.pyindex 6d637b3..ebceaaa 100644--- a/core/source/maintenance/backlog_updator.py+++ b/core/source/maintenance/backlog_updator.py@@ -1,4 +1,7 @@+from __future__ import annotations+import json+import osfrom pathlib import Pathimport sys@@ -44,7 +47,33 @@ def generate_backlog(print("Warning: GITHUB_TOKEN not found. Requests may be rate-limited.")headers = get_github_headers(token)+ # Load fallback cache from existing generated backlog to avoid data loss on failures+ existing_lookup: dict[str, dict] = {}+ if output_path.exists():+ try:+ with open(output_path, "r", encoding="utf-8") as f:+ existing_data = json.load(f)+ for existing_app in existing_data.get("applications", []):+ raw_url = existing_app.get("repo_url", "")+ if raw_url:+ existing_lookup[raw_url.strip()] = existing_app+ existing_lookup[raw_url.strip().lower()] = existing_app+ path = extract_repo_path(raw_url)+ if path:+ existing_lookup[path.lower()] = existing_app+ name = existing_app.get("name", "")+ if name:+ existing_lookup[name.strip().lower()] = existing_app+ except Exception as e:+ print(f"Notice: Could not load existing backlog data for fallback: {e}")+applications = data.get("applications", [])+ if not applications and output_path.exists():+ print(+ f"Warning: No applications found in {input_path}. Preserving existing {output_path} to prevent data loss."+ )+ return {"applications": []}+total = len(applications)repo_paths = [@@ -65,13 +94,33 @@ def generate_backlog(stars = 0repo_path = extract_repo_path(repo_url)++ # Look up fallback from cache+ fallback_app = None+ if repo_path:+ fallback_app = existing_lookup.get(repo_path.lower())+ if not fallback_app and repo_url:+ fallback_app = existing_lookup.get(repo_url.strip()) or existing_lookup.get(+ repo_url.strip().lower()+ )+ if not fallback_app and name:+ fallback_app = existing_lookup.get(name.strip().lower())+if repo_path and repo_path in batch_results and batch_results[repo_path]:repo_data = batch_results[repo_path]stars = repo_data.get("stargazers_count", 0)last_commit = format_commit_date(repo_data.get("pushed_at"))elif repo_path:- last_commit, stars = fetch_repo_summary(repo_path, headers)+ # Batch didn't return data; if fallback cache exists, use it to avoid data loss+ if fallback_app and fallback_app.get("last_commit"):+ last_commit = fallback_app.get("last_commit", "")+ stars = fallback_app.get("stars", 0)+ else:+ last_commit, stars = fetch_repo_summary(repo_path, headers)else:+ if fallback_app:+ last_commit = fallback_app.get("last_commit", "")+ stars = fallback_app.get("stars", 0)print(f"[{idx}/{total}] Skipping {name}: Invalid or non-GitHub URL ({repo_url})")@@ -88,10 +137,18 @@ def generate_backlog(generated_data = {"applications": generated_apps}+ # Atomic write to prevent file corruption or destructive 0-byte truncation on crashoutput_path.parent.mkdir(parents=True, exist_ok=True)- with open(output_path, "w", encoding="utf-8") as f:- json.dump(generated_data, f, indent=4)- f.write("\n")+ temp_path = output_path.with_name(f"{output_path.name}.tmp.{os.getpid()}")+ try:+ with open(temp_path, "w", encoding="utf-8") as f:+ json.dump(generated_data, f, indent=4)+ f.write("\n")+ temp_path.replace(output_path)+ except Exception:+ if temp_path.exists():+ temp_path.unlink()+ raiseprint(f"Successfully generated {output_path} with {len(generated_apps)} applications."diff --git a/core/source/maintenance/stats_updator.py b/core/source/maintenance/stats_updator.pyindex 1ee79f9..088a71a 100644--- a/core/source/maintenance/stats_updator.py+++ b/core/source/maintenance/stats_updator.py@@ -1,6 +1,7 @@from __future__ import annotationsimport json+import osfrom pathlib import Pathimport sys@@ -26,8 +27,46 @@ def update_application_data() -> dict:repo_url = app.get("repo_url", "")repo_name = extract_repo_path(repo_url)+ flags = app.get("flags", [])+if not repo_name:print(f"Skipping {app.get('name', 'Unknown')}: Invalid GitHub URL ({repo_url})")+ if fallback_app:+ if "stars" in fallback_app and fallback_app["stars"] is not None:+ app["stars"] = fallback_app["stars"]+ elif "stars" not in app or app["stars"] is None:+ app["stars"] = 0+ if "language" in fallback_app and fallback_app["language"] is not None:+ app["language"] = fallback_app["language"]+ elif "language" not in app or app["language"] is None:+ app["language"] = ""+ if (+ "homepage_url" in fallback_app+ and fallback_app["homepage_url"] is not None+ ):+ app["homepage_url"] = fallback_app["homepage_url"]+ elif "homepage_url" not in app or app["homepage_url"] is None:+ app["homepage_url"] = ""+ if (+ "last_commit" in fallback_app+ and fallback_app["last_commit"] is not None+ ):+ app["last_commit"] = fallback_app["last_commit"]+ elif "last_commit" not in app or app["last_commit"] is None:+ app["last_commit"] = ""+ else:+ if "stars" not in app or app["stars"] is None:+ app["stars"] = 0+ if "language" not in app or app["language"] is None:+ app["language"] = ""+ if "homepage_url" not in app or app["homepage_url"] is None:+ app["homepage_url"] = ""+ if "last_commit" not in app or app["last_commit"] is None:+ app["last_commit"] = ""+ if "description" not in app or app["description"] is None:+ app["description"] = ""+ if "license" not in app or app["license"] is None:+ app["license"] = ""return appif repo_data is None and headers is not None:@@ -36,45 +75,129 @@ def update_application_data(if repo_data is not None:app["stars"] = repo_data.get("stargazers_count", app.get("stars", 0))- app["language"] = repo_data.get("language", app.get("language", ""))+ app["language"] = repo_data.get("language") or app.get("language", "") or ""# Flags- if "custom-homepage" not in app.get("flags", []):- app["homepage_url"] = repo_data.get("homepage", app.get("homepage_url", ""))+ if "custom-homepage" not in flags:+ app["homepage_url"] = (+ repo_data.get("homepage") or app.get("homepage_url", "") or ""+ )+ elif "homepage_url" not in app or app["homepage_url"] is None:+ app["homepage_url"] = ""- if "custom-description" not in app.get("flags", []):- app["description"] = repo_data.get(- "description", app.get("description", "")+ if "custom-description" not in flags:+ app["description"] = (+ repo_data.get("description", app.get("description", "")) or "")+ elif "description" not in app or app["description"] is None:+ app["description"] = ""- if "custom-license" not in app.get("flags", []):+ if "custom-license" not in flags:license_data = repo_data.get("license")- if license_data is not None:- app["license"] = license_data.get("spdx_id", app.get("license", ""))+ if isinstance(license_data, dict):+ app["license"] = (+ license_data.get("spdx_id") or app.get("license", "") or ""+ )+ elif isinstance(license_data, str):+ app["license"] = license_dataelse:- app["license"] = app.get("license", "")+ app["license"] = app.get("license", "") or ""+ elif "license" not in app or app["license"] is None:+ app["license"] = ""# Checkpushed_at = repo_data.get("pushed_at")if pushed_at:app["last_commit"] = format_commit_date(pushed_at)else:- app["last_commit"] = app.get("last_commit", "")+ app["last_commit"] = app.get("last_commit", "") or ""++ # Ensure all dynamic keys exist and are not None+ if "stars" not in app or app["stars"] is None:+ app["stars"] = 0+ if "language" not in app or app["language"] is None:+ app["language"] = ""+ if "homepage_url" not in app or app["homepage_url"] is None:+ app["homepage_url"] = ""+ if "last_commit" not in app or app["last_commit"] is None:+ app["last_commit"] = ""+ if "description" not in app or app["description"] is None:+ app["description"] = ""+ if "license" not in app or app["license"] is None:+ app["license"] = ""return appelse:- # If fetch failed but fallback_app exists, preserve previous dynamic values+ # If fetch failed but fallback_app exists, preserve previous dynamic values without wipingif fallback_app:- for field in [- "stars",- "language",- "homepage_url",- "description",- "license",- "last_commit",- ]:- if (field not in app or not app.get(field)) and fallback_app.get(field):- app[field] = fallback_app[field]+ # 1. stars+ if "stars" in fallback_app and fallback_app["stars"] is not None:+ app["stars"] = fallback_app["stars"]+ elif "stars" not in app or app["stars"] is None:+ app["stars"] = 0++ # 2. language+ if "language" in fallback_app and fallback_app["language"] is not None:+ app["language"] = fallback_app["language"]+ elif "language" not in app or app["language"] is None:+ app["language"] = ""++ # 3. homepage_url+ if "custom-homepage" not in flags:+ if (+ "homepage_url" in fallback_app+ and fallback_app["homepage_url"] is not None+ ):+ app["homepage_url"] = fallback_app["homepage_url"]+ elif "homepage_url" not in app or app["homepage_url"] is None:+ app["homepage_url"] = ""+ elif "homepage_url" not in app or app["homepage_url"] is None:+ app["homepage_url"] = ""++ # 4. description+ if "custom-description" not in flags:+ if (+ "description" in fallback_app+ and fallback_app["description"] is not None+ ):+ app["description"] = fallback_app["description"]+ elif "description" not in app or app["description"] is None:+ app["description"] = ""+ elif "description" not in app or app["description"] is None:+ app["description"] = ""++ # 5. license+ if "custom-license" not in flags:+ if "license" in fallback_app and fallback_app["license"] is not None:+ app["license"] = fallback_app["license"]+ elif "license" not in app or app["license"] is None:+ app["license"] = ""+ elif "license" not in app or app["license"] is None:+ app["license"] = ""++ # 6. last_commit+ if (+ "last_commit" in fallback_app+ and fallback_app["last_commit"] is not None+ ):+ app["last_commit"] = fallback_app["last_commit"]+ elif "last_commit" not in app or app["last_commit"] is None:+ app["last_commit"] = ""+ else:+ # No fallback available, ensure valid schema defaults so entries are not stripped+ if "stars" not in app or app["stars"] is None:+ app["stars"] = 0+ if "language" not in app or app["language"] is None:+ app["language"] = ""+ if "homepage_url" not in app or app["homepage_url"] is None:+ app["homepage_url"] = ""+ if "description" not in app or app["description"] is None:+ app["description"] = ""+ if "license" not in app or app["license"] is None:+ app["license"] = ""+ if "last_commit" not in app or app["last_commit"] is None:+ app["last_commit"] = ""+return app@@ -112,10 +235,16 @@ def update_all_applications(with open(output_path, "r", encoding="utf-8") as f:existing_data = json.load(f)for existing_app in existing_data.get("applications", []):- if existing_app.get("repo_url"):- existing_lookup[existing_app["repo_url"]] = existing_app- if existing_app.get("name"):- existing_lookup[existing_app["name"]] = existing_app+ raw_url = existing_app.get("repo_url", "")+ if raw_url:+ existing_lookup[raw_url.strip()] = existing_app+ existing_lookup[raw_url.strip().lower()] = existing_app+ path = extract_repo_path(raw_url)+ if path:+ existing_lookup[path.lower()] = existing_app+ name = existing_app.get("name", "")+ if name:+ existing_lookup[name.strip().lower()] = existing_appexcept Exception as e:print(f"Notice: Could not load existing generated data for fallback: {e}")@@ -123,6 +252,12 @@ def update_all_applications(data = json.load(f)applications = data.get("applications", [])+ if not applications and output_path.exists():+ print(+ f"Warning: No applications found in {input_path}. Preserving existing {output_path} to prevent data loss."+ )+ return {"applications": []}+repo_paths: list[str] = []for app in applications:repo_path = extract_repo_path(app.get("repo_url", ""))@@ -139,15 +274,26 @@ def update_all_applications(generated_apps = []for app in applications:- repo_path = extract_repo_path(app.get("repo_url", ""))+ raw_url = app.get("repo_url", "")+ repo_path = extract_repo_path(raw_url)repo_data = batch_results.get(repo_path) if repo_path else None- fallback_app = (- existing_lookup.get(app.get("repo_url", ""))- or existing_lookup.get(app.get("name", ""))- )++ # Look up fallback cache by normalized repo path, exact URL, or normalized name+ fallback_app = None+ if repo_path:+ fallback_app = existing_lookup.get(repo_path.lower())+ if not fallback_app and raw_url:+ fallback_app = existing_lookup.get(raw_url.strip()) or existing_lookup.get(+ raw_url.strip().lower()+ )+ if not fallback_app and app.get("name"):+ fallback_app = existing_lookup.get(app["name"].strip().lower())++ # Do not pass headers=headers here because batch fetch was already performed;+ # this avoids blocking on hundreds of synchronous individual requests if batch failed.updated_app = update_application_data(app.copy(),- headers=headers,+ headers=None,repo_data=repo_data,fallback_app=fallback_app,)@@ -155,10 +301,18 @@ def update_all_applications(generated_data = {"applications": generated_apps}+ # Atomic write to prevent file corruption or destructive 0-byte truncation on crashoutput_path.parent.mkdir(parents=True, exist_ok=True)- with open(output_path, "w", encoding="utf-8") as f:- json.dump(generated_data, f, indent=4)- f.write("\n")+ temp_path = output_path.with_name(f"{output_path.name}.tmp.{os.getpid()}")+ try:+ with open(temp_path, "w", encoding="utf-8") as f:+ json.dump(generated_data, f, indent=4)+ f.write("\n")+ temp_path.replace(output_path)+ except Exception:+ if temp_path.exists():+ temp_path.unlink()+ raiseprint(f"Updated application data successfully. Output written to {output_path}.")return generated_data@@ -166,4 +320,3 @@ def update_all_applications(if __name__ == "__main__":update_all_applications()-diff --git a/core/tests/test_batch_github.py b/core/tests/test_batch_github.pyindex 0e18e07..3b32251 100644--- a/core/tests/test_batch_github.py+++ b/core/tests/test_batch_github.py@@ -11,6 +11,7 @@ CORE_DIR = Path(__file__).resolve().parents[1]if str(CORE_DIR) not in sys.path:sys.path.insert(0, str(CORE_DIR))+from source.maintenance.backlog_updator import generate_backlog # noqa: E402from source.maintenance.stats_updator import ( # noqa: E402update_all_applications,update_application_data,@@ -158,7 +159,10 @@ class TestBatchGitHubUtils(unittest.TestCase):mock_graphql.assert_called_once()mock_rest.assert_called_once_with(- ["owner/renamed"], headers=mock_rest.call_args[1]["headers"], max_workers=8, timeout=30+ ["owner/renamed"],+ headers=mock_rest.call_args[1]["headers"],+ max_workers=8,+ timeout=30,)self.assertEqual(results["owner/good"]["stargazers_count"], 200)self.assertEqual(results["owner/renamed"]["stargazers_count"], 300)@@ -256,6 +260,90 @@ class TestStatsUpdator(unittest.TestCase):self.assertEqual(updated["license"], "MIT")self.assertEqual(updated["last_commit"], "09/20/2026")+ def test_update_application_data_preserves_zero_stars_from_fallback(self):+ app = {+ "name": "Zero Star App",+ "repo_url": "https://github.com/foo/zero",+ "category": "ai",+ "flags": [],+ }+ fallback_app = {+ "name": "Zero Star App",+ "repo_url": "https://github.com/foo/zero",+ "stars": 0,+ "language": "Python",+ "homepage_url": "",+ "description": "Zero stars repo",+ "license": "MIT",+ "last_commit": "09/01/2026",+ }++ updated = update_application_data(+ app.copy(),+ repo_data=None,+ fallback_app=fallback_app,+ )+ self.assertIn("stars", updated)+ self.assertEqual(updated["stars"], 0)+ self.assertEqual(updated["language"], "Python")+ self.assertEqual(updated["description"], "Zero stars repo")+ self.assertEqual(updated["license"], "MIT")+ self.assertEqual(updated["last_commit"], "09/01/2026")++ def test_update_application_data_populates_safe_defaults_when_no_fallback(self):+ app = {+ "name": "Brand New App",+ "repo_url": "https://github.com/new/app",+ "category": "ai",+ "flags": [],+ }++ # Both repo_data and fallback_app are None+ updated = update_application_data(+ app.copy(),+ repo_data=None,+ fallback_app=None,+ )+ self.assertEqual(updated["stars"], 0)+ self.assertEqual(updated["language"], "")+ self.assertEqual(updated["homepage_url"], "")+ self.assertEqual(updated["description"], "")+ self.assertEqual(updated["license"], "")+ self.assertEqual(updated["last_commit"], "")++ def test_update_application_data_preserves_custom_flags_on_fetch_failure(self):+ app = {+ "name": "Custom Flags App",+ "repo_url": "https://github.com/foo/flags",+ "category": "ai",+ "flags": ["custom-description", "custom-license", "custom-homepage"],+ "description": "Curated description",+ "license": "Proprietary",+ "homepage_url": "https://custom.site",+ }+ fallback_app = {+ "name": "Custom Flags App",+ "repo_url": "https://github.com/foo/flags",+ "stars": 42,+ "language": "Go",+ "description": "Old stale description",+ "license": "Old License",+ "homepage_url": "https://old.site",+ "last_commit": "08/15/2026",+ }++ updated = update_application_data(+ app.copy(),+ repo_data=None,+ fallback_app=fallback_app,+ )+ self.assertEqual(updated["description"], "Curated description")+ self.assertEqual(updated["license"], "Proprietary")+ self.assertEqual(updated["homepage_url"], "https://custom.site")+ self.assertEqual(updated["stars"], 42)+ self.assertEqual(updated["language"], "Go")+ self.assertEqual(updated["last_commit"], "08/15/2026")+@patch("source.maintenance.stats_updator.fetch_repos_batch")def test_update_all_applications_flow(self, mock_batch):mock_batch.return_value = {@@ -314,6 +402,116 @@ class TestStatsUpdator(unittest.TestCase):if test_output.exists():test_output.unlink()+ @patch("source.maintenance.stats_updator.fetch_repos_batch")+ def test_update_all_applications_preserves_fallback_cache_on_batch_failure(+ self, mock_batch+ ):+ # Simulate complete batch fetch failure (e.g. rate limit)+ mock_batch.return_value = {"foo/bar": None}++ test_input = CORE_DIR / "tests" / "test_input_failure.json"+ test_output = CORE_DIR / "tests" / "test_output_failure.json"++ input_data = {+ "applications": [+ {+ "name": "FooBar",+ "repo_url": "https://github.com/foo/bar/", # Note trailing slash+ "category": "tools",+ "flags": [],+ }+ ]+ }+ existing_data = {+ "applications": [+ {+ "name": "FooBar",+ "repo_url": "https://github.com/foo/bar",+ "stars": 9999,+ "language": "Rust",+ "homepage_url": "https://foobar.rs",+ "description": "Previously cached description",+ "license": "MIT",+ "last_commit": "09/10/2026",+ }+ ]+ }+ with open(test_input, "w", encoding="utf-8") as f:+ json.dump(input_data, f)+ with open(test_output, "w", encoding="utf-8") as f:+ json.dump(existing_data, f)++ try:+ result = update_all_applications(+ input_file=test_input,+ output_file=test_output,+ batch_size=50,+ )+ self.assertEqual(len(result["applications"]), 1)+ app = result["applications"][0]+ # Verify cached metadata was preserved and not stripped+ self.assertEqual(app["stars"], 9999)+ self.assertEqual(app["language"], "Rust")+ self.assertEqual(app["homepage_url"], "https://foobar.rs")+ self.assertEqual(app["description"], "Previously cached description")+ self.assertEqual(app["license"], "MIT")+ self.assertEqual(app["last_commit"], "09/10/2026")+ finally:+ if test_input.exists():+ test_input.unlink()+ if test_output.exists():+ test_output.unlink()+++class TestBacklogUpdator(unittest.TestCase):+ @patch("source.maintenance.backlog_updator.fetch_repos_batch")+ def test_generate_backlog_preserves_fallback_cache_on_failure(self, mock_batch):+ mock_batch.return_value = {"old/project": None}++ test_input = CORE_DIR / "tests" / "test_backlog_input.json"+ test_output = CORE_DIR / "tests" / "test_backlog_output.json"++ input_data = {+ "applications": [+ {+ "name": "Old Project",+ "repo_url": "https://github.com/old/project/",+ "note": "Potentially abandoned",+ }+ ]+ }+ existing_data = {+ "applications": [+ {+ "name": "Old Project",+ "repo_url": "https://github.com/old/project",+ "note": "Potentially abandoned",+ "last_commit": "05/10/2025",+ "stars": 321,+ }+ ]+ }+ with open(test_input, "w", encoding="utf-8") as f:+ json.dump(input_data, f)+ with open(test_output, "w", encoding="utf-8") as f:+ json.dump(existing_data, f)++ try:+ result = generate_backlog(+ input_file=test_input,+ output_file=test_output,+ )+ self.assertEqual(len(result["applications"]), 1)+ app = result["applications"][0]+ self.assertEqual(app["name"], "Old Project")+ self.assertEqual(app["stars"], 321)+ self.assertEqual(app["last_commit"], "05/10/2025")+ finally:+ if test_input.exists():+ test_input.unlink()+ if test_output.exists():+ test_output.unlink()+if __name__ == "__main__":unittest.main()