Skip to content

The definitive list of the best of (consumer facing) open source.

Added fallback in case of file corruption, api fail, etc

Lucas08d6a09d0e91
diff --git a/core/source/maintenance/backlog_updator.py b/core/source/maintenance/backlog_updator.py
index 6d637b3..ebceaaa 100644
--- a/core/source/maintenance/backlog_updator.py
+++ b/core/source/maintenance/backlog_updator.py
@@ -1,4 +1,7 @@
+from __future__ import annotations
+
import json
+import os
from pathlib import Path
import sys
@@ -44,7 +47,33 @@ def generate_backlog(
print("Warning: GITHUB_TOKEN not found. Requests may be rate-limited.")
headers = get_github_headers(token)
+ # Load fallback cache from existing generated backlog to avoid data loss on failures
+ existing_lookup: dict[str, dict] = {}
+ if output_path.exists():
+ try:
+ with open(output_path, "r", encoding="utf-8") as f:
+ existing_data = json.load(f)
+ for existing_app in existing_data.get("applications", []):
+ raw_url = existing_app.get("repo_url", "")
+ if raw_url:
+ existing_lookup[raw_url.strip()] = existing_app
+ existing_lookup[raw_url.strip().lower()] = existing_app
+ path = extract_repo_path(raw_url)
+ if path:
+ existing_lookup[path.lower()] = existing_app
+ name = existing_app.get("name", "")
+ if name:
+ existing_lookup[name.strip().lower()] = existing_app
+ except Exception as e:
+ print(f"Notice: Could not load existing backlog data for fallback: {e}")
+
applications = data.get("applications", [])
+ if not applications and output_path.exists():
+ print(
+ f"Warning: No applications found in {input_path}. Preserving existing {output_path} to prevent data loss."
+ )
+ return {"applications": []}
+
total = len(applications)
repo_paths = [
@@ -65,13 +94,33 @@ def generate_backlog(
stars = 0
repo_path = extract_repo_path(repo_url)
+
+ # Look up fallback from cache
+ fallback_app = None
+ if repo_path:
+ fallback_app = existing_lookup.get(repo_path.lower())
+ if not fallback_app and repo_url:
+ fallback_app = existing_lookup.get(repo_url.strip()) or existing_lookup.get(
+ repo_url.strip().lower()
+ )
+ if not fallback_app and name:
+ fallback_app = existing_lookup.get(name.strip().lower())
+
if repo_path and repo_path in batch_results and batch_results[repo_path]:
repo_data = batch_results[repo_path]
stars = repo_data.get("stargazers_count", 0)
last_commit = format_commit_date(repo_data.get("pushed_at"))
elif repo_path:
- last_commit, stars = fetch_repo_summary(repo_path, headers)
+ # Batch didn't return data; if fallback cache exists, use it to avoid data loss
+ if fallback_app and fallback_app.get("last_commit"):
+ last_commit = fallback_app.get("last_commit", "")
+ stars = fallback_app.get("stars", 0)
+ else:
+ last_commit, stars = fetch_repo_summary(repo_path, headers)
else:
+ if fallback_app:
+ last_commit = fallback_app.get("last_commit", "")
+ stars = fallback_app.get("stars", 0)
print(
f"[{idx}/{total}] Skipping {name}: Invalid or non-GitHub URL ({repo_url})"
)
@@ -88,10 +137,18 @@ def generate_backlog(
generated_data = {"applications": generated_apps}
+ # Atomic write to prevent file corruption or destructive 0-byte truncation on crash
output_path.parent.mkdir(parents=True, exist_ok=True)
- with open(output_path, "w", encoding="utf-8") as f:
- json.dump(generated_data, f, indent=4)
- f.write("\n")
+ temp_path = output_path.with_name(f"{output_path.name}.tmp.{os.getpid()}")
+ try:
+ with open(temp_path, "w", encoding="utf-8") as f:
+ json.dump(generated_data, f, indent=4)
+ f.write("\n")
+ temp_path.replace(output_path)
+ except Exception:
+ if temp_path.exists():
+ temp_path.unlink()
+ raise
print(
f"Successfully generated {output_path} with {len(generated_apps)} applications."
diff --git a/core/source/maintenance/stats_updator.py b/core/source/maintenance/stats_updator.py
index 1ee79f9..088a71a 100644
--- a/core/source/maintenance/stats_updator.py
+++ b/core/source/maintenance/stats_updator.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import json
+import os
from pathlib import Path
import sys
@@ -26,8 +27,46 @@ def update_application_data(
) -> dict:
repo_url = app.get("repo_url", "")
repo_name = extract_repo_path(repo_url)
+ flags = app.get("flags", [])
+
if not repo_name:
print(f"Skipping {app.get('name', 'Unknown')}: Invalid GitHub URL ({repo_url})")
+ if fallback_app:
+ if "stars" in fallback_app and fallback_app["stars"] is not None:
+ app["stars"] = fallback_app["stars"]
+ elif "stars" not in app or app["stars"] is None:
+ app["stars"] = 0
+ if "language" in fallback_app and fallback_app["language"] is not None:
+ app["language"] = fallback_app["language"]
+ elif "language" not in app or app["language"] is None:
+ app["language"] = ""
+ if (
+ "homepage_url" in fallback_app
+ and fallback_app["homepage_url"] is not None
+ ):
+ app["homepage_url"] = fallback_app["homepage_url"]
+ elif "homepage_url" not in app or app["homepage_url"] is None:
+ app["homepage_url"] = ""
+ if (
+ "last_commit" in fallback_app
+ and fallback_app["last_commit"] is not None
+ ):
+ app["last_commit"] = fallback_app["last_commit"]
+ elif "last_commit" not in app or app["last_commit"] is None:
+ app["last_commit"] = ""
+ else:
+ if "stars" not in app or app["stars"] is None:
+ app["stars"] = 0
+ if "language" not in app or app["language"] is None:
+ app["language"] = ""
+ if "homepage_url" not in app or app["homepage_url"] is None:
+ app["homepage_url"] = ""
+ if "last_commit" not in app or app["last_commit"] is None:
+ app["last_commit"] = ""
+ if "description" not in app or app["description"] is None:
+ app["description"] = ""
+ if "license" not in app or app["license"] is None:
+ app["license"] = ""
return app
if repo_data is None and headers is not None:
@@ -36,45 +75,129 @@ def update_application_data(
if repo_data is not None:
app["stars"] = repo_data.get("stargazers_count", app.get("stars", 0))
- app["language"] = repo_data.get("language", app.get("language", ""))
+ app["language"] = repo_data.get("language") or app.get("language", "") or ""
# Flags
- if "custom-homepage" not in app.get("flags", []):
- app["homepage_url"] = repo_data.get("homepage", app.get("homepage_url", ""))
+ if "custom-homepage" not in flags:
+ app["homepage_url"] = (
+ repo_data.get("homepage") or app.get("homepage_url", "") or ""
+ )
+ elif "homepage_url" not in app or app["homepage_url"] is None:
+ app["homepage_url"] = ""
- if "custom-description" not in app.get("flags", []):
- app["description"] = repo_data.get(
- "description", app.get("description", "")
+ if "custom-description" not in flags:
+ app["description"] = (
+ repo_data.get("description", app.get("description", "")) or ""
)
+ elif "description" not in app or app["description"] is None:
+ app["description"] = ""
- if "custom-license" not in app.get("flags", []):
+ if "custom-license" not in flags:
license_data = repo_data.get("license")
- if license_data is not None:
- app["license"] = license_data.get("spdx_id", app.get("license", ""))
+ if isinstance(license_data, dict):
+ app["license"] = (
+ license_data.get("spdx_id") or app.get("license", "") or ""
+ )
+ elif isinstance(license_data, str):
+ app["license"] = license_data
else:
- app["license"] = app.get("license", "")
+ app["license"] = app.get("license", "") or ""
+ elif "license" not in app or app["license"] is None:
+ app["license"] = ""
# Check
pushed_at = repo_data.get("pushed_at")
if pushed_at:
app["last_commit"] = format_commit_date(pushed_at)
else:
- app["last_commit"] = app.get("last_commit", "")
+ app["last_commit"] = app.get("last_commit", "") or ""
+
+ # Ensure all dynamic keys exist and are not None
+ if "stars" not in app or app["stars"] is None:
+ app["stars"] = 0
+ if "language" not in app or app["language"] is None:
+ app["language"] = ""
+ if "homepage_url" not in app or app["homepage_url"] is None:
+ app["homepage_url"] = ""
+ if "last_commit" not in app or app["last_commit"] is None:
+ app["last_commit"] = ""
+ if "description" not in app or app["description"] is None:
+ app["description"] = ""
+ if "license" not in app or app["license"] is None:
+ app["license"] = ""
return app
else:
- # If fetch failed but fallback_app exists, preserve previous dynamic values
+ # If fetch failed but fallback_app exists, preserve previous dynamic values without wiping
if fallback_app:
- for field in [
- "stars",
- "language",
- "homepage_url",
- "description",
- "license",
- "last_commit",
- ]:
- if (field not in app or not app.get(field)) and fallback_app.get(field):
- app[field] = fallback_app[field]
+ # 1. stars
+ if "stars" in fallback_app and fallback_app["stars"] is not None:
+ app["stars"] = fallback_app["stars"]
+ elif "stars" not in app or app["stars"] is None:
+ app["stars"] = 0
+
+ # 2. language
+ if "language" in fallback_app and fallback_app["language"] is not None:
+ app["language"] = fallback_app["language"]
+ elif "language" not in app or app["language"] is None:
+ app["language"] = ""
+
+ # 3. homepage_url
+ if "custom-homepage" not in flags:
+ if (
+ "homepage_url" in fallback_app
+ and fallback_app["homepage_url"] is not None
+ ):
+ app["homepage_url"] = fallback_app["homepage_url"]
+ elif "homepage_url" not in app or app["homepage_url"] is None:
+ app["homepage_url"] = ""
+ elif "homepage_url" not in app or app["homepage_url"] is None:
+ app["homepage_url"] = ""
+
+ # 4. description
+ if "custom-description" not in flags:
+ if (
+ "description" in fallback_app
+ and fallback_app["description"] is not None
+ ):
+ app["description"] = fallback_app["description"]
+ elif "description" not in app or app["description"] is None:
+ app["description"] = ""
+ elif "description" not in app or app["description"] is None:
+ app["description"] = ""
+
+ # 5. license
+ if "custom-license" not in flags:
+ if "license" in fallback_app and fallback_app["license"] is not None:
+ app["license"] = fallback_app["license"]
+ elif "license" not in app or app["license"] is None:
+ app["license"] = ""
+ elif "license" not in app or app["license"] is None:
+ app["license"] = ""
+
+ # 6. last_commit
+ if (
+ "last_commit" in fallback_app
+ and fallback_app["last_commit"] is not None
+ ):
+ app["last_commit"] = fallback_app["last_commit"]
+ elif "last_commit" not in app or app["last_commit"] is None:
+ app["last_commit"] = ""
+ else:
+ # No fallback available, ensure valid schema defaults so entries are not stripped
+ if "stars" not in app or app["stars"] is None:
+ app["stars"] = 0
+ if "language" not in app or app["language"] is None:
+ app["language"] = ""
+ if "homepage_url" not in app or app["homepage_url"] is None:
+ app["homepage_url"] = ""
+ if "description" not in app or app["description"] is None:
+ app["description"] = ""
+ if "license" not in app or app["license"] is None:
+ app["license"] = ""
+ if "last_commit" not in app or app["last_commit"] is None:
+ app["last_commit"] = ""
+
return app
@@ -112,10 +235,16 @@ def update_all_applications(
with open(output_path, "r", encoding="utf-8") as f:
existing_data = json.load(f)
for existing_app in existing_data.get("applications", []):
- if existing_app.get("repo_url"):
- existing_lookup[existing_app["repo_url"]] = existing_app
- if existing_app.get("name"):
- existing_lookup[existing_app["name"]] = existing_app
+ raw_url = existing_app.get("repo_url", "")
+ if raw_url:
+ existing_lookup[raw_url.strip()] = existing_app
+ existing_lookup[raw_url.strip().lower()] = existing_app
+ path = extract_repo_path(raw_url)
+ if path:
+ existing_lookup[path.lower()] = existing_app
+ name = existing_app.get("name", "")
+ if name:
+ existing_lookup[name.strip().lower()] = existing_app
except Exception as e:
print(f"Notice: Could not load existing generated data for fallback: {e}")
@@ -123,6 +252,12 @@ def update_all_applications(
data = json.load(f)
applications = data.get("applications", [])
+ if not applications and output_path.exists():
+ print(
+ f"Warning: No applications found in {input_path}. Preserving existing {output_path} to prevent data loss."
+ )
+ return {"applications": []}
+
repo_paths: list[str] = []
for app in applications:
repo_path = extract_repo_path(app.get("repo_url", ""))
@@ -139,15 +274,26 @@ def update_all_applications(
generated_apps = []
for app in applications:
- repo_path = extract_repo_path(app.get("repo_url", ""))
+ raw_url = app.get("repo_url", "")
+ repo_path = extract_repo_path(raw_url)
repo_data = batch_results.get(repo_path) if repo_path else None
- fallback_app = (
- existing_lookup.get(app.get("repo_url", ""))
- or existing_lookup.get(app.get("name", ""))
- )
+
+ # Look up fallback cache by normalized repo path, exact URL, or normalized name
+ fallback_app = None
+ if repo_path:
+ fallback_app = existing_lookup.get(repo_path.lower())
+ if not fallback_app and raw_url:
+ fallback_app = existing_lookup.get(raw_url.strip()) or existing_lookup.get(
+ raw_url.strip().lower()
+ )
+ if not fallback_app and app.get("name"):
+ fallback_app = existing_lookup.get(app["name"].strip().lower())
+
+ # Do not pass headers=headers here because batch fetch was already performed;
+ # this avoids blocking on hundreds of synchronous individual requests if batch failed.
updated_app = update_application_data(
app.copy(),
- headers=headers,
+ headers=None,
repo_data=repo_data,
fallback_app=fallback_app,
)
@@ -155,10 +301,18 @@ def update_all_applications(
generated_data = {"applications": generated_apps}
+ # Atomic write to prevent file corruption or destructive 0-byte truncation on crash
output_path.parent.mkdir(parents=True, exist_ok=True)
- with open(output_path, "w", encoding="utf-8") as f:
- json.dump(generated_data, f, indent=4)
- f.write("\n")
+ temp_path = output_path.with_name(f"{output_path.name}.tmp.{os.getpid()}")
+ try:
+ with open(temp_path, "w", encoding="utf-8") as f:
+ json.dump(generated_data, f, indent=4)
+ f.write("\n")
+ temp_path.replace(output_path)
+ except Exception:
+ if temp_path.exists():
+ temp_path.unlink()
+ raise
print(f"Updated application data successfully. Output written to {output_path}.")
return generated_data
@@ -166,4 +320,3 @@ def update_all_applications(
if __name__ == "__main__":
update_all_applications()
-
diff --git a/core/tests/test_batch_github.py b/core/tests/test_batch_github.py
index 0e18e07..3b32251 100644
--- a/core/tests/test_batch_github.py
+++ b/core/tests/test_batch_github.py
@@ -11,6 +11,7 @@ CORE_DIR = Path(__file__).resolve().parents[1]
if str(CORE_DIR) not in sys.path:
sys.path.insert(0, str(CORE_DIR))
+from source.maintenance.backlog_updator import generate_backlog # noqa: E402
from source.maintenance.stats_updator import ( # noqa: E402
update_all_applications,
update_application_data,
@@ -158,7 +159,10 @@ class TestBatchGitHubUtils(unittest.TestCase):
mock_graphql.assert_called_once()
mock_rest.assert_called_once_with(
- ["owner/renamed"], headers=mock_rest.call_args[1]["headers"], max_workers=8, timeout=30
+ ["owner/renamed"],
+ headers=mock_rest.call_args[1]["headers"],
+ max_workers=8,
+ timeout=30,
)
self.assertEqual(results["owner/good"]["stargazers_count"], 200)
self.assertEqual(results["owner/renamed"]["stargazers_count"], 300)
@@ -256,6 +260,90 @@ class TestStatsUpdator(unittest.TestCase):
self.assertEqual(updated["license"], "MIT")
self.assertEqual(updated["last_commit"], "09/20/2026")
+ def test_update_application_data_preserves_zero_stars_from_fallback(self):
+ app = {
+ "name": "Zero Star App",
+ "repo_url": "https://github.com/foo/zero",
+ "category": "ai",
+ "flags": [],
+ }
+ fallback_app = {
+ "name": "Zero Star App",
+ "repo_url": "https://github.com/foo/zero",
+ "stars": 0,
+ "language": "Python",
+ "homepage_url": "",
+ "description": "Zero stars repo",
+ "license": "MIT",
+ "last_commit": "09/01/2026",
+ }
+
+ updated = update_application_data(
+ app.copy(),
+ repo_data=None,
+ fallback_app=fallback_app,
+ )
+ self.assertIn("stars", updated)
+ self.assertEqual(updated["stars"], 0)
+ self.assertEqual(updated["language"], "Python")
+ self.assertEqual(updated["description"], "Zero stars repo")
+ self.assertEqual(updated["license"], "MIT")
+ self.assertEqual(updated["last_commit"], "09/01/2026")
+
+ def test_update_application_data_populates_safe_defaults_when_no_fallback(self):
+ app = {
+ "name": "Brand New App",
+ "repo_url": "https://github.com/new/app",
+ "category": "ai",
+ "flags": [],
+ }
+
+ # Both repo_data and fallback_app are None
+ updated = update_application_data(
+ app.copy(),
+ repo_data=None,
+ fallback_app=None,
+ )
+ self.assertEqual(updated["stars"], 0)
+ self.assertEqual(updated["language"], "")
+ self.assertEqual(updated["homepage_url"], "")
+ self.assertEqual(updated["description"], "")
+ self.assertEqual(updated["license"], "")
+ self.assertEqual(updated["last_commit"], "")
+
+ def test_update_application_data_preserves_custom_flags_on_fetch_failure(self):
+ app = {
+ "name": "Custom Flags App",
+ "repo_url": "https://github.com/foo/flags",
+ "category": "ai",
+ "flags": ["custom-description", "custom-license", "custom-homepage"],
+ "description": "Curated description",
+ "license": "Proprietary",
+ "homepage_url": "https://custom.site",
+ }
+ fallback_app = {
+ "name": "Custom Flags App",
+ "repo_url": "https://github.com/foo/flags",
+ "stars": 42,
+ "language": "Go",
+ "description": "Old stale description",
+ "license": "Old License",
+ "homepage_url": "https://old.site",
+ "last_commit": "08/15/2026",
+ }
+
+ updated = update_application_data(
+ app.copy(),
+ repo_data=None,
+ fallback_app=fallback_app,
+ )
+ self.assertEqual(updated["description"], "Curated description")
+ self.assertEqual(updated["license"], "Proprietary")
+ self.assertEqual(updated["homepage_url"], "https://custom.site")
+ self.assertEqual(updated["stars"], 42)
+ self.assertEqual(updated["language"], "Go")
+ self.assertEqual(updated["last_commit"], "08/15/2026")
+
@patch("source.maintenance.stats_updator.fetch_repos_batch")
def test_update_all_applications_flow(self, mock_batch):
mock_batch.return_value = {
@@ -314,6 +402,116 @@ class TestStatsUpdator(unittest.TestCase):
if test_output.exists():
test_output.unlink()
+ @patch("source.maintenance.stats_updator.fetch_repos_batch")
+ def test_update_all_applications_preserves_fallback_cache_on_batch_failure(
+ self, mock_batch
+ ):
+ # Simulate complete batch fetch failure (e.g. rate limit)
+ mock_batch.return_value = {"foo/bar": None}
+
+ test_input = CORE_DIR / "tests" / "test_input_failure.json"
+ test_output = CORE_DIR / "tests" / "test_output_failure.json"
+
+ input_data = {
+ "applications": [
+ {
+ "name": "FooBar",
+ "repo_url": "https://github.com/foo/bar/", # Note trailing slash
+ "category": "tools",
+ "flags": [],
+ }
+ ]
+ }
+ existing_data = {
+ "applications": [
+ {
+ "name": "FooBar",
+ "repo_url": "https://github.com/foo/bar",
+ "stars": 9999,
+ "language": "Rust",
+ "homepage_url": "https://foobar.rs",
+ "description": "Previously cached description",
+ "license": "MIT",
+ "last_commit": "09/10/2026",
+ }
+ ]
+ }
+ with open(test_input, "w", encoding="utf-8") as f:
+ json.dump(input_data, f)
+ with open(test_output, "w", encoding="utf-8") as f:
+ json.dump(existing_data, f)
+
+ try:
+ result = update_all_applications(
+ input_file=test_input,
+ output_file=test_output,
+ batch_size=50,
+ )
+ self.assertEqual(len(result["applications"]), 1)
+ app = result["applications"][0]
+ # Verify cached metadata was preserved and not stripped
+ self.assertEqual(app["stars"], 9999)
+ self.assertEqual(app["language"], "Rust")
+ self.assertEqual(app["homepage_url"], "https://foobar.rs")
+ self.assertEqual(app["description"], "Previously cached description")
+ self.assertEqual(app["license"], "MIT")
+ self.assertEqual(app["last_commit"], "09/10/2026")
+ finally:
+ if test_input.exists():
+ test_input.unlink()
+ if test_output.exists():
+ test_output.unlink()
+
+
+class TestBacklogUpdator(unittest.TestCase):
+ @patch("source.maintenance.backlog_updator.fetch_repos_batch")
+ def test_generate_backlog_preserves_fallback_cache_on_failure(self, mock_batch):
+ mock_batch.return_value = {"old/project": None}
+
+ test_input = CORE_DIR / "tests" / "test_backlog_input.json"
+ test_output = CORE_DIR / "tests" / "test_backlog_output.json"
+
+ input_data = {
+ "applications": [
+ {
+ "name": "Old Project",
+ "repo_url": "https://github.com/old/project/",
+ "note": "Potentially abandoned",
+ }
+ ]
+ }
+ existing_data = {
+ "applications": [
+ {
+ "name": "Old Project",
+ "repo_url": "https://github.com/old/project",
+ "note": "Potentially abandoned",
+ "last_commit": "05/10/2025",
+ "stars": 321,
+ }
+ ]
+ }
+ with open(test_input, "w", encoding="utf-8") as f:
+ json.dump(input_data, f)
+ with open(test_output, "w", encoding="utf-8") as f:
+ json.dump(existing_data, f)
+
+ try:
+ result = generate_backlog(
+ input_file=test_input,
+ output_file=test_output,
+ )
+ self.assertEqual(len(result["applications"]), 1)
+ app = result["applications"][0]
+ self.assertEqual(app["name"], "Old Project")
+ self.assertEqual(app["stars"], 321)
+ self.assertEqual(app["last_commit"], "05/10/2025")
+ finally:
+ if test_input.exists():
+ test_input.unlink()
+ if test_output.exists():
+ test_output.unlink()
+
if __name__ == "__main__":
unittest.main()