Agent Zero AI framework
Remove backup file count cap
Alessandro71607476071e
diff --git a/api/backup_test.py b/api/backup_test.pyindex b5b43eac8..c7a748ebd 100644--- a/api/backup_test.py+++ b/api/backup_test.py@@ -47,12 +47,13 @@ class BackupTest(ApiHandler):backup_service = BackupService()matched_files = await backup_service.test_patterns(metadata, max_files=max_files)+ truncated = max_files is not None and len(matched_files) >= max_filesreturn {"success": True,"files": matched_files,"total_count": len(matched_files),- "truncated": len(matched_files) >= max_files+ "truncated": truncated}except Exception as e:diff --git a/api/backup_test.py.dox.md b/api/backup_test.py.dox.mdindex c5f01a155..a8e06cc6b 100644--- a/api/backup_test.py.dox.md+++ b/api/backup_test.py.dox.md@@ -25,6 +25,7 @@- `BackupTest` defines `requires_auth(...)`.- `BackupTest` defines `requires_loopback(...)`.- Imported dependency areas include: `helpers.api`, `helpers.backup`.+- The `truncated` response flag is true only when a finite `max_files` limit is supplied and the result reaches that limit.## Key Conceptsdiff --git a/helpers/backup.py b/helpers/backup.pyindex 1aa651893..47e41c5f8 100644--- a/helpers/backup.py+++ b/helpers/backup.py@@ -240,8 +240,12 @@ class BackupService:return translated_patterns- async def test_patterns(self, metadata: Dict[str, Any], max_files: int = 1000) -> List[Dict[str, Any]]:- """Test backup patterns and return list of matched files"""+ async def test_patterns(self, metadata: Dict[str, Any], max_files: Optional[int] = 1000) -> List[Dict[str, Any]]:+ """Test backup patterns and return list of matched files.++ Pass max_files=None for internal flows that must process the complete+ match set, such as backup creation and restore cleanup.+ """include_patterns = metadata.get("include_patterns", [])exclude_patterns = metadata.get("exclude_patterns", [])include_hidden = metadata.get("include_hidden", True)@@ -258,6 +262,7 @@ class BackupService:# Get explicit patterns for hidden file handlingexplicit_patterns = self._get_explicit_patterns(include_patterns)+ has_limit = max_files is not Nonematched_files = []processed_count = 0@@ -285,7 +290,7 @@ class BackupService:dirs[:] = dirs_to_keepfor file in files_list:- if processed_count >= max_files:+ if has_limit and processed_count >= max_files:breakfile_path = os.path.join(root, file)@@ -317,10 +322,10 @@ class BackupService:# Skip files we can't accesscontinue- if processed_count >= max_files:+ if has_limit and processed_count >= max_files:break- if processed_count >= max_files:+ if has_limit and processed_count >= max_files:breakexcept Exception as e:@@ -344,8 +349,10 @@ class BackupService:"include_hidden": include_hidden}- # Get matched files- matched_files = await self.test_patterns(metadata, max_files=50000)+ # Get the complete matched file set. Preview and dry-run callers may+ # cap their scans for UI responsiveness, but the archive itself must be+ # complete.+ matched_files = await self.test_patterns(metadata, max_files=None)if not matched_files:raise Exception("No files matched the backup patterns")@@ -824,7 +831,7 @@ class BackupService:# Find existing files that match the translated user-edited patternstry:- existing_files = await self.test_patterns(metadata, max_files=10000)+ existing_files = await self.test_patterns(metadata, max_files=None)# Convert to delete operations formatfiles_to_delete = []diff --git a/helpers/backup.py.dox.md b/helpers/backup.py.dox.mdindex a72c49377..e08f7a74e 100644--- a/helpers/backup.py.dox.md+++ b/helpers/backup.py.dox.md@@ -13,7 +13,7 @@- Classes:- `BackupService` (no explicit base class)- `get_default_backup_metadata(self) -> Dict[str, Any]`- - `async test_patterns(self, metadata: Dict[str, Any], max_files: int=...) -> List[Dict[str, Any]]`+ - `async test_patterns(self, metadata: Dict[str, Any], max_files: Optional[int]=...) -> List[Dict[str, Any]]`- `async create_backup(self, include_patterns: List[str], exclude_patterns: List[str], include_hidden: bool=..., backup_name: str=...) -> str`- `async inspect_backup(self, backup_file) -> Dict[str, Any]`- `async preview_restore(self, backup_file, restore_include_patterns: Optional[List[str]]=..., restore_exclude_patterns: Optional[List[str]]=..., overwrite_policy: str=..., clean_before_restore: bool=..., user_edited_metadata: Optional[Dict[str, Any]]=...) -> Dict[str, Any]`@@ -25,6 +25,7 @@- Update this file whenever public functions, classes, persistence behavior, path/security assumptions, side effects, or cross-module contracts change.- Observed side-effect areas: filesystem reads, filesystem writes, filesystem deletion, settings/state persistence, secret handling.- Imported dependency areas include: `datetime`, `helpers`, `helpers.localization`, `helpers.print_style`, `json`, `os`, `pathspec`, `platform`, `tempfile`, `typing`, `zipfile`.+- `test_patterns(..., max_files=None)` is the unlimited scan mode. UI preview and dry-run callers may pass bounded limits, but real backup creation and restore clean-before-restore must use unlimited matching so archives and cleanup are not silently truncated.## Key Conceptsdiff --git a/tests/test_backup_large_archives.py b/tests/test_backup_large_archives.pynew file mode 100644index 000000000..7638fbbc5--- /dev/null+++ b/tests/test_backup_large_archives.py@@ -0,0 +1,162 @@+import json+import shutil+import zipfile+from pathlib import Path++import pytest++from helpers.backup import BackupService+++class UploadedBackup:+ def __init__(self, path: Path):+ self.path = path++ def save(self, target: str) -> None:+ shutil.copyfile(self.path, target)+++async def test_pattern_scan_can_run_without_file_limit(tmp_path):+ root = tmp_path / "a0"+ usr = root / "usr"+ usr.mkdir(parents=True)+ for index in range(3):+ (usr / f"file-{index}.txt").write_text(f"{index}\n", encoding="utf-8")++ service = BackupService()+ service.agent_zero_root = str(root)+ service.base_paths = {str(root): str(root)}+ metadata = {+ "include_patterns": [f"{root}/usr/**"],+ "exclude_patterns": [],+ "include_hidden": True,+ }++ capped_files = await service.test_patterns(metadata, max_files=2)+ all_files = await service.test_patterns(metadata, max_files=None)++ assert len(capped_files) == 2+ assert len(all_files) == 3+++async def test_create_backup_uses_unlimited_pattern_scan(tmp_path, monkeypatch):+ source_file = tmp_path / "source.txt"+ source_file.write_text("payload\n", encoding="utf-8")+ captured = {}++ service = BackupService()++ async def fake_test_patterns(metadata, max_files=1000):+ captured["max_files"] = max_files+ return [+ {+ "path": f"{service.agent_zero_root.rstrip('/')}/usr/file-{index}.txt",+ "real_path": str(source_file),+ "size": source_file.stat().st_size,+ "modified": "2026-06-26T00:00:00+00:00",+ "type": "file",+ }+ for index in range(3)+ ]++ async def fake_info():+ return {}++ async def fake_author():+ return "test"++ monkeypatch.setattr(service, "test_patterns", fake_test_patterns)+ monkeypatch.setattr(service, "_get_system_info", fake_info)+ monkeypatch.setattr(service, "_get_environment_info", fake_info)+ monkeypatch.setattr(service, "_get_backup_author", fake_author)++ zip_path = await service.create_backup(+ include_patterns=[f"{service.agent_zero_root}/usr/**"],+ exclude_patterns=[],+ include_hidden=True,+ backup_name="large-backup",+ )++ assert captured["max_files"] is None+ with zipfile.ZipFile(zip_path) as archive:+ metadata = json.loads(archive.read("metadata.json").decode("utf-8"))+ assert metadata["total_files"] == 3+ assert (+ f"{service.agent_zero_root.rstrip('/').lstrip('/')}/usr/file-2.txt"+ in archive.namelist()+ )+++async def test_restore_can_reach_files_after_50000_archive_entries(tmp_path):+ old_root = "/old-a0"+ archive_root = old_root.lstrip("/")+ file_count = 50_001+ last_index = file_count - 1+ zip_path = tmp_path / "large-backup.zip"++ metadata = {+ "environment_info": {"agent_zero_root": old_root},+ "include_patterns": [f"{old_root}/usr/large/**"],+ "exclude_patterns": [],+ "include_hidden": True,+ }+ with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_STORED) as archive:+ archive.writestr("metadata.json", json.dumps(metadata))+ for index in range(file_count):+ payload = "tail payload\n" if index == last_index else ""+ archive.writestr(+ f"{archive_root}/usr/large/file-{index:05d}.txt",+ payload,+ )++ service = BackupService()+ service.agent_zero_root = str(tmp_path / "restored-a0")++ result = await service.restore_backup(+ backup_file=UploadedBackup(zip_path),+ restore_include_patterns=[+ f"{old_root}/usr/large/file-{last_index:05d}.txt"+ ],+ restore_exclude_patterns=[],+ overwrite_policy="overwrite",+ )++ restored_path = (+ Path(service.agent_zero_root)+ / "usr"+ / "large"+ / f"file-{last_index:05d}.txt"+ )+ assert len(result["restored_files"]) == 1+ assert len(result["skipped_files"]) == file_count - 1+ assert result["errors"] == []+ assert restored_path.read_text(encoding="utf-8") == "tail payload\n"+++async def test_restore_clean_before_restore_uses_unlimited_pattern_scan(monkeypatch):+ service = BackupService()+ captured = {}++ async def fake_test_patterns(metadata, max_files=1000):+ captured["max_files"] = max_files+ return []++ monkeypatch.setattr(service, "test_patterns", fake_test_patterns)++ result = await service._find_files_to_clean_with_user_metadata(+ user_metadata={+ "include_patterns": [f"{service.agent_zero_root}/usr/**"],+ "exclude_patterns": [],+ "include_hidden": True,+ },+ original_metadata={+ "environment_info": {"agent_zero_root": service.agent_zero_root}+ },+ )++ assert result == []+ assert captured["max_files"] is None