Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix(cos): preserve concurrent shared Git data during recovery (#12731)
[._.]/011605e43925
diff --git a/scripts/lib/initialized_checkout_archive.py b/scripts/lib/initialized_checkout_archive.pyindex 2839fa9f71..2f4c43c7e0 100644--- a/scripts/lib/initialized_checkout_archive.py+++ b/scripts/lib/initialized_checkout_archive.py@@ -1209,15 +1209,181 @@ def verify(request):**({'metadataRecoveryAccepted': True} if recovering else {})}-def verify_common_metadata(saved, mutable_shared):+def unrelated_tracking_changes(root, saved, common, expected, observed):+ """Concurrent branch creation/deletion may rewrite config; never undo it.++ Only whole tracking-only stanzas for unrelated local branches may appear+ or disappear, with present/absent refs proven respectively. Existing values,+ target/global settings, security metadata and archive bytes remain strict.+ Capture and ordinary moves never use this.+ """+ if root is None or len(expected) != 1 or len(observed) != 1:+ return False+ old, current = expected[0], observed[0]+ variable = {'bytes', 'sha256', 'mtimeNs', 'filesystemIdentity'}+ if old.get('kind') != 'file' or {k: v for k, v in old.items() if k not in variable} != {+ k: v for k, v in current.items() if k not in variable}:+ return False+ if max(old.get('bytes', 0), current.get('bytes', 0)) > 16 * 1024 ** 2:+ return False+ archived = root / 'archive/common/config'+ if sha(archived) != old['sha256']:+ fail('shared-tracking-archive-changed')+ def entries(path):+ raw = git(common, Path(saved['facts']['repoPath']), 'config', '--null',+ '--no-includes', '--file', str(path), '--list')+ records = []+ for value in raw.split('\0'):+ if not value:+ continue+ key, separator, content = value.partition('\n')+ # Preserve boolean/no-value distinctions and repeated-value order.+ records.append((key, separator, content))+ return records+ before, after = entries(archived), entries(common / 'config')+ old_keys, new_keys = {r[0] for r in before}, {r[0] for r in after}+ removed, added = old_keys - new_keys, new_keys - old_keys+ removed_branches, added_branches = set(), set()+ for key in removed | added:+ match = re.fullmatch(r'branch\.(.+)\.(remote|merge)', key)+ if not match or match[1] == saved['facts']['release']['branch']:+ return False+ (removed_branches if key in removed else added_branches).add(match[1])+ if not removed_branches and not added_branches:+ return False+ for branch in removed_branches:+ prefix = 'branch.' + branch + '.'+ # Removing only part of a stanza, or non-tracking attributes, is not+ # the narrowly understood unrelated-branch cleanup operation.+ if any(key.startswith(prefix) for key in new_keys) or any(+ key.startswith(prefix) and key not in removed for key in old_keys):+ return False+ ref = 'refs/heads/' + branch+ git(common, Path(saved['facts']['repoPath']), 'check-ref-format', ref)+ try:+ require_absent_local_ref(saved['facts'], ref)+ except RuntimeError:+ return False+ for branch in added_branches:+ prefix = 'branch.' + branch + '.'+ if any(key.startswith(prefix) for key in old_keys) or {+ key for key in new_keys if key.startswith(prefix)} != {+ prefix + 'remote', prefix + 'merge'}:+ return False+ ref = 'refs/heads/' + branch+ try:+ git(common, Path(saved['facts']['repoPath']), 'check-ref-format', ref)+ git(common, Path(saved['facts']['repoPath']),+ 'symbolic-ref', '--quiet', ref, expected_returncode=1)+ head = git(common, Path(saved['facts']['repoPath']),+ 'rev-parse', '--verify', ref + '^{commit}')+ if git(common, Path(saved['facts']['repoPath']),+ 'show-ref', '--verify', ref) != head + ' ' + ref:+ return False+ git(common, Path(saved['facts']['repoPath']),+ 'symbolic-ref', '--quiet', ref, expected_returncode=1)+ except RuntimeError:+ return False+ if not re.fullmatch(r'[a-f0-9]{' + str(len(saved['facts']['parent']['head'])) + '}', head):+ return False+ if [r for r in before if r[0] not in removed] != [r for r in after if r[0] not in added]:+ return False+ # The parser and ref observations must describe the same file incarnation.+ if manifest(common / 'config', True) != observed or sha(archived) != old['sha256']:+ fail('shared-tracking-observation-changed')+ return True+++def verify_live_resolution_cache(path, expected):+ """Recovery leaves this shared conflict-resolution cache entirely live.++ Its original bytes remain checked in verify_saved_archive. Concurrent Git+ resolutions may update or prune entries; none are read by recovery's Git+ commands. Keep the root incarnation/security and a bounded no-follow shape.+ """+ original = next((row for row in expected if row['path'] == ''), None)+ def root_security():+ info = path.lstat()+ return {'path': '', 'kind': 'directory' if stat.S_ISDIR(info.st_mode) else 'unsafe',+ 'mode': stat.S_IMODE(info.st_mode), 'uid': info.st_uid, 'gid': info.st_gid,+ 'flags': info.st_flags, 'acl': acl(path),+ 'xattrs': {k: hashlib.sha256(v).hexdigest() for k, v in raw_xattrs(path).items()},+ 'filesystemIdentity': identity(path)}+ pinned = {k: v for k, v in (original or {}).items() if k != 'mtimeNs'}+ if root_security() != pinned:+ fail('shared-resolution-cache-root-changed')+ fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)+ try:+ held = os.fstat(fd)+ if (held.st_dev, held.st_ino) != (pinned['filesystemIdentity']['dev'], pinned['filesystemIdentity']['ino']):+ fail('shared-resolution-cache-root-changed')+ count, size = 0, 0+ def failed_walk(_error):+ fail('shared-resolution-cache-unreadable')+ for _, directories, files, directory_fd in os.fwalk('.', dir_fd=fd, follow_symlinks=False, onerror=failed_walk):+ for name in directories + files:+ info = os.stat(name, dir_fd=directory_fd, follow_symlinks=False)+ count += 1+ if not (stat.S_ISDIR(info.st_mode) or stat.S_ISREG(info.st_mode)):+ fail('shared-resolution-cache-unsafe-entry')+ if stat.S_ISREG(info.st_mode):+ size += info.st_size+ if info.st_nlink != 1:+ fail('shared-resolution-cache-hardlink')+ if count > MAX_FILES or size > MAX_BYTES:+ fail('shared-resolution-cache-budget-exceeded')+ if root_security() != pinned:+ fail('shared-resolution-cache-root-changed')+ finally:+ os.close(fd)+++def shared_ref_advertisement_changed(common, expected, observed):+ """Only info/refs is generated data; every sibling remains a control.++ Local recovery plumbing does not consume this dumb-transport advertisement+ and never rewrites it. Its original bytes/provenance remain in the exact+ archive; atomic Git regeneration may replace the live file and provenance.+ """+ old, current = ({row['path']: row for row in rows} for rows in (expected, observed))+ if old.keys() != current.keys() or not {'', 'refs'} <= old.keys():+ return False+ if old['']['kind'] != 'directory' or {k: v for k, v in old[''].items() if k != 'mtimeNs'} != {+ k: v for k, v in current[''].items() if k != 'mtimeNs'}:+ return False+ before, after = old['refs'], current['refs']+ variable = {'bytes', 'sha256', 'mtimeNs', 'filesystemIdentity', 'xattrs'}+ if (before['kind'] != 'file' or after['kind'] != 'file'+ or 'hardlinkGroup' in before or 'hardlinkGroup' in after+ or max(before['bytes'], after['bytes']) > 16 * 1024 ** 2+ or {k: v for k, v in before.items() if k not in variable} != {+ k: v for k, v in after.items() if k not in variable}+ or {k: v for k, v in before['xattrs'].items() if k != 'com.apple.provenance'} != {+ k: v for k, v in after['xattrs'].items() if k != 'com.apple.provenance'}):+ return False+ if any(old[name] != current[name] for name in old.keys() - {'', 'refs'}):+ return False+ if manifest(common / 'info', True) != observed:+ fail('shared-ref-advertisement-observation-changed')+ return True+++def verify_common_metadata(saved, mutable_shared, root=None):facts = saved['facts']common = safe_path(facts['commonGitDir'])check_admin(common, common=True)if identity(common) != facts['identity']['common']:fail('common-store-incarnation-changed')for name, expected in facts['sources']['common'].items():- if name not in mutable_shared and manifest(common / name, True) != expected:- fail('common-store-changed:' + name)+ if name not in mutable_shared:+ if name == 'rr-cache' and mutable_shared:+ verify_live_resolution_cache(common / name, expected)+ continue+ observed = manifest(common / name, True)+ if observed != expected and not (mutable_shared and (+ (name == 'config' and unrelated_tracking_changes(root, saved, common, expected, observed))+ or (name == 'info' and shared_ref_advertisement_changed(common, expected, observed)))):+ fail('common-store-changed:' + name)if sorted(p.name for p in common.iterdir() if p.name in COMMON_PARTS and p.name not in mutable_shared) != sorted(name for name in facts['sources']['common'] if name not in mutable_shared):fail('common-store-layout-changed')@@ -1240,7 +1406,7 @@ def verify_live_object_closure(root, common):def verify_live_common(root, saved, restoring):- common = verify_common_metadata(saved, {'objects', 'refs', 'logs', 'packed-refs'} if restoring else set())+ common = verify_common_metadata(saved, {'objects', 'refs', 'logs', 'packed-refs'} if restoring else set(), root)if restoring:# Other branches may legitimately advance after quarantine. Rollback# never rewrites shared refs or objects: prove the original private index@@ -1306,19 +1472,8 @@ def require_no_packed_ref(common, ref, oid_length):fail('deleted-branch-packed-refs-changed')-def verify_deleted_parent(saved, request):- facts = saved['facts']- receipt = request.get('confirmedDeletion')- # The service derives this binding only from its validated durable receipt,- # never from ref absence or a deletion intent. This command writes no refs.- if (not isinstance(receipt, dict) or set(receipt) != {'journalDigest', 'branch', 'head', 'phase'}- or not isinstance(receipt['journalDigest'], str)- or not re.fullmatch(r'[0-9a-f]{64}', receipt['journalDigest'])- or receipt['branch'] != facts['release']['branch']- or receipt['head'] != facts['parent']['head']- or receipt['phase'] not in ('branch-retired', 'retired')):- fail('quarantine-resumption-confirmed-deletion-required')- ref = 'refs/heads/' + facts['release']['branch']+def require_absent_local_ref(facts, ref):+ """Prove absence independently of Git omitting broken loose/packed refs."""def require_no_loose_ref():current = safe_path(facts['commonGitDir'])if identity(current) != facts['identity']['common']:@@ -1349,6 +1504,21 @@ def verify_deleted_parent(saved, request):require_no_loose_ref()+def verify_deleted_parent(saved, request):+ facts = saved['facts']+ receipt = request.get('confirmedDeletion')+ # The service derives this binding only from its validated durable receipt,+ # never from ref absence or a deletion intent. This command writes no refs.+ if (not isinstance(receipt, dict) or set(receipt) != {'journalDigest', 'branch', 'head', 'phase'}+ or not isinstance(receipt['journalDigest'], str)+ or not re.fullmatch(r'[0-9a-f]{64}', receipt['journalDigest'])+ or receipt['branch'] != facts['release']['branch']+ or receipt['head'] != facts['parent']['head']+ or receipt['phase'] not in ('branch-retired', 'retired')):+ fail('quarantine-resumption-confirmed-deletion-required')+ require_absent_local_ref(facts, 'refs/heads/' + facts['release']['branch'])++def verify_deleted_branch(request):if not re.fullmatch(r'[0-9a-f]{64}', request.get('manifestSha256', '')):fail('deleted-branch-manifest-proof-required')@@ -1374,7 +1544,7 @@ def quarantine_observation(request, retired=False):verify_deleted_parent(saved, request)# Receipt replay restores nothing. The strict standalone archive proof# above retains closure even after legitimate live-store garbage collection.- verify_common_metadata(saved, {'objects', 'refs', 'logs', 'packed-refs'})+ verify_common_metadata(saved, {'objects', 'refs', 'logs', 'packed-refs'}, root)else:verify_live_common(root, saved, restoring=True)if original_sources(root, saved['facts']) != (located, sources):diff --git a/scripts/lib/initialized_checkout_archive_test.py b/scripts/lib/initialized_checkout_archive_test.pyindex 384fdea194..cac1f0b711 100644--- a/scripts/lib/initialized_checkout_archive_test.py+++ b/scripts/lib/initialized_checkout_archive_test.py@@ -1166,6 +1166,196 @@ class NativeArchive(unittest.TestCase):'only in index')self.assertNotEqual(self.git(tree, 'rev-parse', recovery_ref), self.git(tree, 'rev-parse', 'HEAD'))+ def test_quarantine_recovery_preserves_unrelated_tracking_creation_and_cleanup(self):+ cache = self.repo / '.git/rr-cache'+ conflict = cache / ('a' * 40)+ conflict.mkdir(parents=True)+ (conflict / 'preimage').write_text('old conflict\n')+ (conflict / 'postimage').write_text('old resolution\n')+ self.git(self.repo, 'branch', 'other/finished')+ self.git(self.repo, 'config', 'branch.other/finished.remote', 'origin')+ self.git(self.repo, 'config', '--add', 'branch.other/finished.merge', 'refs/heads/other/finished')+ self.git(self.repo, 'config', '--add', 'branch.other/finished.merge', 'refs/heads/other/second')+ self.git(self.repo, 'update-server-info')+ request = self.quarantine_resumption_fixture()+ archived = self.txn / 'archive/common/config'+ archived_bytes = archived.read_bytes()+ self.git(self.repo, 'branch', '-D', 'other/finished')+ active_head = self.git(self.repo, 'commit-tree', 'HEAD^{tree}', '-p', 'HEAD',+ '-m', 'new unrelated work')+ self.git(self.repo, 'update-ref', 'refs/heads/other/active', active_head)+ self.git(self.repo, 'config', 'branch.other/active.remote', 'origin')+ self.git(self.repo, 'config', 'branch.other/active.merge', 'refs/heads/other/active')+ self.git(self.repo, 'update-server-info')+ current_info = archive.manifest(self.repo / '.git/info', True)+ (conflict / 'thisimage').write_text('concurrent conflict\n')+ (conflict / 'postimage').write_text('new resolution\n')+ (conflict / 'preimage').unlink()+ current_cache = archive.manifest(cache, True)+ current = archive.manifest(self.repo / '.git/config', True)+ # This is a real atomic Git config rewrite, not a mocked metadata delta.+ self.assertNotEqual(archived_bytes, (self.repo / '.git/config').read_bytes())+ rollback = archive.review_metadata_recovery(self.request)['metadataRecovery']+ self.assertTrue(archive.verify({+ **self.request, 'mode': 'restore', 'metadataRecovery': rollback})['verified'])+ self.assertTrue(archive.verify_quarantine_resumption(request)['verified'])+ self.assertTrue(archive.resume_quarantine(request)['moved'])+ self.assertTrue(archive.verify_completed_quarantine(request)['verified'])+ self.assertEqual(archive.manifest(self.repo / '.git/config', True), current)+ self.assertEqual(archive.manifest(self.repo / '.git/info', True), current_info)+ self.assertEqual(archive.manifest(cache, True), current_cache)+ archived_cache = self.txn / 'archive/common/rr-cache' / ('a' * 40)+ self.assertEqual((archived_cache / 'preimage').read_text(), 'old conflict\n')+ self.assertEqual((archived_cache / 'postimage').read_text(), 'old resolution\n')+ self.assertEqual(self.git(self.repo, 'rev-parse', 'refs/heads/other/active'), active_head)+ self.assertEqual(archived.read_bytes(), archived_bytes)+ self.assertTrue(archive.verify({**self.request, 'mode': 'standalone'})['standalone'])+ _, saved = archive.load_transaction(request)+ # Ordinary preservation still demands exact common metadata.+ with self.assertRaisesRegex(RuntimeError, 'common-store-changed'):+ archive.verify_common_metadata(saved, set(), self.txn)++ def test_generated_ref_advertisement_keeps_controls_security_and_archive_strict(self):+ self.git(self.repo, 'update-server-info')+ request = self.quarantine_resumption_fixture()+ info = self.repo / '.git/info'+ self.git(self.repo, 'branch', 'other/advertised')+ self.git(self.repo, 'update-server-info')+ self.assertTrue(archive.verify_quarantine_resumption(request)['verified'])+ control = info / 'exclude'+ original, stamp = control.read_bytes(), control.stat()+ control.write_bytes(original + b'new-policy\n')+ with self.assertRaisesRegex(RuntimeError, 'common-store-changed:info'):+ archive.verify_quarantine_resumption(request)+ control.write_bytes(original)+ os.utime(control, ns=(stamp.st_atime_ns, stamp.st_mtime_ns))+ mode = info.stat().st_mode & 0o777+ os.chmod(info, mode ^ 0o040)+ with self.assertRaisesRegex(RuntimeError, 'common-store-changed:info'):+ archive.verify_quarantine_resumption(request)+ os.chmod(info, mode)+ advertisement = info / 'refs'+ subprocess.run(['/usr/bin/xattr', '-w', 'com.portos.fixture', 'unknown metadata', str(advertisement)], check=True)+ with self.assertRaisesRegex(RuntimeError, 'common-store-changed:info'):+ archive.verify_quarantine_resumption(request)+ subprocess.run(['/usr/bin/xattr', '-d', 'com.portos.fixture', str(advertisement)], check=True)+ retained = self.scratch / 'retained-advertisement'+ advertisement.rename(retained)+ advertisement.symlink_to(retained)+ with self.assertRaisesRegex(RuntimeError, 'git-administration-symlink'):+ archive.verify_quarantine_resumption(request)+ advertisement.unlink()+ retained.rename(advertisement)+ self.assertTrue(archive.verify_quarantine_resumption(request)['verified'])+ (self.txn / 'archive/common/info/refs').write_text('tampered\n')+ with self.assertRaisesRegex(RuntimeError, 'common-archive-changed:info'):+ archive.verify_quarantine_resumption(request)++ def test_shared_resolution_cache_recovery_keeps_archive_root_and_structure_strict(self):+ cache = self.repo / '.git/rr-cache'+ conflict = cache / ('a' * 40)+ conflict.mkdir(parents=True)+ (conflict / 'preimage').write_text('original\n')+ request = self.quarantine_resumption_fixture()+ (conflict / 'thisimage').write_text('concurrent\n')+ _, saved = archive.load_transaction(request)+ with self.assertRaisesRegex(RuntimeError, 'common-store-changed:rr-cache'):+ archive.verify_common_metadata(saved, set(), self.txn)+ mode = cache.stat().st_mode & 0o777+ os.chmod(cache, mode ^ 0o040)+ with self.assertRaisesRegex(RuntimeError, 'shared-resolution-cache-root-changed'):+ archive.verify_quarantine_resumption(request)+ os.chmod(cache, mode)+ retained = self.scratch / 'retained-cache'+ cache.rename(retained)+ cache.mkdir()+ with self.assertRaisesRegex(RuntimeError, 'shared-resolution-cache-root-changed'):+ archive.verify_quarantine_resumption(request)+ cache.rmdir()+ retained.rename(cache)+ unsafe = conflict / 'unsafe'+ unsafe.symlink_to(self.scratch)+ with self.assertRaisesRegex(RuntimeError, 'git-administration-symlink'):+ archive.verify_quarantine_resumption(request)+ unsafe.unlink()+ os.mkfifo(unsafe)+ with self.assertRaisesRegex(RuntimeError, 'special-git-file-unsupported'):+ archive.verify_quarantine_resumption(request)+ unsafe.unlink()+ self.assertTrue(archive.verify_quarantine_resumption(request)['verified'])+ archived = self.txn / 'archive/common/rr-cache' / ('a' * 40) / 'preimage'+ archived.write_text('tampered\n')+ with self.assertRaisesRegex(RuntimeError, 'common-archive-changed:rr-cache'):+ archive.verify_quarantine_resumption(request)++ def test_shared_tracking_exception_rejects_target_global_additions_live_refs_metadata_and_races(self):+ self.git(self.repo, 'branch', 'other/finished')+ for name in ('other/finished', 'feature/fixture'):+ self.git(self.repo, 'config', 'branch.' + name + '.remote', 'origin')+ self.git(self.repo, 'config', 'branch.' + name + '.merge', 'refs/heads/' + name)+ self.preserve()+ _, saved = archive.load_transaction(self.request)+ common = self.repo / '.git'+ expected = saved['facts']['sources']['common']['config']+ self.git(self.repo, 'branch', '-D', 'other/finished')+ valid = (common / 'config').read_bytes()+ mode = (common / 'config').stat().st_mode & 0o777+ def check():+ return archive.unrelated_tracking_changes(+ self.txn, saved, common, expected, archive.manifest(common / 'config', True))+ self.assertTrue(check())+ for label, arguments in (+ ('target removed', ('--remove-section', 'branch.feature/fixture')),+ ('global changed', ('core.fsmonitor', 'changed')),+ ('added tracking', ('branch.new-work.remote', 'origin')),+ ('partly removed stanza', ('branch.other/finished.remote', 'origin')),+ ('duplicate changed', ('--add', 'branch.feature/fixture.merge', 'refs/heads/other')),+ ):+ with self.subTest(change=label):+ (common / 'config').write_bytes(valid)+ self.git(self.repo, 'config', *arguments)+ self.assertFalse(check())+ (common / 'config').write_bytes(valid)+ self.git(self.repo, 'config', 'branch.new-work.remote', 'origin')+ self.git(self.repo, 'config', 'branch.new-work.merge', 'refs/heads/new-work')+ self.assertFalse(check()) # A complete stanza without its branch is held.+ added_ref = common / 'refs/heads/new-work'+ for contents in ('not-an-object\n', 'ref: refs/heads/missing-target\n',+ 'ref: refs/heads/feature/fixture\n'):+ with self.subTest(unknownAddedRef=contents):+ added_ref.write_text(contents)+ self.assertFalse(check())+ added_ref.unlink()+ (common / 'config').write_bytes(valid)+ loose = common / 'refs/heads/other/finished'+ loose.parent.mkdir(parents=True, exist_ok=True)+ for contents in ('not-an-object\n', 'ref: refs/heads/missing-target\n'):+ with self.subTest(unknownRef=contents):+ loose.write_text(contents)+ self.assertFalse(check())+ loose.unlink()+ self.git(self.repo, 'branch', 'other/finished')+ self.assertFalse(check())+ self.git(self.repo, 'branch', '-D', 'other/finished')+ (common / 'config').write_bytes(valid)+ os.chmod(common / 'config', mode ^ 0o040)+ self.assertFalse(check())+ os.chmod(common / 'config', mode)+ original_git = archive.git+ def change_during_parse(admin, work, *arguments, **kwargs):+ result = original_git(admin, work, *arguments, **kwargs)+ if arguments[:1] == ('config',) and str(common / 'config') in arguments:+ self.git(self.repo, 'config', 'core.editor', 'concurrent')+ return result+ with patch.object(archive, 'git', change_during_parse):+ with self.assertRaisesRegex(RuntimeError, 'shared-tracking-observation-changed'):+ check()+ (common / 'config').write_bytes(valid)+ archived = self.txn / 'archive/common/config'+ archived.write_bytes(archived.read_bytes() + b'\n')+ with self.assertRaisesRegex(RuntimeError, 'shared-tracking-archive-changed'):+ check()+def test_execution_lock_lasts_until_parent_stdin_closes(self):holder = subprocess.Popen([sys.executable, str(Path(archive.__file__))],stdin=subprocess.PIPE, stdout=subprocess.PIPE,