Peach: Local-first video metadata scraper and media manager.
fix(test): 校准补充用例覆盖与输入失效
longmeidao073317aac9fa
diff --git a/docs/TESTING.md b/docs/TESTING.mdindex a5b60e632..4af58648b 100644--- a/docs/TESTING.md+++ b/docs/TESTING.md@@ -212,7 +212,7 @@ macOS Release 使用 `macos-15`(Apple Silicon)和 `macos-15-intel`(Intel`auto` 扫描同环境的有效记录,按测试文件合并 24 小时内、未受输入差异影响的成功结果;记录数量没有固定搜索上限。每项保留原通过时间,最新的失败或未完成状态阻止旧成功结果恢复。缺少、损坏、过期或未来时间的证明不计入覆盖;单份损坏记录不影响其他候选。`-Fresh` 与显式 `full` 始终实际执行请求范围。-正式父运行在开始时原子保存待验证状态,每个分片完成后立即保存检查点。失败批次仍保留成功文件及其时间;快速门槛失败时,其余文件保持未完成。进程中断后,下一次 `auto` 补齐缺少的文件,串行与并行使用相同结果模型。分片报告必须完整且格式有效;异常退出不能签发该片的整体成功。分片写出完成报告后仍未退出的进程,等待 15 秒后回收该片的进程树,不给尚在执行的测试设置额外时限。+正式父运行在开始时原子保存待验证状态,每个分片完成后立即保存检查点。失败批次仍保留成功文件、成功补充用例及其时间;整文件状态不覆盖已取得的单项结果。共享输入变化和 `Fresh` 同时作用于文件及其补充用例。快速门槛失败时,其余文件保持未完成。进程中断后,下一次 `auto` 在所需范围内补齐缺少的文件,串行与并行使用相同结果模型。分片报告必须完整且格式有效;异常退出不能签发该片的整体成功。分片写出完成报告后仍未退出的进程,等待 15 秒后回收该片的进程树,不给尚在执行的测试设置额外时限。浏览器与 Vitest 容器内按外部测试文件保存完成结果,只有逐条输出不能证明整个文件完成。恢复计划绑定当前输入快照、完整文件集合及每个省略项的成功时间。Python 容器用例自身也必须有成功证明,才能连同其外部文件一起省略;容器的后置检查失败仍需补测。外部选择仅由正式父运行生成,`Focus` 和独立 CI 分片不签发正式记录。diff --git a/scripts/test_resume.py b/scripts/test_resume.pyindex b33afd164..558880012 100644--- a/scripts/test_resume.py+++ b/scripts/test_resume.py@@ -93,6 +93,7 @@ def affected(root: Path, record: dict, context: dict, all_units: set[str], resolelse:scopes, _ = resolve([path], kinds={path: kinds[path]} if path in kinds else {})invalid.update(unit for scope in scopes for unit in scope_units(root, scope, select))+ invalid.update(name for name in all_units if name.startswith('case:') and owner(name) in invalid)return invalid & all_units@@ -112,6 +113,9 @@ def eligible(root: Path, record: dict, context: dict, all_units: set[str], resolfor scope in record.get('failed_scopes', ()):units.update({name: {'outcome': 'fail', 'at': record.get('finished', 0)}for name in recorded_scope(root, scope, select)})+ for name in all_units:+ if name.startswith('case:') and name not in units and owner(name) in units:+ units[name] = units[owner(name)]return {name: row for name, row in units.items() if name in all_units and name not in invalid}@@ -130,10 +134,6 @@ def combine(root: Path, records, context: dict, all_units: set[str], resolve, serank = (row['at'], row['outcome'] != 'pass')if previous is None or rank > (previous['at'], previous['outcome'] != 'pass'):merged[name] = row- for name in all_units:- parent = merged.get(owner(name)) if name.startswith('case:') else None- if parent is not None and (name not in merged or parent['at'] >= merged[name]['at']):- merged[name] = parentreturn merged@@ -141,6 +141,11 @@ def passed_units(units: dict) -> set[str]:return {name for name, row in units.items() if row.get('outcome') == 'pass'}+def without_requested(units: dict, required: set[str]) -> dict:+ modules = {name for name in required if name.endswith('.py')}+ return {name: row for name, row in units.items() if name not in required and owner(name) not in modules}++def validated(units: dict, requirements: dict[str, set[str]], *, allow_full=False) -> dict:successful = passed_units(units)return {scope: min(units[name]['at'] for name in names)@@ -167,12 +172,15 @@ def result_cases(identity: str, cases):return ()-def report_units(assigned, report: dict, *, clean_exit: bool, external=None, cases=(), stamp=None) -> dict:- """分片完整报告中有明确结果的文件;缺失或异常报告不产生成功证明。"""- stamp = time.time() if stamp is None else stamp+def unattributed_failure(assigned, report, cases):+ return any(row.get('outcome') in {'fail', 'error'} and row_unit(str(row.get('id', '')), assigned) is None+ and not result_cases(str(row.get('id', '')), cases) for row in report.get('results', ()))+++def grouped_results(assigned, report, cases):+ """归集叶子结果、补充用例和外部文件终态。"""grouped = {}completed_files = set()- assigned = {Path(item).name for item in assigned}for row in report.get('results', ()):identity = str(row.get('id', ''))for case in result_cases(identity, cases):@@ -182,6 +190,14 @@ def report_units(assigned, report: dict, *, clean_exit: bool, external=None, casgrouped.setdefault(unit, []).append(row.get('outcome'))if row.get('file_complete') is True and row.get('outcome') == 'pass':completed_files.add(unit)+ return grouped, completed_files+++def report_units(assigned, report: dict, *, clean_exit: bool, external=None, cases=(), stamp=None) -> dict:+ """分片完整报告中有明确结果的文件;缺失或异常报告不产生成功证明。"""+ stamp = time.time() if stamp is None else stamp+ assigned = {Path(item).name for item in assigned}+ grouped, completed_files = grouped_results(assigned, report, cases)result = {}for unit, outcomes in grouped.items():failed = any(outcome in {'fail', 'error'} for outcome in outcomes)@@ -199,8 +215,10 @@ def report_units(assigned, report: dict, *, clean_exit: bool, external=None, casresult[unit] = {'outcome': 'pass', 'at': stamp}elif module not in grouped:result[module] = {'outcome': 'unknown', 'at': stamp}- if report.get('success') and not clean_exit:- result = {name: {'outcome': 'unknown', 'at': stamp} for name in result}+ if ((report.get('success') and not clean_exit) or report.get('count', 0) <= 0+ or unattributed_failure(assigned, report, cases)):+ result = {name: {**row, 'outcome': 'unknown'} if row['outcome'] == 'pass' else row+ for name, row in result.items()}return resultdiff --git a/scripts/test_run_progress.py b/scripts/test_run_progress.pyindex d94c5e513..2e987ebbe 100644--- a/scripts/test_run_progress.py+++ b/scripts/test_run_progress.py@@ -31,7 +31,7 @@ class RunProgress:if row.get('cohort') == self.cohort or row.get('outcome') != 'pass'}self.observed = 0if force:- self.units = {name: row for name, row in self.units.items() if name not in self.required}+ self.units = resume.without_requested(self.units, self.required)for case in resume.CONTAINERS:if resume.container_files(root, case) - resume.passed_units(self.units):self.units.pop('case:' + case, None)@@ -56,7 +56,7 @@ class RunProgress:current = evidence.inputs(self.root)invalid = resume.affected(self.root, self.context, current, self.all_units, self.resolve, self.select)self.units.update({name: {**row, 'cohort': self.cohort}- for name, row in completed.items() if name in self.required and name not in invalid})+ for name, row in completed.items() if name in self.all_units and name not in invalid})self.persist()return all(self.units.get(name, {}).get('outcome') == 'pass' for name in assigned)@@ -67,14 +67,19 @@ class RunProgress:names = set()for selector in reruns:name = selector if selector.startswith('frontend/') else selector.split('.', 1)[0] + '.py'- names.update({name, resume.owner(name), 'case:' + selector})if '.' not in selector and not selector.startswith('frontend/'):- names.update(unit for unit in self.required if resume.owner(unit) == name)- names.update('case:' + case for case in resume.CONTAINERS- if name in resume.container_files(self.root, case))+ names.update(unit for unit in self.all_units if resume.owner(unit) == name)+ else:+ names.add('case:' + selector)+ if selector.startswith('frontend/'):+ names.add(name)+ names.update('case:' + case for case in resume.CONTAINERS+ if name in resume.container_files(self.root, case))+ if resume.owner(name) in self.required:+ names.add(resume.owner(name))stamp = time.time()self.units.update({name: {'outcome': 'pass', 'at': stamp, 'cohort': self.cohort}- for name in names if name in self.required})+ for name in names if name in self.all_units})def complete(self):return self.required <= resume.passed_units(evidence.live_units(self.units) or {})diff --git a/tests/test_test_evidence.py b/tests/test_test_evidence.pyindex 0df6560cc..aeceacce1 100644--- a/tests/test_test_evidence.py+++ b/tests/test_test_evidence.py@@ -298,6 +298,42 @@ class ReceiptTests(ReceiptFixture, unittest.TestCase):self.assertEqual(units['case:' + case]['outcome'], 'fail')self.assertEqual(units['test_web_e2e.py']['outcome'], 'fail')+ def test_explicit_case_success_survives_an_unrelated_failure_in_its_module(self):+ case = 'case:test_catalog.Suite.test_value'+ proof = {'test_catalog.py': self.unit('fail'), case: self.unit()}+ kept = test_resume.combine(self.root, [self.record(proof)], self.context, set(proof),+ self.resolve, self.mapping.__getitem__)+ self.assertEqual(kept[case]['outcome'], 'pass')++ def test_module_aggregation_does_not_refresh_an_explicit_case_timestamp(self):+ case = 'case:test_catalog.Suite.test_value'+ proof = {'test_catalog.py': self.unit(), case: self.unit(offset=-100)}+ kept = test_resume.combine(self.root, [self.record(proof)], self.context, set(proof),+ self.resolve, self.mapping.__getitem__)+ self.assertEqual(kept[case]['at'], self.now - 100)++ def test_shared_inputs_invalidate_supplemental_cases_with_their_module(self):+ case = 'case:test_catalog.Suite.test_value'+ proof = {'test_catalog.py': self.unit(), case: self.unit()}+ context = {**self.context, 'manifest': {'files': {'scripts/test_runner.py': [420, 'changed', '100644']}}}+ kept = test_resume.combine(self.root, [self.record(proof)], context, set(proof),+ lambda paths, **kwargs: (('full',), 'fixture'), self.mapping.__getitem__)+ self.assertEqual(kept, {})++ def test_zero_executed_tests_do_not_produce_positive_coverage(self):+ case = 'test_catalog.Suite.test_value'+ report = {'success': False, 'count': 0, 'results': [+ {'id': 'setUpClass (test_catalog.Suite)', 'outcome': 'skip'}]}+ units = test_resume.report_units(['test_catalog.py'], report, clean_exit=False, cases={case})+ self.assertEqual(test_resume.passed_units(units), set())++ def test_unattributed_failure_prevents_success_from_being_inferred(self):+ report = {'success': False, 'count': 1, 'results': [+ {'id': 'test_catalog.Suite.test_value', 'outcome': 'pass'},+ {'id': 'executor bootstrap', 'outcome': 'error'}]}+ units = test_resume.report_units(['test_catalog.py'], report, clean_exit=False)+ self.assertEqual(units['test_catalog.py']['outcome'], 'unknown')+class CheckpointTests(ReceiptFixture, unittest.TestCase):def setUp(self):@@ -473,6 +509,36 @@ class CheckpointTests(ReceiptFixture, unittest.TestCase):progress.accept_success(['test_web_e2e'])self.assertTrue(progress.complete())+ def test_failed_full_file_preserves_the_successful_supplement_it_executed(self):+ case = 'test_extra.Suite.test_value'+ self.mapping['full'] += (Path('test_extra.py'),)+ supplements = {'checks': (case,)}+ progress = self.progress(supplements=supplements)+ progress.start()+ progress.checkpoint(['test_extra.py'], {'complete': True, 'success': False, 'count': 2, 'results': [+ {'id': case, 'outcome': 'pass'}, {'id': 'test_extra.Suite.test_other', 'outcome': 'fail'}]}, True)+ saved = evidence.read(self.root, self.context['state'])+ self.assertEqual(saved['units']['case:' + case]['outcome'], 'pass')+ resumed = self.progress(('checks',), records=[saved], supplements=supplements)+ self.assertTrue(resumed.complete())++ def test_fresh_file_verification_also_invalidates_its_supplement_receipts(self):+ case = 'test_catalog.Suite.test_value'+ record = self.record({'test_catalog.py': self.unit(), 'case:' + case: self.unit()})+ progress = self.progress(('catalog',), records=[record], supplements={'checks': (case,)}, force=True)+ self.assertNotIn('case:' + case, progress.units)++ def test_case_only_rerun_does_not_certify_its_entire_module(self):+ case = 'test_extra.Suite.test_value'+ self.mapping['full'] += (Path('test_extra.py'),)+ progress = self.progress(('catalog',), supplements={'catalog': (case,)})+ progress.start()+ progress.checkpoint(['test_catalog.py'], {'complete': True, 'success': False, 'count': 2, 'results': [+ {'id': 'test_catalog.Suite.test_value', 'outcome': 'pass'}, {'id': case, 'outcome': 'fail'}]}, True)+ progress.accept_success([case])+ self.assertTrue(progress.complete())+ self.assertNotIn('test_extra.py', progress.units)+class ShardReportTests(unittest.TestCase):def test_malformed_reports_are_failures(self):@@ -512,6 +578,72 @@ class ShardReportTests(unittest.TestCase):test_shard_results.stop_process_tree(process)+class SubprocessResumeTests(unittest.TestCase):+ def invoke(self, repo, scope):+ env = {**os.environ, 'GITHUB_ACTIONS': 'false', 'CI': ''}+ for key in ('PEACH_TEST_FOCUS', 'PEACH_E2E_FILES', 'PEACH_VITEST_FILES'):+ env.pop(key, None)+ process = subprocess.Popen([sys.executable, '-X', 'utf8', str(repo / 'entry.py'),+ '--scope', scope, '--jobs', 'auto'], cwd=repo, env=env,+ stdout=subprocess.PIPE, stderr=subprocess.STDOUT,+ text=True, encoding='utf8')+ try:+ output, _ = process.communicate(timeout=180)+ finally:+ if process.poll() is None:+ test_shard_results.stop_process_tree(process)+ return process.returncode, output++ def test_real_parent_and_children_resume_only_the_failed_file(self):+ source = Path(__file__).resolve().parents[1]+ modules = ('test_runner', 'test_evidence', 'test_history', 'test_resume', 'test_run_progress',+ 'test_shard_results', 'test_environment_identity', 'test_focus', 'change_impact')+ files = {'scripts/__init__.py': '', '.gitignore': 'build/\nruntime/\n__pycache__/\n*.pyc\n',+ '.gitattributes': '* text=auto eol=lf\n'}+ files.update({'scripts/' + name + '.py': (source / 'scripts' / (name + '.py')).read_text(encoding='utf8')+ for name in modules})+ files['entry.py'] = (+ 'from pathlib import Path\nfrom unittest import mock\nfrom scripts import test_runner as runner\n'+ 'with mock.patch.object(runner.test_evidence, "machine_lock_path", '+ 'return_value=Path.cwd()/"runtime/machine.lock"):\n raise SystemExit(runner.main())\n')+ sample = ('from pathlib import Path\nimport unittest\nclass Suite(unittest.TestCase):\n'+ ' def test_value(self):\n root=Path(__file__).resolve().parents[1]/"runtime"\n'+ ' with (root/(__name__+".calls")).open("a", encoding="utf8") as output:\n'+ ' output.write("called\\n")\n'+ ' if __name__ == "test_web_library_shortcuts":\n'+ ' self.assertFalse((root/"fail").exists())\n')+ for name in ('test_ad_judgement', 'test_web_library_shortcuts'):+ files['tests/' + name + '.py'] = sample+ with tempfile.TemporaryDirectory() as folder:+ repo = seed_repository(Path(folder).resolve() / 'repo', files, 'fixture')+ evidence.git(repo, 'checkout', '-b', 'fixture')+ (repo / 'shared-input').write_text('fixture', encoding='utf8')+ evidence.git(repo, 'add', 'shared-input')+ evidence.git(repo, 'commit', '-m', 'fixture')+ runtime = repo / 'runtime'+ runtime.mkdir()+ marker = runtime / 'fail'+ marker.touch()+ code, output = self.invoke(repo, 'full')+ self.assertEqual(code, 1, output)+ record_file = next(path for path in (repo / 'build/agent-verification').glob('*.json')+ if len(path.stem) == 64)+ before = json.loads(record_file.read_text(encoding='utf8'))+ self.assertEqual(before['units']['test_ad_judgement.py']['outcome'], 'pass')+ stamp = before['units']['test_ad_judgement.py']['at']+ successful_calls = (runtime / 'test_ad_judgement.calls').read_text(encoding='utf8')+ failed_calls = (runtime / 'test_web_library_shortcuts.calls').read_text(encoding='utf8').splitlines()+ marker.unlink()+ code, output = self.invoke(repo, 'auto')+ self.assertEqual(code, 0, output)+ after = json.loads(record_file.read_text(encoding='utf8'))+ self.assertEqual(after['units']['test_ad_judgement.py']['at'], stamp)+ self.assertEqual(after['validated']['full'], stamp)+ self.assertEqual((runtime / 'test_ad_judgement.calls').read_text(encoding='utf8'), successful_calls)+ self.assertEqual(len((runtime / 'test_web_library_shortcuts.calls').read_text(encoding='utf8').splitlines()),+ len(failed_calls) + 1)++class VerificationTests(unittest.TestCase):def setUp(self):# 这组用例模拟本机验收,运行它的 CI job 身份不属于临时仓库。