Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

fix(test): 校准补充用例覆盖与输入失效

longmeidao073317aac9fa
diff --git a/docs/TESTING.md b/docs/TESTING.md
index a5b60e632..4af58648b 100644
--- a/docs/TESTING.md
+++ b/docs/TESTING.md
@@ -212,7 +212,7 @@ macOS Release 使用 `macos-15`(Apple Silicon)和 `macos-15-intel`(Intel
`auto` 扫描同环境的有效记录,按测试文件合并 24 小时内、未受输入差异影响的成功结果;记录数量没有固定搜索上限。每项保留原通过时间,最新的失败或未完成状态阻止旧成功结果恢复。缺少、损坏、过期或未来时间的证明不计入覆盖;单份损坏记录不影响其他候选。`-Fresh` 与显式 `full` 始终实际执行请求范围。
-正式父运行在开始时原子保存待验证状态,每个分片完成后立即保存检查点。失败批次仍保留成功文件及其时间;快速门槛失败时,其余文件保持未完成。进程中断后,下一次 `auto` 补齐缺少的文件,串行与并行使用相同结果模型。分片报告必须完整且格式有效;异常退出不能签发该片的整体成功。分片写出完成报告后仍未退出的进程,等待 15 秒后回收该片的进程树,不给尚在执行的测试设置额外时限。
+正式父运行在开始时原子保存待验证状态,每个分片完成后立即保存检查点。失败批次仍保留成功文件、成功补充用例及其时间;整文件状态不覆盖已取得的单项结果。共享输入变化和 `Fresh` 同时作用于文件及其补充用例。快速门槛失败时,其余文件保持未完成。进程中断后,下一次 `auto` 在所需范围内补齐缺少的文件,串行与并行使用相同结果模型。分片报告必须完整且格式有效;异常退出不能签发该片的整体成功。分片写出完成报告后仍未退出的进程,等待 15 秒后回收该片的进程树,不给尚在执行的测试设置额外时限。
浏览器与 Vitest 容器内按外部测试文件保存完成结果,只有逐条输出不能证明整个文件完成。恢复计划绑定当前输入快照、完整文件集合及每个省略项的成功时间。Python 容器用例自身也必须有成功证明,才能连同其外部文件一起省略;容器的后置检查失败仍需补测。外部选择仅由正式父运行生成,`Focus` 和独立 CI 分片不签发正式记录。
diff --git a/scripts/test_resume.py b/scripts/test_resume.py
index b33afd164..558880012 100644
--- a/scripts/test_resume.py
+++ b/scripts/test_resume.py
@@ -93,6 +93,7 @@ def affected(root: Path, record: dict, context: dict, all_units: set[str], resol
else:
scopes, _ = resolve([path], kinds={path: kinds[path]} if path in kinds else {})
invalid.update(unit for scope in scopes for unit in scope_units(root, scope, select))
+ invalid.update(name for name in all_units if name.startswith('case:') and owner(name) in invalid)
return invalid & all_units
@@ -112,6 +113,9 @@ def eligible(root: Path, record: dict, context: dict, all_units: set[str], resol
for scope in record.get('failed_scopes', ()):
units.update({name: {'outcome': 'fail', 'at': record.get('finished', 0)}
for name in recorded_scope(root, scope, select)})
+ for name in all_units:
+ if name.startswith('case:') and name not in units and owner(name) in units:
+ units[name] = units[owner(name)]
return {name: row for name, row in units.items() if name in all_units and name not in invalid}
@@ -130,10 +134,6 @@ def combine(root: Path, records, context: dict, all_units: set[str], resolve, se
rank = (row['at'], row['outcome'] != 'pass')
if previous is None or rank > (previous['at'], previous['outcome'] != 'pass'):
merged[name] = row
- for name in all_units:
- parent = merged.get(owner(name)) if name.startswith('case:') else None
- if parent is not None and (name not in merged or parent['at'] >= merged[name]['at']):
- merged[name] = parent
return merged
@@ -141,6 +141,11 @@ def passed_units(units: dict) -> set[str]:
return {name for name, row in units.items() if row.get('outcome') == 'pass'}
+def without_requested(units: dict, required: set[str]) -> dict:
+ modules = {name for name in required if name.endswith('.py')}
+ return {name: row for name, row in units.items() if name not in required and owner(name) not in modules}
+
+
def validated(units: dict, requirements: dict[str, set[str]], *, allow_full=False) -> dict:
successful = passed_units(units)
return {scope: min(units[name]['at'] for name in names)
@@ -167,12 +172,15 @@ def result_cases(identity: str, cases):
return ()
-def report_units(assigned, report: dict, *, clean_exit: bool, external=None, cases=(), stamp=None) -> dict:
- """分片完整报告中有明确结果的文件;缺失或异常报告不产生成功证明。"""
- stamp = time.time() if stamp is None else stamp
+def unattributed_failure(assigned, report, cases):
+ return any(row.get('outcome') in {'fail', 'error'} and row_unit(str(row.get('id', '')), assigned) is None
+ and not result_cases(str(row.get('id', '')), cases) for row in report.get('results', ()))
+
+
+def grouped_results(assigned, report, cases):
+ """归集叶子结果、补充用例和外部文件终态。"""
grouped = {}
completed_files = set()
- assigned = {Path(item).name for item in assigned}
for row in report.get('results', ()):
identity = str(row.get('id', ''))
for case in result_cases(identity, cases):
@@ -182,6 +190,14 @@ def report_units(assigned, report: dict, *, clean_exit: bool, external=None, cas
grouped.setdefault(unit, []).append(row.get('outcome'))
if row.get('file_complete') is True and row.get('outcome') == 'pass':
completed_files.add(unit)
+ return grouped, completed_files
+
+
+def report_units(assigned, report: dict, *, clean_exit: bool, external=None, cases=(), stamp=None) -> dict:
+ """分片完整报告中有明确结果的文件;缺失或异常报告不产生成功证明。"""
+ stamp = time.time() if stamp is None else stamp
+ assigned = {Path(item).name for item in assigned}
+ grouped, completed_files = grouped_results(assigned, report, cases)
result = {}
for unit, outcomes in grouped.items():
failed = any(outcome in {'fail', 'error'} for outcome in outcomes)
@@ -199,8 +215,10 @@ def report_units(assigned, report: dict, *, clean_exit: bool, external=None, cas
result[unit] = {'outcome': 'pass', 'at': stamp}
elif module not in grouped:
result[module] = {'outcome': 'unknown', 'at': stamp}
- if report.get('success') and not clean_exit:
- result = {name: {'outcome': 'unknown', 'at': stamp} for name in result}
+ if ((report.get('success') and not clean_exit) or report.get('count', 0) <= 0
+ or unattributed_failure(assigned, report, cases)):
+ result = {name: {**row, 'outcome': 'unknown'} if row['outcome'] == 'pass' else row
+ for name, row in result.items()}
return result
diff --git a/scripts/test_run_progress.py b/scripts/test_run_progress.py
index d94c5e513..2e987ebbe 100644
--- a/scripts/test_run_progress.py
+++ b/scripts/test_run_progress.py
@@ -31,7 +31,7 @@ class RunProgress:
if row.get('cohort') == self.cohort or row.get('outcome') != 'pass'}
self.observed = 0
if force:
- self.units = {name: row for name, row in self.units.items() if name not in self.required}
+ self.units = resume.without_requested(self.units, self.required)
for case in resume.CONTAINERS:
if resume.container_files(root, case) - resume.passed_units(self.units):
self.units.pop('case:' + case, None)
@@ -56,7 +56,7 @@ class RunProgress:
current = evidence.inputs(self.root)
invalid = resume.affected(self.root, self.context, current, self.all_units, self.resolve, self.select)
self.units.update({name: {**row, 'cohort': self.cohort}
- for name, row in completed.items() if name in self.required and name not in invalid})
+ for name, row in completed.items() if name in self.all_units and name not in invalid})
self.persist()
return all(self.units.get(name, {}).get('outcome') == 'pass' for name in assigned)
@@ -67,14 +67,19 @@ class RunProgress:
names = set()
for selector in reruns:
name = selector if selector.startswith('frontend/') else selector.split('.', 1)[0] + '.py'
- names.update({name, resume.owner(name), 'case:' + selector})
if '.' not in selector and not selector.startswith('frontend/'):
- names.update(unit for unit in self.required if resume.owner(unit) == name)
- names.update('case:' + case for case in resume.CONTAINERS
- if name in resume.container_files(self.root, case))
+ names.update(unit for unit in self.all_units if resume.owner(unit) == name)
+ else:
+ names.add('case:' + selector)
+ if selector.startswith('frontend/'):
+ names.add(name)
+ names.update('case:' + case for case in resume.CONTAINERS
+ if name in resume.container_files(self.root, case))
+ if resume.owner(name) in self.required:
+ names.add(resume.owner(name))
stamp = time.time()
self.units.update({name: {'outcome': 'pass', 'at': stamp, 'cohort': self.cohort}
- for name in names if name in self.required})
+ for name in names if name in self.all_units})
def complete(self):
return self.required <= resume.passed_units(evidence.live_units(self.units) or {})
diff --git a/tests/test_test_evidence.py b/tests/test_test_evidence.py
index 0df6560cc..aeceacce1 100644
--- a/tests/test_test_evidence.py
+++ b/tests/test_test_evidence.py
@@ -298,6 +298,42 @@ class ReceiptTests(ReceiptFixture, unittest.TestCase):
self.assertEqual(units['case:' + case]['outcome'], 'fail')
self.assertEqual(units['test_web_e2e.py']['outcome'], 'fail')
+ def test_explicit_case_success_survives_an_unrelated_failure_in_its_module(self):
+ case = 'case:test_catalog.Suite.test_value'
+ proof = {'test_catalog.py': self.unit('fail'), case: self.unit()}
+ kept = test_resume.combine(self.root, [self.record(proof)], self.context, set(proof),
+ self.resolve, self.mapping.__getitem__)
+ self.assertEqual(kept[case]['outcome'], 'pass')
+
+ def test_module_aggregation_does_not_refresh_an_explicit_case_timestamp(self):
+ case = 'case:test_catalog.Suite.test_value'
+ proof = {'test_catalog.py': self.unit(), case: self.unit(offset=-100)}
+ kept = test_resume.combine(self.root, [self.record(proof)], self.context, set(proof),
+ self.resolve, self.mapping.__getitem__)
+ self.assertEqual(kept[case]['at'], self.now - 100)
+
+ def test_shared_inputs_invalidate_supplemental_cases_with_their_module(self):
+ case = 'case:test_catalog.Suite.test_value'
+ proof = {'test_catalog.py': self.unit(), case: self.unit()}
+ context = {**self.context, 'manifest': {'files': {'scripts/test_runner.py': [420, 'changed', '100644']}}}
+ kept = test_resume.combine(self.root, [self.record(proof)], context, set(proof),
+ lambda paths, **kwargs: (('full',), 'fixture'), self.mapping.__getitem__)
+ self.assertEqual(kept, {})
+
+ def test_zero_executed_tests_do_not_produce_positive_coverage(self):
+ case = 'test_catalog.Suite.test_value'
+ report = {'success': False, 'count': 0, 'results': [
+ {'id': 'setUpClass (test_catalog.Suite)', 'outcome': 'skip'}]}
+ units = test_resume.report_units(['test_catalog.py'], report, clean_exit=False, cases={case})
+ self.assertEqual(test_resume.passed_units(units), set())
+
+ def test_unattributed_failure_prevents_success_from_being_inferred(self):
+ report = {'success': False, 'count': 1, 'results': [
+ {'id': 'test_catalog.Suite.test_value', 'outcome': 'pass'},
+ {'id': 'executor bootstrap', 'outcome': 'error'}]}
+ units = test_resume.report_units(['test_catalog.py'], report, clean_exit=False)
+ self.assertEqual(units['test_catalog.py']['outcome'], 'unknown')
+
class CheckpointTests(ReceiptFixture, unittest.TestCase):
def setUp(self):
@@ -473,6 +509,36 @@ class CheckpointTests(ReceiptFixture, unittest.TestCase):
progress.accept_success(['test_web_e2e'])
self.assertTrue(progress.complete())
+ def test_failed_full_file_preserves_the_successful_supplement_it_executed(self):
+ case = 'test_extra.Suite.test_value'
+ self.mapping['full'] += (Path('test_extra.py'),)
+ supplements = {'checks': (case,)}
+ progress = self.progress(supplements=supplements)
+ progress.start()
+ progress.checkpoint(['test_extra.py'], {'complete': True, 'success': False, 'count': 2, 'results': [
+ {'id': case, 'outcome': 'pass'}, {'id': 'test_extra.Suite.test_other', 'outcome': 'fail'}]}, True)
+ saved = evidence.read(self.root, self.context['state'])
+ self.assertEqual(saved['units']['case:' + case]['outcome'], 'pass')
+ resumed = self.progress(('checks',), records=[saved], supplements=supplements)
+ self.assertTrue(resumed.complete())
+
+ def test_fresh_file_verification_also_invalidates_its_supplement_receipts(self):
+ case = 'test_catalog.Suite.test_value'
+ record = self.record({'test_catalog.py': self.unit(), 'case:' + case: self.unit()})
+ progress = self.progress(('catalog',), records=[record], supplements={'checks': (case,)}, force=True)
+ self.assertNotIn('case:' + case, progress.units)
+
+ def test_case_only_rerun_does_not_certify_its_entire_module(self):
+ case = 'test_extra.Suite.test_value'
+ self.mapping['full'] += (Path('test_extra.py'),)
+ progress = self.progress(('catalog',), supplements={'catalog': (case,)})
+ progress.start()
+ progress.checkpoint(['test_catalog.py'], {'complete': True, 'success': False, 'count': 2, 'results': [
+ {'id': 'test_catalog.Suite.test_value', 'outcome': 'pass'}, {'id': case, 'outcome': 'fail'}]}, True)
+ progress.accept_success([case])
+ self.assertTrue(progress.complete())
+ self.assertNotIn('test_extra.py', progress.units)
+
class ShardReportTests(unittest.TestCase):
def test_malformed_reports_are_failures(self):
@@ -512,6 +578,72 @@ class ShardReportTests(unittest.TestCase):
test_shard_results.stop_process_tree(process)
+class SubprocessResumeTests(unittest.TestCase):
+ def invoke(self, repo, scope):
+ env = {**os.environ, 'GITHUB_ACTIONS': 'false', 'CI': ''}
+ for key in ('PEACH_TEST_FOCUS', 'PEACH_E2E_FILES', 'PEACH_VITEST_FILES'):
+ env.pop(key, None)
+ process = subprocess.Popen([sys.executable, '-X', 'utf8', str(repo / 'entry.py'),
+ '--scope', scope, '--jobs', 'auto'], cwd=repo, env=env,
+ stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+ text=True, encoding='utf8')
+ try:
+ output, _ = process.communicate(timeout=180)
+ finally:
+ if process.poll() is None:
+ test_shard_results.stop_process_tree(process)
+ return process.returncode, output
+
+ def test_real_parent_and_children_resume_only_the_failed_file(self):
+ source = Path(__file__).resolve().parents[1]
+ modules = ('test_runner', 'test_evidence', 'test_history', 'test_resume', 'test_run_progress',
+ 'test_shard_results', 'test_environment_identity', 'test_focus', 'change_impact')
+ files = {'scripts/__init__.py': '', '.gitignore': 'build/\nruntime/\n__pycache__/\n*.pyc\n',
+ '.gitattributes': '* text=auto eol=lf\n'}
+ files.update({'scripts/' + name + '.py': (source / 'scripts' / (name + '.py')).read_text(encoding='utf8')
+ for name in modules})
+ files['entry.py'] = (
+ 'from pathlib import Path\nfrom unittest import mock\nfrom scripts import test_runner as runner\n'
+ 'with mock.patch.object(runner.test_evidence, "machine_lock_path", '
+ 'return_value=Path.cwd()/"runtime/machine.lock"):\n raise SystemExit(runner.main())\n')
+ sample = ('from pathlib import Path\nimport unittest\nclass Suite(unittest.TestCase):\n'
+ ' def test_value(self):\n root=Path(__file__).resolve().parents[1]/"runtime"\n'
+ ' with (root/(__name__+".calls")).open("a", encoding="utf8") as output:\n'
+ ' output.write("called\\n")\n'
+ ' if __name__ == "test_web_library_shortcuts":\n'
+ ' self.assertFalse((root/"fail").exists())\n')
+ for name in ('test_ad_judgement', 'test_web_library_shortcuts'):
+ files['tests/' + name + '.py'] = sample
+ with tempfile.TemporaryDirectory() as folder:
+ repo = seed_repository(Path(folder).resolve() / 'repo', files, 'fixture')
+ evidence.git(repo, 'checkout', '-b', 'fixture')
+ (repo / 'shared-input').write_text('fixture', encoding='utf8')
+ evidence.git(repo, 'add', 'shared-input')
+ evidence.git(repo, 'commit', '-m', 'fixture')
+ runtime = repo / 'runtime'
+ runtime.mkdir()
+ marker = runtime / 'fail'
+ marker.touch()
+ code, output = self.invoke(repo, 'full')
+ self.assertEqual(code, 1, output)
+ record_file = next(path for path in (repo / 'build/agent-verification').glob('*.json')
+ if len(path.stem) == 64)
+ before = json.loads(record_file.read_text(encoding='utf8'))
+ self.assertEqual(before['units']['test_ad_judgement.py']['outcome'], 'pass')
+ stamp = before['units']['test_ad_judgement.py']['at']
+ successful_calls = (runtime / 'test_ad_judgement.calls').read_text(encoding='utf8')
+ failed_calls = (runtime / 'test_web_library_shortcuts.calls').read_text(encoding='utf8').splitlines()
+ marker.unlink()
+ code, output = self.invoke(repo, 'auto')
+ self.assertEqual(code, 0, output)
+ after = json.loads(record_file.read_text(encoding='utf8'))
+ self.assertEqual(after['units']['test_ad_judgement.py']['at'], stamp)
+ self.assertEqual(after['validated']['full'], stamp)
+ self.assertEqual((runtime / 'test_ad_judgement.calls').read_text(encoding='utf8'), successful_calls)
+ self.assertEqual(len((runtime / 'test_web_library_shortcuts.calls').read_text(encoding='utf8').splitlines()),
+ len(failed_calls) + 1)
+
+
class VerificationTests(unittest.TestCase):
def setUp(self):
# 这组用例模拟本机验收,运行它的 CI job 身份不属于临时仓库。