Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

test(scenarios): resolve every catalog pointer to a test that exists, and fix the dead ones (#2562)

Call White449792b84571
diff --git a/standards/scenario-testing/STANDARD.md b/standards/scenario-testing/STANDARD.md
index 0dff6b72e..d2f21beae 100644
--- a/standards/scenario-testing/STANDARD.md
+++ b/standards/scenario-testing/STANDARD.md
@@ -191,7 +191,10 @@ this row leaves the row reading `covered` with nothing executable tied to it.
That is not hypothetical — it is how four dead citations survived a test-file
rewrite unnoticed for five days. A pointer key such as `related_tests` may name
bare files precisely because it claims no coverage; anything that does claim
-coverage names its case.
+coverage names its case. A pointer that does name a case must still name one
+that exists, or a rename leaves it leading nowhere. A pointer may also name a
+group: a test class, or, for a vitest file, every case whose name contains the
+term it names, such as an ID.
## 5. Harness Design Rules
diff --git a/tests/scenarios/model_hub/catalog.yaml b/tests/scenarios/model_hub/catalog.yaml
index b0dc24b76..75efd1856 100644
--- a/tests/scenarios/model_hub/catalog.yaml
+++ b/tests/scenarios/model_hub/catalog.yaml
@@ -1125,7 +1125,7 @@ scenarios:
- tests/test_model_hub_background_discovery.py::test_schedule_covers_hub_sources_and_skips_native_and_waiting_credentials
- tests/test_model_hub_background_discovery.py::test_stale_sources_catch_up_together_then_refresh_at_different_times
- tests/test_model_hub_background_discovery.py::test_listing_that_would_strand_a_menu_model_is_left_to_manual_refresh
- - tests/test_model_hub_background_discovery.py::test_stop_retires_the_background_schedule
+ - tests/test_model_hub_background_discovery.py::test_stop_retires_the_background_schedule_without_waiting_for_a_listing
- id: MH-QUOTA-005
name: Each subscription's quota is re-read at most every five minutes and concurrent reads share one fetch
status: covered
@@ -1477,7 +1477,7 @@ scenarios:
layer: scenario
test: tests/scenarios/model_hub/test_model_hub_live_resolution_scenarios.py::test_mh_unlisted_001_removed_model_fails_its_next_turn_with_one_clear_reason
related_tests:
- - tests/scenarios/model_hub/test_model_hub_live_resolution_scenarios.py::test_only_an_unlisted_model_holds_its_input_for_an_explicit_retry
+ - tests/scenarios/model_hub/test_model_hub_live_resolution_scenarios.py::test_a_refusal_only_models_can_clear_holds_its_input_for_an_explicit_retry
- tests/scenarios/model_hub/test_model_hub_live_resolution_scenarios.py::test_opencode_refuses_an_unlisted_model_its_retained_route_cannot_address
- tests/test_claude_cli_path.py::test_claude_launch_on_an_unlisted_model_holds_the_input
- tests/test_claude_session_recovery.py::test_unlisted_hub_model_shows_the_shared_refusal_once
@@ -1526,7 +1526,7 @@ scenarios:
related_tests:
- ui/src/components/onboarding/providers/ProvidersScreen.test.tsx::counts the takeable group beside it, and only that one
- ui/src/components/settings/models/migrationScan.test.ts::counts an OpenCode key, which never arrives as kind api_key
- - ui/src/components/settings/models/MigrationDialog.test.tsx::applies only complete backend groups with one migration action
+ - ui/src/components/settings/models/MigrationDialog.test.tsx::applies every carried row with one migration action, leaving the rest native
- id: MH-MIG-005
name: Persisted migration ignores runtime authentication and preserves shared-file custody
status: covered
@@ -1575,7 +1575,7 @@ scenarios:
- tests/test_model_hub_native_takeover.py::test_mode_only_adoption_refuses_a_native_config_the_cli_cannot_parse
- tests/test_model_hub_native_takeover.py::test_mode_switch_refuses_an_unreadable_completed_receipt_before_interrupting
- tests/test_opencode_restore_polls.py::test_forced_refresh_neither_waits_on_nor_strands_a_stuck_restored_poll
- - tests/test_agent_auth_service.py::AgentAuthServiceTests::test_opencode_agent_force_refresh_restarts_after_global_config_change
+ - tests/test_opencode_generations.py::test_strict_retirement_retries_a_stop_that_declined_while_a_request_ran
- id: MH-MIG-010
name: A refused switch to the gateway is reported where it was made and retried in place
status: covered
@@ -2002,7 +2002,7 @@ scenarios:
test: tests/e2e/test_model_hub_migration_persisted.py::test_f4_persisted_configuration_migrates_with_runtime_auth_present
related_tests:
- tests/e2e/test_model_hub_migration_persisted.py::test_f4_unproven_shell_key_is_not_removed
- - tests/e2e/test_model_hub_migration_persisted.py::test_f4_custom_api_key_transport_is_refused_without_cleanup
+ - tests/e2e/test_model_hub_migration_persisted.py::test_f4_custom_api_key_refusing_bearer_is_kept_without_cleanup
- tests/e2e/test_model_hub_migration_persisted.py::test_f4_explicit_dynamic_writer_refuses_http_apply_without_proof
- tests/e2e/test_model_hub_migration_persisted.py::test_f4_replace_migrated_bearer_without_old_secret_retains_cpa_transport
- tests/e2e/test_model_hub_migration_persisted.py::test_f4_oauth_history_fence_survives_later_api_batch_over_http
diff --git a/tests/scenarios/runtime_generations/catalog.yaml b/tests/scenarios/runtime_generations/catalog.yaml
index 86a60cbeb..7b6b1a872 100644
--- a/tests/scenarios/runtime_generations/catalog.yaml
+++ b/tests/scenarios/runtime_generations/catalog.yaml
@@ -39,7 +39,7 @@ scenarios:
backend: shared
test: tests/test_runtime_generations.py::test_runtime_gen_003_at_the_cap_the_oldest_retiring_generation_gives_way
related_tests:
- - tests/test_opencode_generations.py::test_a_cap_forced_stop_settles_the_activities_of_the_sessions_it_interrupts
+ - tests/test_opencode_generations.py::test_a_cap_forced_stop_settles_the_turns_and_activities_of_its_process
- tests/test_opencode_generations.py::test_the_cap_stops_the_oldest_adopted_generation_whatever_its_record_order
- id: RUNTIME-GEN-004
name: A routine restart request renews in place and never drains, holds a turn, or interrupts
diff --git a/tests/scenarios/test_catalog_citations.py b/tests/scenarios/test_catalog_citations.py
new file mode 100644
index 000000000..a624b5683
--- /dev/null
+++ b/tests/scenarios/test_catalog_citations.py
@@ -0,0 +1,102 @@
+"""Every case a scenario catalog's pointer keys name still exists, in every catalog.
+
+``related_tests`` and ``supporting_tests`` point a reader from a scenario to
+neighbouring cases. They claim no coverage, so nothing read them, and a renamed
+or deleted test left the pointer leading nowhere: six had, in two catalogs.
+Evidence keys are another check's subject: MH-CATALOG-001 resolves Model Hub's,
+and the UI gate resolves every catalog's UI evidence.
+
+A pointer is a list. Each entry names a file, which must exist, and may name a
+case in it: ``file::Class::method``, or ``file::name`` naming a module-level
+test function, a test class, or a test method of a class in the file. pytest's
+default naming rules decide what is a test, so a helper or a plain class is
+not. This reads source and matches exact names; it does not emulate
+collection-time exclusions such as ``__test__ = False``, which #2565 tracks.
+
+UI pointers are resolved against vitest's own collection by
+``ui/scripts/validate-scenario-catalog.mjs``; here only their file is checked.
+"""
+
+from __future__ import annotations
+
+import ast
+from functools import cache
+from pathlib import Path
+
+import pytest
+import yaml
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+SCENARIO_ROOT = REPO_ROOT / "tests" / "scenarios"
+_UI_SUFFIXES = {".ts", ".tsx", ".mts", ".mjs"}
+_POINTER_KEYS = ("related_tests", "supporting_tests")
+
+
+@cache
+def _test_names(path: Path) -> frozenset[str]:
+ """Every name in ``path`` that is a test pytest's default rules collect, or a test class.
+
+ A module-level ``test*`` function; a ``Test*`` class or a ``unittest.TestCase``
+ subclass, directly or through another class in the file; and each ``test*``
+ method of such a class, by itself and as ``Class::method``.
+ """
+
+ tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
+ classes = {node.name: node for node in tree.body if isinstance(node, ast.ClassDef)}
+
+ def is_test_case(name: str, seen: frozenset[str] = frozenset()) -> bool:
+ node = classes.get(name)
+ if node is None or name in seen:
+ return False
+ bases = [base.attr if isinstance(base, ast.Attribute) else getattr(base, "id", "") for base in node.bases]
+ return any(base.endswith("TestCase") or is_test_case(base, seen | {name}) for base in bases)
+
+ def test_functions(body: list[ast.stmt]) -> set[str]:
+ return {
+ node.name
+ for node in body
+ if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name.startswith("test")
+ }
+
+ names = test_functions(tree.body)
+ for name, node in classes.items():
+ if name.startswith("Test") or is_test_case(name):
+ names.add(name)
+ for method in test_functions(node.body):
+ names.update({method, f"{name}::{method}"})
+ return frozenset(names)
+
+
+def _catalogs() -> list[Path]:
+ catalogs = sorted(SCENARIO_ROOT.glob("*/catalog.yaml"))
+ assert catalogs, f"no scenario catalogs under {SCENARIO_ROOT}"
+ return catalogs
+
+
[email protected]("catalog", _catalogs(), ids=lambda path: path.parent.name)
+def test_every_pointer_names_a_file_or_case_that_exists(catalog: Path) -> None:
+ rows = yaml.safe_load(catalog.read_text(encoding="utf-8")).get("scenarios") or []
+ dead: list[str] = []
+ for row in rows:
+ for key in _POINTER_KEYS:
+ pointers = row.get(key)
+ if pointers is None:
+ continue
+ if not isinstance(pointers, list):
+ dead.append(f"{row['id']} {key} is not a list")
+ continue
+ for reference in pointers:
+ where = f"{row['id']} {key}: {reference!r}"
+ if not isinstance(reference, str):
+ dead.append(f"{where} (not a reference)")
+ continue
+ path_text, separator, name = reference.partition("::")
+ path = REPO_ROOT / path_text
+ if not path.is_file():
+ dead.append(f"{where} (no such file)")
+ elif separator and not name.strip():
+ dead.append(f"{where} (names an empty case)")
+ elif separator and path.suffix not in _UI_SUFFIXES:
+ if name.partition("[")[0] not in _test_names(path):
+ dead.append(f"{where} (no such test in the file)")
+ assert not dead, f"{catalog.relative_to(REPO_ROOT)} has pointers that lead nowhere:\n" + "\n".join(dead)
diff --git a/ui/scripts/scenarioCatalog.mjs b/ui/scripts/scenarioCatalog.mjs
index 749fe20e6..3619710fa 100644
--- a/ui/scripts/scenarioCatalog.mjs
+++ b/ui/scripts/scenarioCatalog.mjs
@@ -34,6 +34,9 @@ const citedPath = (testRef) => testRef.split('::')[0];
export const isUiEvidence = (testRef) => UI_SUFFIXES.has(path.extname(citedPath(testRef)));
+/** Keys that point a reader at neighbouring cases without claiming the row's coverage. */
+const POINTER_KEYS = ['related_tests', 'supporting_tests'];
+
/**
* Every UI citation the catalog's rows make, whatever key it sits in.
*
@@ -52,10 +55,13 @@ export const isUiEvidence = (testRef) => UI_SUFFIXES.has(path.extname(citedPath(
* another key. It cost four of those five: three named cases #1401 deleted on
* 2026-08-14, unnoticed because nothing read them.
*
- * `related_tests` and `canonical_tests` stay unread, and that is the same rule
- * rather than an exception to it: they name a file and no case, so they cannot
- * evidence a row, and they do not claim to — "related" is a pointer, not a
- * coverage claim.
+ * `canonical_tests` stays unread, and so does a pointer key's bare file, and
+ * that is the same rule rather than an exception to it: they name a file and no
+ * case, so they cannot evidence a row, and they do not claim to. A pointer
+ * (`related_tests`, `supporting_tests`) that names a case is read, though. It
+ * claims no coverage, but a case renamed out from under it leaves the reader a
+ * reference to nothing, which is what one of MH-MIG-004's pointers had become.
+ * Such a citation carries `pointer`, the key it sits in.
*
* `cited` is null for a citation that names a file and no case in it, which
* `resolveUiEvidence` reports rather than resolves.
@@ -79,6 +85,18 @@ export const uiEvidenceRows = (catalog) =>
inputs: (contract.inputs ?? []).map(String),
});
}
+ for (const pointer of POINTER_KEYS) {
+ for (const reference of row[pointer] ?? []) {
+ const separator = typeof reference === 'string' ? reference.indexOf('::') : -1;
+ if (separator < 0 || !isUiEvidence(reference)) continue;
+ citations.push({
+ id: row.id,
+ file: reference.slice(0, separator),
+ cited: reference.slice(separator + 2),
+ pointer,
+ });
+ }
+ }
return citations;
});
@@ -152,7 +170,8 @@ export const resolveUiEvidence = (rows, collected, ids = new Set(rows.map((row)
);
continue;
}
- if (row.cited !== row.id && ids.has(row.cited)) {
+ // A pointer claims no evidence, so naming a case another row owns borrows nothing.
+ if (!row.pointer && row.cited !== row.id && ids.has(row.cited)) {
problems.push(
`${where} cites \`${row.cited}\`, which is another row's ID; a row resolves through evidence named for itself`,
);
@@ -162,7 +181,17 @@ export const resolveUiEvidence = (rows, collected, ids = new Set(rows.map((row)
const matches = collected
.filter((entry) => sameFile(entry.file, row.file))
.filter((entry) => citableNames(entry).some((name) => terms.every((term) => name.includes(term))));
- if (matches.length !== 1) {
+ if (row.pointer) {
+ // A pointer only has to lead somewhere live: it may name a group of cases,
+ // such as every case tagged with an ID, because it identifies no evidence.
+ // An empty name is contained in every case's, so it leads nowhere either.
+ if (matches.length === 0 || !row.cited.trim()) {
+ problems.push(
+ `${where}: its ${row.pointer} entry names \`${terms.join('` + `')}\` in ${row.file}, and vitest collects no `
+ + 'case by that name — a renamed or removed case leaves the pointer leading nowhere',
+ );
+ }
+ } else if (matches.length !== 1) {
problems.push(
`${where}: vitest collects ${matches.length} cases named \`${terms.join('` + `')}\` in ${row.file}, expected exactly one — `
+ 'a commented-out, skipped, unreachable or merely quoted case is not executable evidence, and a name '
diff --git a/ui/scripts/scenarioCatalog.test.mjs b/ui/scripts/scenarioCatalog.test.mjs
index fdd04b385..51e98c4e6 100644
--- a/ui/scripts/scenarioCatalog.test.mjs
+++ b/ui/scripts/scenarioCatalog.test.mjs
@@ -113,6 +113,13 @@ describe('UI evidence resolution', () => {
// particular — whatever else cites the file keeps it collected, so the row
// would read `covered` with nothing tied to itself. Reported, not resolved.
expect(resolveUiEvidence([row('MH-ROW-E', { cited: null })], collected)).toHaveLength(1);
+ // A pointer identifies no evidence, so it only has to lead to a live case: a
+ // group of cases is fine, and so is a case another row owns, but nothing is not.
+ const pointer = (id, cited) => row(id, { cited, pointer: 'related_tests' });
+ expect(resolveUiEvidence([pointer('MH-ROW-G', 'MH-ROW-C')], collected)).toEqual([]);
+ expect(resolveUiEvidence([pointer('MH-ROW-D', 'MH-ROW-A')], collected, ids)).toEqual([]);
+ expect(resolveUiEvidence([pointer('MH-ROW-G', 'MH-ROW-B')], collected)).toHaveLength(1);
+ expect(resolveUiEvidence([pointer('MH-ROW-G', '')], collected)).toHaveLength(1);
});
it('resolves a parameterized case through the inputs the citation names', () => {
@@ -158,9 +165,18 @@ describe('UI evidence resolution', () => {
// row and does not say it does. Unread for the same reason a file-only
// citation is rejected, rather than as an exception to it.
{ id: 'MH-RELATED', status: 'covered', test: 'tests/test_c.py::test_c', related_tests: ['ui/src/d.test.tsx'] },
+ // A pointer that names a case is asked, in either pointer key: still no
+ // claim, but a renamed case would leave it pointing at nothing.
+ {
+ id: 'MH-POINTER',
+ status: 'covered',
+ test: 'tests/test_d.py::test_d',
+ related_tests: ['ui/src/e.test.tsx::names its neighbour', 'tests/test_e.py::test_e'],
+ supporting_tests: ['ui/src/f.test.tsx::supports it'],
+ },
{ id: 'MH-GAP', status: 'gap', test: null },
];
- const asked = ['MH-UI', 'MH-SCRIPT', 'MH-FILE', 'MH-CONTRACT'];
+ const asked = ['MH-UI', 'MH-SCRIPT', 'MH-FILE', 'MH-CONTRACT', 'MH-POINTER', 'MH-POINTER'];
const rowsOf = (catalog) => uiEvidenceRows({ ...catalog, scenarios });
expect(rowsOf({ status_legend: { covered: { test_required: true } } }).map((row) => row.id)).toEqual(asked);
@@ -172,6 +188,10 @@ describe('UI evidence resolution', () => {
file: 'ui/src/c.test.tsx',
cited: 'holds the frozen contract',
});
+ expect(rowsOf({}).filter((row) => row.id === 'MH-POINTER')).toEqual([
+ { id: 'MH-POINTER', file: 'ui/src/e.test.tsx', cited: 'names its neighbour', pointer: 'related_tests' },
+ { id: 'MH-POINTER', file: 'ui/src/f.test.tsx', cited: 'supports it', pointer: 'supporting_tests' },
+ ]);
expect(isUiEvidence('tests/test_a.py::test_a')).toBe(false);
});