Graph-Native Infrastructure for Context and Accountable AI Systems
Python agent-memoryaiai-governanceai-infrastructureartificial-intelligencecontext-engineeringcontext-graphsdata-engineeringdecision-intelligencedeveloper-toolsexplainable-aigenerative-aigraph-ragknowledge-graphllmontologyprovenancepythonreasoningsemantic-search
feat(context): add trust tier calculation for graph facts (#1583)
Kevinc5e2fbffca38
diff --git a/semantica/context/tiers.py b/semantica/context/tiers.pynew file mode 100644index 00000000..351617d2--- /dev/null+++ b/semantica/context/tiers.py@@ -0,0 +1,319 @@+"""+Trust Tiers for Graph Facts++Retrieval results carry a relevance score but no evidence-quality signal, so a+fact corroborated by three ingested sources and a singleton extracted at low+confidence arrive at the agent looking identical. This module grades a fact+into a discrete trust tier using signals that already exist inside semantica:++ - Corroboration count: how many sources back the entity, as reported by+ ProvenanceManager.get_all_sources.+ - Extraction confidence: the value graph_builder persisted on the node via+ getattr(item, "confidence", 1.0).++Tiers are derived on demand at retrieval time and never persisted, so they stay+accurate as source sets evolve and existing storage layouts are untouched.++Algorithms Used:++ - Primary signal: corroboration count. Confidence alone cannot separate+ "1.0 because it was earned" from "1.0 because nothing set it": the+ extraction dataclasses in semantic_extract.types default confidence to+ 1.0, and graph_builder falls back to 1.0 for objects that lack the+ attribute, so a stored 1.0 usually means the fact was never scored.+ - Secondary signal: extraction confidence, used to keep or downgrade the+ tier produced by corroboration. Values listed in treat_as_missing are+ treated as absent, which by default covers that 1.0 placeholder.+ - Missing confidence degrades the tier instead of defaulting high, so+ absent data is never trusted as if it had been measured.++Tiers:++ - gold: multiple independent sources plus usable confidence.+ - silver: partial support, a single source, or a higher band pulled down+ by absent or low confidence.+ - bronze: no corroboration at all, held up only by usable confidence.+ - quarantine: no corroboration and no usable confidence; needs review.++Example:+ >>> from semantica.context.tiers import TierCalculator, TrustTier+ >>> calculator = TierCalculator()+ >>> calculator.calculate(3, 0.9) is TrustTier.GOLD+ True+ >>> calculator.calculate(0, None) is TrustTier.QUARANTINE+ True+"""++import math+from enum import Enum+from typing import Any, Iterable, Optional+++class TrustTier(Enum):+ """+ Evidence-quality tier for a retrieved graph fact.++ Members are ordered from least to most trustworthy. Values are stable+ strings so a tier can be written straight into RetrievedContext.metadata+ for downstream consumers, while ordering is exposed through rank and meets+ rather than through the value type.+ """++ QUARANTINE = "quarantine"+ BRONZE = "bronze"+ SILVER = "silver"+ GOLD = "gold"++ @property+ def rank(self) -> int:+ """+ Return the position of this tier in the trust ordering.++ Returns:+ Integer rank: 0 for quarantine through 3 for gold.+ """+ return _TIER_RANKS[self]++ def meets(self, minimum: "TrustTier") -> bool:+ """+ Return whether this tier is at least as trustworthy as minimum.++ Args:+ minimum: Lowest acceptable tier.++ Returns:+ True when this tier ranks at or above minimum.++ Raises:+ TypeError: If minimum is not a TrustTier.++ Example:+ >>> TrustTier.GOLD.meets(TrustTier.SILVER)+ True+ >>> TrustTier.BRONZE.meets(TrustTier.SILVER)+ False+ """+ if not isinstance(minimum, TrustTier):+ raise TypeError(+ f"minimum must be a TrustTier, got {type(minimum).__name__}"+ )+ return self.rank >= minimum.rank++ def downgrade(self) -> "TrustTier":+ """+ Return the next lower tier, flooring at quarantine.++ Used when a secondary signal (extraction confidence) is missing or+ unusable, so the corroboration-based tier cannot be kept as it stands.++ Returns:+ The tier one step less trustworthy, or quarantine at the floor.++ Example:+ >>> TrustTier.GOLD.downgrade()+ <TrustTier.SILVER: 'silver'>+ >>> TrustTier.QUARANTINE.downgrade() is TrustTier.QUARANTINE+ True+ """+ if self is TrustTier.GOLD:+ return TrustTier.SILVER+ if self is TrustTier.SILVER:+ return TrustTier.BRONZE+ return TrustTier.QUARANTINE+++_TIER_RANKS = {+ TrustTier.QUARANTINE: 0,+ TrustTier.BRONZE: 1,+ TrustTier.SILVER: 2,+ TrustTier.GOLD: 3,+}++# Stored confidence that means "never scored" rather than "scored perfectly".+# semantic_extract.types.Entity defaults confidence to 1.0 and graph_builder+# keeps 1.0 for objects without the attribute, so the value is not evidence.+_DEFAULT_TREAT_AS_MISSING = frozenset({1.0})+++class TierCalculator:+ """+ Derive a trust tier from corroboration count and extraction confidence.++ Args:+ min_corroboration_for_gold: Number of corroborating sources required+ before a fact can reach gold (default: 2).+ min_confidence: Confidence at or above which the corroboration-based+ tier is kept; below it the tier is downgraded (default: 0.7).+ missing_confidence_degrades: Downgrade one tier when no confidence+ signal is available, instead of treating absent data as high+ confidence (default: True).+ treat_as_missing: Confidence values that carry no measurement and must+ therefore be handled as absent (default: 1.0). The extraction+ dataclasses in semantic_extract.types default confidence to 1.0 and+ graph_builder falls back to 1.0, so an unstored score is+ indistinguishable from a perfect one at the storage layer. Pass an+ empty iterable to disable the substitution, or add 0.9 to also+ treat the schema default as missing.++ Raises:+ ValueError: If min_corroboration_for_gold is below 1, min_confidence is+ outside [0.0, 1.0], or a treat_as_missing value is outside+ [0.0, 1.0].+ TypeError: If treat_as_missing is not iterable.++ Example:+ >>> TierCalculator().calculate(2, 0.8) is TrustTier.GOLD+ True+ >>> TierCalculator().calculate(2, None) is TrustTier.SILVER+ True+ >>> TierCalculator().calculate(2, 1.0) is TrustTier.SILVER+ True+ """++ def __init__(+ self,+ min_corroboration_for_gold: int = 2,+ min_confidence: float = 0.7,+ missing_confidence_degrades: bool = True,+ treat_as_missing: Optional[Iterable[Any]] = _DEFAULT_TREAT_AS_MISSING,+ ) -> None:+ if min_corroboration_for_gold < 1:+ raise ValueError("min_corroboration_for_gold must be at least 1")+ if not 0.0 <= min_confidence <= 1.0:+ raise ValueError("min_confidence must be within [0.0, 1.0]")++ self.min_corroboration_for_gold = int(min_corroboration_for_gold)+ self.min_confidence = float(min_confidence)+ self.missing_confidence_degrades = bool(missing_confidence_degrades)+ self.treat_as_missing = self._normalize_treat_as_missing(treat_as_missing)++ def calculate(+ self,+ corroboration_count: Optional[int],+ confidence: Optional[float] = None,+ ) -> TrustTier:+ """+ Calculate the trust tier for a single fact.++ Corroboration count sets the tier, confidence keeps or downgrades it,+ and a complete absence of both signals yields quarantine. Confidence+ values configured in treat_as_missing count as absent, so a placeholder+ score cannot earn a high tier.++ Args:+ corroboration_count: Number of independent sources backing the+ fact. None, negatives and unusable values count as zero.+ confidence: Extraction confidence recorded for the fact, or None+ when no confidence signal is available.++ Returns:+ The TrustTier for the fact.++ Example:+ >>> calculator = TierCalculator()+ >>> calculator.calculate(3, 0.9) is TrustTier.GOLD+ True+ >>> calculator.calculate(1, 0.9) is TrustTier.SILVER+ True+ >>> calculator.calculate(3, None) is TrustTier.SILVER+ True+ >>> calculator.calculate(0, None) is TrustTier.QUARANTINE+ True+ """+ count = self._normalize_count(corroboration_count)+ confidence_is_missing = self._is_absent_confidence(confidence)+ has_usable_confidence = (+ self._is_usable_confidence(confidence) and not confidence_is_missing+ )++ if count >= self.min_corroboration_for_gold:+ tier = TrustTier.GOLD+ elif count >= 1:+ tier = TrustTier.SILVER+ elif has_usable_confidence:+ tier = TrustTier.BRONZE+ else:+ tier = TrustTier.QUARANTINE++ if confidence_is_missing:+ if self.missing_confidence_degrades:+ return tier.downgrade()+ return tier++ if has_usable_confidence:+ return tier++ return tier.downgrade()++ def _is_absent_confidence(self, confidence: Optional[float]) -> bool:+ """+ Return whether a confidence reading carries no measurement at all.++ None is always absent. A value listed in treat_as_missing is absent+ too, because the storage layer writes the same placeholder whether or+ not anything scored the fact.+ """+ if confidence is None:+ return True+ if not self.treat_as_missing:+ return False+ try:+ value = float(confidence)+ except (TypeError, ValueError):+ return False+ return value in self.treat_as_missing++ @staticmethod+ def _normalize_treat_as_missing(values: Iterable[Any]) -> frozenset:+ """Validate configured missing markers into a set of floats."""+ try:+ candidates = tuple(values)+ except TypeError as exc:+ raise TypeError(+ "treat_as_missing must be an iterable of confidence values"+ ) from exc++ normalized = set()+ for candidate in candidates:+ try:+ value = float(candidate)+ except (TypeError, ValueError) as exc:+ raise ValueError(+ "treat_as_missing values must be numbers, got " f"{candidate!r}"+ ) from exc+ if not 0.0 <= value <= 1.0:+ raise ValueError(+ "treat_as_missing values must be within [0.0, 1.0], got "+ f"{candidate!r}"+ )+ normalized.add(value)+ return frozenset(normalized)++ @staticmethod+ def _normalize_count(corroboration_count: Optional[int]) -> int:+ """Coerce a corroboration count into a non-negative integer."""+ if corroboration_count is None:+ return 0+ try:+ count = int(corroboration_count)+ except (TypeError, ValueError, OverflowError):+ return 0+ return max(0, count)++ def _is_usable_confidence(self, confidence: Optional[float]) -> bool:+ """Return whether confidence is a finite number at least+ min_confidence.++ Non-finite values (positive or negative infinity) are not real+ measurements, so they count as unusable rather than earning the top+ tier through the lower-bound comparison alone.+ """+ if confidence is None:+ return False+ try:+ value = float(confidence)+ except (TypeError, ValueError):+ return False+ if not math.isfinite(value):+ return False+ return value >= self.min_confidencediff --git a/tests/context/test_tiers.py b/tests/context/test_tiers.pynew file mode 100644index 00000000..cbdab3a3--- /dev/null+++ b/tests/context/test_tiers.py@@ -0,0 +1,245 @@+"""+Unit tests for trust tiers (semantica.context.tiers).++Covers tier ordering, the corroboration-led thresholds, the degradation rules+for missing or unusable confidence, the placeholder confidence values that+count as absent, and the tolerance for messy inputs.+"""++import json+import unittest++from semantica.context.tiers import TierCalculator, TrustTier+++class TestTrustTierOrdering(unittest.TestCase):+ """TrustTier ordering helpers used by min_tier filtering."""++ def test_rank_increases_with_trust(self):+ self.assertLess(TrustTier.QUARANTINE.rank, TrustTier.BRONZE.rank)+ self.assertLess(TrustTier.BRONZE.rank, TrustTier.SILVER.rank)+ self.assertLess(TrustTier.SILVER.rank, TrustTier.GOLD.rank)++ def test_meets_is_inclusive(self):+ self.assertTrue(TrustTier.GOLD.meets(TrustTier.GOLD))+ self.assertTrue(TrustTier.GOLD.meets(TrustTier.QUARANTINE))+ self.assertFalse(TrustTier.BRONZE.meets(TrustTier.SILVER))++ def test_meets_rejects_non_tier(self):+ with self.assertRaises(TypeError):+ TrustTier.GOLD.meets("gold")++ def test_values_are_stable_strings(self):+ self.assertEqual(TrustTier.GOLD.value, "gold")+ self.assertEqual(TrustTier.SILVER.value, "silver")+ self.assertEqual(TrustTier.BRONZE.value, "bronze")+ self.assertEqual(TrustTier.QUARANTINE.value, "quarantine")++ def test_downgrade_steps_down_one_tier(self):+ self.assertIs(TrustTier.GOLD.downgrade(), TrustTier.SILVER)+ self.assertIs(TrustTier.SILVER.downgrade(), TrustTier.BRONZE)+ self.assertIs(TrustTier.BRONZE.downgrade(), TrustTier.QUARANTINE)++ def test_downgrade_floors_at_quarantine(self):+ self.assertIs(TrustTier.QUARANTINE.downgrade(), TrustTier.QUARANTINE)+++class TestTierCalculatorThresholds(unittest.TestCase):+ """Corroboration count sets the tier; confidence adjusts it."""++ def setUp(self):+ self.calculator = TierCalculator()++ def test_multiple_sources_with_usable_confidence_is_gold(self):+ self.assertIs(self.calculator.calculate(3, 0.9), TrustTier.GOLD)++ def test_source_count_at_gold_threshold_is_gold(self):+ self.assertIs(self.calculator.calculate(2, 0.7), TrustTier.GOLD)++ def test_confidence_below_threshold_downgrades_gold(self):+ self.assertIs(self.calculator.calculate(2, 0.69), TrustTier.SILVER)++ def test_single_source_with_usable_confidence_is_silver(self):+ self.assertIs(self.calculator.calculate(1, 0.9), TrustTier.SILVER)++ def test_single_source_with_weak_confidence_is_bronze(self):+ self.assertIs(self.calculator.calculate(1, 0.5), TrustTier.BRONZE)++ def test_no_sources_with_usable_confidence_is_bronze(self):+ self.assertIs(self.calculator.calculate(0, 0.9), TrustTier.BRONZE)++ def test_no_sources_and_no_confidence_is_quarantine(self):+ self.assertIs(self.calculator.calculate(0, None), TrustTier.QUARANTINE)++ def test_no_sources_with_weak_confidence_is_quarantine(self):+ self.assertIs(self.calculator.calculate(0, 0.2), TrustTier.QUARANTINE)++ def test_many_sources_with_weak_confidence_is_silver(self):+ self.assertIs(self.calculator.calculate(5, 0.3), TrustTier.SILVER)++ def test_confidence_is_optional(self):+ self.assertIs(self.calculator.calculate(3), TrustTier.SILVER)+++class TestMissingConfidenceDegradation(unittest.TestCase):+ """Absent confidence degrades a tier rather than defaulting high."""++ def setUp(self):+ self.calculator = TierCalculator()++ def test_missing_confidence_downgrades_gold(self):+ self.assertIs(self.calculator.calculate(2, None), TrustTier.SILVER)++ def test_missing_confidence_downgrades_silver(self):+ self.assertIs(self.calculator.calculate(1, None), TrustTier.BRONZE)++ def test_missing_confidence_does_not_earn_bronze(self):+ # No corroboration and no confidence is quarantine, not bronze.+ self.assertIs(self.calculator.calculate(0, None), TrustTier.QUARANTINE)++ def test_missing_confidence_never_yields_gold(self):+ for count in (2, 5, 50):+ self.assertIsNot(self.calculator.calculate(count, None), TrustTier.GOLD)++ def test_degradation_can_be_disabled(self):+ calculator = TierCalculator(missing_confidence_degrades=False)+ self.assertIs(calculator.calculate(2, None), TrustTier.GOLD)+ self.assertIs(calculator.calculate(0, None), TrustTier.QUARANTINE)+++class TestInputHandling(unittest.TestCase):+ """Messy inputs must not raise or inflate a tier."""++ def setUp(self):+ self.calculator = TierCalculator()++ def test_none_count_is_treated_as_zero(self):+ self.assertIs(self.calculator.calculate(None, 0.9), TrustTier.BRONZE)++ def test_negative_count_is_treated_as_zero(self):+ self.assertIs(self.calculator.calculate(-4, 0.9), TrustTier.BRONZE)++ def test_non_numeric_count_is_treated_as_zero(self):+ self.assertIs(self.calculator.calculate("many", 0.9), TrustTier.BRONZE)++ def test_numeric_string_count_is_accepted(self):+ self.assertIs(self.calculator.calculate("3", 0.9), TrustTier.GOLD)++ def test_float_count_is_truncated(self):+ self.assertIs(self.calculator.calculate(2.9, 0.9), TrustTier.GOLD)+ self.assertIs(self.calculator.calculate(1.5, 0.9), TrustTier.SILVER)++ def test_unusable_confidence_downgrades(self):+ self.assertIs(self.calculator.calculate(2, "high"), TrustTier.SILVER)++ def test_nan_confidence_downgrades(self):+ tier = self.calculator.calculate(2, float("nan"))+ self.assertIs(tier, TrustTier.SILVER)++ def test_infinite_confidence_downgrades(self):+ self.assertIs(self.calculator.calculate(2, float("inf")), TrustTier.SILVER)+ self.assertIs(self.calculator.calculate(2, float("-inf")), TrustTier.SILVER)++ def test_infinite_count_is_treated_as_zero(self):+ self.assertIs(self.calculator.calculate(float("inf"), 0.9), TrustTier.BRONZE)+ self.assertIs(self.calculator.calculate(float("-inf"), 0.9), TrustTier.BRONZE)++ def test_nan_count_is_treated_as_zero(self):+ self.assertIs(+ self.calculator.calculate(float("nan"), 0.5), TrustTier.QUARANTINE+ )++ def test_confidence_above_one_is_still_usable(self):+ self.assertIs(self.calculator.calculate(2, 1.4), TrustTier.GOLD)+++class TestTierCalculatorConfiguration(unittest.TestCase):+ """Thresholds are configurable but invalid ones are rejected."""++ def test_gold_threshold_must_be_positive(self):+ with self.assertRaises(ValueError):+ TierCalculator(min_corroboration_for_gold=0)++ def test_min_confidence_must_be_a_probability(self):+ with self.assertRaises(ValueError):+ TierCalculator(min_confidence=1.5)+ with self.assertRaises(ValueError):+ TierCalculator(min_confidence=-0.1)++ def test_lower_gold_threshold_promotes_single_source(self):+ calculator = TierCalculator(min_corroboration_for_gold=1)+ self.assertIs(calculator.calculate(1, 0.9), TrustTier.GOLD)++ def test_lower_min_confidence_keeps_weaker_confidence(self):+ calculator = TierCalculator(min_confidence=0.5)+ self.assertIs(calculator.calculate(1, 0.6), TrustTier.SILVER)+ self.assertIs(calculator.calculate(1, 0.4), TrustTier.BRONZE)+++class TestMetadataFriendly(unittest.TestCase):+ """Tier values are ready to drop into RetrievedContext.metadata."""++ def test_tier_value_is_json_serialisable(self):+ tier = TierCalculator().calculate(3, 0.9)+ payload = json.dumps({"trust_tier": tier.value})+ self.assertEqual(payload, '{"trust_tier": "gold"}')+++class TestTreatAsMissing(unittest.TestCase):+ """Placeholder confidence is handled as absent, not as a perfect score."""++ def setUp(self):+ self.calculator = TierCalculator()++ def test_answer_placeholder_cannot_earn_gold(self):+ for count in (2, 5, 50):+ self.assertIsNot(self.calculator.calculate(count, 1.0), TrustTier.GOLD)++ def test_answer_placeholder_downgrades_gold(self):+ self.assertIs(self.calculator.calculate(2, 1.0), TrustTier.SILVER)++ def test_integer_one_is_also_a_placeholder(self):+ # The dataclass default can arrive as int 1 depending on the source.+ self.assertIs(self.calculator.calculate(2, 1), TrustTier.SILVER)++ def test_placeholder_without_corroboration_is_quarantine(self):+ self.assertIs(self.calculator.calculate(0, 1.0), TrustTier.QUARANTINE)++ def test_earned_confidence_is_not_treated_as_missing(self):+ # 0.9 is the value the issue discusses; the repo default is 1.0, so+ # 0.9 stays usable unless it is configured as a placeholder.+ self.assertIs(self.calculator.calculate(2, 0.9), TrustTier.GOLD)++ def test_default_placeholder_is_exposed(self):+ self.assertEqual(self.calculator.treat_as_missing, frozenset({1.0}))++ def test_substitution_can_be_disabled(self):+ calculator = TierCalculator(treat_as_missing=())+ self.assertIs(calculator.calculate(2, 1.0), TrustTier.GOLD)++ def test_additional_placeholders_can_be_configured(self):+ calculator = TierCalculator(treat_as_missing=[0.9, 1.0])+ self.assertIs(calculator.calculate(2, 0.9), TrustTier.SILVER)+ self.assertIs(calculator.calculate(2, 1.0), TrustTier.SILVER)+ self.assertIs(calculator.calculate(2, 0.95), TrustTier.GOLD)++ def test_degradation_flag_applies_to_placeholders(self):+ calculator = TierCalculator(missing_confidence_degrades=False)+ self.assertIs(calculator.calculate(2, 1.0), TrustTier.GOLD)+ self.assertIs(calculator.calculate(0, 1.0), TrustTier.QUARANTINE)++ def test_placeholder_outside_probability_range_is_rejected(self):+ with self.assertRaises(ValueError):+ TierCalculator(treat_as_missing=[1.5])++ def test_non_numeric_placeholder_is_rejected(self):+ with self.assertRaises(ValueError):+ TierCalculator(treat_as_missing=["high"])++ def test_non_iterable_placeholder_is_rejected(self):+ with self.assertRaises(TypeError):+ TierCalculator(treat_as_missing=1.0)+++if __name__ == "__main__":+ unittest.main()