Skip to content

Graph-Native Infrastructure for Context and Accountable AI Systems

feat(context): add trust tier calculation for graph facts (#1583)

Kevinc5e2fbffca38
diff --git a/semantica/context/tiers.py b/semantica/context/tiers.py
new file mode 100644
index 00000000..351617d2
--- /dev/null
+++ b/semantica/context/tiers.py
@@ -0,0 +1,319 @@
+"""
+Trust Tiers for Graph Facts
+
+Retrieval results carry a relevance score but no evidence-quality signal, so a
+fact corroborated by three ingested sources and a singleton extracted at low
+confidence arrive at the agent looking identical. This module grades a fact
+into a discrete trust tier using signals that already exist inside semantica:
+
+ - Corroboration count: how many sources back the entity, as reported by
+ ProvenanceManager.get_all_sources.
+ - Extraction confidence: the value graph_builder persisted on the node via
+ getattr(item, "confidence", 1.0).
+
+Tiers are derived on demand at retrieval time and never persisted, so they stay
+accurate as source sets evolve and existing storage layouts are untouched.
+
+Algorithms Used:
+
+ - Primary signal: corroboration count. Confidence alone cannot separate
+ "1.0 because it was earned" from "1.0 because nothing set it": the
+ extraction dataclasses in semantic_extract.types default confidence to
+ 1.0, and graph_builder falls back to 1.0 for objects that lack the
+ attribute, so a stored 1.0 usually means the fact was never scored.
+ - Secondary signal: extraction confidence, used to keep or downgrade the
+ tier produced by corroboration. Values listed in treat_as_missing are
+ treated as absent, which by default covers that 1.0 placeholder.
+ - Missing confidence degrades the tier instead of defaulting high, so
+ absent data is never trusted as if it had been measured.
+
+Tiers:
+
+ - gold: multiple independent sources plus usable confidence.
+ - silver: partial support, a single source, or a higher band pulled down
+ by absent or low confidence.
+ - bronze: no corroboration at all, held up only by usable confidence.
+ - quarantine: no corroboration and no usable confidence; needs review.
+
+Example:
+ >>> from semantica.context.tiers import TierCalculator, TrustTier
+ >>> calculator = TierCalculator()
+ >>> calculator.calculate(3, 0.9) is TrustTier.GOLD
+ True
+ >>> calculator.calculate(0, None) is TrustTier.QUARANTINE
+ True
+"""
+
+import math
+from enum import Enum
+from typing import Any, Iterable, Optional
+
+
+class TrustTier(Enum):
+ """
+ Evidence-quality tier for a retrieved graph fact.
+
+ Members are ordered from least to most trustworthy. Values are stable
+ strings so a tier can be written straight into RetrievedContext.metadata
+ for downstream consumers, while ordering is exposed through rank and meets
+ rather than through the value type.
+ """
+
+ QUARANTINE = "quarantine"
+ BRONZE = "bronze"
+ SILVER = "silver"
+ GOLD = "gold"
+
+ @property
+ def rank(self) -> int:
+ """
+ Return the position of this tier in the trust ordering.
+
+ Returns:
+ Integer rank: 0 for quarantine through 3 for gold.
+ """
+ return _TIER_RANKS[self]
+
+ def meets(self, minimum: "TrustTier") -> bool:
+ """
+ Return whether this tier is at least as trustworthy as minimum.
+
+ Args:
+ minimum: Lowest acceptable tier.
+
+ Returns:
+ True when this tier ranks at or above minimum.
+
+ Raises:
+ TypeError: If minimum is not a TrustTier.
+
+ Example:
+ >>> TrustTier.GOLD.meets(TrustTier.SILVER)
+ True
+ >>> TrustTier.BRONZE.meets(TrustTier.SILVER)
+ False
+ """
+ if not isinstance(minimum, TrustTier):
+ raise TypeError(
+ f"minimum must be a TrustTier, got {type(minimum).__name__}"
+ )
+ return self.rank >= minimum.rank
+
+ def downgrade(self) -> "TrustTier":
+ """
+ Return the next lower tier, flooring at quarantine.
+
+ Used when a secondary signal (extraction confidence) is missing or
+ unusable, so the corroboration-based tier cannot be kept as it stands.
+
+ Returns:
+ The tier one step less trustworthy, or quarantine at the floor.
+
+ Example:
+ >>> TrustTier.GOLD.downgrade()
+ <TrustTier.SILVER: 'silver'>
+ >>> TrustTier.QUARANTINE.downgrade() is TrustTier.QUARANTINE
+ True
+ """
+ if self is TrustTier.GOLD:
+ return TrustTier.SILVER
+ if self is TrustTier.SILVER:
+ return TrustTier.BRONZE
+ return TrustTier.QUARANTINE
+
+
+_TIER_RANKS = {
+ TrustTier.QUARANTINE: 0,
+ TrustTier.BRONZE: 1,
+ TrustTier.SILVER: 2,
+ TrustTier.GOLD: 3,
+}
+
+# Stored confidence that means "never scored" rather than "scored perfectly".
+# semantic_extract.types.Entity defaults confidence to 1.0 and graph_builder
+# keeps 1.0 for objects without the attribute, so the value is not evidence.
+_DEFAULT_TREAT_AS_MISSING = frozenset({1.0})
+
+
+class TierCalculator:
+ """
+ Derive a trust tier from corroboration count and extraction confidence.
+
+ Args:
+ min_corroboration_for_gold: Number of corroborating sources required
+ before a fact can reach gold (default: 2).
+ min_confidence: Confidence at or above which the corroboration-based
+ tier is kept; below it the tier is downgraded (default: 0.7).
+ missing_confidence_degrades: Downgrade one tier when no confidence
+ signal is available, instead of treating absent data as high
+ confidence (default: True).
+ treat_as_missing: Confidence values that carry no measurement and must
+ therefore be handled as absent (default: 1.0). The extraction
+ dataclasses in semantic_extract.types default confidence to 1.0 and
+ graph_builder falls back to 1.0, so an unstored score is
+ indistinguishable from a perfect one at the storage layer. Pass an
+ empty iterable to disable the substitution, or add 0.9 to also
+ treat the schema default as missing.
+
+ Raises:
+ ValueError: If min_corroboration_for_gold is below 1, min_confidence is
+ outside [0.0, 1.0], or a treat_as_missing value is outside
+ [0.0, 1.0].
+ TypeError: If treat_as_missing is not iterable.
+
+ Example:
+ >>> TierCalculator().calculate(2, 0.8) is TrustTier.GOLD
+ True
+ >>> TierCalculator().calculate(2, None) is TrustTier.SILVER
+ True
+ >>> TierCalculator().calculate(2, 1.0) is TrustTier.SILVER
+ True
+ """
+
+ def __init__(
+ self,
+ min_corroboration_for_gold: int = 2,
+ min_confidence: float = 0.7,
+ missing_confidence_degrades: bool = True,
+ treat_as_missing: Optional[Iterable[Any]] = _DEFAULT_TREAT_AS_MISSING,
+ ) -> None:
+ if min_corroboration_for_gold < 1:
+ raise ValueError("min_corroboration_for_gold must be at least 1")
+ if not 0.0 <= min_confidence <= 1.0:
+ raise ValueError("min_confidence must be within [0.0, 1.0]")
+
+ self.min_corroboration_for_gold = int(min_corroboration_for_gold)
+ self.min_confidence = float(min_confidence)
+ self.missing_confidence_degrades = bool(missing_confidence_degrades)
+ self.treat_as_missing = self._normalize_treat_as_missing(treat_as_missing)
+
+ def calculate(
+ self,
+ corroboration_count: Optional[int],
+ confidence: Optional[float] = None,
+ ) -> TrustTier:
+ """
+ Calculate the trust tier for a single fact.
+
+ Corroboration count sets the tier, confidence keeps or downgrades it,
+ and a complete absence of both signals yields quarantine. Confidence
+ values configured in treat_as_missing count as absent, so a placeholder
+ score cannot earn a high tier.
+
+ Args:
+ corroboration_count: Number of independent sources backing the
+ fact. None, negatives and unusable values count as zero.
+ confidence: Extraction confidence recorded for the fact, or None
+ when no confidence signal is available.
+
+ Returns:
+ The TrustTier for the fact.
+
+ Example:
+ >>> calculator = TierCalculator()
+ >>> calculator.calculate(3, 0.9) is TrustTier.GOLD
+ True
+ >>> calculator.calculate(1, 0.9) is TrustTier.SILVER
+ True
+ >>> calculator.calculate(3, None) is TrustTier.SILVER
+ True
+ >>> calculator.calculate(0, None) is TrustTier.QUARANTINE
+ True
+ """
+ count = self._normalize_count(corroboration_count)
+ confidence_is_missing = self._is_absent_confidence(confidence)
+ has_usable_confidence = (
+ self._is_usable_confidence(confidence) and not confidence_is_missing
+ )
+
+ if count >= self.min_corroboration_for_gold:
+ tier = TrustTier.GOLD
+ elif count >= 1:
+ tier = TrustTier.SILVER
+ elif has_usable_confidence:
+ tier = TrustTier.BRONZE
+ else:
+ tier = TrustTier.QUARANTINE
+
+ if confidence_is_missing:
+ if self.missing_confidence_degrades:
+ return tier.downgrade()
+ return tier
+
+ if has_usable_confidence:
+ return tier
+
+ return tier.downgrade()
+
+ def _is_absent_confidence(self, confidence: Optional[float]) -> bool:
+ """
+ Return whether a confidence reading carries no measurement at all.
+
+ None is always absent. A value listed in treat_as_missing is absent
+ too, because the storage layer writes the same placeholder whether or
+ not anything scored the fact.
+ """
+ if confidence is None:
+ return True
+ if not self.treat_as_missing:
+ return False
+ try:
+ value = float(confidence)
+ except (TypeError, ValueError):
+ return False
+ return value in self.treat_as_missing
+
+ @staticmethod
+ def _normalize_treat_as_missing(values: Iterable[Any]) -> frozenset:
+ """Validate configured missing markers into a set of floats."""
+ try:
+ candidates = tuple(values)
+ except TypeError as exc:
+ raise TypeError(
+ "treat_as_missing must be an iterable of confidence values"
+ ) from exc
+
+ normalized = set()
+ for candidate in candidates:
+ try:
+ value = float(candidate)
+ except (TypeError, ValueError) as exc:
+ raise ValueError(
+ "treat_as_missing values must be numbers, got " f"{candidate!r}"
+ ) from exc
+ if not 0.0 <= value <= 1.0:
+ raise ValueError(
+ "treat_as_missing values must be within [0.0, 1.0], got "
+ f"{candidate!r}"
+ )
+ normalized.add(value)
+ return frozenset(normalized)
+
+ @staticmethod
+ def _normalize_count(corroboration_count: Optional[int]) -> int:
+ """Coerce a corroboration count into a non-negative integer."""
+ if corroboration_count is None:
+ return 0
+ try:
+ count = int(corroboration_count)
+ except (TypeError, ValueError, OverflowError):
+ return 0
+ return max(0, count)
+
+ def _is_usable_confidence(self, confidence: Optional[float]) -> bool:
+ """Return whether confidence is a finite number at least
+ min_confidence.
+
+ Non-finite values (positive or negative infinity) are not real
+ measurements, so they count as unusable rather than earning the top
+ tier through the lower-bound comparison alone.
+ """
+ if confidence is None:
+ return False
+ try:
+ value = float(confidence)
+ except (TypeError, ValueError):
+ return False
+ if not math.isfinite(value):
+ return False
+ return value >= self.min_confidence
diff --git a/tests/context/test_tiers.py b/tests/context/test_tiers.py
new file mode 100644
index 00000000..cbdab3a3
--- /dev/null
+++ b/tests/context/test_tiers.py
@@ -0,0 +1,245 @@
+"""
+Unit tests for trust tiers (semantica.context.tiers).
+
+Covers tier ordering, the corroboration-led thresholds, the degradation rules
+for missing or unusable confidence, the placeholder confidence values that
+count as absent, and the tolerance for messy inputs.
+"""
+
+import json
+import unittest
+
+from semantica.context.tiers import TierCalculator, TrustTier
+
+
+class TestTrustTierOrdering(unittest.TestCase):
+ """TrustTier ordering helpers used by min_tier filtering."""
+
+ def test_rank_increases_with_trust(self):
+ self.assertLess(TrustTier.QUARANTINE.rank, TrustTier.BRONZE.rank)
+ self.assertLess(TrustTier.BRONZE.rank, TrustTier.SILVER.rank)
+ self.assertLess(TrustTier.SILVER.rank, TrustTier.GOLD.rank)
+
+ def test_meets_is_inclusive(self):
+ self.assertTrue(TrustTier.GOLD.meets(TrustTier.GOLD))
+ self.assertTrue(TrustTier.GOLD.meets(TrustTier.QUARANTINE))
+ self.assertFalse(TrustTier.BRONZE.meets(TrustTier.SILVER))
+
+ def test_meets_rejects_non_tier(self):
+ with self.assertRaises(TypeError):
+ TrustTier.GOLD.meets("gold")
+
+ def test_values_are_stable_strings(self):
+ self.assertEqual(TrustTier.GOLD.value, "gold")
+ self.assertEqual(TrustTier.SILVER.value, "silver")
+ self.assertEqual(TrustTier.BRONZE.value, "bronze")
+ self.assertEqual(TrustTier.QUARANTINE.value, "quarantine")
+
+ def test_downgrade_steps_down_one_tier(self):
+ self.assertIs(TrustTier.GOLD.downgrade(), TrustTier.SILVER)
+ self.assertIs(TrustTier.SILVER.downgrade(), TrustTier.BRONZE)
+ self.assertIs(TrustTier.BRONZE.downgrade(), TrustTier.QUARANTINE)
+
+ def test_downgrade_floors_at_quarantine(self):
+ self.assertIs(TrustTier.QUARANTINE.downgrade(), TrustTier.QUARANTINE)
+
+
+class TestTierCalculatorThresholds(unittest.TestCase):
+ """Corroboration count sets the tier; confidence adjusts it."""
+
+ def setUp(self):
+ self.calculator = TierCalculator()
+
+ def test_multiple_sources_with_usable_confidence_is_gold(self):
+ self.assertIs(self.calculator.calculate(3, 0.9), TrustTier.GOLD)
+
+ def test_source_count_at_gold_threshold_is_gold(self):
+ self.assertIs(self.calculator.calculate(2, 0.7), TrustTier.GOLD)
+
+ def test_confidence_below_threshold_downgrades_gold(self):
+ self.assertIs(self.calculator.calculate(2, 0.69), TrustTier.SILVER)
+
+ def test_single_source_with_usable_confidence_is_silver(self):
+ self.assertIs(self.calculator.calculate(1, 0.9), TrustTier.SILVER)
+
+ def test_single_source_with_weak_confidence_is_bronze(self):
+ self.assertIs(self.calculator.calculate(1, 0.5), TrustTier.BRONZE)
+
+ def test_no_sources_with_usable_confidence_is_bronze(self):
+ self.assertIs(self.calculator.calculate(0, 0.9), TrustTier.BRONZE)
+
+ def test_no_sources_and_no_confidence_is_quarantine(self):
+ self.assertIs(self.calculator.calculate(0, None), TrustTier.QUARANTINE)
+
+ def test_no_sources_with_weak_confidence_is_quarantine(self):
+ self.assertIs(self.calculator.calculate(0, 0.2), TrustTier.QUARANTINE)
+
+ def test_many_sources_with_weak_confidence_is_silver(self):
+ self.assertIs(self.calculator.calculate(5, 0.3), TrustTier.SILVER)
+
+ def test_confidence_is_optional(self):
+ self.assertIs(self.calculator.calculate(3), TrustTier.SILVER)
+
+
+class TestMissingConfidenceDegradation(unittest.TestCase):
+ """Absent confidence degrades a tier rather than defaulting high."""
+
+ def setUp(self):
+ self.calculator = TierCalculator()
+
+ def test_missing_confidence_downgrades_gold(self):
+ self.assertIs(self.calculator.calculate(2, None), TrustTier.SILVER)
+
+ def test_missing_confidence_downgrades_silver(self):
+ self.assertIs(self.calculator.calculate(1, None), TrustTier.BRONZE)
+
+ def test_missing_confidence_does_not_earn_bronze(self):
+ # No corroboration and no confidence is quarantine, not bronze.
+ self.assertIs(self.calculator.calculate(0, None), TrustTier.QUARANTINE)
+
+ def test_missing_confidence_never_yields_gold(self):
+ for count in (2, 5, 50):
+ self.assertIsNot(self.calculator.calculate(count, None), TrustTier.GOLD)
+
+ def test_degradation_can_be_disabled(self):
+ calculator = TierCalculator(missing_confidence_degrades=False)
+ self.assertIs(calculator.calculate(2, None), TrustTier.GOLD)
+ self.assertIs(calculator.calculate(0, None), TrustTier.QUARANTINE)
+
+
+class TestInputHandling(unittest.TestCase):
+ """Messy inputs must not raise or inflate a tier."""
+
+ def setUp(self):
+ self.calculator = TierCalculator()
+
+ def test_none_count_is_treated_as_zero(self):
+ self.assertIs(self.calculator.calculate(None, 0.9), TrustTier.BRONZE)
+
+ def test_negative_count_is_treated_as_zero(self):
+ self.assertIs(self.calculator.calculate(-4, 0.9), TrustTier.BRONZE)
+
+ def test_non_numeric_count_is_treated_as_zero(self):
+ self.assertIs(self.calculator.calculate("many", 0.9), TrustTier.BRONZE)
+
+ def test_numeric_string_count_is_accepted(self):
+ self.assertIs(self.calculator.calculate("3", 0.9), TrustTier.GOLD)
+
+ def test_float_count_is_truncated(self):
+ self.assertIs(self.calculator.calculate(2.9, 0.9), TrustTier.GOLD)
+ self.assertIs(self.calculator.calculate(1.5, 0.9), TrustTier.SILVER)
+
+ def test_unusable_confidence_downgrades(self):
+ self.assertIs(self.calculator.calculate(2, "high"), TrustTier.SILVER)
+
+ def test_nan_confidence_downgrades(self):
+ tier = self.calculator.calculate(2, float("nan"))
+ self.assertIs(tier, TrustTier.SILVER)
+
+ def test_infinite_confidence_downgrades(self):
+ self.assertIs(self.calculator.calculate(2, float("inf")), TrustTier.SILVER)
+ self.assertIs(self.calculator.calculate(2, float("-inf")), TrustTier.SILVER)
+
+ def test_infinite_count_is_treated_as_zero(self):
+ self.assertIs(self.calculator.calculate(float("inf"), 0.9), TrustTier.BRONZE)
+ self.assertIs(self.calculator.calculate(float("-inf"), 0.9), TrustTier.BRONZE)
+
+ def test_nan_count_is_treated_as_zero(self):
+ self.assertIs(
+ self.calculator.calculate(float("nan"), 0.5), TrustTier.QUARANTINE
+ )
+
+ def test_confidence_above_one_is_still_usable(self):
+ self.assertIs(self.calculator.calculate(2, 1.4), TrustTier.GOLD)
+
+
+class TestTierCalculatorConfiguration(unittest.TestCase):
+ """Thresholds are configurable but invalid ones are rejected."""
+
+ def test_gold_threshold_must_be_positive(self):
+ with self.assertRaises(ValueError):
+ TierCalculator(min_corroboration_for_gold=0)
+
+ def test_min_confidence_must_be_a_probability(self):
+ with self.assertRaises(ValueError):
+ TierCalculator(min_confidence=1.5)
+ with self.assertRaises(ValueError):
+ TierCalculator(min_confidence=-0.1)
+
+ def test_lower_gold_threshold_promotes_single_source(self):
+ calculator = TierCalculator(min_corroboration_for_gold=1)
+ self.assertIs(calculator.calculate(1, 0.9), TrustTier.GOLD)
+
+ def test_lower_min_confidence_keeps_weaker_confidence(self):
+ calculator = TierCalculator(min_confidence=0.5)
+ self.assertIs(calculator.calculate(1, 0.6), TrustTier.SILVER)
+ self.assertIs(calculator.calculate(1, 0.4), TrustTier.BRONZE)
+
+
+class TestMetadataFriendly(unittest.TestCase):
+ """Tier values are ready to drop into RetrievedContext.metadata."""
+
+ def test_tier_value_is_json_serialisable(self):
+ tier = TierCalculator().calculate(3, 0.9)
+ payload = json.dumps({"trust_tier": tier.value})
+ self.assertEqual(payload, '{"trust_tier": "gold"}')
+
+
+class TestTreatAsMissing(unittest.TestCase):
+ """Placeholder confidence is handled as absent, not as a perfect score."""
+
+ def setUp(self):
+ self.calculator = TierCalculator()
+
+ def test_answer_placeholder_cannot_earn_gold(self):
+ for count in (2, 5, 50):
+ self.assertIsNot(self.calculator.calculate(count, 1.0), TrustTier.GOLD)
+
+ def test_answer_placeholder_downgrades_gold(self):
+ self.assertIs(self.calculator.calculate(2, 1.0), TrustTier.SILVER)
+
+ def test_integer_one_is_also_a_placeholder(self):
+ # The dataclass default can arrive as int 1 depending on the source.
+ self.assertIs(self.calculator.calculate(2, 1), TrustTier.SILVER)
+
+ def test_placeholder_without_corroboration_is_quarantine(self):
+ self.assertIs(self.calculator.calculate(0, 1.0), TrustTier.QUARANTINE)
+
+ def test_earned_confidence_is_not_treated_as_missing(self):
+ # 0.9 is the value the issue discusses; the repo default is 1.0, so
+ # 0.9 stays usable unless it is configured as a placeholder.
+ self.assertIs(self.calculator.calculate(2, 0.9), TrustTier.GOLD)
+
+ def test_default_placeholder_is_exposed(self):
+ self.assertEqual(self.calculator.treat_as_missing, frozenset({1.0}))
+
+ def test_substitution_can_be_disabled(self):
+ calculator = TierCalculator(treat_as_missing=())
+ self.assertIs(calculator.calculate(2, 1.0), TrustTier.GOLD)
+
+ def test_additional_placeholders_can_be_configured(self):
+ calculator = TierCalculator(treat_as_missing=[0.9, 1.0])
+ self.assertIs(calculator.calculate(2, 0.9), TrustTier.SILVER)
+ self.assertIs(calculator.calculate(2, 1.0), TrustTier.SILVER)
+ self.assertIs(calculator.calculate(2, 0.95), TrustTier.GOLD)
+
+ def test_degradation_flag_applies_to_placeholders(self):
+ calculator = TierCalculator(missing_confidence_degrades=False)
+ self.assertIs(calculator.calculate(2, 1.0), TrustTier.GOLD)
+ self.assertIs(calculator.calculate(0, 1.0), TrustTier.QUARANTINE)
+
+ def test_placeholder_outside_probability_range_is_rejected(self):
+ with self.assertRaises(ValueError):
+ TierCalculator(treat_as_missing=[1.5])
+
+ def test_non_numeric_placeholder_is_rejected(self):
+ with self.assertRaises(ValueError):
+ TierCalculator(treat_as_missing=["high"])
+
+ def test_non_iterable_placeholder_is_rejected(self):
+ with self.assertRaises(TypeError):
+ TierCalculator(treat_as_missing=1.0)
+
+
+if __name__ == "__main__":
+ unittest.main()