Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

feat: LLM-assisted conflict detection (Pass 3) with judge trait

riley0227cd5a2d3ead48
diff --git a/memory-layer/src/main/scala/ix/memory/context/ConflictDetector.scala b/memory-layer/src/main/scala/ix/memory/context/ConflictDetector.scala
index c3ac7816..e47a01c4 100644
--- a/memory-layer/src/main/scala/ix/memory/context/ConflictDetector.scala
+++ b/memory-layer/src/main/scala/ix/memory/context/ConflictDetector.scala
@@ -1,13 +1,16 @@
package ix.memory.context
import java.util.UUID
+
+import cats.effect.IO
+import cats.implicits._
import ix.memory.model._
trait ConflictDetector {
def detect(claims: Vector[ScoredClaim]): Vector[ConflictReport]
}
-class ConflictDetectorImpl extends ConflictDetector {
+class ConflictDetectorImpl(llmJudge: Option[LlmJudge] = None) extends ConflictDetector {
/**
* Multi-pass conflict detection.
@@ -47,6 +50,54 @@ class ConflictDetectorImpl extends ConflictDetector {
}.toVector
}
+ /**
+ * Pass 3: LLM-assisted conflict detection.
+ *
+ * Takes the conflicts found by Pass 1+2 and asks the LLM to evaluate
+ * whether each pair truly conflicts semantically. This can:
+ * - Upgrade the reason with LLM explanation
+ * - Dismiss false positives (remove conflict)
+ * - Provide a preferred claim recommendation
+ *
+ * Only runs if an LlmJudge is provided.
+ */
+ def refineWithLlm(
+ conflicts: Vector[ConflictReport],
+ claims: Vector[ScoredClaim]
+ ): IO[Vector[ConflictReport]] = {
+ llmJudge match {
+ case None => IO.pure(conflicts) // No LLM, return as-is
+ case Some(judge) =>
+ conflicts.traverse { conflict =>
+ val claimA = claims.find(_.claim.id == conflict.claimA).map(_.claim)
+ val claimB = claims.find(_.claim.id == conflict.claimB).map(_.claim)
+
+ (claimA, claimB) match {
+ case (Some(a), Some(b)) =>
+ judge.judge(a, b).map {
+ case Some(judgment) if judgment.isConflict =>
+ Some(conflict.copy(
+ reason = s"${conflict.reason} [LLM: ${judgment.explanation}]",
+ recommendation = judgment.preferredClaim
+ .flatMap(id => claims.find(_.claim.id == id))
+ .map(sc => s"LLM recommends '${sc.claim.statement}' (LLM confidence: ${judgment.confidence})")
+ .getOrElse(conflict.recommendation)
+ ))
+ case Some(_) =>
+ // LLM says not a conflict — dismiss
+ None
+ case None =>
+ // LLM didn't respond — keep original
+ Some(conflict)
+ }
+ case _ =>
+ // Claims not found — keep original conflict
+ IO.pure(Some(conflict))
+ }
+ }.map(_.flatten)
+ }
+ }
+
/** Check whether two statement strings share at least one 4-char stem. */
private def keywordsOverlap(a: String, b: String): Boolean = {
val stemsA = tokenize(a).map(_.take(4).toLowerCase).toSet
diff --git a/memory-layer/src/main/scala/ix/memory/context/ContextService.scala b/memory-layer/src/main/scala/ix/memory/context/ContextService.scala
index 7388b5e4..9fed36b5 100644
--- a/memory-layer/src/main/scala/ix/memory/context/ContextService.scala
+++ b/memory-layer/src/main/scala/ix/memory/context/ContextService.scala
@@ -48,8 +48,14 @@ class ContextService(
))
}
- // 6. Detect conflicts among scored claims
- conflicts = conflictDetector.detect(scored)
+ // 6. Detect conflicts among scored claims (Pass 1+2)
+ pass12Conflicts = conflictDetector.detect(scored)
+
+ // 6.5. Refine conflicts with LLM (Pass 3, if available)
+ conflicts <- conflictDetector match {
+ case impl: ConflictDetectorImpl => impl.refineWithLlm(pass12Conflicts, scored)
+ case _ => IO.pure(pass12Conflicts)
+ }
// 7. Relevance score: weight claims by hop distance from seeds
relevant = RelevanceScorer.score(scored, seeds.map(_.id).toSet, expanded.edges)
diff --git a/memory-layer/src/main/scala/ix/memory/context/LlmJudge.scala b/memory-layer/src/main/scala/ix/memory/context/LlmJudge.scala
new file mode 100644
index 00000000..9c6fede6
--- /dev/null
+++ b/memory-layer/src/main/scala/ix/memory/context/LlmJudge.scala
@@ -0,0 +1,36 @@
+package ix.memory.context
+
+import cats.effect.IO
+import ix.memory.model._
+
+/**
+ * Trait for LLM-based conflict judgment.
+ * Implementations should send the two claim statements to an LLM
+ * and return a judgment on whether they truly conflict.
+ */
+trait LlmJudge {
+ /**
+ * Ask the LLM whether two claims conflict.
+ * @return Some(judgment) if the LLM detects a semantic conflict, None if no conflict
+ */
+ def judge(claimA: Claim, claimB: Claim): IO[Option[LlmJudgment]]
+}
+
+/**
+ * The LLM's judgment about a pair of claims.
+ */
+case class LlmJudgment(
+ isConflict: Boolean,
+ confidence: Double, // 0.0 to 1.0 — how confident the LLM is in its judgment
+ explanation: String, // Why the LLM thinks they do or don't conflict
+ preferredClaim: Option[ClaimId] // Which claim the LLM prefers, if any
+)
+
+/**
+ * Stub implementation — always returns None (no LLM available).
+ * Replace with real implementation when LLM integration is ready.
+ */
+class StubLlmJudge extends LlmJudge {
+ def judge(claimA: Claim, claimB: Claim): IO[Option[LlmJudgment]] =
+ IO.pure(None)
+}
diff --git a/memory-layer/src/test/scala/ix/memory/context/LlmJudgeSpec.scala b/memory-layer/src/test/scala/ix/memory/context/LlmJudgeSpec.scala
new file mode 100644
index 00000000..045bb470
--- /dev/null
+++ b/memory-layer/src/test/scala/ix/memory/context/LlmJudgeSpec.scala
@@ -0,0 +1,102 @@
+package ix.memory.context
+
+import java.time.Instant
+import java.util.UUID
+
+import cats.effect.IO
+import cats.effect.testing.scalatest.AsyncIOSpec
+import org.scalatest.flatspec.AsyncFlatSpec
+import org.scalatest.matchers.should.Matchers
+
+import ix.memory.model._
+
+class LlmJudgeSpec extends AsyncFlatSpec with AsyncIOSpec with Matchers {
+
+ private def makeScoredClaim(statement: String, entityId: NodeId, score: Double = 0.8): ScoredClaim = {
+ val claim = Claim(
+ ClaimId(UUID.randomUUID()), entityId, statement,
+ ClaimStatus.Active,
+ Provenance("test://src", None, "test", SourceType.Code, Instant.now()),
+ Rev(1L), None
+ )
+ ScoredClaim(claim, ConfidenceBreakdown(
+ Factor(score, "ok"), Factor(1.0, "ok"), Factor(1.0, "ok"),
+ Factor(1.0, "ok"), Factor(1.0, "ok"), Factor(1.0, "ok")
+ ), relevance = 1.0, finalScore = score)
+ }
+
+ val entityId: NodeId = NodeId(UUID.randomUUID())
+
+ // Test 1: StubLlmJudge returns None
+ "StubLlmJudge" should "always return None" in {
+ val judge = new StubLlmJudge()
+ val c1 = makeScoredClaim("handles billing", entityId).claim
+ val c2 = makeScoredClaim("does not handle billing", entityId).claim
+ judge.judge(c1, c2).asserting(_ shouldBe None)
+ }
+
+ // Test 2: ConflictDetectorImpl without LLM — refineWithLlm returns conflicts unchanged
+ "ConflictDetectorImpl" should "return conflicts unchanged when no LLM is provided" in {
+ val detector = new ConflictDetectorImpl(llmJudge = None)
+ val sc1 = makeScoredClaim("handles billing", entityId)
+ val sc2 = makeScoredClaim("does not handle billing", entityId)
+ val conflicts = detector.detect(Vector(sc1, sc2))
+ conflicts should not be empty
+
+ detector.refineWithLlm(conflicts, Vector(sc1, sc2)).asserting { refined =>
+ refined shouldBe conflicts
+ }
+ }
+
+ // Test 3: ConflictDetectorImpl with LLM that confirms conflict
+ it should "upgrade conflict reason with LLM explanation" in {
+ val mockJudge = new LlmJudge {
+ def judge(a: Claim, b: Claim): IO[Option[LlmJudgment]] =
+ IO.pure(Some(LlmJudgment(
+ isConflict = true,
+ confidence = 0.95,
+ explanation = "These claims directly contradict each other",
+ preferredClaim = Some(a.id)
+ )))
+ }
+ val detector = new ConflictDetectorImpl(llmJudge = Some(mockJudge))
+ val sc1 = makeScoredClaim("handles billing", entityId)
+ val sc2 = makeScoredClaim("does not handle billing", entityId)
+ val conflicts = detector.detect(Vector(sc1, sc2))
+
+ detector.refineWithLlm(conflicts, Vector(sc1, sc2)).asserting { refined =>
+ refined should have length conflicts.length.toLong
+ refined.head.reason should include("[LLM:")
+ refined.head.recommendation should include("LLM recommends")
+ }
+ }
+
+ // Test 4: ConflictDetectorImpl with LLM that dismisses conflict
+ it should "dismiss conflicts when LLM says no conflict" in {
+ val mockJudge = new LlmJudge {
+ def judge(a: Claim, b: Claim): IO[Option[LlmJudgment]] =
+ IO.pure(Some(LlmJudgment(
+ isConflict = false,
+ confidence = 0.9,
+ explanation = "These are complementary, not contradictory",
+ preferredClaim = None
+ )))
+ }
+ val detector = new ConflictDetectorImpl(llmJudge = Some(mockJudge))
+ val sc1 = makeScoredClaim("handles billing", entityId)
+ val sc2 = makeScoredClaim("billing service handler", entityId)
+ val conflicts = detector.detect(Vector(sc1, sc2))
+
+ detector.refineWithLlm(conflicts, Vector(sc1, sc2)).asserting { refined =>
+ refined shouldBe empty
+ }
+ }
+
+ // Test 5: Empty conflicts — nothing to refine
+ it should "return empty when no conflicts to refine" in {
+ val detector = new ConflictDetectorImpl(llmJudge = Some(new StubLlmJudge()))
+ detector.refineWithLlm(Vector.empty, Vector.empty).asserting { refined =>
+ refined shouldBe empty
+ }
+ }
+}