A self-improving RLM agent for coding workflows and long-running autonomous tasks.
Rust
[RSI, other] Rank digest and harness search by IDF-weighted terms (#2392)
Seth Karten7f39eb825cca
diff --git a/packages/coding-agent/.changes/cb-fix-idf-rank.md b/packages/coding-agent/.changes/cb-fix-idf-rank.mdnew file mode 100644index 0000000000..2b73eab105--- /dev/null+++ b/packages/coding-agent/.changes/cb-fix-idf-rank.md@@ -0,0 +1 @@+- Harness digest ranking and harness search now weigh matched query terms by document frequency (tf-idf style), so rare distinctive terms rank above entries dense in ubiquitous words.diff --git a/packages/coding-agent/src/core/refinement/refinement.ts b/packages/coding-agent/src/core/refinement/refinement.tsindex 7060c1606e..84557210f9 100644--- a/packages/coding-agent/src/core/refinement/refinement.ts+++ b/packages/coding-agent/src/core/refinement/refinement.ts@@ -468,8 +468,9 @@ export function formatRefinementNoticeBody(result: RefinementResult): string {/*** Query terms for relevance-ranked harness digests: term -> weight.* Built by the caller from task signal (goal objective, recent- * messages). The ranking is a pure weighted-term overlap over the- * entry's searchable fields.+ * messages). The ranking is weighted term overlap over the entry's+ * searchable fields, discounted per term by document frequency in the+ * ranked corpus, so rare distinctive terms outweigh ubiquitous ones.*/export type HarnessQueryTerms = Map<string, number>;@@ -518,8 +519,45 @@ export function harnessQueryTerms(text: string): string[] {return [...new Set(terms)];}-/** Score one harness entry against query terms: weighted term overlap. */-export function scoreHarnessEntryForQuery(entry: HarnessEntry, terms: HarnessQueryTerms): number {+/**+ * Inverse document frequency per query term over the entries being+ * ranked: `log(1 + documents / matches)`. A term present in every entry+ * still weighs `log(2)`, while a term in one entry of N weighs+ * `log(1 + N)`, so rare distinctive terms outrank ubiquitous ones.+ * Terms matching no entry are absent (they cannot score anything).+ */+export function harnessQueryTermIdf(entries: HarnessEntry[], terms: HarnessQueryTerms): Map<string, number> {+ const idf = new Map<string, number>();+ if (terms.size === 0) return idf;+ let documents = 0;+ const matches = new Map<string, number>();+ for (const entry of entries) {+ documents += 1;+ const title = searchableField(entry.title);+ const content = searchableField(entry.content);+ const identifier = `${searchableField(entry.path)} ${searchableField(entry.id)}`;+ for (const term of terms.keys()) {+ if (title.includes(term) || content.includes(term) || identifier.includes(term)) {+ matches.set(term, (matches.get(term) ?? 0) + 1);+ }+ }+ }+ for (const [term, documentFrequency] of matches) {+ idf.set(term, Math.log(1 + documents / documentFrequency));+ }+ return idf;+}++/**+ * Score one harness entry against query terms: weighted term overlap,+ * with each matched term's weight discounted by its document frequency+ * in the ranked corpus (`idf`; a missing map weights every term at 1).+ */+export function scoreHarnessEntryForQuery(+ entry: HarnessEntry,+ terms: HarnessQueryTerms,+ idf?: Map<string, number>,+): number {if (terms.size === 0) return 0;const title = searchableField(entry.title);const content = searchableField(entry.content);@@ -534,13 +572,20 @@ export function scoreHarnessEntryForQuery(entry: HarnessEntry, terms: HarnessQueif (title.includes(term)) fields += 1;if (content.includes(term)) fields += 1;if (identifier.includes(term)) fields += 1;- if (fields > 0) score += weight * (1 + (fields - 1) * 0.5);+ if (fields > 0) {+ score += weight * (idf?.get(term) ?? 1) * (1 + (fields - 1) * 0.5);+ }}return score;}-function compareRankedHarnessEntries(a: HarnessEntry, b: HarnessEntry, terms: HarnessQueryTerms): number {- const scoreDifference = scoreHarnessEntryForQuery(b, terms) - scoreHarnessEntryForQuery(a, terms);+function compareRankedHarnessEntries(+ a: HarnessEntry,+ b: HarnessEntry,+ terms: HarnessQueryTerms,+ idf?: Map<string, number>,+): number {+ const scoreDifference = scoreHarnessEntryForQuery(b, terms, idf) - scoreHarnessEntryForQuery(a, terms, idf);if (scoreDifference !== 0) return scoreDifference;// Recency breaks ties; alphabetical order keeps selection deterministic.const recencyDifference = (Date.parse(b.updated_at) || 0) - (Date.parse(a.updated_at) || 0);@@ -590,9 +635,14 @@ export function formatHarnessStateForPrompt(const queryTerms = options.queryTerms;let totalEntries = 0;for (const kind of Object.keys(state.entries) as RefinementKind[]) {- const entries = Object.values(state.entries[kind]).sort((a, b) =>+ // The ranked corpus is the kind's own entries: they compete for the+ // same top-k slots, so document frequency discounts terms ubiquitous+ // within the kind rather than across unrelated kinds.+ const ranked = Object.values(state.entries[kind]);+ const idf = queryTerms !== undefined && queryTerms.size > 0 ? harnessQueryTermIdf(ranked, queryTerms) : undefined;+ const entries = ranked.sort((a, b) =>queryTerms !== undefined && queryTerms.size > 0- ? compareRankedHarnessEntries(a, b, queryTerms)+ ? compareRankedHarnessEntries(a, b, queryTerms, idf): [a.path, a.title, a.id].join("\0").localeCompare([b.path, b.title, b.id].join("\0")),);totalEntries += entries.length;diff --git a/packages/coding-agent/test/refinement.test.ts b/packages/coding-agent/test/refinement.test.tsindex e8f7a4bad9..61714f05fc 100644--- a/packages/coding-agent/test/refinement.test.ts+++ b/packages/coding-agent/test/refinement.test.ts@@ -16,6 +16,7 @@ import {getRefinementHistoryPath,type HarnessEntry,type HarnessState,+ harnessQueryTermIdf,harnessQueryTerms,inferRefinementResultScope,loadGlobalRefinementHistory,@@ -900,6 +901,28 @@ describe("harness digest relevance ranking", () => {expect(scoreHarnessEntryForQuery(entry, new Map())).toBe(0);});+ it("computes document-frequency discounts over the ranked corpus", () => {+ const at = "2026-08-01T00:00:00.000Z";+ const terms = new Map([+ ["session", 1],+ ["quantum", 1],+ ["missing", 1],+ ]);+ const common = ["common0", "common1"].map((id) => makeEntry(id, "Session notes", "session text", at));+ const rare = makeEntry("rare", "Quantum note", "quantum text", at);+ // "session" matches 2 of 3 entries, "quantum" 1 of 3, "missing" none.+ const idf = harnessQueryTermIdf([...common, rare], terms);+ expect([...idf.keys()]).toEqual(["session", "quantum"]);+ expect(idf.get("session")).toBeCloseTo(Math.log(1 + 3 / 2));+ expect(idf.get("quantum")).toBeCloseTo(Math.log(1 + 3 / 1));+ // The discount scales the weighted overlap: "quantum" covers 2 fields of 1 entry.+ expect(scoreHarnessEntryForQuery(rare, terms, idf)).toBeCloseTo(Math.log(1 + 3 / 1) * 1.5);+ // A term in every entry still weighs log(2); degenerate corpora stay inert.+ expect(harnessQueryTermIdf([rare], terms).get("quantum")).toBeCloseTo(Math.log(2));+ expect(harnessQueryTermIdf([], terms).size).toBe(0);+ expect(harnessQueryTermIdf([...common, rare], new Map()).size).toBe(0);+ });+it.each<{ label: string; query: string; winner: string; loser: string; entries: Record<string, HarnessEntry> }>([{label: "relevance over alphabetical order",@@ -912,13 +935,25 @@ describe("harness digest relevance ranking", () => {},},{+ // Equal discounts keep the recency tie-break: alphabetical order would pick aa_older.label: "recency as the tie-break for equal scores",query: "worktree",- winner: "newer",- loser: "older",+ winner: "zz_newer",+ loser: "aa_older",+ entries: {+ aa_older: makeEntry("aa_older", "Worktree policy", "Same worktree signal.", "2026-08-01T00:00:00.000Z"),+ zz_newer: makeEntry("zz_newer", "Worktree policy", "Same worktree signal.", "2026-09-01T00:00:00.000Z"),+ },+ },+ {+ label: "a rare distinctive term over a common-term-dense entry",+ query: "session quantum",+ winner: "rare",+ loser: "common1",entries: {- older: makeEntry("older", "Worktree policy", "Same worktree signal.", "2026-08-01T00:00:00.000Z"),- newer: makeEntry("newer", "Worktree policy 2", "Same worktree signal.", "2026-09-01T00:00:00.000Z"),+ common0: makeEntry("common0", "Session notes", "Session state notes.", "2026-08-01T00:00:00.000Z"),+ common1: makeEntry("common1", "Session notes", "Session state notes.", "2026-08-02T00:00:00.000Z"),+ rare: makeEntry("rare", "Quantum note", "Only quantum annealing matters.", "2026-07-01T00:00:00.000Z"),},},{@@ -931,16 +966,6 @@ describe("harness digest relevance ranking", () => {tea: makeEntry("tea", "Tea notes", "All about oolong brewing.", "2026-08-02T00:00:00.000Z"),},},- {- label: "real terms rather than incidental query punctuation",- query: "worktree?",- winner: "worktree",- loser: "question",- entries: {- worktree: makeEntry("worktree", "Branch hygiene", "Use git worktrees.", "2026-08-01T00:00:00.000Z"),- question: makeEntry("question", "Question", "Anything else left open?", "2026-08-02T00:00:00.000Z"),- },- },])("ranks by $label", ({ query, winner, loser, entries }) => {const ranked = rank(entries, query);diff --git a/prime-agent-runtime/src/rlm/harness.py b/prime-agent-runtime/src/rlm/harness.pyindex cba1c7ed0c..5040a2fcff 100644--- a/prime-agent-runtime/src/rlm/harness.py+++ b/prime-agent-runtime/src/rlm/harness.py@@ -10,6 +10,7 @@ Execution still belongs to Prime Agent's TypeScript host and the existingfrom __future__ import annotationsimport json+import mathimport osimport reimport stat@@ -870,7 +871,10 @@ class HarnessState:"""Return harness entries ranked by weighted term overlap with *query*.Terms are scored against an entry's title, content, path, and id;- matches in more distinct fields count more.+ matches in more distinct fields count more. Each matched term is+ discounted by its document frequency across the ranked corpus+ (tf-idf style, ``weight * log(1 + N / df)``), so a rare,+ distinctive term outranks terms present in most entries."""if target := self._global_target(global_, kwargs):return target.search(query, kind=kind, limit=limit)@@ -883,21 +887,38 @@ class HarnessState:if not terms:return []+ entries = self.list(kind, **kwargs) if kind is not None else self.list(None, **kwargs)++ # Document frequency per term over the ranked corpus: a term in+ # every entry weighs log(2), a term in one entry of N weighs+ # log(1 + N), so rare distinctive terms outrank ubiquitous ones.+ matches: dict[str, int] = {term: 0 for term in terms}+ for entry in entries:+ title = entry.title.lower()+ content = entry.content.lower()+ path_and_id = f"{entry.path} {entry.id}".lower()+ for term in terms:+ if term in title or term in content or term in path_and_id:+ matches[term] += 1+ term_idf = {+ term: math.log(1 + len(entries) / count)+ for term, count in matches.items()+ if count > 0+ }+def score(entry: HarnessEntry) -> float:title = entry.title.lower()content = entry.content.lower()path_and_id = f"{entry.path} {entry.id}".lower()total = 0.0- for term in terms:+ for term, idf in term_idf.items():fields = (1 if term in title else 0) + (1 if term in content else 0) + (1 if term in path_and_id else 0)if fields:- total += 1 + (fields - 1) * 0.5+ total += idf * (1 + (fields - 1) * 0.5)return total- entries = self.list(kind, **kwargs) if kind is not None else self.list(None, **kwargs)-def recency(entry: HarnessEntry) -> str:return entry.updated_at if isinstance(entry.updated_at, str) else ""diff --git a/prime-agent-runtime/test/test_harness.py b/prime-agent-runtime/test/test_harness.pyindex 6e2d6e70ec..fcd3be2f07 100644--- a/prime-agent-runtime/test/test_harness.py+++ b/prime-agent-runtime/test/test_harness.py@@ -1135,3 +1135,25 @@ class HarnessSearchTest(unittest.TestCase):with self.assertRaises(TypeError):state.search("worktree", limit=0)+ def test_search_discounts_common_terms_and_keeps_frequency_ties(self) -> None:+ with tempfile.TemporaryDirectory() as temp_dir:+ state = HarnessState(Path(temp_dir) / "harness_state.json")++ # An empty corpus scores nothing, and a lone entry (N=1, df=1 -> log(2)) still scores.+ self.assertEqual(state.search("session"), [])+ state.create_memory("Session notes", "Session signal.", id="solo")+ self.assertEqual([entry.id for entry in state.search("session")], ["solo"])+ state.entries["memory"]["solo"].updated_at = "2026-07-01T00:00:00+00:00"++ # Equal frequency discounts every match alike, so recency still orders them: id order+ # alone would put aa_older first.+ for entry_id, day in (("aa_older", "08-01"), ("zz_newer", "09-01")):+ state.create_memory("Session notes", "Same session signal.", id=entry_id)+ state.entries["memory"][entry_id].updated_at = f"2026-{day}T00:00:00+00:00"+ self.assertEqual([entry.id for entry in state.search("session")], ["zz_newer", "aa_older", "solo"])++ # "session" matches 3 of 4 (log(1 + 4/3)), "quantum" 1 of 4 (log(1 + 4)): rare ranks first.+ state.create_memory("Quantum note", "Only quantum annealing matters once.", id="rare")+ state.entries["memory"]["rare"].updated_at = "2026-07-01T00:00:00+00:00"+ self.assertEqual([entry.id for entry in state.search("session quantum")], ["rare", "zz_newer", "aa_older", "solo"])+