Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

[RSI, other] Rank digest and harness search by IDF-weighted terms (#2392)

Seth Karten7f39eb825cca
diff --git a/packages/coding-agent/.changes/cb-fix-idf-rank.md b/packages/coding-agent/.changes/cb-fix-idf-rank.md
new file mode 100644
index 0000000000..2b73eab105
--- /dev/null
+++ b/packages/coding-agent/.changes/cb-fix-idf-rank.md
@@ -0,0 +1 @@
+- Harness digest ranking and harness search now weigh matched query terms by document frequency (tf-idf style), so rare distinctive terms rank above entries dense in ubiquitous words.
diff --git a/packages/coding-agent/src/core/refinement/refinement.ts b/packages/coding-agent/src/core/refinement/refinement.ts
index 7060c1606e..84557210f9 100644
--- a/packages/coding-agent/src/core/refinement/refinement.ts
+++ b/packages/coding-agent/src/core/refinement/refinement.ts
@@ -468,8 +468,9 @@ export function formatRefinementNoticeBody(result: RefinementResult): string {
/**
* Query terms for relevance-ranked harness digests: term -> weight.
* Built by the caller from task signal (goal objective, recent
- * messages). The ranking is a pure weighted-term overlap over the
- * entry's searchable fields.
+ * messages). The ranking is weighted term overlap over the entry's
+ * searchable fields, discounted per term by document frequency in the
+ * ranked corpus, so rare distinctive terms outweigh ubiquitous ones.
*/
export type HarnessQueryTerms = Map<string, number>;
@@ -518,8 +519,45 @@ export function harnessQueryTerms(text: string): string[] {
return [...new Set(terms)];
}
-/** Score one harness entry against query terms: weighted term overlap. */
-export function scoreHarnessEntryForQuery(entry: HarnessEntry, terms: HarnessQueryTerms): number {
+/**
+ * Inverse document frequency per query term over the entries being
+ * ranked: `log(1 + documents / matches)`. A term present in every entry
+ * still weighs `log(2)`, while a term in one entry of N weighs
+ * `log(1 + N)`, so rare distinctive terms outrank ubiquitous ones.
+ * Terms matching no entry are absent (they cannot score anything).
+ */
+export function harnessQueryTermIdf(entries: HarnessEntry[], terms: HarnessQueryTerms): Map<string, number> {
+ const idf = new Map<string, number>();
+ if (terms.size === 0) return idf;
+ let documents = 0;
+ const matches = new Map<string, number>();
+ for (const entry of entries) {
+ documents += 1;
+ const title = searchableField(entry.title);
+ const content = searchableField(entry.content);
+ const identifier = `${searchableField(entry.path)} ${searchableField(entry.id)}`;
+ for (const term of terms.keys()) {
+ if (title.includes(term) || content.includes(term) || identifier.includes(term)) {
+ matches.set(term, (matches.get(term) ?? 0) + 1);
+ }
+ }
+ }
+ for (const [term, documentFrequency] of matches) {
+ idf.set(term, Math.log(1 + documents / documentFrequency));
+ }
+ return idf;
+}
+
+/**
+ * Score one harness entry against query terms: weighted term overlap,
+ * with each matched term's weight discounted by its document frequency
+ * in the ranked corpus (`idf`; a missing map weights every term at 1).
+ */
+export function scoreHarnessEntryForQuery(
+ entry: HarnessEntry,
+ terms: HarnessQueryTerms,
+ idf?: Map<string, number>,
+): number {
if (terms.size === 0) return 0;
const title = searchableField(entry.title);
const content = searchableField(entry.content);
@@ -534,13 +572,20 @@ export function scoreHarnessEntryForQuery(entry: HarnessEntry, terms: HarnessQue
if (title.includes(term)) fields += 1;
if (content.includes(term)) fields += 1;
if (identifier.includes(term)) fields += 1;
- if (fields > 0) score += weight * (1 + (fields - 1) * 0.5);
+ if (fields > 0) {
+ score += weight * (idf?.get(term) ?? 1) * (1 + (fields - 1) * 0.5);
+ }
}
return score;
}
-function compareRankedHarnessEntries(a: HarnessEntry, b: HarnessEntry, terms: HarnessQueryTerms): number {
- const scoreDifference = scoreHarnessEntryForQuery(b, terms) - scoreHarnessEntryForQuery(a, terms);
+function compareRankedHarnessEntries(
+ a: HarnessEntry,
+ b: HarnessEntry,
+ terms: HarnessQueryTerms,
+ idf?: Map<string, number>,
+): number {
+ const scoreDifference = scoreHarnessEntryForQuery(b, terms, idf) - scoreHarnessEntryForQuery(a, terms, idf);
if (scoreDifference !== 0) return scoreDifference;
// Recency breaks ties; alphabetical order keeps selection deterministic.
const recencyDifference = (Date.parse(b.updated_at) || 0) - (Date.parse(a.updated_at) || 0);
@@ -590,9 +635,14 @@ export function formatHarnessStateForPrompt(
const queryTerms = options.queryTerms;
let totalEntries = 0;
for (const kind of Object.keys(state.entries) as RefinementKind[]) {
- const entries = Object.values(state.entries[kind]).sort((a, b) =>
+ // The ranked corpus is the kind's own entries: they compete for the
+ // same top-k slots, so document frequency discounts terms ubiquitous
+ // within the kind rather than across unrelated kinds.
+ const ranked = Object.values(state.entries[kind]);
+ const idf = queryTerms !== undefined && queryTerms.size > 0 ? harnessQueryTermIdf(ranked, queryTerms) : undefined;
+ const entries = ranked.sort((a, b) =>
queryTerms !== undefined && queryTerms.size > 0
- ? compareRankedHarnessEntries(a, b, queryTerms)
+ ? compareRankedHarnessEntries(a, b, queryTerms, idf)
: [a.path, a.title, a.id].join("\0").localeCompare([b.path, b.title, b.id].join("\0")),
);
totalEntries += entries.length;
diff --git a/packages/coding-agent/test/refinement.test.ts b/packages/coding-agent/test/refinement.test.ts
index e8f7a4bad9..61714f05fc 100644
--- a/packages/coding-agent/test/refinement.test.ts
+++ b/packages/coding-agent/test/refinement.test.ts
@@ -16,6 +16,7 @@ import {
getRefinementHistoryPath,
type HarnessEntry,
type HarnessState,
+ harnessQueryTermIdf,
harnessQueryTerms,
inferRefinementResultScope,
loadGlobalRefinementHistory,
@@ -900,6 +901,28 @@ describe("harness digest relevance ranking", () => {
expect(scoreHarnessEntryForQuery(entry, new Map())).toBe(0);
});
+ it("computes document-frequency discounts over the ranked corpus", () => {
+ const at = "2026-08-01T00:00:00.000Z";
+ const terms = new Map([
+ ["session", 1],
+ ["quantum", 1],
+ ["missing", 1],
+ ]);
+ const common = ["common0", "common1"].map((id) => makeEntry(id, "Session notes", "session text", at));
+ const rare = makeEntry("rare", "Quantum note", "quantum text", at);
+ // "session" matches 2 of 3 entries, "quantum" 1 of 3, "missing" none.
+ const idf = harnessQueryTermIdf([...common, rare], terms);
+ expect([...idf.keys()]).toEqual(["session", "quantum"]);
+ expect(idf.get("session")).toBeCloseTo(Math.log(1 + 3 / 2));
+ expect(idf.get("quantum")).toBeCloseTo(Math.log(1 + 3 / 1));
+ // The discount scales the weighted overlap: "quantum" covers 2 fields of 1 entry.
+ expect(scoreHarnessEntryForQuery(rare, terms, idf)).toBeCloseTo(Math.log(1 + 3 / 1) * 1.5);
+ // A term in every entry still weighs log(2); degenerate corpora stay inert.
+ expect(harnessQueryTermIdf([rare], terms).get("quantum")).toBeCloseTo(Math.log(2));
+ expect(harnessQueryTermIdf([], terms).size).toBe(0);
+ expect(harnessQueryTermIdf([...common, rare], new Map()).size).toBe(0);
+ });
+
it.each<{ label: string; query: string; winner: string; loser: string; entries: Record<string, HarnessEntry> }>([
{
label: "relevance over alphabetical order",
@@ -912,13 +935,25 @@ describe("harness digest relevance ranking", () => {
},
},
{
+ // Equal discounts keep the recency tie-break: alphabetical order would pick aa_older.
label: "recency as the tie-break for equal scores",
query: "worktree",
- winner: "newer",
- loser: "older",
+ winner: "zz_newer",
+ loser: "aa_older",
+ entries: {
+ aa_older: makeEntry("aa_older", "Worktree policy", "Same worktree signal.", "2026-08-01T00:00:00.000Z"),
+ zz_newer: makeEntry("zz_newer", "Worktree policy", "Same worktree signal.", "2026-09-01T00:00:00.000Z"),
+ },
+ },
+ {
+ label: "a rare distinctive term over a common-term-dense entry",
+ query: "session quantum",
+ winner: "rare",
+ loser: "common1",
entries: {
- older: makeEntry("older", "Worktree policy", "Same worktree signal.", "2026-08-01T00:00:00.000Z"),
- newer: makeEntry("newer", "Worktree policy 2", "Same worktree signal.", "2026-09-01T00:00:00.000Z"),
+ common0: makeEntry("common0", "Session notes", "Session state notes.", "2026-08-01T00:00:00.000Z"),
+ common1: makeEntry("common1", "Session notes", "Session state notes.", "2026-08-02T00:00:00.000Z"),
+ rare: makeEntry("rare", "Quantum note", "Only quantum annealing matters.", "2026-07-01T00:00:00.000Z"),
},
},
{
@@ -931,16 +966,6 @@ describe("harness digest relevance ranking", () => {
tea: makeEntry("tea", "Tea notes", "All about oolong brewing.", "2026-08-02T00:00:00.000Z"),
},
},
- {
- label: "real terms rather than incidental query punctuation",
- query: "worktree?",
- winner: "worktree",
- loser: "question",
- entries: {
- worktree: makeEntry("worktree", "Branch hygiene", "Use git worktrees.", "2026-08-01T00:00:00.000Z"),
- question: makeEntry("question", "Question", "Anything else left open?", "2026-08-02T00:00:00.000Z"),
- },
- },
])("ranks by $label", ({ query, winner, loser, entries }) => {
const ranked = rank(entries, query);
diff --git a/prime-agent-runtime/src/rlm/harness.py b/prime-agent-runtime/src/rlm/harness.py
index cba1c7ed0c..5040a2fcff 100644
--- a/prime-agent-runtime/src/rlm/harness.py
+++ b/prime-agent-runtime/src/rlm/harness.py
@@ -10,6 +10,7 @@ Execution still belongs to Prime Agent's TypeScript host and the existing
from __future__ import annotations
import json
+import math
import os
import re
import stat
@@ -870,7 +871,10 @@ class HarnessState:
"""Return harness entries ranked by weighted term overlap with *query*.
Terms are scored against an entry's title, content, path, and id;
- matches in more distinct fields count more.
+ matches in more distinct fields count more. Each matched term is
+ discounted by its document frequency across the ranked corpus
+ (tf-idf style, ``weight * log(1 + N / df)``), so a rare,
+ distinctive term outranks terms present in most entries.
"""
if target := self._global_target(global_, kwargs):
return target.search(query, kind=kind, limit=limit)
@@ -883,21 +887,38 @@ class HarnessState:
if not terms:
return []
+ entries = self.list(kind, **kwargs) if kind is not None else self.list(None, **kwargs)
+
+ # Document frequency per term over the ranked corpus: a term in
+ # every entry weighs log(2), a term in one entry of N weighs
+ # log(1 + N), so rare distinctive terms outrank ubiquitous ones.
+ matches: dict[str, int] = {term: 0 for term in terms}
+ for entry in entries:
+ title = entry.title.lower()
+ content = entry.content.lower()
+ path_and_id = f"{entry.path} {entry.id}".lower()
+ for term in terms:
+ if term in title or term in content or term in path_and_id:
+ matches[term] += 1
+ term_idf = {
+ term: math.log(1 + len(entries) / count)
+ for term, count in matches.items()
+ if count > 0
+ }
+
def score(entry: HarnessEntry) -> float:
title = entry.title.lower()
content = entry.content.lower()
path_and_id = f"{entry.path} {entry.id}".lower()
total = 0.0
- for term in terms:
+ for term, idf in term_idf.items():
fields = (1 if term in title else 0) + (1 if term in content else 0) + (
1 if term in path_and_id else 0
)
if fields:
- total += 1 + (fields - 1) * 0.5
+ total += idf * (1 + (fields - 1) * 0.5)
return total
- entries = self.list(kind, **kwargs) if kind is not None else self.list(None, **kwargs)
-
def recency(entry: HarnessEntry) -> str:
return entry.updated_at if isinstance(entry.updated_at, str) else ""
diff --git a/prime-agent-runtime/test/test_harness.py b/prime-agent-runtime/test/test_harness.py
index 6e2d6e70ec..fcd3be2f07 100644
--- a/prime-agent-runtime/test/test_harness.py
+++ b/prime-agent-runtime/test/test_harness.py
@@ -1135,3 +1135,25 @@ class HarnessSearchTest(unittest.TestCase):
with self.assertRaises(TypeError):
state.search("worktree", limit=0)
+ def test_search_discounts_common_terms_and_keeps_frequency_ties(self) -> None:
+ with tempfile.TemporaryDirectory() as temp_dir:
+ state = HarnessState(Path(temp_dir) / "harness_state.json")
+
+ # An empty corpus scores nothing, and a lone entry (N=1, df=1 -> log(2)) still scores.
+ self.assertEqual(state.search("session"), [])
+ state.create_memory("Session notes", "Session signal.", id="solo")
+ self.assertEqual([entry.id for entry in state.search("session")], ["solo"])
+ state.entries["memory"]["solo"].updated_at = "2026-07-01T00:00:00+00:00"
+
+ # Equal frequency discounts every match alike, so recency still orders them: id order
+ # alone would put aa_older first.
+ for entry_id, day in (("aa_older", "08-01"), ("zz_newer", "09-01")):
+ state.create_memory("Session notes", "Same session signal.", id=entry_id)
+ state.entries["memory"][entry_id].updated_at = f"2026-{day}T00:00:00+00:00"
+ self.assertEqual([entry.id for entry in state.search("session")], ["zz_newer", "aa_older", "solo"])
+
+ # "session" matches 3 of 4 (log(1 + 4/3)), "quantum" 1 of 4 (log(1 + 4)): rare ranks first.
+ state.create_memory("Quantum note", "Only quantum annealing matters once.", id="rare")
+ state.entries["memory"]["rare"].updated_at = "2026-07-01T00:00:00+00:00"
+ self.assertEqual([entry.id for entry in state.search("session quantum")], ["rare", "zz_newer", "aa_older", "solo"])
+