The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development for Claude Code, Codex, Opencode, Cursor and beyond.
feat(session-start): rank injected instincts by project/stack relevance (#2466)
Gaurav Dubey5987bd4dc6d6
diff --git a/README.md b/README.mdindex 03d336224..442e220f8 100644--- a/README.md+++ b/README.md@@ -1434,6 +1434,13 @@ export ECC_MAX_INJECTED_INSTINCTS=6# Minimum confidence an instinct needs to be injected, 0-1 (default: 0.7)export ECC_INSTINCT_CONFIDENCE_THRESHOLD=0.7+# SessionStart ranks injected instincts by confidence + project/stack relevance+# (default: on). Project-scoped instincts, and instincts whose domain/trigger+# matches the detected stack (languages, frameworks, plus terraform/dbt markers),+# get a small ranking boost so they surface above unrelated higher-confidence+# ones. Set to off/false/0/no to rank by confidence alone.+export ECC_INSTINCT_RELEVANCE_RANKING=on+# Keep context/scope/loop warnings but suppress API-rate cost estimatesexport ECC_CONTEXT_MONITOR_COST_WARNINGS=off```diff --git a/scripts/hooks/session-start.js b/scripts/hooks/session-start.jsindex 4cfc443ec..63854aff1 100644--- a/scripts/hooks/session-start.js+++ b/scripts/hooks/session-start.js@@ -24,6 +24,11 @@ const { resolveProjectContext, writeSessionLease, resolveSessionId, getHomunculuconst { getPackageManager, getSelectionPrompt } = require('../lib/package-manager');const { listAliases } = require('../lib/session-aliases');const { detectProjectType } = require('../lib/project-detect');+const {+ isRelevanceRankingEnabled,+ detectStackKeywords,+ computeRelevanceBoost,+} = require('../lib/instinct-relevance');const path = require('path');const fs = require('fs');@@ -422,6 +427,20 @@ function summarizeActiveInstincts(observerContext) {const confidenceThreshold = getInstinctConfidenceThreshold();const maxInjected = getMaxInjectedInstincts();+ // Relevance ranking (issue #2371 part b): at SessionStart there is no user+ // task yet, so relevance is location/stack based. Project-scoped and+ // stack-matching instincts get a small additive boost over their confidence.+ // Gated by ECC_INSTINCT_RELEVANCE_RANKING (default on); when off, or when no+ // stack is detected and nothing is project-scoped, every boost is 0 and the+ // ranking collapses to confidence-only (unchanged behaviour).+ // Detect the stack from the real project source tree (projectRoot), not the+ // homunculus state dir (projectDir). In a global session projectRoot is empty,+ // so detectStackKeywords falls back to process.cwd().+ const relevanceEnabled = isRelevanceRankingEnabled();+ const stackKeywords = relevanceEnabled+ ? detectStackKeywords(observerContext.projectRoot || undefined)+ : new Set();+const deduped = new Map();for (const instinct of scopedInstincts) {if (!instinct.id || instinct.confidence < confidenceThreshold) continue;@@ -435,10 +454,17 @@ function summarizeActiveInstincts(observerContext) {.map(instinct => ({...instinct,action: extractInstinctAction(instinct.content),+ _relevance: relevanceEnabled ? computeRelevanceBoost(instinct, stackKeywords) : 0,})).filter(instinct => instinct.action).sort((left, right) => {- if (right.confidence !== left.confidence) return right.confidence - left.confidence;+ // Primary: combined confidence + relevance. When relevance is off every+ // _relevance is 0, so this reduces to the prior confidence-only ordering.+ // Tie-breaks on a genuinely equal combined score: project scope first,+ // then id (deterministic).+ const leftScore = left.confidence + left._relevance;+ const rightScore = right.confidence + right._relevance;+ if (rightScore !== leftScore) return rightScore - leftScore;if (left._scopeLabel !== right._scopeLabel) return left._scopeLabel === 'project' ? -1 : 1;return String(left.id).localeCompare(String(right.id));})diff --git a/scripts/lib/instinct-relevance.js b/scripts/lib/instinct-relevance.jsnew file mode 100644index 000000000..81dde7e79--- /dev/null+++ b/scripts/lib/instinct-relevance.js@@ -0,0 +1,173 @@+/**+ * Instinct relevance ranking for SessionStart.+ *+ * At SessionStart there is no user task yet, so "relevance" is location/stack+ * relevance: instincts scoped to the current project, or whose domain/trigger+ * matches the detected stack, get a small additive boost on top of their+ * confidence when ranking which instincts to inject. The confidence >=+ * threshold floor and the injection cap are enforced by the caller; this+ * module only computes the additive boost and the stack keyword set. When+ * nothing is project-scoped and no stack is detected, every boost is 0 and the+ * ranking degrades to confidence-only (unchanged behaviour).+ *+ * Resolves part (b) of:+ * https://github.com/affaan-m/everything-claude-code/issues/2371+ */++const fs = require('fs');+const path = require('path');+const { detectProjectType } = require('./project-detect');++// Additive ranking boosts. These are intentionally NOT env-configurable: part+// (b) of the issue asks for relevance ranking, not more tunable knobs (part (a)+// already made the injection count + confidence threshold configurable). The+// values are chosen so a project-scoped 0.7 instinct (0.7 + 0.25 = 0.95) can+// surface above an unrelated global 0.9, and a stack-matching 0.75 instinct+// (0.75 + 0.2 = 0.95) can surface above an unrelated 0.9.+const DEFAULT_PROJECT_SCOPE_BOOST = 0.25;+const DEFAULT_STACK_MATCH_BOOST = 0.2;++/**+ * Whether a file with any of the given extensions exists directly in the root+ * (non-recursive, top-level only — kept cheap for a blocking SessionStart hook).+ * @param {string} root - Project root directory.+ * @param {string[]} extensions - Extensions to look for (e.g. ['.tf']).+ * @returns {boolean}+ */+function hasFileWithExtension(root, extensions) {+ try {+ return fs.readdirSync(root, { withFileTypes: true }).some(+ (entry) => entry.isFile() && extensions.includes(path.extname(entry.name))+ );+ } catch {+ return false;+ }+}++/**+ * Whether a named file exists directly in the root.+ * @param {string} root - Project root directory.+ * @param {string} name - File name relative to root.+ * @returns {boolean}+ */+function fileExists(root, name) {+ try {+ return fs.existsSync(path.join(root, name));+ } catch {+ return false;+ }+}++/**+ * Resolve whether relevance ranking is enabled. Default on; opt out by setting+ * `ECC_INSTINCT_RELEVANCE_RANKING` to `off`, `false`, `0`, or `no`+ * (case-insensitive). Any other value (including unset) keeps ranking on.+ * @returns {boolean}+ */+function isRelevanceRankingEnabled() {+ const raw = process.env.ECC_INSTINCT_RELEVANCE_RANKING;+ if (raw === undefined || raw === null || raw === '') return true;+ const normalized = String(raw).trim().toLowerCase();+ return !['off', 'false', '0', 'no'].includes(normalized);+}++/**+ * Cheap, non-recursive stack-keyword detection for the project root. Reuses+ * detectProjectType (languages + frameworks) and layers the extra IaC/data+ * markers issue #2371 calls out that detectProjectType does not cover+ * (`*.tf` / `*.tfvars` -> terraform, `dbt_project.yml` -> dbt).+ * @param {string} [projectRoot] - Defaults to process.cwd().+ * @param {{languages?: string[], frameworks?: string[]}} [projectInfo] -+ * Optional precomputed detectProjectType() result, to avoid a second pass.+ * @returns {Set<string>} Lowercase keyword set (may be empty).+ */+function detectStackKeywords(projectRoot, projectInfo) {+ const root = projectRoot || process.cwd();+ const keywords = new Set();++ let info = projectInfo;+ if (!info) {+ try {+ info = detectProjectType(root);+ } catch {+ info = { languages: [], frameworks: [] };+ }+ }+ for (const language of info.languages || []) keywords.add(String(language).toLowerCase());+ for (const framework of info.frameworks || []) keywords.add(String(framework).toLowerCase());++ if (hasFileWithExtension(root, ['.tf', '.tfvars'])) keywords.add('terraform');+ if (fileExists(root, 'dbt_project.yml')) keywords.add('dbt');++ return keywords;+}++/**+ * Tokenize a free-text field into lowercase word tokens (split on+ * non-alphanumerics). Token-set matching avoids substring false positives such+ * as the keyword `go` matching the word `good`.+ * @param {string} value+ * @returns {string[]}+ */+function tokenize(value) {+ return String(value || '')+ .toLowerCase()+ .split(/[^a-z0-9]+/)+ .filter(Boolean);+}++/**+ * Whether an instinct's domain/trigger/stack fields intersect the stack+ * keywords by whole-token match.+ * @param {object} instinct - Parsed instinct (frontmatter fields as properties).+ * @param {Set<string>} stackKeywords+ * @returns {boolean}+ */+function instinctMatchesStack(instinct, stackKeywords) {+ if (!instinct || !stackKeywords || stackKeywords.size === 0) return false;+ const tokens = new Set([+ ...tokenize(instinct.domain),+ ...tokenize(instinct.trigger),+ ...tokenize(instinct.stack),+ ]);+ for (const keyword of stackKeywords) {+ if (tokens.has(keyword)) return true;+ }+ return false;+}++/**+ * Additive relevance boost for ranking. Deterministic and pure. A+ * project-scoped instinct (location-relevant by construction) and a+ * stack-matching instinct each contribute their boost; both can apply.+ * @param {object} instinct - Must carry `_scopeLabel` ('project'|'global') and+ * optional `domain`/`trigger`/`stack` fields.+ * @param {Set<string>} stackKeywords+ * @param {{projectBoost?: number, stackBoost?: number}} [opts]+ * @returns {number}+ */+function computeRelevanceBoost(instinct, stackKeywords, opts) {+ const options = opts || {};+ const projectBoost = Number.isFinite(options.projectBoost)+ ? options.projectBoost+ : DEFAULT_PROJECT_SCOPE_BOOST;+ const stackBoost = Number.isFinite(options.stackBoost)+ ? options.stackBoost+ : DEFAULT_STACK_MATCH_BOOST;++ let boost = 0;+ if (instinct && instinct._scopeLabel === 'project') boost += projectBoost;+ if (instinctMatchesStack(instinct, stackKeywords)) boost += stackBoost;+ return boost;+}++module.exports = {+ DEFAULT_PROJECT_SCOPE_BOOST,+ DEFAULT_STACK_MATCH_BOOST,+ isRelevanceRankingEnabled,+ detectStackKeywords,+ instinctMatchesStack,+ computeRelevanceBoost,+ // Exported for testing.+ tokenize,+};diff --git a/tests/hooks/hooks.test.js b/tests/hooks/hooks.test.jsindex 07d48dcd5..49d6f1e23 100644--- a/tests/hooks/hooks.test.js+++ b/tests/hooks/hooks.test.js@@ -600,6 +600,64 @@ async function runTests() {passed++;else failed++;+ if (+ await asyncTest('ranks stack-relevant instincts above higher-confidence unrelated ones (#2371)', async () => {+ const isoHome = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-instinct-relevance-'));+ const homunculusDir = path.join(isoHome, 'homunculus');+ const instinctsDir = path.join(homunculusDir, 'instincts', 'personal');+ fs.mkdirSync(instinctsDir, { recursive: true });+ // A stack-matching 0.75 instinct and an unrelated higher-confidence 0.9.+ fs.writeFileSync(+ path.join(instinctsDir, 'terraform-first.md'),+ '---\nid: terraform-first\nconfidence: 0.75\ndomain: terraform\n---\n## Action\nRun terraform plan before every apply.\n'+ );+ fs.writeFileSync(+ path.join(instinctsDir, 'unrelated-high.md'),+ '---\nid: unrelated-high\nconfidence: 0.9\ndomain: python\n---\n## Action\nPin Python dependencies in requirements.txt.\n'+ );+ // A project root that detects as terraform via a *.tf marker.+ const projectRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-tf-project-'));+ fs.writeFileSync(path.join(projectRoot, 'main.tf'), 'resource "null_resource" "x" {}\n');++ const baseEnv = {+ HOME: isoHome,+ USERPROFILE: isoHome,+ CLV2_HOMUNCULUS_DIR: homunculusDir,+ CLAUDE_PROJECT_DIR: projectRoot,+ ECC_INSTINCT_RELEVANCE_RANKING: 'on',+ ECC_INSTINCT_CONFIDENCE_THRESHOLD: '0.7',+ ECC_MAX_INJECTED_INSTINCTS: '6',+ };++ try {+ const on = await runScript(path.join(scriptsDir, 'session-start.js'), '', baseEnv);+ assert.strictEqual(on.code, 0);+ const ctxOn = getSessionStartAdditionalContext(on.stdout);+ const tfOn = ctxOn.indexOf('Run terraform plan before every apply.');+ const pyOn = ctxOn.indexOf('Pin Python dependencies in requirements.txt.');+ assert.ok(tfOn !== -1 && pyOn !== -1, `both instincts should inject, ctx: ${ctxOn}`);+ assert.ok(tfOn < pyOn, `stack-matching 0.75 should rank above unrelated 0.9 when relevance is on, ctx: ${ctxOn}`);++ // Opting out restores pure confidence ordering (0.9 before 0.75).+ const off = await runScript(path.join(scriptsDir, 'session-start.js'), '', {+ ...baseEnv,+ ECC_INSTINCT_RELEVANCE_RANKING: 'off',+ });+ assert.strictEqual(off.code, 0);+ const ctxOff = getSessionStartAdditionalContext(off.stdout);+ const tfOff = ctxOff.indexOf('Run terraform plan before every apply.');+ const pyOff = ctxOff.indexOf('Pin Python dependencies in requirements.txt.');+ assert.ok(tfOff !== -1 && pyOff !== -1, `both instincts should still inject, ctx: ${ctxOff}`);+ assert.ok(pyOff < tfOff, `with ranking off, higher-confidence 0.9 should rank first, ctx: ${ctxOff}`);+ } finally {+ fs.rmSync(isoHome, { recursive: true, force: true });+ fs.rmSync(projectRoot, { recursive: true, force: true });+ }+ })+ )+ passed++;+ else failed++;+if (await asyncTest('disables session-start additional context when requested', async () => {const isoHome = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-disabled-start-'));diff --git a/tests/lib/instinct-relevance.test.js b/tests/lib/instinct-relevance.test.jsnew file mode 100644index 000000000..a3920c4cd--- /dev/null+++ b/tests/lib/instinct-relevance.test.js@@ -0,0 +1,231 @@+/**+ * Tests for scripts/lib/instinct-relevance.js+ *+ * Run with: node tests/lib/instinct-relevance.test.js+ */++const assert = require('assert');+const path = require('path');+const fs = require('fs');+const os = require('os');++const {+ DEFAULT_PROJECT_SCOPE_BOOST,+ DEFAULT_STACK_MATCH_BOOST,+ isRelevanceRankingEnabled,+ detectStackKeywords,+ instinctMatchesStack,+ computeRelevanceBoost,+ tokenize,+} = require('../../scripts/lib/instinct-relevance');++function test(name, fn) {+ try {+ fn();+ console.log(` ✓ ${name}`);+ return true;+ } catch (err) {+ console.log(` ✗ ${name}`);+ console.log(` ${err.message}`);+ return false;+ }+}++function createTempDir() {+ return fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-instinct-relevance-'));+}++function cleanupDir(dir) {+ try {+ fs.rmSync(dir, { recursive: true, force: true });+ } catch {+ /* ignore */+ }+}++function writeFile(dir, name, content) {+ fs.writeFileSync(path.join(dir, name), content);+}++function runTests() {+ let passed = 0;+ let failed = 0;++ console.log('\nInstinct relevance ranking tests\n');++ // --- tokenize ---------------------------------------------------------+ if (test('tokenize splits on non-alphanumerics and lowercases', () => {+ assert.deepStrictEqual(tokenize('Terraform-AWS_infra'), ['terraform', 'aws', 'infra']);+ assert.deepStrictEqual(tokenize('when editing hooks'), ['when', 'editing', 'hooks']);+ assert.deepStrictEqual(tokenize(''), []);+ assert.deepStrictEqual(tokenize(undefined), []);+ })) passed++; else failed++;++ // --- detectStackKeywords ---------------------------------------------+ if (test('detectStackKeywords returns empty set for an empty directory', () => {+ const dir = createTempDir();+ try {+ const kw = detectStackKeywords(dir);+ assert.ok(kw instanceof Set, 'should return a Set');+ assert.strictEqual(kw.size, 0);+ } finally {+ cleanupDir(dir);+ }+ })) passed++; else failed++;++ if (test('detectStackKeywords picks up a Rust project (Cargo.toml)', () => {+ const dir = createTempDir();+ try {+ writeFile(dir, 'Cargo.toml', '[package]\nname = "x"\n');+ const kw = detectStackKeywords(dir);+ assert.ok(kw.has('rust'), `expected rust in ${[...kw].join(',')}`);+ } finally {+ cleanupDir(dir);+ }+ })) passed++; else failed++;++ if (test('detectStackKeywords picks up a Go project (go.mod)', () => {+ const dir = createTempDir();+ try {+ writeFile(dir, 'go.mod', 'module example.com/x\n\ngo 1.21\n');+ const kw = detectStackKeywords(dir);+ assert.ok(kw.has('golang'), `expected golang in ${[...kw].join(',')}`);+ } finally {+ cleanupDir(dir);+ }+ })) passed++; else failed++;++ if (test('detectStackKeywords adds terraform for *.tf / *.tfvars files', () => {+ const dir = createTempDir();+ try {+ writeFile(dir, 'main.tf', 'resource "null_resource" "x" {}\n');+ const kw = detectStackKeywords(dir);+ assert.ok(kw.has('terraform'), `expected terraform in ${[...kw].join(',')}`);+ } finally {+ cleanupDir(dir);+ }+ })) passed++; else failed++;++ if (test('detectStackKeywords adds dbt for dbt_project.yml', () => {+ const dir = createTempDir();+ try {+ writeFile(dir, 'dbt_project.yml', "name: 'demo'\n");+ const kw = detectStackKeywords(dir);+ assert.ok(kw.has('dbt'), `expected dbt in ${[...kw].join(',')}`);+ } finally {+ cleanupDir(dir);+ }+ })) passed++; else failed++;++ if (test('detectStackKeywords accepts a precomputed projectInfo', () => {+ const kw = detectStackKeywords('/nonexistent', {+ languages: ['python'],+ frameworks: ['django'],+ });+ assert.ok(kw.has('python') && kw.has('django'));+ })) passed++; else failed++;++ // --- instinctMatchesStack --------------------------------------------+ if (test('instinctMatchesStack matches on domain token', () => {+ const kw = new Set(['terraform']);+ assert.strictEqual(instinctMatchesStack({ domain: 'terraform' }, kw), true);+ assert.strictEqual(instinctMatchesStack({ domain: 'terraform-aws' }, kw), true);+ })) passed++; else failed++;++ if (test('instinctMatchesStack matches on trigger token', () => {+ const kw = new Set(['python']);+ assert.strictEqual(+ instinctMatchesStack({ trigger: 'when writing python tests' }, kw),+ true+ );+ })) passed++; else failed++;++ if (test('instinctMatchesStack avoids substring false positives (go != good)', () => {+ const kw = new Set(['go']);+ assert.strictEqual(instinctMatchesStack({ domain: 'good practices' }, kw), false);+ })) passed++; else failed++;++ if (test('instinctMatchesStack is false with empty keyword set or fields', () => {+ assert.strictEqual(instinctMatchesStack({ domain: 'terraform' }, new Set()), false);+ assert.strictEqual(instinctMatchesStack({}, new Set(['terraform'])), false);+ assert.strictEqual(instinctMatchesStack(null, new Set(['terraform'])), false);+ })) passed++; else failed++;++ // --- computeRelevanceBoost -------------------------------------------+ if (test('computeRelevanceBoost gives project boost only for project scope', () => {+ const kw = new Set();+ assert.strictEqual(+ computeRelevanceBoost({ _scopeLabel: 'project' }, kw),+ DEFAULT_PROJECT_SCOPE_BOOST+ );+ assert.strictEqual(computeRelevanceBoost({ _scopeLabel: 'global' }, kw), 0);+ })) passed++; else failed++;++ if (test('computeRelevanceBoost gives stack boost only on a stack match', () => {+ const kw = new Set(['rust']);+ assert.strictEqual(+ computeRelevanceBoost({ _scopeLabel: 'global', domain: 'rust' }, kw),+ DEFAULT_STACK_MATCH_BOOST+ );+ assert.strictEqual(+ computeRelevanceBoost({ _scopeLabel: 'global', domain: 'python' }, kw),+ 0+ );+ })) passed++; else failed++;++ if (test('computeRelevanceBoost stacks project + stack boosts', () => {+ const kw = new Set(['rust']);+ const boost = computeRelevanceBoost({ _scopeLabel: 'project', domain: 'rust' }, kw);+ assert.strictEqual(boost, DEFAULT_PROJECT_SCOPE_BOOST + DEFAULT_STACK_MATCH_BOOST);+ })) passed++; else failed++;++ if (test('computeRelevanceBoost honours custom boost overrides', () => {+ const kw = new Set(['rust']);+ const boost = computeRelevanceBoost(+ { _scopeLabel: 'project', domain: 'rust' },+ kw,+ { projectBoost: 1, stackBoost: 2 }+ );+ assert.strictEqual(boost, 3);+ })) passed++; else failed++;++ if (test('a project 0.7 instinct outranks an unrelated global 0.9 with boosts', () => {+ // Confirms the boost magnitudes satisfy the issue's motivating example.+ const kw = new Set();+ const projectScore = 0.7 + computeRelevanceBoost({ _scopeLabel: 'project' }, kw);+ const globalScore = 0.9 + computeRelevanceBoost({ _scopeLabel: 'global' }, kw);+ assert.ok(projectScore > globalScore, `${projectScore} !> ${globalScore}`);+ })) passed++; else failed++;++ if (test('a stack-matching 0.75 instinct outranks an unrelated 0.9 with boosts', () => {+ const kw = new Set(['terraform']);+ const matchScore = 0.75 + computeRelevanceBoost({ _scopeLabel: 'global', domain: 'terraform' }, kw);+ const otherScore = 0.9 + computeRelevanceBoost({ _scopeLabel: 'global', domain: 'python' }, kw);+ assert.ok(matchScore > otherScore, `${matchScore} !> ${otherScore}`);+ })) passed++; else failed++;++ // --- isRelevanceRankingEnabled ---------------------------------------+ if (test('isRelevanceRankingEnabled defaults on and honours the opt-out toggle', () => {+ const original = process.env.ECC_INSTINCT_RELEVANCE_RANKING;+ try {+ delete process.env.ECC_INSTINCT_RELEVANCE_RANKING;+ assert.strictEqual(isRelevanceRankingEnabled(), true, 'unset should be on');+ for (const off of ['off', 'OFF', 'false', '0', 'no']) {+ process.env.ECC_INSTINCT_RELEVANCE_RANKING = off;+ assert.strictEqual(isRelevanceRankingEnabled(), false, `${off} should be off`);+ }+ for (const on of ['on', '1', 'true', 'yes', 'anything']) {+ process.env.ECC_INSTINCT_RELEVANCE_RANKING = on;+ assert.strictEqual(isRelevanceRankingEnabled(), true, `${on} should be on`);+ }+ } finally {+ if (original === undefined) delete process.env.ECC_INSTINCT_RELEVANCE_RANKING;+ else process.env.ECC_INSTINCT_RELEVANCE_RANKING = original;+ }+ })) passed++; else failed++;++ console.log(`\n=== Results: ${passed} passed, ${failed} failed ===\n`);+ process.exit(failed > 0 ? 1 : 0);+}++runTests();