Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

fix(context): keep main's default bundle size; charge only what is emitted (CL-06)

KageBinary07eae6629f83
diff --git a/ix-cli/src/cli/__tests__/context-pick-validation.test.ts b/ix-cli/src/cli/__tests__/context-pick-validation.test.ts
index 099fe8f2..579f08fb 100644
--- a/ix-cli/src/cli/__tests__/context-pick-validation.test.ts
+++ b/ix-cli/src/cli/__tests__/context-pick-validation.test.ts
@@ -116,8 +116,8 @@ describe("ix context --max-* validation", () => {
for (const [flag, text, fallback, min, max] of [
["--max-entities", "Maximum entities in the bundle", 50, 1, 500],
["--max-relationships", "Maximum relationships in the bundle", 100, 1, 1000],
- ["--max-evidence", "Maximum evidence items in the bundle; --max-tokens normally binds first", 200, 1, 200],
- ["--max-tokens", "Maximum tokens of evidence output", 3000, 500, 200000],
+ ["--max-evidence", "Maximum evidence items in the bundle", 25, 1, 200],
+ ["--max-tokens", "Token budget for the printed output (llm: all of it; json and text: the evidence)", 3000, 500, 200000],
] as const) {
// One assertion per flag, so a failure names which one lost its default
// rather than reporting that some string was missing from a wall of help.
@@ -130,7 +130,7 @@ describe("ix context --max-* validation", () => {
// it no longer has: its absence produces whatever --max-tokens works out
// to, not 12000.
expect(flat, "help for --max-chars").toContain(
- "--max-chars <n> Maximum characters of evidence output (overrides --max-tokens; clamped to 1000-1000000)",
+ "--max-chars <n> Character budget for the printed output (llm: all of it; json and text: the evidence) (overrides --max-tokens; clamped to 1000-1000000)",
);
});
});
diff --git a/ix-cli/src/cli/__tests__/context-token-budget.test.ts b/ix-cli/src/cli/__tests__/context-token-budget.test.ts
index 0a85780f..1f552d51 100644
--- a/ix-cli/src/cli/__tests__/context-token-budget.test.ts
+++ b/ix-cli/src/cli/__tests__/context-token-budget.test.ts
@@ -14,17 +14,21 @@ describe("--max-tokens", () => {
it("is what bounds a default bundle, not 12,000 characters", () => {
const budgets = clampBudgets({});
expect(budgets.maxTokens).toBe(3000);
- expect(budgets.maxChars).toBe(8100);
- expect(clampBudgets({}, "llm").maxChars).toBe(10_500);
- expect(clampBudgets({}, "text").maxChars).toBe(10_500);
+ // The default size is main's, in every format: what the budget is charged
+ // against changed, not how big it is. Larger defaults wait on a re-measure.
+ for (const format of ["json", "llm", "text"] as const) {
+ expect(clampBudgets({}, format).maxChars, format).toBe(6420);
+ expect(clampBudgets({}, format).maxEvidence, format).toBe(25);
+ }
// The old character default was nearly twice this.
expect(budgets.maxChars).toBeLessThan(12_000);
});
it("moves the character budget with it", () => {
expect(clampBudgets({ maxTokens: 3000 }, "json").maxChars).toBe(Math.round(3000 * CHARS_PER_TOKEN.json));
- expect(clampBudgets({ maxTokens: 500 }, "llm").maxChars).toBe(1750);
- expect(clampBudgets({ maxTokens: 500 }).maxChars).toBe(1350);
+ expect(clampBudgets({ maxTokens: 3000 }, "llm").maxChars).toBe(Math.round(3000 * CHARS_PER_TOKEN.llm));
+ expect(clampBudgets({ maxTokens: 500 }, "llm").maxChars).toBe(1070);
+ expect(clampBudgets({ maxTokens: 500 }).maxChars).toBe(1070);
});
it("is clamped to its range before it is converted", () => {
diff --git a/ix-cli/src/cli/__tests__/context.test.ts b/ix-cli/src/cli/__tests__/context.test.ts
index d17fbb83..d602ef34 100644
--- a/ix-cli/src/cli/__tests__/context.test.ts
+++ b/ix-cli/src/cli/__tests__/context.test.ts
@@ -14,7 +14,8 @@ import type {
} from "../../client/types.js";
import type { EntityFacts } from "../explain/facts.js";
import {
- buildBundle, bundleFormat, clampBudgets, renderBundle, sanitizeId, textEvidenceLines,
+ buildBundle, bundleFormat, bundleLlmLines, clampBudgets, llmNextByCut, llmNextLines, renderBundle, sanitizeId,
+ textEvidenceLines,
} from "../commands/context.js";
function makeFacts(overrides: Partial<EntityFacts> = {}): EntityFacts {
@@ -459,6 +460,19 @@ describe("ix context budget is charged against the emitted text", () => {
return { ...input(), facts, context: makeContext({ nodes, edges }) };
};
+ // Members with ranges, two sharing one, and some without: the shapes the
+ // `next` rows skip or dedupe.
+ const readableMembers = () => {
+ const memberRefs = Array.from({ length: 12 }, (_, i) => ({
+ id: `m-${i}`, name: `member_${i}`, kind: "function", path: "src/widget.ts",
+ ...(i % 3 === 1 ? {} : { lineStart: i < 2 ? 10 : 10 + i * 20, lineEnd: i < 2 ? 20 : 20 + i * 20 }),
+ }));
+ const facts = makeFacts({
+ path: "src/widget.ts", memberRefs, members: memberRefs.map((m) => m.name), memberCount: memberRefs.length,
+ });
+ return { ...input(), facts };
+ };
+
const emitted = (bundle: ReturnType<typeof buildBundle>, format: string): string => {
const lines: string[] = [];
const orig = console.log;
@@ -491,25 +505,59 @@ describe("ix context budget is charged against the emitted text", () => {
});
}
- // Every budget in a sweep, so some settle within a few characters of
- // maxChars: the header's `truncated_chars` digits must be the ones measured,
- // not a placeholder that grows after the fit was checked.
+ // A sample of budgets, each probed at its own boundary: the output it
+ // settled at, and one character under. Landing exactly on maxChars is where
+ // a `truncated_chars` placeholder that grows after the fit was checked would
+ // overrun, so each sample tests the tightest budgets rather than hoping a
+ // dense sweep lands on them. Every 25th value keeps this to ~120 builds.
it("never emits past maxChars in llm once the truncated_chars header is final", () => {
delete process.env.IX_CALLER;
- const base = clampBudgets({}, "llm");
- let cut = 0;
- for (let maxChars = 1500; maxChars <= 2500; maxChars += 1) {
- const budgets = { ...base, maxChars };
- const bundle = buildBundle({ ...manyMembers(), budgets, format: "llm" });
- if (bundle.truncation.evidenceTruncated > 0) cut += 1;
+ const base = clampBudgets({ maxEvidence: 200 }, "llm");
+ const build = (maxChars: number) => {
+ const bundle = buildBundle({ ...manyMembers(), budgets: { ...base, maxChars }, format: "llm" });
+ expect(bundle.truncation.evidenceTruncated, `cut at ${maxChars}`).toBeGreaterThan(0);
expect(bundle.truncation.charactersTruncated).toBeGreaterThan(0);
- expect(emitted(bundle, "llm").length).toBeLessThanOrEqual(maxChars);
+ const out = emitted(bundle, "llm").length;
+ expect(out, `emitted at maxChars=${maxChars}`).toBeLessThanOrEqual(maxChars);
+ return { bundle, out };
+ };
+ let samples = 0;
+ for (let maxChars = 1500; maxChars <= 2500; maxChars += 25) {
+ const { bundle, out } = build(maxChars);
+ // Exactly at the size it settled at, it keeps the same rows.
+ expect(build(out).bundle.evidence.length).toBe(bundle.evidence.length);
+ // One under, it must drop a row rather than overrun.
+ build(out - 1);
+ samples += 1;
}
- expect(cut).toBe(1001);
+ expect(samples).toBe(41);
});
+ // The fitting loop measures the llm output at many cuts with the `next`
+ // rows looked up, not recomputed (#834 makes them read files). The lookup
+ // must give the lines a fresh computation would at every cut, or the size
+ // measured is not the size printed.
+ for (const caller of [undefined, "mcp"]) {
+ it(`looks up the same next lines a fresh computation gives, at every cut (IX_CALLER=${caller ?? "unset"})`, () => {
+ if (caller) process.env.IX_CALLER = caller;
+ else delete process.env.IX_CALLER;
+ const big = clampBudgets({ maxChars: 100000, maxEvidence: 200 }, "llm");
+ const bundle = buildBundle({ ...readableMembers(), budgets: big, format: "llm" });
+ const evidence = bundle.evidence;
+ // Rows without a range and rows repeating one are in the list, so the
+ // breakpoints are not simply the first three rows.
+ expect(evidence.filter((e) => e.location?.lineStart === undefined).length).toBeGreaterThan(0);
+ const nextAt = llmNextByCut(bundle, evidence);
+ for (let count = 0; count <= evidence.length; count++) {
+ const prefix = { ...bundle, evidence: evidence.slice(0, count) };
+ expect(nextAt(count), `cut at ${count}`).toEqual(llmNextLines(prefix));
+ expect(bundleLlmLines(prefix, nextAt(count))).toEqual(bundleLlmLines(prefix));
+ }
+ });
+ }
+
it("keeps every row when the whole bundle fits exactly (no reserved diagnostic)", () => {
- const big = clampBudgets({ maxChars: 100000 }, "llm");
+ const big = clampBudgets({ maxChars: 100000, maxEvidence: 200 }, "llm");
const full = buildBundle({ ...manyMembers(), budgets: big, format: "llm" });
expect(full.truncation.evidenceTruncated).toBe(0);
const exact = emitted(full, "llm").length;
@@ -549,7 +597,7 @@ describe("ix context budget is charged against the emitted text", () => {
});
it("reports charactersTruncated as full minus kept in the format's own unit", () => {
- const budgets = clampBudgets({ maxTokens: 600 }, "json");
+ const budgets = clampBudgets({ maxTokens: 600, maxEvidence: 200 }, "json");
const cut = buildBundle({ ...manyMembers(), budgets, format: "json" });
const wide = buildBundle({
...manyMembers(), format: "json", budgets: { ...budgets, maxChars: 1_000_000 },
diff --git a/ix-cli/src/cli/commands/context.ts b/ix-cli/src/cli/commands/context.ts
index 47d75b7d..fa5e0277 100644
--- a/ix-cli/src/cli/commands/context.ts
+++ b/ix-cli/src/cli/commands/context.ts
@@ -84,9 +84,9 @@ const MAX_CONFLICTS = 10;
const BUDGETS = [
{ key: "maxEntities", flag: "--max-entities", label: "entities", help: "Maximum entities in the bundle", min: 1, max: 500, fallback: 50 },
{ key: "maxRelationships", flag: "--max-relationships", label: "relationships", help: "Maximum relationships in the bundle", min: 1, max: 1000, fallback: 100 },
- { key: "maxEvidence", flag: "--max-evidence", label: "evidence", help: "Maximum evidence items in the bundle; --max-tokens normally binds first", min: 1, max: 200, fallback: 200 },
- { key: "maxTokens", flag: "--max-tokens", label: "tokens", help: "Maximum tokens of evidence output", min: 500, max: 200_000, fallback: 3_000 },
- { key: "maxChars", flag: "--max-chars", label: "chars", help: "Maximum characters of evidence output", min: 1000, max: 1_000_000, fallback: 12_000 },
+ { key: "maxEvidence", flag: "--max-evidence", label: "evidence", help: "Maximum evidence items in the bundle", min: 1, max: 200, fallback: 25 },
+ { key: "maxTokens", flag: "--max-tokens", label: "tokens", help: "Token budget for the printed output (llm: all of it; json and text: the evidence)", min: 500, max: 200_000, fallback: 3_000 },
+ { key: "maxChars", flag: "--max-chars", label: "chars", help: "Character budget for the printed output (llm: all of it; json and text: the evidence)", min: 1000, max: 1_000_000, fallback: 12_000 },
] as const satisfies ReadonlyArray<{
key: keyof BudgetSnapshot;
flag: string;
@@ -143,17 +143,17 @@ export function bundleFormat(fmt: string | undefined): BundleFormat {
* Characters per token, by the format the bundle is printed in.
*
* `--max-tokens` is charged against the text that is actually emitted, so the
- * ratio is per format. Measured with cl100k: llm 3.73 and json 2.69 on the
- * audit fixture; the constants sit just under, so a bundle comes in at or under
- * its budget. `text` measured 4.09 on a live `context.ts` bundle and shares
- * the llm value, which keeps it on the safe side.
+ * ratio is per format. Every format still uses 2.14, the value measured across
+ * 41 recorded bundles (JSON plus prompt) and the one saved investigations were
+ * sized at, so the default budget is the 6,420 characters it has always been.
+ * cl100k measured llm 3.73 and json 2.69 on the audit fixture, so 2.14 is
+ * conservative for both: a bundle comes in under its token budget, never over.
*
- * The 2.14 this replaced was measured across 41 recorded bundles (JSON plus
- * prompt) and is the value legacy saved bundles were sized at; their stored
- * `maxChars` is kept as it is. Re-measure before raising any of these, never
- * raise one to make a bundle fit.
+ * Raising a ratio makes every default bundle larger (the ix-bench result that
+ * all expected files appear at 3,000 tokens was measured at 2.14), so it waits
+ * until that is re-measured. Never raise one to make a bundle fit.
*/
-export const CHARS_PER_TOKEN: Record<BundleFormat, number> = { llm: 3.5, json: 2.7, text: 3.5 };
+export const CHARS_PER_TOKEN: Record<BundleFormat, number> = { llm: 2.14, json: 2.14, text: 2.14 };
/**
* Apply the table's range and default to whatever the caller supplied, and
@@ -2227,10 +2227,14 @@ export function buildBundle(input: BuildInput): ContextBundle {
if (cut.length > 0) bundle.truncation.cut = cut;
else delete bundle.truncation.cut;
};
+ // The llm output at the current cut, with the `next` rows looked up rather
+ // than recomputed: this is measured once per fitting step.
+ const nextAt = format === "llm" ? llmNextByCut(bundle, evidence) : undefined;
+ const llmSize = (): number => bundleLlmLines(bundle, nextAt!(bundle.evidence.length)).join("\n").length;
let overhead = 0;
if (format === "llm") {
applyCut(0);
- overhead = bundleLlmLines(bundle).join("\n").length;
+ overhead = llmSize();
} else if (format === "text") {
overhead = TEXT_HEADER_ESTIMATE;
}
@@ -2247,7 +2251,7 @@ export function buildBundle(input: BuildInput): ContextBundle {
if (format === "llm") {
// The header digits, the bundle_truncated diagnostic and the `next` lines
// all depend on the cut, so settle it by measuring the real output.
- while (kept > 1 && bundleLlmLines(bundle).join("\n").length > budgets.maxChars) {
+ while (kept > 1 && llmSize() > budgets.maxChars) {
kept -= 1;
applyCut(kept);
}
@@ -2259,7 +2263,7 @@ export function buildBundle(input: BuildInput): ContextBundle {
// the nearly-complete ones do not: try the whole list before growing.
if (kept < limit) {
applyCut(limit);
- if (bundleLlmLines(bundle).join("\n").length <= budgets.maxChars) {
+ if (llmSize() <= budgets.maxChars) {
kept = limit;
} else {
applyCut(kept);
@@ -2267,7 +2271,7 @@ export function buildBundle(input: BuildInput): ContextBundle {
}
while (kept < limit) {
applyCut(kept + 1);
- if (bundleLlmLines(bundle).join("\n").length > budgets.maxChars) {
+ if (llmSize() > budgets.maxChars) {
applyCut(kept);
break;
}
@@ -2537,11 +2541,64 @@ function issueEvidence(issue: IssueBundleInput): EvidenceItem[] {
return items;
}
+/**
+ * The `next` lines `--format llm` closes with, for the bundle's evidence as it
+ * stands. Draws only on the first few rows with a readable range; see
+ * {@link llmNextByCut}.
+ */
+export function llmNextLines(bundle: ContextBundle): string[] {
+ return forMcp()
+ ? nextToolCalls(bundle).map((step) => llmLine("next", { cmd: step.cmd, why: step.why }))
+ : nextReads(bundle).map((cmd) => llmLine("next", { cmd }));
+}
+
+/**
+ * Indexes of the first `limit` evidence rows with a distinct readable range
+ * (path, start and end line), in evidence order. These are the only rows
+ * {@link nextReads} and {@link nextToolCalls} read, so the `next` lines can
+ * only change at a cut that drops one of them.
+ */
+function readableRowIndexes(evidence: EvidenceItem[], limit: number): number[] {
+ const seen = new Set<string>();
+ const out: number[] = [];
+ for (let i = 0; i < evidence.length && out.length < limit; i++) {
+ const loc = evidence[i]!.location;
+ if (!loc?.path || loc.lineStart === undefined || loc.lineEnd === undefined) continue;
+ const range = `${loc.path}:${loc.lineStart}-${loc.lineEnd}`;
+ if (seen.has(range)) continue;
+ seen.add(range);
+ out.push(i);
+ }
+ return out;
+}
+
+/**
+ * The `next` lines for every cut of `evidence`, computed once per bundle.
+ *
+ * The budget-fitting loop in `buildBundle` measures the llm output at many cut
+ * counts. Recomputing the `next` rows at each would repeat whatever they cost
+ * (a stale-row check that reads files, say) once per iteration. They only
+ * change at the cuts that drop one of the first three readable rows, so they
+ * are computed for those few prefixes, up front, and looked up by count after.
+ */
+export function llmNextByCut(bundle: ContextBundle, evidence: EvidenceItem[]): (count: number) => string[] {
+ // Each breakpoint is a prefix length at which the next lines can change.
+ const breaks = [0, ...readableRowIndexes(evidence, 3).map((i) => i + 1)];
+ const table = breaks.map((at) => llmNextLines({ ...bundle, evidence: evidence.slice(0, at) }));
+ return (count) => {
+ let k = 0;
+ while (k + 1 < breaks.length && breaks[k + 1]! <= count) k += 1;
+ return table[k]!;
+ };
+}
+
/**
* The lines `--format llm` prints, in order. Pure, so `buildBundle` can measure
- * exactly what a budget will be charged for.
+ * exactly what a budget will be charged for. `next` is the closing lines when
+ * the caller already has them (`buildBundle` takes them from
+ * {@link llmNextByCut}); otherwise they are computed from the bundle.
*/
-export function bundleLlmLines(bundle: ContextBundle): string[] {
+export function bundleLlmLines(bundle: ContextBundle, next: string[] = llmNextLines(bundle)): string[] {
// Every line through `llmLine`, none through a template literal: see the
// history of this block. Absent sections are null and filtered out.
const lines: Array<string | null> = [
@@ -2589,9 +2646,7 @@ export function bundleLlmLines(bundle: ContextBundle): string[] {
// `--format json` carries the rest for a caller that wants it.
...bundle.evidence.map(evidenceRecord(undefined)),
// Last, so it reads as the closing instruction it is.
- ...(forMcp()
- ? nextToolCalls(bundle).map((step) => llmLine("next", { cmd: step.cmd, why: step.why }))
- : nextReads(bundle).map((cmd) => llmLine("next", { cmd }))),
+ ...next,
];
return lines.filter((line): line is string => line !== null);
}
diff --git a/skills/ix/references/flags.md b/skills/ix/references/flags.md
index 2190db2e..22e057c4 100644
--- a/skills/ix/references/flags.md
+++ b/skills/ix/references/flags.md
@@ -49,19 +49,22 @@ surfaces (#575).
auto-paginates on its own; `--offset` or `--regions` turns that off, and
`--limit` only sets the page size.
- **`ix context` budgets the text it emits, in tokens.** `--max-tokens`
- defaults to 3,000 and is converted to characters per output format: 3.5 per
- token for `--format llm` and text, 2.7 for `--format json` (measured with
- cl100k; the old flat 2.14 was measured on recorded bundles and is what saved
- investigations from before this were sized at). For `llm` the budget covers
- the whole output: header, diagnostics, evidence rows and `next` lines. For
- `json` it bounds the evidence list only; entities and relationships keep
- their own `--max-entities` and `--max-relationships` counts. `--max-evidence`
- defaults to 200 so the token budget normally binds first. A saved
+ defaults to 3,000 and is converted at 2.14 characters per token, measured
+ across 41 recorded bundles, in every output format. At 1,500 the evidence
+ block held about ten items, and for a file target those were its own
+ members: ix-bench agents were shown 0.59 of the expected files ix had found,
+ against all of them at 3,000, for about 320 more tokens. For `--format llm`
+ the budget covers the whole output: header, diagnostics, evidence rows and
+ `next` lines. For `json` it bounds the serialized evidence list, and for text
+ the evidence lines plus a reserve for the header; entities and relationships
+ keep their own `--max-entities` and `--max-relationships` counts.
+ `--max-evidence` defaults to 25. cl100k counts about 3.7 characters per token
+ in llm output and 2.7 in JSON, so 2.14 keeps a bundle under its token budget;
+ larger defaults are deferred until they are measured on ix-bench. A saved
investigation records the format its budget was sized for, and `--diff`
- rebuilds with it. The ix-bench result that all expected files appear at 3,000
- tokens was measured under the old sizing. `--max-chars` is still there for
- a caller who needs exact bytes and overrides it; passing both is refused
- rather than silently ranked, because the output does not say which one won.
+ rebuilds with it. `--max-chars` is still there for a caller who needs exact
+ bytes and overrides it; passing both is refused rather than silently ranked,
+ because the output does not say which one won.
- **`ix mcp` advertises five tools by default**, not twenty-six:
`ix_context`, `ix_search`, `ix_neighbors`, `ix_impact` and `ix_read`, one
per question the graph answers better than Grep/Read. `ix_neighbors{relation}`
@@ -250,9 +253,9 @@ Build a bounded, deterministic context bundle for a symbol, file, or entity (or
| `--as-of-rev` | `<n>` | — | Historical context as of a graph revision |
| `--max-entities` | `<n>` | — | Maximum entities in the bundle (default: 50, clamped to 1-500) |
| `--max-relationships` | `<n>` | — | Maximum relationships in the bundle (default: 100, clamped to 1-1000) |
-| `--max-evidence` | `<n>` | — | Maximum evidence items in the bundle; `--max-tokens` normally binds first (default: 200, clamped to 1-200) |
-| `--max-tokens` | `<n>` | `3000` | Maximum tokens of evidence output (clamped to 500-200000) |
-| `--max-chars` | `<n>` | — | Maximum characters of evidence output; overrides `--max-tokens` (clamped to 1000-1000000) |
+| `--max-evidence` | `<n>` | — | Maximum evidence items in the bundle (default: 25, clamped to 1-200) |
+| `--max-tokens` | `<n>` | `3000` | Token budget for the printed output (llm: all of it; json and text: the evidence) (clamped to 500-200000) |
+| `--max-chars` | `<n>` | — | Character budget for the printed output (llm: all of it; json and text: the evidence); overrides `--max-tokens` (clamped to 1000-1000000) |
| `--format` | `text\|json\|llm` | `text` | Output format — see [output-formats.md](output-formats.md) |
| `--pretty` | — | off | Indent JSON output; the default only when stdout is a terminal |
| `--quiet` | — | off | Drop headers, section titles and advisory hints |