Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
fix(text): stream ripgrep output within the result limit (#660)
Hiro-Chibaea026a1a613e
diff --git a/ix-cli/knip.json b/ix-cli/knip.jsonindex e3616b42..16c783f9 100644--- a/ix-cli/knip.json+++ b/ix-cli/knip.json@@ -3,5 +3,5 @@"entry": ["src/cli/main.ts", "src/index.ts", "test/parser.smoke.mjs"],"project": ["src/**/*.ts"],"ignore": ["test/fixtures/**", "test/parser.stress.mjs"],- "ignoreBinaries": ["ix", "cygpath", "tar", "start", "xdg-open", "mkfifo"]+ "ignoreBinaries": ["ix", "rg", "cygpath", "tar", "start", "xdg-open", "mkfifo"]}diff --git a/ix-cli/src/cli/__tests__/text-arg-injection.test.ts b/ix-cli/src/cli/__tests__/text-arg-injection.test.tsindex 687ca9ca..c22f48cc 100644--- a/ix-cli/src/cli/__tests__/text-arg-injection.test.ts+++ b/ix-cli/src/cli/__tests__/text-arg-injection.test.ts@@ -1,18 +1,7 @@import { beforeEach, describe, expect, it, vi } from "vitest";import { Command } from "commander";-const execFile = vi.hoisted(() =>- vi.fn((_cmd: string, _args: string[], _opts: unknown, cb: (e: unknown, r: unknown) => void) =>- cb(null, { stdout: "", stderr: "" }),- ),-);--// Spread the real module: `config.ts`, reached through this command, imports-// `execSync` from it.-vi.mock("node:child_process", async (orig) => ({- ...(await orig<Record<string, unknown>>()),- execFile,-}));+const runRipgrep = vi.fn(async (_args: string[], _limit: number) => ({ stdout: "" }));import { registerTextCommand } from "../commands/text.js";@@ -25,14 +14,14 @@ import { registerTextCommand } from "../commands/text.js";*/describe("ix text does not let the search term become an rg flag", () => {beforeEach(() => {- execFile.mockClear();+ runRipgrep.mockClear();});async function argvFor(term: string, extra: string[] = []): Promise<string[]> {const program = new Command();- registerTextCommand(program);+ registerTextCommand(program, runRipgrep);await program.parseAsync(["text", ...extra, "--", term], { from: "user" });- return execFile.mock.calls[0]?.[1] as string[];+ return runRipgrep.mock.calls[0]?.[0] as string[];}it.each(["--files", "--pre=/bin/sh", "--file=/etc/passwd", "-l", "--no-ignore"])(diff --git a/ix-cli/src/cli/__tests__/text-stream-integration.test.ts b/ix-cli/src/cli/__tests__/text-stream-integration.test.tsnew file mode 100644index 00000000..83d064a7--- /dev/null+++ b/ix-cli/src/cli/__tests__/text-stream-integration.test.ts@@ -0,0 +1,37 @@+import { afterEach, describe, expect, it, vi } from "vitest";+import { Command } from "commander";+import { execFileSync, spawnSync } from "node:child_process";+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";+import { tmpdir } from "node:os";+import path from "node:path";+import { registerTextCommand } from "../commands/text.js";++const hasRipgrep = spawnSync("rg", ["--version"]).status === 0;+const fixtures: string[] = [];+afterEach(() => {+ vi.restoreAllMocks();+ for (const fixture of fixtures.splice(0)) rmSync(fixture, { recursive: true, force: true });+});++describe.skipIf(!hasRipgrep)("text streaming with real ripgrep", () => {+ it("returns the requested matches when per-file output exceeds 10 MiB", async () => {+ const root = mkdtempSync(path.join(tmpdir(), "ix-text-stream-"));+ fixtures.push(root);+ for (let i = 0; i < 128; i++) {+ writeFileSync(path.join(root, `${i}.ts`), `needle ${"x".repeat(100_000)}\n`);+ }+ const raw = execFileSync("rg", ["--json", "--max-count", "20", "--", "needle", root], {+ maxBuffer: 32 * 1024 * 1024,+ });+ expect(raw.length).toBeGreaterThan(10 * 1024 * 1024);+ const log = vi.spyOn(console, "log").mockImplementation(() => {});+ vi.spyOn(process, "exit").mockImplementation(() => { throw new Error("unexpected exit"); });+ const program = new Command();+ registerTextCommand(program);+ await program.parseAsync(["text", "needle", "--root", root, "--format", "json"], { from: "user" });+ const results = JSON.parse(log.mock.calls.map(call => call.join(" ")).join("\n"));+ expect(results).toHaveLength(20);+ expect(new Set(results.map((r: { path: string }) => r.path)).size).toBe(20);+ expect(results[0]).toMatchObject({ engine: "ripgrep", line_start: 1 });+ });+});diff --git a/ix-cli/src/cli/__tests__/text-stream.test.ts b/ix-cli/src/cli/__tests__/text-stream.test.tsnew file mode 100644index 00000000..8d42a000--- /dev/null+++ b/ix-cli/src/cli/__tests__/text-stream.test.ts@@ -0,0 +1,102 @@+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import { EventEmitter } from "node:events";+import { PassThrough } from "node:stream";+import { Command } from "commander";++const spawn = vi.hoisted(() => vi.fn());+vi.mock("node:child_process", async (original) => ({+ ...(await original<Record<string, unknown>>()), spawn,+}));+import { registerTextCommand } from "../commands/text.js";++function match(snippet = "needle") {+ return JSON.stringify({ type: "match", data: {+ path: { text: "sample.ts" }, line_number: 1, lines: { text: snippet },+ } }) + "\n";+}++describe("text ripgrep stream lifecycle", () => {+ let child: EventEmitter & { stdout: PassThrough; stderr: PassThrough; kill: ReturnType<typeof vi.fn> };+ let log: ReturnType<typeof vi.spyOn>;+ beforeEach(() => {+ child = Object.assign(new EventEmitter(), {+ stdout: new PassThrough(), stderr: new PassThrough(), kill: vi.fn(() => true),+ });+ spawn.mockReturnValue(child);+ log = vi.spyOn(console, "log").mockImplementation(() => {});+ vi.spyOn(process.stderr, "write").mockImplementation(() => true);+ vi.spyOn(process, "exit").mockImplementation(() => { throw new Error("exit 1"); });+ });+ afterEach(() => {+ child.stdout.destroy();+ child.stderr.destroy();+ vi.restoreAllMocks();+ spawn.mockReset();+ });+ function run(limit = "2", format = "json") {+ const program = new Command();+ registerTextCommand(program);+ return program.parseAsync(["text", "needle", "--limit", limit, "--format", format], { from: "user" });+ }+ function close(code: number | null, signal: string | null = null) {+ child.stdout.end();+ child.stderr.end();+ child.emit("close", code, signal);+ }+ it("handles split UTF-8 records, ignores metadata, and drains until the child exits", async () => {+ const pending = run();+ const bytes = Buffer.from(match("日本語"));+ const split = bytes.indexOf(Buffer.from("日本")) + 1;+ child.stdout.write('{"type":"begin"}\nnot JSON\n');+ child.stdout.write(bytes.subarray(0, split));+ child.stdout.write(bytes.subarray(split));+ child.stdout.write(match("second") + match("third"));+ expect(child.kill).not.toHaveBeenCalled();+ expect(log).not.toHaveBeenCalled();+ close(0);+ await pending;+ expect(JSON.parse(log.mock.calls[0][0]).map((r: { snippet: string }) => r.snippet)).toEqual(["日本語", "second"]);+ expect(spawn).toHaveBeenCalledWith("rg", expect.any(Array), { stdio: ["ignore", "pipe", "pipe"] });+ });+ it.each([0, 1])("accepts normal exit %s without killing below the limit", async code => {+ const pending = run();+ if (code === 0) child.stdout.write(match());+ close(code);+ await pending;+ expect(JSON.parse(log.mock.calls[0][0])).toHaveLength(code === 0 ? 1 : 0);+ expect(child.kill).not.toHaveBeenCalled();+ });+ it("accepts diagnostic stderr on a successful search", async () => {+ const pending = run("1");+ child.stdout.write(match());+ child.stderr.write("DEBUG globset: built glob set\n");+ close(0);+ await pending;+ expect(JSON.parse(log.mock.calls[0][0])).toHaveLength(1);+ });+ it.each(["text", "llm"])("preserves %s formatting", async format => {+ const pending = run("1", format);+ child.stdout.write(match());+ close(0);+ await pending;+ expect(log.mock.calls.flat().join("\n")).toContain("needle");+ });+ it.each(["stderr before limit", "stderr after limit", "exit 2", "unexpected signal"])("does not hide %s", async scenario => {+ const pending = run("1");+ const rejected = expect(pending).rejects.toThrow("exit 1");+ if (scenario === "stderr before limit") child.stderr.write("read error\n");+ child.stdout.write(match());+ if (scenario === "stderr after limit") child.stderr.write("read error\n");+ close(scenario === "unexpected signal" ? null : 2, scenario === "unexpected signal" ? "SIGKILL" : null);+ await rejected;+ expect(log).not.toHaveBeenCalled();+ });+ it("preserves the missing-ripgrep error", async () => {+ const pending = run();+ const rejected = expect(pending).rejects.toThrow("exit 1");+ child.emit("error", Object.assign(new Error("missing"), { code: "ENOENT" }));+ close(-2);+ await rejected;+ expect(process.stderr.write).toHaveBeenCalledWith(expect.stringContaining("is not installed"));+ });+});diff --git a/ix-cli/src/cli/__tests__/text-workspace-boundary.test.ts b/ix-cli/src/cli/__tests__/text-workspace-boundary.test.tsindex ec159a43..9403785d 100644--- a/ix-cli/src/cli/__tests__/text-workspace-boundary.test.ts+++ b/ix-cli/src/cli/__tests__/text-workspace-boundary.test.ts@@ -60,7 +60,7 @@ describe("text workspace boundary", () => {await run(["text", "boundaryNeedle", "--root", workspace, "--path", "src", "--format", "json"]);- expect(runRipgrep).toHaveBeenCalledWith(expect.arrayContaining(["boundaryNeedle", join(workspace, "src")]));+ expect(runRipgrep).toHaveBeenCalledWith(expect.arrayContaining(["boundaryNeedle", join(workspace, "src")]), 20);expect(JSON.parse(logs.join("\n"))).toMatchObject([{ path: "src/inside.ts" }]);expect(process.exitCode).toBeUndefined();});diff --git a/ix-cli/src/cli/commands/text.ts b/ix-cli/src/cli/commands/text.tsindex 55423575..e3109b1e 100644--- a/ix-cli/src/cli/commands/text.ts+++ b/ix-cli/src/cli/commands/text.ts@@ -1,5 +1,5 @@-import { execFile } from "node:child_process";-import { promisify } from "node:util";+import { spawn } from "node:child_process";+import { createInterface } from "node:readline";import path from "node:path";import type { Command } from "commander";import { formatTextResults, type TextResult } from "../format.js";@@ -7,12 +7,43 @@ import { isPathInsideResolvedRoot, resolveWorkspaceRoot } from "../config.js";import { stderr } from "../stderr.js";import { llmError } from "../llm.js";-const execFileAsync = promisify(execFile);+type RunRipgrep = (args: string[], limit: number) => Promise<{ stdout: string }>;-type RunRipgrep = (args: string[]) => Promise<{ stdout: string }>;+async function runRipgrep(args: string[], limit: number): Promise<{ stdout: string }> {+ return new Promise((resolve, reject) => {+ const child = spawn("rg", args, { stdio: ["ignore", "pipe", "pipe"] });+ const lines = createInterface({ input: child.stdout, crlfDelay: Infinity });+ const matches: string[] = [];+ let errorOutput = "";-async function runRipgrep(args: string[]): Promise<{ stdout: string }> {- return execFileAsync("rg", args, { maxBuffer: 10 * 1024 * 1024 });+ child.stderr.setEncoding("utf8");+ child.stderr.on("data", (chunk: string) => {+ errorOutput = (errorOutput + chunk).slice(0, 64 * 1024);+ });+ lines.on("line", (line: string) => {+ if (matches.length >= limit) return;+ try {+ if (JSON.parse(line).type !== "match") return;+ } catch {+ return;+ }+ matches.push(line);+ });+ child.on("error", reject);+ child.on("close", (code, signal) => {+ lines.close();+ // Drain through exit so errors after the retained matches are not hidden.+ if (code === 0 || code === 1) {+ resolve({ stdout: matches.join("\n") });+ } else {+ reject(Object.assign(new Error("ripgrep failed"), {+ code,+ signal,+ stderr: errorOutput,+ }));+ }+ });+ });}export function resolveTextSearchPath(root: string, searchPath: string): string {@@ -68,7 +99,7 @@ export function registerTextCommand(program: Command, executeRipgrep: RunRipgrep// other tool until the timeout and leaves the child process behind.rgArgs.push("--", term, searchPath);- const { stdout } = await executeRipgrep(rgArgs);+ const { stdout } = await executeRipgrep(rgArgs, limit);const results: TextResult[] = [];for (const line of stdout.split("\n")) {