Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

fix(text): stream ripgrep output within the result limit (#660)

Hiro-Chibaea026a1a613e
diff --git a/ix-cli/knip.json b/ix-cli/knip.json
index e3616b42..16c783f9 100644
--- a/ix-cli/knip.json
+++ b/ix-cli/knip.json
@@ -3,5 +3,5 @@
"entry": ["src/cli/main.ts", "src/index.ts", "test/parser.smoke.mjs"],
"project": ["src/**/*.ts"],
"ignore": ["test/fixtures/**", "test/parser.stress.mjs"],
- "ignoreBinaries": ["ix", "cygpath", "tar", "start", "xdg-open", "mkfifo"]
+ "ignoreBinaries": ["ix", "rg", "cygpath", "tar", "start", "xdg-open", "mkfifo"]
}
diff --git a/ix-cli/src/cli/__tests__/text-arg-injection.test.ts b/ix-cli/src/cli/__tests__/text-arg-injection.test.ts
index 687ca9ca..c22f48cc 100644
--- a/ix-cli/src/cli/__tests__/text-arg-injection.test.ts
+++ b/ix-cli/src/cli/__tests__/text-arg-injection.test.ts
@@ -1,18 +1,7 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import { Command } from "commander";
-const execFile = vi.hoisted(() =>
- vi.fn((_cmd: string, _args: string[], _opts: unknown, cb: (e: unknown, r: unknown) => void) =>
- cb(null, { stdout: "", stderr: "" }),
- ),
-);
-
-// Spread the real module: `config.ts`, reached through this command, imports
-// `execSync` from it.
-vi.mock("node:child_process", async (orig) => ({
- ...(await orig<Record<string, unknown>>()),
- execFile,
-}));
+const runRipgrep = vi.fn(async (_args: string[], _limit: number) => ({ stdout: "" }));
import { registerTextCommand } from "../commands/text.js";
@@ -25,14 +14,14 @@ import { registerTextCommand } from "../commands/text.js";
*/
describe("ix text does not let the search term become an rg flag", () => {
beforeEach(() => {
- execFile.mockClear();
+ runRipgrep.mockClear();
});
async function argvFor(term: string, extra: string[] = []): Promise<string[]> {
const program = new Command();
- registerTextCommand(program);
+ registerTextCommand(program, runRipgrep);
await program.parseAsync(["text", ...extra, "--", term], { from: "user" });
- return execFile.mock.calls[0]?.[1] as string[];
+ return runRipgrep.mock.calls[0]?.[0] as string[];
}
it.each(["--files", "--pre=/bin/sh", "--file=/etc/passwd", "-l", "--no-ignore"])(
diff --git a/ix-cli/src/cli/__tests__/text-stream-integration.test.ts b/ix-cli/src/cli/__tests__/text-stream-integration.test.ts
new file mode 100644
index 00000000..83d064a7
--- /dev/null
+++ b/ix-cli/src/cli/__tests__/text-stream-integration.test.ts
@@ -0,0 +1,37 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { Command } from "commander";
+import { execFileSync, spawnSync } from "node:child_process";
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { registerTextCommand } from "../commands/text.js";
+
+const hasRipgrep = spawnSync("rg", ["--version"]).status === 0;
+const fixtures: string[] = [];
+afterEach(() => {
+ vi.restoreAllMocks();
+ for (const fixture of fixtures.splice(0)) rmSync(fixture, { recursive: true, force: true });
+});
+
+describe.skipIf(!hasRipgrep)("text streaming with real ripgrep", () => {
+ it("returns the requested matches when per-file output exceeds 10 MiB", async () => {
+ const root = mkdtempSync(path.join(tmpdir(), "ix-text-stream-"));
+ fixtures.push(root);
+ for (let i = 0; i < 128; i++) {
+ writeFileSync(path.join(root, `${i}.ts`), `needle ${"x".repeat(100_000)}\n`);
+ }
+ const raw = execFileSync("rg", ["--json", "--max-count", "20", "--", "needle", root], {
+ maxBuffer: 32 * 1024 * 1024,
+ });
+ expect(raw.length).toBeGreaterThan(10 * 1024 * 1024);
+ const log = vi.spyOn(console, "log").mockImplementation(() => {});
+ vi.spyOn(process, "exit").mockImplementation(() => { throw new Error("unexpected exit"); });
+ const program = new Command();
+ registerTextCommand(program);
+ await program.parseAsync(["text", "needle", "--root", root, "--format", "json"], { from: "user" });
+ const results = JSON.parse(log.mock.calls.map(call => call.join(" ")).join("\n"));
+ expect(results).toHaveLength(20);
+ expect(new Set(results.map((r: { path: string }) => r.path)).size).toBe(20);
+ expect(results[0]).toMatchObject({ engine: "ripgrep", line_start: 1 });
+ });
+});
diff --git a/ix-cli/src/cli/__tests__/text-stream.test.ts b/ix-cli/src/cli/__tests__/text-stream.test.ts
new file mode 100644
index 00000000..8d42a000
--- /dev/null
+++ b/ix-cli/src/cli/__tests__/text-stream.test.ts
@@ -0,0 +1,102 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { EventEmitter } from "node:events";
+import { PassThrough } from "node:stream";
+import { Command } from "commander";
+
+const spawn = vi.hoisted(() => vi.fn());
+vi.mock("node:child_process", async (original) => ({
+ ...(await original<Record<string, unknown>>()), spawn,
+}));
+import { registerTextCommand } from "../commands/text.js";
+
+function match(snippet = "needle") {
+ return JSON.stringify({ type: "match", data: {
+ path: { text: "sample.ts" }, line_number: 1, lines: { text: snippet },
+ } }) + "\n";
+}
+
+describe("text ripgrep stream lifecycle", () => {
+ let child: EventEmitter & { stdout: PassThrough; stderr: PassThrough; kill: ReturnType<typeof vi.fn> };
+ let log: ReturnType<typeof vi.spyOn>;
+ beforeEach(() => {
+ child = Object.assign(new EventEmitter(), {
+ stdout: new PassThrough(), stderr: new PassThrough(), kill: vi.fn(() => true),
+ });
+ spawn.mockReturnValue(child);
+ log = vi.spyOn(console, "log").mockImplementation(() => {});
+ vi.spyOn(process.stderr, "write").mockImplementation(() => true);
+ vi.spyOn(process, "exit").mockImplementation(() => { throw new Error("exit 1"); });
+ });
+ afterEach(() => {
+ child.stdout.destroy();
+ child.stderr.destroy();
+ vi.restoreAllMocks();
+ spawn.mockReset();
+ });
+ function run(limit = "2", format = "json") {
+ const program = new Command();
+ registerTextCommand(program);
+ return program.parseAsync(["text", "needle", "--limit", limit, "--format", format], { from: "user" });
+ }
+ function close(code: number | null, signal: string | null = null) {
+ child.stdout.end();
+ child.stderr.end();
+ child.emit("close", code, signal);
+ }
+ it("handles split UTF-8 records, ignores metadata, and drains until the child exits", async () => {
+ const pending = run();
+ const bytes = Buffer.from(match("日本語"));
+ const split = bytes.indexOf(Buffer.from("日本")) + 1;
+ child.stdout.write('{"type":"begin"}\nnot JSON\n');
+ child.stdout.write(bytes.subarray(0, split));
+ child.stdout.write(bytes.subarray(split));
+ child.stdout.write(match("second") + match("third"));
+ expect(child.kill).not.toHaveBeenCalled();
+ expect(log).not.toHaveBeenCalled();
+ close(0);
+ await pending;
+ expect(JSON.parse(log.mock.calls[0][0]).map((r: { snippet: string }) => r.snippet)).toEqual(["日本語", "second"]);
+ expect(spawn).toHaveBeenCalledWith("rg", expect.any(Array), { stdio: ["ignore", "pipe", "pipe"] });
+ });
+ it.each([0, 1])("accepts normal exit %s without killing below the limit", async code => {
+ const pending = run();
+ if (code === 0) child.stdout.write(match());
+ close(code);
+ await pending;
+ expect(JSON.parse(log.mock.calls[0][0])).toHaveLength(code === 0 ? 1 : 0);
+ expect(child.kill).not.toHaveBeenCalled();
+ });
+ it("accepts diagnostic stderr on a successful search", async () => {
+ const pending = run("1");
+ child.stdout.write(match());
+ child.stderr.write("DEBUG globset: built glob set\n");
+ close(0);
+ await pending;
+ expect(JSON.parse(log.mock.calls[0][0])).toHaveLength(1);
+ });
+ it.each(["text", "llm"])("preserves %s formatting", async format => {
+ const pending = run("1", format);
+ child.stdout.write(match());
+ close(0);
+ await pending;
+ expect(log.mock.calls.flat().join("\n")).toContain("needle");
+ });
+ it.each(["stderr before limit", "stderr after limit", "exit 2", "unexpected signal"])("does not hide %s", async scenario => {
+ const pending = run("1");
+ const rejected = expect(pending).rejects.toThrow("exit 1");
+ if (scenario === "stderr before limit") child.stderr.write("read error\n");
+ child.stdout.write(match());
+ if (scenario === "stderr after limit") child.stderr.write("read error\n");
+ close(scenario === "unexpected signal" ? null : 2, scenario === "unexpected signal" ? "SIGKILL" : null);
+ await rejected;
+ expect(log).not.toHaveBeenCalled();
+ });
+ it("preserves the missing-ripgrep error", async () => {
+ const pending = run();
+ const rejected = expect(pending).rejects.toThrow("exit 1");
+ child.emit("error", Object.assign(new Error("missing"), { code: "ENOENT" }));
+ close(-2);
+ await rejected;
+ expect(process.stderr.write).toHaveBeenCalledWith(expect.stringContaining("is not installed"));
+ });
+});
diff --git a/ix-cli/src/cli/__tests__/text-workspace-boundary.test.ts b/ix-cli/src/cli/__tests__/text-workspace-boundary.test.ts
index ec159a43..9403785d 100644
--- a/ix-cli/src/cli/__tests__/text-workspace-boundary.test.ts
+++ b/ix-cli/src/cli/__tests__/text-workspace-boundary.test.ts
@@ -60,7 +60,7 @@ describe("text workspace boundary", () => {
await run(["text", "boundaryNeedle", "--root", workspace, "--path", "src", "--format", "json"]);
- expect(runRipgrep).toHaveBeenCalledWith(expect.arrayContaining(["boundaryNeedle", join(workspace, "src")]));
+ expect(runRipgrep).toHaveBeenCalledWith(expect.arrayContaining(["boundaryNeedle", join(workspace, "src")]), 20);
expect(JSON.parse(logs.join("\n"))).toMatchObject([{ path: "src/inside.ts" }]);
expect(process.exitCode).toBeUndefined();
});
diff --git a/ix-cli/src/cli/commands/text.ts b/ix-cli/src/cli/commands/text.ts
index 55423575..e3109b1e 100644
--- a/ix-cli/src/cli/commands/text.ts
+++ b/ix-cli/src/cli/commands/text.ts
@@ -1,5 +1,5 @@
-import { execFile } from "node:child_process";
-import { promisify } from "node:util";
+import { spawn } from "node:child_process";
+import { createInterface } from "node:readline";
import path from "node:path";
import type { Command } from "commander";
import { formatTextResults, type TextResult } from "../format.js";
@@ -7,12 +7,43 @@ import { isPathInsideResolvedRoot, resolveWorkspaceRoot } from "../config.js";
import { stderr } from "../stderr.js";
import { llmError } from "../llm.js";
-const execFileAsync = promisify(execFile);
+type RunRipgrep = (args: string[], limit: number) => Promise<{ stdout: string }>;
-type RunRipgrep = (args: string[]) => Promise<{ stdout: string }>;
+async function runRipgrep(args: string[], limit: number): Promise<{ stdout: string }> {
+ return new Promise((resolve, reject) => {
+ const child = spawn("rg", args, { stdio: ["ignore", "pipe", "pipe"] });
+ const lines = createInterface({ input: child.stdout, crlfDelay: Infinity });
+ const matches: string[] = [];
+ let errorOutput = "";
-async function runRipgrep(args: string[]): Promise<{ stdout: string }> {
- return execFileAsync("rg", args, { maxBuffer: 10 * 1024 * 1024 });
+ child.stderr.setEncoding("utf8");
+ child.stderr.on("data", (chunk: string) => {
+ errorOutput = (errorOutput + chunk).slice(0, 64 * 1024);
+ });
+ lines.on("line", (line: string) => {
+ if (matches.length >= limit) return;
+ try {
+ if (JSON.parse(line).type !== "match") return;
+ } catch {
+ return;
+ }
+ matches.push(line);
+ });
+ child.on("error", reject);
+ child.on("close", (code, signal) => {
+ lines.close();
+ // Drain through exit so errors after the retained matches are not hidden.
+ if (code === 0 || code === 1) {
+ resolve({ stdout: matches.join("\n") });
+ } else {
+ reject(Object.assign(new Error("ripgrep failed"), {
+ code,
+ signal,
+ stderr: errorOutput,
+ }));
+ }
+ });
+ });
}
export function resolveTextSearchPath(root: string, searchPath: string): string {
@@ -68,7 +99,7 @@ export function registerTextCommand(program: Command, executeRipgrep: RunRipgrep
// other tool until the timeout and leaves the child process behind.
rgArgs.push("--", term, searchPath);
- const { stdout } = await executeRipgrep(rgArgs);
+ const { stdout } = await executeRipgrep(rgArgs, limit);
const results: TextResult[] = [];
for (const line of stdout.split("\n")) {