Skip to content

#1 Persistent memory for AI coding agents based on real-world benchmarks

fix(replay,cli): clamp maxFiles to upper bound + cap traversal by entries visited

Rohit Ghumareaef1dbbfe857
diff --git a/src/cli.ts b/src/cli.ts
index 6c8de462..0d632637 100644
--- a/src/cli.ts
+++ b/src/cli.ts
@@ -36,7 +36,8 @@ Commands:
upgrade Upgrade local deps + iii runtime (best effort)
mcp Start standalone MCP server (no engine required)
import-jsonl [p] Import Claude Code JSONL transcripts (default: ~/.claude/projects)
- Use --max-files <N> or --max-files=<N> to override the 200-file scan cap (default: 200)
+ --max-files <N> | --max-files=<N>: override scan cap (default 200, max 1000;
+ out-of-range is rejected; for trees >1000 files, batch by subdirectory)
Options:
--help, -h Show this help
@@ -961,6 +962,11 @@ async function runMcp(): Promise<void> {
}
async function runImportJsonl(): Promise<void> {
+ // Long-form flags that take a value. Their value tokens must be
+ // consumed alongside the flag so they don't leak into positional
+ // args (e.g. `--port 3112 import-jsonl` would otherwise turn
+ // 3112 into pathArg).
+ const VALUE_FLAGS = new Set(["--port", "--tools"]);
let maxFiles: number | undefined;
const tail = args.slice(1);
const positional: string[] = [];
@@ -987,6 +993,10 @@ async function runImportJsonl(): Promise<void> {
}
continue;
}
+ if (VALUE_FLAGS.has(a)) {
+ i++;
+ continue;
+ }
if (a.startsWith("-")) continue;
positional.push(a);
}
@@ -1046,7 +1056,9 @@ async function runImportJsonl(): Promise<void> {
observations?: number;
discovered?: number;
truncated?: boolean;
+ traversalCapped?: boolean;
maxFiles?: number;
+ maxFilesUpperBound?: number;
} = {};
if (text.length > 0) {
try {
@@ -1086,12 +1098,30 @@ async function runImportJsonl(): Promise<void> {
);
if (json.truncated) {
const cap = json.maxFiles ?? 200;
- const skipped = (json.discovered ?? 0) - (json.imported ?? 0);
- p.log.warn(
- `Hit the ${cap}-file scan cap; ${skipped} of ${json.discovered ?? "?"} discovered file(s) were skipped. ` +
- `Re-run with --max-files=<N> (e.g. --max-files=${Math.max((json.discovered ?? cap) + 100, cap * 2)}) ` +
- `or batch by subdirectory.`,
- );
+ const upper = json.maxFilesUpperBound ?? 1000;
+ const discovered = json.discovered ?? 0;
+ const skipped = discovered - (json.imported ?? 0);
+ const discoveredLabel = json.traversalCapped
+ ? `${discovered}+ (traversal halted at safety cap)`
+ : String(discovered);
+ const baseMsg = `Hit the ${cap}-file scan cap; ${skipped} of ${discoveredLabel} discovered file(s) were skipped.`;
+ // If we already saw more than the server's hard cap (or the
+ // walker stopped early), bumping --max-files won't help on its
+ // own — recommend batching by subdirectory.
+ if (discovered > upper || json.traversalCapped) {
+ p.log.warn(
+ `${baseMsg} Tree exceeds the server's --max-files limit of ${upper}; ` +
+ `batch by subdirectory (run import-jsonl once per project under ~/.claude/projects).`,
+ );
+ } else {
+ const suggested = Math.min(
+ Math.max((discovered || cap) + 100, cap * 2),
+ upper,
+ );
+ p.log.warn(
+ `${baseMsg} Re-run with --max-files=${suggested} (max ${upper}) or batch by subdirectory.`,
+ );
+ }
}
if (json.sessionIds && json.sessionIds.length > 0) {
p.log.info(`View at ${getViewerUrl()} → Replay tab`);
diff --git a/src/functions/replay.ts b/src/functions/replay.ts
index e89a5411..25ccc439 100644
--- a/src/functions/replay.ts
+++ b/src/functions/replay.ts
@@ -18,6 +18,9 @@ import { buildSyntheticCompression } from "./compress-synthetic.js";
import { getSearchIndex } from "./search.js";
import { logger } from "../logger.js";
+export const MAX_FILES_DEFAULT = 200;
+export const MAX_FILES_UPPER_BOUND = 1000;
+
const SENSITIVE_PATH_PATTERNS: RegExp[] = [
/(^|[\\/_.-])secret([\\/_.-]|s?$)/i,
/(^|[\\/_.-])credentials?([\\/_.-]|$)/i,
@@ -203,16 +206,23 @@ async function loadObservations(
async function findJsonlFiles(
root: string,
limit = 200,
-): Promise<{ files: string[]; truncated: boolean; discovered: number }> {
+): Promise<{
+ files: string[];
+ truncated: boolean;
+ discovered: number;
+ traversalCapped: boolean;
+}> {
const out: string[] = [];
let discovered = 0;
- // Cap how far we walk past the user's requested limit. We keep counting
- // beyond `limit` so we can report `truncated`/`discovered` accurately,
- // but we stop the walk well before million-file trees can lock the
- // 30s function timeout.
- const traversalCap = Math.max(limit * 10, 10_000);
+ let walked = 0;
+ // Hard bound on entries visited (regardless of extension) so trees
+ // dominated by non-jsonl files (node_modules, lockfiles, etc.) cannot
+ // lock the 30s function timeout. `discovered` may underrepresent the
+ // true count when traversalCapped fires — callers should surface that
+ // distinction to the user.
+ const traversalCap = Math.max(limit * 50, 50_000);
async function walk(dir: string) {
- if (discovered >= traversalCap) return;
+ if (walked >= traversalCap) return;
let names: string[];
try {
names = await readdir(dir);
@@ -220,7 +230,8 @@ async function findJsonlFiles(
return;
}
for (const name of names) {
- if (discovered >= traversalCap) return;
+ if (walked >= traversalCap) return;
+ walked++;
const full = join(dir, name);
let st;
try {
@@ -238,7 +249,13 @@ async function findJsonlFiles(
}
}
await walk(root);
- return { files: out, truncated: discovered > out.length, discovered };
+ const traversalCapped = walked >= traversalCap;
+ return {
+ files: out,
+ truncated: discovered > out.length || traversalCapped,
+ discovered,
+ traversalCapped,
+ };
}
export function registerReplayFunctions(sdk: ISdk, kv: StateKV): void {
@@ -279,7 +296,9 @@ export function registerReplayFunctions(sdk: ISdk, kv: StateKV): void {
observations: number;
discovered: number;
truncated: boolean;
+ traversalCapped: boolean;
maxFiles: number;
+ maxFilesUpperBound: number;
}
| { success: false; error: string }
> => {
@@ -306,22 +325,24 @@ export function registerReplayFunctions(sdk: ISdk, kv: StateKV): void {
return { success: false, error: "path not found" };
}
- const MAX_FILES_DEFAULT = 200;
- const MAX_FILES_UPPER_BOUND = 1000;
+ // Valid integer requests are clamped to MAX_FILES_UPPER_BOUND so
+ // callers see a stable maxFiles in the response. Non-integer or
+ // <= 0 falls back to the safe default. The HTTP layer rejects
+ // out-of-range up front; this is the SDK-callable safety net.
const maxFiles =
- Number.isInteger(data.maxFiles) &&
- (data.maxFiles as number) > 0 &&
- (data.maxFiles as number) <= MAX_FILES_UPPER_BOUND
- ? (data.maxFiles as number)
+ Number.isInteger(data.maxFiles) && (data.maxFiles as number) > 0
+ ? Math.min(data.maxFiles as number, MAX_FILES_UPPER_BOUND)
: MAX_FILES_DEFAULT;
let files: string[] = [];
let truncated = false;
let discovered = 0;
+ let traversalCapped = false;
if (stat.isDirectory()) {
const found = await findJsonlFiles(abs, maxFiles);
files = found.files;
truncated = found.truncated;
discovered = found.discovered;
+ traversalCapped = found.traversalCapped;
} else if (stat.isFile() && abs.endsWith(".jsonl")) {
files = [abs];
discovered = 1;
@@ -337,7 +358,9 @@ export function registerReplayFunctions(sdk: ISdk, kv: StateKV): void {
observations: 0,
discovered,
truncated,
+ traversalCapped,
maxFiles,
+ maxFilesUpperBound: MAX_FILES_UPPER_BOUND,
};
}
@@ -436,7 +459,9 @@ export function registerReplayFunctions(sdk: ISdk, kv: StateKV): void {
observations: observationCount,
discovered,
truncated,
+ traversalCapped,
maxFiles,
+ maxFilesUpperBound: MAX_FILES_UPPER_BOUND,
};
},
);
diff --git a/src/triggers/api.ts b/src/triggers/api.ts
index 75549a9a..6a65f233 100644
--- a/src/triggers/api.ts
+++ b/src/triggers/api.ts
@@ -8,6 +8,7 @@ import type { ResilientProvider } from "../providers/resilient.js";
import { VERSION } from "../version.js";
import { timingSafeCompare } from "../auth.js";
import { renderViewerDocument } from "../viewer/document.js";
+import { MAX_FILES_UPPER_BOUND } from "../functions/replay.js";
import {
isGraphExtractionEnabled,
isConsolidationEnabled,
@@ -486,13 +487,20 @@ export function registerApiTriggers(
payload.path = body.path.trim();
}
if (body.maxFiles !== undefined) {
- if (!Number.isInteger(body.maxFiles) || (body.maxFiles as number) < 1) {
+ const n = body.maxFiles as number;
+ if (
+ !Number.isInteger(n) ||
+ n < 1 ||
+ n > MAX_FILES_UPPER_BOUND
+ ) {
return {
status_code: 400,
- body: { error: "maxFiles must be a positive integer" },
+ body: {
+ error: `maxFiles must be an integer between 1 and ${MAX_FILES_UPPER_BOUND}`,
+ },
};
}
- payload.maxFiles = body.maxFiles as number;
+ payload.maxFiles = n;
}
const result = await sdk.trigger({
function_id: "mem::replay::import-jsonl",