Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
fix: remove raw NUL bytes that made the two largest sources unsearchable (#334)
KageBinaryddf5cff54bce
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 58993745..23cea901 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -25,6 +25,34 @@ jobs:- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1with:persist-credentials: false+ # A raw NUL byte makes a file "binary" to grep/ripgrep, which then skips it+ # silently — including `ix text`, which shells out to rg. Two of the+ # largest sources in this repo carried them and were invisible to search.+ # Needs no toolchain, so it runs first and fails in seconds.+ - name: No raw NUL bytes in tracked sources+ # `shell: bash` for -o pipefail: without it the `tr | wc` pipeline below+ # reports wc's exit status, so a failed read would score as "0 NULs".+ shell: bash+ run: |+ status=0+ checked=0+ # -z / read -d '': a path containing a space would otherwise word-split,+ # the redirect would fail, and `wc -c` would still print 0 — passing the+ # file as clean. A guard against silent skipping must not skip silently.+ while IFS= read -r -d '' f; do+ checked=$((checked + 1))+ n=$(tr -dc '\000' < "$f" | wc -c)+ if [ "$n" -gt 0 ]; then+ echo "::error file=$f::$n raw NUL byte(s) — use the \\x00 escape instead; a raw NUL makes this file binary to grep/ripgrep, which silently skips it"+ status=1+ fi+ done < <(git ls-files -z '*.ts' '*.tsx' '*.mts' '*.cts' '*.js' '*.jsx' '*.mjs' '*.cjs')+ if [ "$checked" -eq 0 ]; then+ echo "::error::NUL-byte guard matched no files — the pathspecs or working directory are wrong, so this check is protecting nothing"+ exit 1+ fi+ echo "checked $checked tracked sources"+ exit $status- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0with:node-version: "22"diff --git a/core-ingestion/src/index.ts b/core-ingestion/src/index.tsindex 8dcc0eca..e5d22723 100644--- a/core-ingestion/src/index.ts+++ b/core-ingestion/src/index.ts@@ -508,7 +508,7 @@ function extractJsExportPublicNames(rootNode: any): ExportPublicName[] {const local = localN.text;const pub = pubN && pubN.text ? pubN.text : local;if (pub !== local) {- const k = `${local} ${pub}`;+ const k = `${local}\x00${pub}`;if (!seen.has(k)) { seen.add(k); out.push({ local, public: pub }); }}}@@ -518,7 +518,7 @@ function extractJsExportPublicNames(rootNode: any): ExportPublicName[] {const nameNode = (decl.childForFieldName && decl.childForFieldName('name')) || decl.namedChild(0);const nm = nameNode?.text;if (nm) {- const k = `${nm} default`;+ const k = `${nm}\x00default`;if (!seen.has(k)) { seen.add(k); out.push({ local: nm, public: 'default' }); }}}@@ -1193,7 +1193,7 @@ function parseLatexFile(filePath: string, source: string): FileParseResult {const envName = (args[0] ?? '').trim();if (envName) {const cont = container();- const key = `${cont} ${envName}`;+ const key = `${cont}\x00${envName}`;if (!seenEnv.has(key)) {seenEnv.add(key);entities.push({ name: envName, kind: 'environment', lineStart: line, lineEnd: line, language, container: cont === fileName ? undefined : cont });diff --git a/ix-cli/src/cli/commands/ingest.ts b/ix-cli/src/cli/commands/ingest.tsindex d4c29778..cf768ae6 100644--- a/ix-cli/src/cli/commands/ingest.ts+++ b/ix-cli/src/cli/commands/ingest.ts@@ -541,7 +541,7 @@ export async function ingestFiles(const pkg = specToPkg(typeof rel.importRaw === 'string' ? rel.importRaw : rel.dstName);if (!pkg || !stitchProdDeps.has(pkg)) continue;const consumerNodeId = fileNodeId(workspaceId, p.filePath);- const k = `${pkg} ${consumerNodeId}`;+ const k = `${pkg}\u0000${consumerNodeId}`;if (stitchConsumeSeen.has(k)) continue;stitchConsumeSeen.add(k);stitchConsumes.push({ name: pkg, consumerNodeId });@@ -1299,7 +1299,7 @@ export async function ingestFiles(const def = stitchDefs.get(a.local);if (!def) continue;const nodeId = symbolNodeId(workspaceId, def.filePath, def.qk);- const ek = `${a.pub} ${nodeId}`;+ const ek = `${a.pub}\u0000${nodeId}`;if (stitchExportAliasSeen.has(ek)) continue;stitchExportAliasSeen.add(ek);stitchExports.push({ name: a.pub, nodeId });@@ -1327,7 +1327,7 @@ export async function ingestFiles(if (stitchDefs.has(c.symbol)) continue;}const callerNodeId = symbolNodeId(workspaceId, callerDef.filePath, callerDef.qk);- const sk = `${symbol} ${pkg ?? ''} ${callerNodeId}`;+ const sk = `${symbol}\u0000${pkg ?? ''}\u0000${callerNodeId}`;if (symbolConsumeSeen.has(sk)) continue;symbolConsumeSeen.add(sk);symbolConsumes.push({ symbol, callerNodeId, pkg });