Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

fix(ingest): keep import.ts, runImport and runConnectorSync within their size caps; regenerate the skills manifest and plugin tree (#7085 #7689)

garrytan7ea8c39ad5be
diff --git a/plugin/skills/chat-connectors/SKILL.md b/plugin/skills/chat-connectors/SKILL.md
index 3f1ee25050..b663058f94 100644
--- a/plugin/skills/chat-connectors/SKILL.md
+++ b/plugin/skills/chat-connectors/SKILL.md
@@ -177,7 +177,8 @@ This skill guarantees:
- Sync is incremental: a durable config-scalar watermark plus a trailing-window
re-list; the watermark advances ONLY on a fully clean run (no fetch errors, no
`--limit` cap, clean ingest), so a partial run never creates a silent gap.
-- `--dry-run` fetches nothing and writes nothing; `--limit N` caps a run and does
+- `--dry-run` fetches nothing and writes nothing; a listing that stopped early
+ reports `partial` with `listComplete: false`. `--limit N` caps a run and does
not advance the watermark.
- Scheduled auto-sync is opt-in per provider (`connectors.<provider>.auto_sync`),
daily by default, credential-gated and auth-error-gated; it fires for nobody
diff --git a/skills/skills.lock.json b/skills/skills.lock.json
index ad82dfcdd5..8859f1b01c 100644
--- a/skills/skills.lock.json
+++ b/skills/skills.lock.json
@@ -31,7 +31,7 @@
"bulk-ingestion/SKILL.md": "48c10ec94fc23cd6f1ecfa4ec7254cc1cb3f5ea8cc43daf2fbe109c59ce1cda4",
"bulk-ingestion/routing-eval.jsonl": "2bf8ffaf90b8ef761fa3cef54bf3af2eb8cc3ef736d054b5bf8b9361f6448d5a",
"capture/SKILL.md": "11434d995ae4d2d40211f3593f75fa48b3b0e6b6c5445547476462510f63da7f",
- "chat-connectors/SKILL.md": "44b63309c779b5394c0be8ec6cabff8381099cfb8c0944143a859c48594ddf01",
+ "chat-connectors/SKILL.md": "1dd6aa8c30b28a0f7d460bc7c6cdb93720661c54b4782651acd19e1c04414fae",
"chat-connectors/routing-eval.jsonl": "688b6bab2020a9628535fb739019f1ac54098e17327e6b1c8828c0d0c4a58be6",
"citation-fixer/SKILL.md": "22dde60db9d5df0a859e131e7e2d7347d5205a3d20ea1ed0d9c4eb373fb11cc9",
"citation-fixer/routing-eval.jsonl": "52b23b71e66fdc18aee67d0576099b0c83997d648cf4ecf8fe7753b91b6c9c53",
diff --git a/src/commands/import.ts b/src/commands/import.ts
index 0e82650b5c..e7580919f5 100644
--- a/src/commands/import.ts
+++ b/src/commands/import.ts
@@ -42,13 +42,12 @@ import { realpathOrResolve } from '../core/path-confine.ts';
import { slog } from '../core/console-prefix.ts';
import { refreshProjectionStatistics } from '../core/search/projection-statistics.ts';
import { importAnalyzeEveryPages, maybeRefreshPlannerStats, PLANNER_STATS_REPAIR_COMMAND } from '../core/planner-stats.ts';
-import { importManagedFile, nextImportBatch, settleManagedImportBatch } from '../core/persistence/import-mutations.ts';
+import { importManagedFile, managedImportWriteThroughRefusal, nextImportBatch, settleManagedImportBatch } from '../core/persistence/import-mutations.ts';
import { acceptedPendingReceipt } from '../core/persistence/accepted-pending.ts';
import { estimateCostFromChars, lookupEmbeddingPrice } from '../core/embedding-pricing.ts';
import { getEmbeddingModel } from '../core/ai/gateway.ts';
import { managedRootMarkerFor } from '../core/persistence/root-registry.ts';
import { readManagedMode } from '../core/persistence/ownership.ts';
-import { writeThroughConfigured } from '../core/persistence/switches.ts';
import { readPageBodyPolicy } from '../core/page-body/limits.ts';
/** Return a refusal when an import target lies outside every admitted root. */
@@ -169,20 +168,6 @@ function symlinkedRootAbort(dirArg: string, dir: string): ImportAbortError {
return new ImportAbortError('managed import refuses a symlinked input root');
}
-/**
- * #7085: with `sync.write_through` off a page write records no worktree, and a managed import publishes only through
- * the source's canonical owner, so every file would be admitted and then refused. Refuse once, before any admission.
- */
-function writeThroughOffAbort(sourceId: string): ImportAbortError {
- const error = opError('owner_unavailable', 'Managed import publishes through the canonical owner, and sync.write_through is off on this brain.',
- `Nothing was imported and no write was admitted. Copy the files into source ${sourceId}'s checkout, commit them, then run gbrain sync --source ${sourceId}.`,
- { detail: 'disabled_by_config', retryable: false, fix: { argv: ['gbrain', 'sync', '--source', sourceId], consent: [], actor: 'agent',
- why: `Imports the files committed in source ${sourceId}'s checkout; run it after committing them.`, requires_exclusive: false } });
- console.error(`Error [${error.code}]: ${error.message}`);
- console.error(`Fix: ${error.suggestion}`);
- return new ImportAbortError('sync.write_through is off, so a managed import cannot publish', 1, undefined, error);
-}
-
/** A refused source filesystem lock admission: print its code, message and fix, and keep it as the cause. */
function lockAdmissionAbort(dirArg: string, dir: string, error: unknown): ImportAbortError {
const message = error instanceof Error ? error.message : String(error);
@@ -497,11 +482,7 @@ export async function runImport(
// v0.22.13 (PR #490 Q2): shared parseWorkers helper rejects bad input
// (--workers 0, -3, "foo") with a loud error instead of silently falling
// through to 1. Mirrors sync.ts's flag handling.
- const {
- clampWorkersForConnectionBudget,
- parseWorkers,
- resolveMaxConnections,
- } = await import('../core/sync-concurrency.ts');
+ const { clampWorkersForConnectionBudget, parseWorkers, resolveMaxConnections } = await import('../core/sync-concurrency.ts');
let workerCount: number;
try {
workerCount = parseWorkers(workersArg ?? undefined) ?? 1;
@@ -529,15 +510,16 @@ export async function runImport(
try {
dir = resolveImportTargetDir(dirArg);
} catch (e) {
- const msg = e instanceof Error ? e.message : String(e);
- console.error(`Import target is not readable: ${dirArg} (${msg})`);
+ console.error(`Import target is not readable: ${dirArg} (${e instanceof Error ? e.message : String(e)})`);
throw new ImportAbortError(`import target not readable: ${dirArg}`);
}
const managedImport = await readManagedMode(engine);
if (managedImport && dir !== resolve(dirArg)) throw symlinkedRootAbort(dirArg, dir);
- if (managedImport && !opts.managedBookmark && !currentCompanyBrainSync(sourceId) && !writeThroughConfigured(await engine.getConfig('sync.write_through'))) {
- throw writeThroughOffAbort(sourceId ?? 'default');
+ const preAdmission = managedImport && !opts.managedBookmark && !currentCompanyBrainSync(sourceId) ? await managedImportWriteThroughRefusal(engine, sourceId ?? 'default') : null;
+ if (preAdmission) {
+ console.error(`Error [${preAdmission.code}]: ${preAdmission.message}\nFix: ${preAdmission.suggestion}`);
+ throw new ImportAbortError(preAdmission.message, 1, undefined, preAdmission);
}
const singleFile = managedImport && lstatSync(dir).isFile();
const importRoot = singleFile ? dirname(dir) : dir;
diff --git a/src/core/connectors/sync.ts b/src/core/connectors/sync.ts
index 507b1a1ca3..234711dcb3 100644
--- a/src/core/connectors/sync.ts
+++ b/src/core/connectors/sync.ts
@@ -343,17 +343,8 @@ export async function runConnectorSync(
const toFetch = [...plan.retries, ...plan.listed];
if (opts.dryRun) {
- return {
- ...base,
- status: listErrored ? 'partial' : 'dry_run',
- listed: stubs.length,
- unresolved: Object.keys(failed).length,
- listComplete: !listErrored,
- wouldFetch: toFetch.length,
- hint: listErrored
- ? `the listing stopped early; ${toFetch.length} conversation(s) listed so far would be fetched`
- : `${toFetch.length} conversation(s) would be fetched`,
- };
+ const hint = listErrored ? `the listing stopped early; ${toFetch.length} conversation(s) listed so far would be fetched` : `${toFetch.length} conversation(s) would be fetched`;
+ return { ...base, status: listErrored ? 'partial' : 'dry_run', listed: stubs.length, unresolved: Object.keys(failed).length, listComplete: !listErrored, wouldFetch: toFetch.length, hint };
}
// FETCH details + spool + ingest, in batches (memory-bounded; banks progress).
diff --git a/src/core/persistence/import-mutations.ts b/src/core/persistence/import-mutations.ts
index afd54b13d0..633214c1be 100644
--- a/src/core/persistence/import-mutations.ts
+++ b/src/core/persistence/import-mutations.ts
@@ -27,6 +27,7 @@ import { pageSnapshotKey } from '../page-snapshot-batch.ts';
import type { WorktreeBinding } from './ownership.ts';
import type { PageSnapshot } from '../page-state/types.ts';
import { readFix, trustedCliRequired } from '../ops/op-fix.ts';
+import { writeThroughConfigured } from './switches.ts';
/** One file of a managed import batch: its path and the source-relative path its slug is derived from. */
export interface ManagedImportFile { filePath: string; sourcePath: string }
@@ -75,6 +76,18 @@ type PendingImport = ManagedImportIntent & { request_id: string; source_id: stri
*/
type StoredImport = Omit<PendingImport, 'content'> & { content?: string };
+/**
+ * #7085: with `sync.write_through` off a page write records no worktree, and a managed import publishes only through
+ * the source's canonical owner, so every file would be admitted and then refused. The import refuses once, before any admission.
+ */
+export async function managedImportWriteThroughRefusal(engine: BrainEngine, sourceId: string): Promise<OperationError | null> {
+ if (writeThroughConfigured(await engine.getConfig('sync.write_through'))) return null;
+ return opError('owner_unavailable', 'Managed import publishes through the canonical owner, and sync.write_through is off on this brain.',
+ `Nothing was imported and no write was admitted. Copy the files into source ${sourceId}'s checkout, commit them, then run gbrain sync --source ${sourceId}.`,
+ { detail: 'disabled_by_config', retryable: false, fix: { argv: ['gbrain', 'sync', '--source', sourceId], consent: [], actor: 'agent',
+ why: `Imports the files committed in source ${sourceId}'s checkout; run it after committing them.`, requires_exclusive: false } });
+}
+
export async function importManagedFile(engine: BrainEngine, filePath: string, sourcePath: string,
opts: { sourceId?: string; noEmbed?: boolean; activePack?: ImportPack; signal?: AbortSignal; slugRoot?: string } = {}): Promise<ImportResult> {
const [settled] = await importManagedFiles(engine, [{ filePath, sourcePath }], opts);