Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
import { withConnectorSync, rethrowConnectorWriteError, pendingConnectorResult, type ManagedConnectorSync } from './persistence/connector-sync.ts';
import { resolveGitHubAccount } from './persistence/connector-account.ts';
import { isValidRepoName } from './github-source-config.ts';
import { connectorRender } from './connectors/connector-text.ts';
import { ConnectorHoldSession, connectorHoldsResult } from './connectors/connector-hold-session.ts';
import { connectorDryRunResult } from './connectors/api-source.ts';
export { isValidRepoName, parseGitHubSourceConfig } from './github-source-config.ts';
export { AppTokenProvider, mintAppInstallationToken } from './github-app-token.ts';
import { AppTokenProvider } from './github-app-token.ts';
import { slugifyPath } from './sync.ts';
/**
* github-source — GitHub issues/PR sync for the `github` source kind.
*
* Opt-in source kind (v0.46): a source registered with kind=github mirrors
* issues, pull requests, comments, reviews, review comments, labels,
* assignees, milestones and open-PR checks summaries into markdown pages
* under the source's managed directory. Pages flow through the standard
* import pipeline (chunks, embeds, link extraction, dream-cycle atoms), so
* the brain's existing machinery does the rest.
*
* Freshness model (three layers, cheapest to fastest):
* 1. `gbrain sync --source <id>` — delta sweep via the `since` filter.
* Picks up everything changed since the last sweep. Zero standing infra.
* 2. `gbrain sync --source <id> --full` — full reconcile: re-enumerates
* every item, refreshes stale pages, deletes pages for vanished items.
* Run nightly via cron or autopilot.
* 3. Webhook (`POST /webhooks/github`) — instant targeted refresh of the
* single item that changed. Optional accelerator; the webhook handler
* submits a `sync` job with `github_item` and this module refreshes
* exactly that item.
*
* Conventions:
* - Slug per item: gh/<owner>/<repo>/<n> — GitHub numbers are unique per
* repo across issues AND PRs, so one namespace per repo is correct.
* - Repo card slug: gh/<owner>/<repo>.
* - Every `#<n>` mention in a body or comment becomes a wikilink to the
* item page, and Closes/Fixes/Resolves references become explicit links.
*
* Rate limits: the client honors x-ratelimit headers, backs off on 403/429
* and is resumable by construction (each page carries the API updated_at in
* frontmatter; a re-run skips items whose page is already fresh).
*/
import { readFileSync, writeFileSync, mkdirSync, existsSync, rmSync, renameSync } from 'node:fs';
import { dirname, join, relative } from 'node:path';
import type { BrainEngine } from './engine.ts';
import type { SyncOpts } from '../commands/sync.ts';
import { isWriteTargetContained } from './path-confine.ts';
import { createProgress, startHeartbeat } from './progress.ts';
import { getCliOptions, cliOptsToProgressOptions } from './cli-options.ts';
import { replaceableFile } from './facts/file-body-guard.ts';
import { assertInlineBody } from './page-body/page-shape.ts';
import { OperationError } from './ops/contract.ts';
import { isWriteThroughDisabled } from './write-through.ts';
import { isSourceDbOnlySlug } from './persistence/source-storage.ts';
import { neutralizeOutsideText } from './outside-content.ts';
// ── Types ────────────────────────────────────────────────────────────────────
export interface GitHubAppConfig {
/** GitHub App ID (Settings -> Developer settings -> GitHub Apps). */
appId: number;
/** Path to the app's private key PEM (rsa). */
pemPath: string;
/** Installation id; when absent the first installation of the app is used. */
installId?: number;
}
export interface GitHubSourceConfig {
/** Env var holding the token (default GH_TOKEN). Ignored when `app` is set. */
tokenEnv: string;
/** GitHub App credentials; when set, the sync mints hourly installation tokens itself. */
app: GitHubAppConfig | null;
/** 'auto' = owner + collaborator + org-member repos; 'repos' = explicit list. */
scope: 'auto' | 'repos';
/** owner/name list (lowercase), only when scope === 'repos'. */
repos: string[];
/** Managed dir where pages are materialized. */
dir: string;
}
export interface GitHubItemRef {
repo: string; // owner/name
number: number;
kind: 'issue' | 'pr';
deleted?: boolean;
}
export interface GitHubSyncSummary {
status: 'synced' | 'up_to_date' | 'first_sync' | 'partial';
added: number;
modified: number;
deleted: number;
chunksCreated: number;
embedded: number;
pagesAffected: string[];
itemsSeen: number;
itemDetailFetches: number;
failedFiles: number;
}
// ── Config ───────────────────────────────────────────────────────────────────
const GH_KIND = 'github';
export function isGitHubSourceConfig(config: Record<string, unknown>): boolean {
return config.kind === GH_KIND;
}
export function gitHubStateFile(dir: string): string {
return join(dir, '.github-source.json');
}
interface GitHubState {
last_sweep_at: string | null;
/** owner/name -> default branch (for check fetches we only need head sha, so this stays small). */
repos: string[];
/** Fix wave 4: connector item holds (src/core/connectors/item-holds.ts). */
item_holds?: unknown;
}
function readState(dir: string): GitHubState {
try {
const raw = readFileSync(gitHubStateFile(dir), 'utf-8');
const parsed = JSON.parse(raw) as Partial<GitHubState>;
return {
last_sweep_at: typeof parsed.last_sweep_at === 'string' ? parsed.last_sweep_at : null,
// Legacy state files may carry canonical-case names; the module keys
// on lowercase everywhere.
repos: Array.isArray(parsed.repos)
? parsed.repos.filter((r): r is string => typeof r === 'string').map((r) => r.toLowerCase())
: [],
...(parsed.item_holds ? { item_holds: parsed.item_holds } : {}),
};
} catch {
return { last_sweep_at: null, repos: [] };
}
}
function writeState(dir: string, state: GitHubState): void {
mkdirSync(dir, { recursive: true });
writeFileSync(gitHubStateFile(dir), JSON.stringify(state, null, 2), 'utf-8');
}
// ── HTTP client (rate-limit aware, injectable for tests) ─────────────────────
export type FetchImpl = (url: string, init: RequestInit) => Promise<Response>;
interface RateInfo {
remaining: number | null;
resetAt: number | null;
}
// ── Token acquisition (PAT or GitHub App) ────────────────────────────────────
/** A credential source the client can refresh mid-run (apps mint hourly tokens). */
export interface GitHubTokenProvider {
getToken(): Promise<string>;
refresh(): Promise<string>;
}
export class GitHubClient {
constructor(
private readonly token: string | GitHubTokenProvider,
private readonly fetchImpl: FetchImpl = fetch,
public readonly log: (msg: string) => void = () => {},
) {}
private rate: RateInfo = { remaining: null, resetAt: null };
private apiUrl(path: string): string {
return `https://api.github.com${path}`;
}
/** Wait for the rate-limit reset when we are near the bucket edge. */
private async waitForBucket(signal: AbortSignal | undefined): Promise<void> {
const { remaining, resetAt } = this.rate;
if (remaining === null || resetAt === null) return;
if (remaining > 20) return;
const waitMs = Math.max(0, resetAt - Date.now()) + 1000;
if (waitMs <= 0) return;
this.log(`[github] rate bucket low (${remaining} left); waiting ${Math.round(waitMs / 1000)}s`);
await new Promise<void>((resolve) => {
const t = setTimeout(resolve, waitMs);
if (signal) {
signal.addEventListener('abort', () => {
clearTimeout(t);
resolve();
}, { once: true });
}
});
}
private trackRate(res: Response): void {
const remaining = res.headers.get('x-ratelimit-remaining');
const reset = res.headers.get('x-ratelimit-reset');
if (remaining !== null) this.rate.remaining = Number(remaining);
if (reset !== null) this.rate.resetAt = Number(reset) * 1000;
}
async fetchJSON<T>(
path: string,
opts: { signal?: AbortSignal; retries?: number } = {},
): Promise<T> {
const { data } = await this.fetchJSONWithMeta<T>(path, opts);
return data;
}
private async tokenValue(): Promise<string> {
return typeof this.token === 'string' ? this.token : this.token.getToken();
}
private async fetchJSONWithMeta<T>(
pathOrUrl: string,
opts: { signal?: AbortSignal; retries?: number } = {},
): Promise<{ data: T; link: string | null }> {
const url = pathOrUrl.startsWith('http') ? pathOrUrl : this.apiUrl(pathOrUrl);
const retries = opts.retries ?? 1;
for (let attempt = 0; attempt <= retries; attempt++) {
await this.waitForBucket(opts.signal);
const res = await this.fetchImpl(url, {
headers: {
accept: 'application/vnd.github+json',
authorization: `Bearer ${await this.tokenValue()}`,
'x-github-api-version': '2022-11-28',
},
signal: opts.signal,
});
this.trackRate(res);
if (res.status === 401 && typeof this.token !== 'string' && attempt < retries) {
// Installation tokens expire hourly; a 401 mid-run means the minted
// token lapsed. Refresh once and retry the same request.
this.log('[github] HTTP 401; refreshing token');
await this.token.refresh();
continue;
}
if (res.status === 403 || res.status === 429) {
const retryAfterMs = parseRetryAfterMs(res.headers.get('retry-after'));
// A 403 with no Retry-After and budget left in the rate bucket is a
// permission/abuse error, not a rate limit — sleeping until the
// bucket reset can't fix it.
const rateExhausted = this.rate.remaining !== null && this.rate.remaining <= 2;
if (res.status === 403 && retryAfterMs === null && !rateExhausted) {
throw new Error(`GitHub API HTTP 403 on ${pathOrUrl} (not rate-limited; check token permissions)`);
}
const waitMs = retryAfterMs !== null ? retryAfterMs : this.rate.resetAt !== null
? Math.max(0, this.rate.resetAt - Date.now()) + 1000
: 60_000;
if (attempt < retries) {
this.log(`[github] HTTP ${res.status}; retrying in ${Math.round(waitMs / 1000)}s`);
await new Promise<void>((resolve) => {
const t = setTimeout(resolve, waitMs);
if (opts.signal) {
opts.signal.addEventListener('abort', () => { clearTimeout(t); resolve(); }, { once: true });
}
});
continue;
}
throw new Error(`GitHub API HTTP ${res.status} on ${pathOrUrl}`);
}
if (!res.ok) {
throw new Error(`GitHub API HTTP ${res.status} on ${pathOrUrl}`);
}
return { data: (await res.json()) as T, link: res.headers.get('link') };
}
throw new Error(`GitHub API unreachable on ${pathOrUrl}`);
}
/**
* GET all pages of a paginated list, concatenated. Follows the Link
* header (rel="next"), which GitHub sends for every paginated endpoint.
* Throws GitHubPaginationError when the safety cap is hit so callers
* can treat the enumeration as incomplete (never reconcile a truncated
* list against the brain).
*/
async fetchAllPages<T>(
path: string,
opts: { signal?: AbortSignal; perPage?: number; field?: string } = {},
): Promise<T[]> {
const perPage = opts.perPage ?? 100;
const out: T[] = [];
let url = `${path}${path.includes('?') ? '&' : '?'}per_page=${perPage}&page=1`;
let pages = 0;
for (;;) {
const { data, link } = await this.fetchJSONWithMeta<Record<string, unknown> | unknown[]>(url, opts);
const raw = Array.isArray(data) ? data : opts.field ? (data as Record<string, unknown>)[opts.field] : data;
// Fail loud on shape drift: silently treating a non-array payload as
// empty would let a reconcile see "no items" where the API sent some.
if (!Array.isArray(raw)) {
throw new Error(
`GitHub API returned a non-array payload on ${path}${opts.field ? ` (field "${opts.field}")` : ''}`,
);
}
const batch = raw as T[];
out.push(...batch);
const next = link !== null ? linkNextUrl(link) : null;
if (next === null) break;
// Never follow a Link header off api.github.com: the bearer token
// must not leak to an arbitrary host (codex LOW, round 3).
if (!next.startsWith('https://api.github.com/')) {
throw new GitHubPaginationError(`Link header points off api.github.com: ${next}`);
}
// Cap counts only pages that actually continue: a complete 500-page
// enumeration is valid, a 501st page is not (codex LOW, round 4).
pages++;
if (pages >= 500) {
throw new GitHubPaginationError(`pagination cap (500 pages) hit on ${path}; refusing to treat a truncated list as complete`);
}
url = next;
}
return out;
}
}
/**
* Retry-After is either delta-seconds or an HTTP-date (RFC 9110 §10.2.3).
* Returns milliseconds to wait, or null when absent/unparseable — never NaN,
* which would turn the backoff sleep into a hot retry loop.
*/
export function parseRetryAfterMs(value: string | null, nowMs: number = Date.now()): number | null {
if (value === null || value.trim() === '') return null;
const secs = Number(value);
if (Number.isFinite(secs)) return Math.max(0, secs * 1000);
const dateMs = Date.parse(value);
return Number.isFinite(dateMs) ? Math.max(0, dateMs - nowMs) : null;
}
/** Thrown when pagination hits the safety cap; callers treat enumeration as incomplete. */
export class GitHubPaginationError extends Error {
constructor(message: string) {
super(message);
this.name = 'GitHubPaginationError';
}
}
/** Parse the Link header and return the rel="next" URL, or null. */
export function linkNextUrl(link: string): string | null {
for (const part of link.split(',')) {
const m = part.match(/<([^>]+)>;\s*rel="next"/);
if (m) return m[1];
}
return null;
}
// ── Scope resolution ─────────────────────────────────────────────────────────
interface RawRepo {
full_name: string;
private: boolean;
archived: boolean;
default_branch: string;
description: string | null;
}
/**
* Expand the source's scope to the concrete owner/name list (lowercase —
* GitHub repo names are case-insensitive and the whole module keys on the
* lowercase form).
* auto = affiliation owner,collaborator,organization_member (paginated).
*
* NEVER persists state.repos: only a sweep that fully succeeded for a repo
* may state-list it (see runGitHubSync). Persisting at discovery time — in
* particular during a webhook single-item refresh — would mark unswept repos
* as already bootstrapped and silently skip their history on the next sweep.
*/
export async function resolveScopeRepos(
cfg: GitHubSourceConfig,
client: GitHubClient,
signal?: AbortSignal,
): Promise<string[]> {
if (cfg.scope === 'repos') {
return [...new Set(cfg.repos.map((r) => r.toLowerCase()))];
}
// Installation tokens cannot call /user/repos, so the app path resolves
// through /installation/repositories (object-shaped: { repositories }).
const names = cfg.app
? (
await client.fetchAllPages<RawRepo>('/installation/repositories', {
signal,
field: 'repositories',
})
).map((r) => r.full_name.toLowerCase())
: (
await client.fetchAllPages<RawRepo>(
'/user/repos?affiliation=owner,collaborator,organization_member&sort=full_name',
{ signal },
)
).map((r) => r.full_name.toLowerCase());
return [...new Set(names)].sort();
}
// ── Item enumeration ─────────────────────────────────────────────────────────
/**
* List-payload shapes. The enumeration endpoints return most issue fields
* (body, labels, assignees, author, milestone, dates) without extra calls,
* so pass 1 of a sync materializes pages from these alone.
*/
interface RawIssueListItem {
number: number;
title: string;
state: 'open' | 'closed';
updated_at: string;
pull_request?: { url: string };
body?: string | null;
html_url?: string;
created_at?: string;
closed_at?: string | null;
labels?: { name: string }[];
assignees?: { login: string }[];
milestone?: RawMilestone | null;
user?: { login: string } | null;
draft?: boolean;
}
interface RawPullListItem {
number: number;
title: string;
state: 'open' | 'closed';
updated_at: string;
head: { sha: string };
body?: string | null;
html_url?: string;
created_at?: string;
closed_at?: string | null;
labels?: { name: string }[];
assignees?: { login: string }[];
milestone?: RawMilestone | null;
user?: { login: string } | null;
draft?: boolean;
merged?: boolean;
mergeable_state?: string | null;
}
/**
* Enumerate items for one repo. `since` (ISO) restricts to items updated
* after it; when absent the full history is enumerated.
* Returns { issues, prs } where prs carry head sha for open-PR checks.
*/
export async function enumerateRepoItems(
repo: string,
client: GitHubClient,
opts: { since?: string; signal?: AbortSignal } = {},
): Promise<{ issues: RawIssueListItem[]; prs: RawPullListItem[] }> {
const sinceQuery = opts.since ? `&since=${encodeURIComponent(opts.since)}` : '';
// The issues endpoint returns PRs too; we classify by the pull_request key.
const all = await client.fetchAllPages<RawIssueListItem>(
`/repos/${repo}/issues?state=all${sinceQuery}`,
opts,
);
const issues = all.filter((i) => !i.pull_request);
const prsFromIssues = all.filter((i) => i.pull_request);
const updatedByNumber = new Map(prsFromIssues.map((i) => [i.number, i.updated_at]));
const prNumbers = prsFromIssues.map((i) => i.number);
// Open PRs get head sha so we can refresh checks cheaply.
const openPrs = await client.fetchAllPages<RawPullListItem>(
`/repos/${repo}/pulls?state=open`,
opts,
);
const openByNumber = new Map(openPrs.map((p) => [p.number, p]));
const prs: RawPullListItem[] = [];
for (const n of prNumbers) {
const open = openByNumber.get(n);
if (open) {
prs.push(open);
} else {
// Keep the real updated_at from the issues list so delta sweeps
// re-fetch closed PRs that changed (comments, merge, review), and
// carry the issue-list fields so a pass-1 render is never blank.
const src = prsFromIssues.find((i) => i.number === n);
prs.push({
number: n,
title: src?.title ?? '',
state: 'closed',
updated_at: updatedByNumber.get(n) ?? '',
head: { sha: '' },
body: src?.body ?? null,
html_url: src?.html_url ?? '',
created_at: src?.created_at,
closed_at: src?.closed_at ?? null,
labels: src?.labels ?? [],
assignees: src?.assignees ?? [],
milestone: src?.milestone ?? null,
user: src?.user ?? null,
draft: src?.draft,
});
}
}
// Open PRs whose updated_at predates `since` never appear in the issues
// list, but their check state can change without bumping updated_at (a new
// check run doesn't touch it). Union them in so the per-sweep open-PR
// refresh actually sees every open PR, not just recently-updated ones.
const sinceFiltered = new Set(prNumbers);
for (const p of openPrs) {
if (!sinceFiltered.has(p.number)) prs.push(p);
}
return { issues, prs };
}
// ── Detail fetching ──────────────────────────────────────────────────────────
interface RawComment {
user: { login: string } | null;
body: string;
created_at: string;
}
interface RawReview {
user: { login: string } | null;
state: string;
body: string;
submitted_at: string | null;
}
interface RawReviewComment {
user: { login: string } | null;
body: string;
created_at: string;
path: string;
line: number | null;
original_line: number | null;
}
interface RawCheckRun {
name: string;
status: string;
conclusion: string | null;
}
interface RawCheckRuns {
total_count: number;
check_runs: RawCheckRun[];
}
interface RawStatusEntry {
state: string;
context: string;
}
interface RawMilestone {
title: string;
state: string;
}
interface RawIssueDetail {
number: number;
title: string;
state: 'open' | 'closed';
state_reason: string | null;
body: string | null;
created_at: string;
updated_at: string;
closed_at: string | null;
labels: { name: string }[];
assignees: { login: string }[];
milestone: RawMilestone | null;
html_url: string;
draft?: boolean;
user: { login: string } | null;
}
interface RawPullDetail extends RawIssueDetail {
merged: boolean;
mergeable_state: string | null;
review_decision: string | null;
head: { sha: string; ref: string };
}
export interface GitHubItemData {
repo: string;
number: number;
kind: 'issue' | 'pr';
detail: RawIssueDetail;
comments: RawComment[];
reviews: RawReview[];
reviewComments: RawReviewComment[];
checks: { pass: number; fail: number; pending: number; failing: string[] } | null;
/** Item numbers referenced via Closes/Fixes/Resolves in the description. */
linked: number[];
}
export async function fetchItemData(
repo: string,
number: number,
kind: 'issue' | 'pr',
client: GitHubClient,
opts: { signal?: AbortSignal } = {},
): Promise<GitHubItemData> {
const detail = await client.fetchJSON<RawIssueDetail>(
`/repos/${repo}/issues/${number}`,
opts,
);
const comments = await client.fetchAllPages<RawComment>(
`/repos/${repo}/issues/${number}/comments`,
opts,
);
let reviews: RawReview[] = [];
let reviewComments: RawReviewComment[] = [];
let checks: GitHubItemData['checks'] = null;
if (kind === 'pr') {
const prDetail = detail as RawPullDetail;
if (prDetail.merged === undefined) {
const fetched = await client.fetchJSON<RawPullDetail>(`/repos/${repo}/pulls/${number}`, opts);
Object.assign(detail, fetched);
}
reviews = await client.fetchAllPages<RawReview>(`/repos/${repo}/pulls/${number}/reviews`, opts);
reviewComments = await client.fetchAllPages<RawReviewComment>(
`/repos/${repo}/pulls/${number}/comments`,
opts,
);
if ((detail as RawPullDetail).state === 'open' && (detail as RawPullDetail).head?.sha) {
checks = await fetchChecks(repo, (detail as RawPullDetail).head.sha, client, opts);
}
}
const linked = extractLinkedNumbers(detail.body ?? '');
return {
repo,
number,
kind,
detail,
comments,
reviews,
reviewComments,
checks,
linked,
};
}
async function fetchChecks(
repo: string,
headSha: string,
client: GitHubClient,
opts: { signal?: AbortSignal },
): Promise<GitHubItemData['checks']> {
try {
const [runs, status] = await Promise.all([
client.fetchAllPages<RawCheckRun>(`/repos/${repo}/commits/${headSha}/check-runs`, { ...opts, field: 'check_runs' }),
client.fetchAllPages<RawStatusEntry>(`/repos/${repo}/commits/${headSha}/status`, { ...opts, field: 'statuses' }),
]);
let pass = 0;
let fail = 0;
let pending = 0;
const failing: string[] = [];
for (const run of runs) {
if (run.status !== 'completed') {
pending++;
} else if (run.conclusion === 'success' || run.conclusion === 'neutral' || run.conclusion === 'skipped') {
pass++;
} else {
fail++;
failing.push(run.name);
}
}
for (const s of status) {
if (s.state === 'success') pass++;
else if (s.state === 'pending') pending++;
else {
fail++;
failing.push(s.context);
}
}
return { pass, fail, pending, failing: [...new Set(failing)].slice(0, 20) };
} catch {
return null; // checks are best-effort
}
}
const LINK_RE = /\b(?:close|closes|closed|fix|fixes|fixed|resolve|resolves|resolved)\s+#(\d+)/gi;
export function extractLinkedNumbers(body: string): number[] {
const out = new Set<number>();
const re = new RegExp(LINK_RE.source, 'gi');
let m: RegExpExecArray | null;
while ((m = re.exec(body)) !== null) {
out.add(Number(m[1]));
}
return [...out].sort((a, b) => a - b);
}
const MENTION_RE = /(^|\s)#(\d{1,7})(?![A-Za-z0-9_])/g;
/** Replace #n mentions with wikilinks to the item pages. */
export function linkifyMentions(body: string, repo: string): string {
return body.replace(MENTION_RE, (_all, lead: string, num: string) => {
return `${lead}[[gh/${repo}/${num}|#${num}]]`;
});
}
// ── Page rendering ───────────────────────────────────────────────────────────
function yamlStr(v: string): string {
return JSON.stringify(v);
}
function yamlList(v: string[]): string {
if (v.length === 0) return '[]';
return `\n${v.map((s) => ` - ${yamlStr(s)}`).join('\n')}`;
}
export function itemPagePath(dir: string, repo: string, number: number): string {
const p = join(dir, 'gh', repo, `${number}.md`);
assertContained(dir, p, repo);
return p;
}
export function repoCardPath(dir: string, repo: string): string {
const p = join(dir, 'gh', repo, 'index.md');
assertContained(dir, p, repo);
return p;
}
/**
* Path-containment guard: a crafted repo name must never escape the managed
* dir. Symlink-safe: `isWriteTargetContained` realpaths the deepest existing
* ancestor, so a planted `gh/` symlink pointing outside the managed dir is
* caught where a lexical prefix check would pass.
*/
function assertContained(dir: string, path: string, repo: string): void {
if (!isValidRepoName(repo)) {
throw new Error(`Invalid GitHub repo name: "${repo}"`);
}
if (!isWriteTargetContained(path, dir)) {
throw new Error(`Path escapes managed dir: "${path}"`);
}
}
export function renderRepoCard(repo: string, data: RawRepo, syncedAt: string | null = new Date().toISOString()): string {
return [
'---',
`kind: repo`,
`repo: ${yamlStr(repo)}`,
`url: ${yamlStr(`https://github.com/${repo}`)}`,
`description: ${yamlStr(data.description ?? '')}`,
`default_branch: ${yamlStr(data.default_branch)}`,
`archived: ${data.archived}`,
`private: ${data.private}`,
...(syncedAt === null ? [] : [`synced_at: ${yamlStr(syncedAt)}`]),
'---',
neutralizeOutsideText(['', `# ${repo}`, '', data.description ?? '', '', `Default branch: ${data.default_branch}`, ''].join('\n')),
].join('\n');
}
function checksSummaryLines(checks: GitHubItemData['checks']): string[] {
if (!checks) return [];
const line = `**Checks:** ${checks.pass} passing, ${checks.fail} failing, ${checks.pending} pending`;
const failing = checks.failing.length > 0 ? `\n\nFailing: ${checks.failing.join(', ')}` : '';
return ['', line + failing, ''];
}
export function renderItemPage(data: GitHubItemData, detailFetched = true, syncedAt: string | null = new Date().toISOString()): string {
const d = data.detail;
const isPr = data.kind === 'pr';
const pr = d as RawPullDetail;
const status = isPr
? pr.merged ? 'merged' : d.state === 'open' ? (d.draft ? 'draft' : 'open') : 'closed'
: d.state;
const reviewDecision = isPr && d.state === 'open' ? (pr.review_decision ?? '') : '';
const frontmatter: string[] = [
'---',
`kind: ${isPr ? 'pr' : 'issue'}`,
`repo: ${yamlStr(data.repo)}`,
`number: ${d.number}`,
`title: ${yamlStr(d.title)}`,
`state: ${d.state}`,
`status: ${yamlStr(status)}`,
`url: ${yamlStr(d.html_url)}`,
`author: ${yamlStr(d.user?.login ?? '')}`,
`created_at: ${yamlStr(d.created_at)}`,
`updated_at: ${yamlStr(d.updated_at)}`,
`closed_at: ${yamlStr(d.closed_at ?? '')}`,
...(syncedAt === null ? [] : [`synced_at: ${yamlStr(syncedAt)}`]),
`detail_fetched: ${detailFetched}`,
`labels: ${yamlList(d.labels.map((l) => l.name))}`,
`assignees: ${yamlList(d.assignees.map((a) => a.login))}`,
`milestone: ${yamlStr(d.milestone?.title ?? '')}`,
`linked: ${yamlList(data.linked.map((n) => String(n)))}`,
];
if (isPr) {
frontmatter.push(
`merged: ${pr.merged}`,
`mergeable_state: ${yamlStr(pr.mergeable_state ?? '')}`,
`review_decision: ${yamlStr(reviewDecision)}`,
`head_ref: ${yamlStr(pr.head?.ref ?? '')}`,
);
if (data.checks) {
frontmatter.push(
`checks_pass: ${data.checks.pass}`,
`checks_fail: ${data.checks.fail}`,
`checks_pending: ${data.checks.pending}`,
);
}
}
frontmatter.push('---');
const body: string[] = [];
body.push(`# ${d.title}`, '');
body.push(`[${isPr ? 'PR' : 'Issue'} #${d.number}](${d.html_url}) · ${d.state}${isPr && pr.merged ? ' · merged' : ''}`, '');
if (data.checks && data.checks.fail > 0) {
body.push(...checksSummaryLines(data.checks));
}
body.push('## Description', '');
if (d.body) {
body.push(linkifyMentions(d.body, data.repo), '');
} else {
body.push('_no description_', '');
// Empty-body items render almost nothing matchable (title alone), which
// makes them near-unrecallable by label/milestone/assignee queries.
// Surface the structured context so retrieval has tokens to hit.
const ctx: string[] = [];
if (d.labels.length) ctx.push(`labels: ${d.labels.map((l) => l.name).join(', ')}`);
if (d.milestone) ctx.push(`milestone: ${d.milestone.title}`);
if (d.assignees.length) ctx.push(`assignees: ${d.assignees.map((a) => a.login).join(', ')}`);
ctx.push(`repo: ${data.repo}`);
body.push('', '## Context', '', ...ctx.map((c) => `- ${c}`), '');
}
if (data.linked.length > 0) {
body.push(
'## Linked',
'',
...data.linked.map((n) => `- [[gh/${data.repo}/${n}|#${n}]]`),
'',
);
}
if (data.comments.length > 0) {
body.push('## Comments', '');
for (const c of data.comments) {
body.push(`### ${c.user?.login ?? 'ghost'} · ${c.created_at}`, '');
body.push(linkifyMentions(c.body, data.repo), '');
}
}
if (data.reviews.length > 0) {
body.push('## Reviews', '');
for (const r of data.reviews) {
body.push(`### ${r.user?.login ?? 'ghost'} · ${r.state}${r.submitted_at ? ` · ${r.submitted_at}` : ''}`, '');
if (r.body) body.push(linkifyMentions(r.body, data.repo), '');
}
}
if (data.reviewComments.length > 0) {
body.push('## Review comments', '');
for (const rc of data.reviewComments) {
const loc = rc.path + (rc.line ? `:${rc.line}` : '');
body.push(`### ${rc.user?.login ?? 'ghost'} · ${loc} · ${rc.created_at}`, '');
body.push(linkifyMentions(rc.body, data.repo), '');
}
}
return frontmatter.join('\n') + '\n' + neutralizeOutsideText(body.join('\n')) + '\n';
}
/**
* Build the pass-1 page from list payloads alone (no per-item detail calls).
* Approximate for PRs: review_decision/merged/mergeable_state come from the
* pulls list when available; comments, reviews and checks are filled by pass 2.
*/
export function renderListItemPage(
repo: string,
kind: 'issue' | 'pr',
item: RawIssueListItem | RawPullListItem,
syncedAt?: string | null,
): string {
const isPr = kind === 'pr';
const pr = item as RawPullListItem;
const detail: RawIssueDetail = {
number: item.number,
title: item.title,
state: item.state,
state_reason: null,
body: item.body ?? null,
created_at: item.created_at ?? '',
updated_at: item.updated_at,
closed_at: item.closed_at ?? null,
labels: item.labels ?? [],
assignees: item.assignees ?? [],
milestone: item.milestone ?? null,
html_url: item.html_url ?? '',
draft: item.draft,
user: item.user ?? null,
};
if (isPr) {
// merged stays null until pass 2: the list payloads do not carry it for
// closed PRs, and a wrong `merged: false` on a merged PR is worse than
// an unknown one.
Object.assign(detail, {
merged: pr.merged ?? null,
mergeable_state: pr.mergeable_state ?? null,
review_decision: null,
head: { sha: pr.head?.sha ?? '', ref: '' },
});
}
const data: GitHubItemData = {
repo,
number: item.number,
kind,
detail,
comments: [],
reviews: [],
reviewComments: [],
checks: null,
linked: extractLinkedNumbers(detail.body ?? ''),
};
return renderItemPage(data, false, syncedAt);
}
// ── Page freshness helpers ───────────────────────────────────────────────────
interface StoredFrontmatter {
updated_at?: string;
}
function readStoredUpdatedAt(filePath: string): string | null {
try {
const raw = readFileSync(filePath, 'utf-8');
const m = raw.match(/^updated_at:\s*"([^"]+)"/m);
return m ? m[1] : null;
} catch {
return null;
}
}
/** True when the on-disk page is already at least as fresh as the API value. */
export function isPageFresh(filePath: string, apiUpdatedAt: string): boolean {
if (!existsSync(filePath)) return false;
const stored = readStoredUpdatedAt(filePath);
if (stored === null) return false;
return stored >= apiUpdatedAt; // ISO-8601 strings compare lexicographically
}
/**
* True when the page went through a detail fetch (comments, reviews, checks).
* Pages written before the two-pass change have no marker and ARE complete,
* so a missing marker reads as true; only `detail_fetched: false` (pass-1
* list render) is treated as pending detail.
*/
export function pageHasDetail(filePath: string): boolean {
// A missing file has no detail yet: pass 1 materializes the cheap list
// page first, pass 2 enriches it. Only markerless EXISTING pages (written
// before the two-pass change) are treated as complete.
if (!existsSync(filePath)) return false;
try {
const raw = readFileSync(filePath, 'utf-8');
const m = raw.match(/^detail_fetched:\s*(true|false)/m);
return m ? m[1] === 'true' : true;
} catch {
return true;
}
}
// ── Sync runner ──────────────────────────────────────────────────────────────
interface GitHubSyncDeps {
managed: ManagedConnectorSync | null;
engine: BrainEngine;
sourceId: string;
cfg: GitHubSourceConfig;
opts: SyncOpts;
client: GitHubClient;
}
async function materializePage(deps: GitHubSyncDeps, filePath: string, content: string,
activePack: Parameters<typeof importPage>[2]) {
const rel = relative(deps.cfg.dir, filePath).replace(/\\/g, '/');
content = connectorRender(content, { path: rel });
if (deps.managed) return deps.managed.importMarkdown(rel, content);
const slug = slugifyPath(rel);
const existing = await deps.engine.getPage(slug, { sourceId: deps.sourceId });
if (existing) assertInlineBody(existing, 'GitHub connector materialization');
const databaseOnly = await isWriteThroughDisabled(deps.engine) || isSourceDbOnlySlug(deps.cfg.dir, slug, 'refuse');
if (databaseOnly) return { ...await importPage(deps, filePath, activePack, rel, content), created: !existing };
const guard = await replaceableFile(deps.engine, deps.sourceId, slug, filePath);
const drift = () => new OperationError('file_database_drift',
`${slug}: the checkout file has an edit not retained in the database, or changed during import. The file was not replaced. If the file is newer, run gbrain sync --source ${deps.sourceId}; if the database is newer, re-save its page with write-through on; if both changed, ask the user which version to keep before retrying.`,
'Import the newer file with gbrain sync, or ask the user which version to keep; the item is held until then.');
if (!guard.ok) throw drift();
const created = guard.expectedFileBytes === null;
mkdirSync(dirname(filePath), { recursive: true });
const tmpPath = `${filePath}.tmp`;
writeFileSync(tmpPath, content, 'utf-8');
try {
const result = await importPage(deps, tmpPath, activePack, rel);
const current = existsSync(filePath) ? readFileSync(filePath, 'utf8') : null;
if (current !== guard.expectedFileBytes) throw drift();
renameSync(tmpPath, filePath);
return { ...result, created };
} finally { rmSync(tmpPath, { force: true }); }
}
async function storedPage(deps: GitHubSyncDeps, filePath: string, apiUpdatedAt: string) {
const slug = relative(deps.cfg.dir, filePath).replace(/\\/g, '/').replace(/\.md$/, '');
if (!deps.managed && !await isWriteThroughDisabled(deps.engine) && !isSourceDbOnlySlug(deps.cfg.dir, slugifyPath(slug), 'refuse')) {
return { fresh: isPageFresh(filePath, apiUpdatedAt), hasDetail: pageHasDetail(filePath), exists: existsSync(filePath) };
}
const page = deps.managed ? await deps.managed.page(slug) : await deps.engine.getPage(slugifyPath(slug), { sourceId: deps.sourceId });
return { exists: !!page, fresh: typeof page?.frontmatter.updated_at === 'string' && page.frontmatter.updated_at >= apiUpdatedAt,
hasDetail: !!page && page.frontmatter.detail_fetched !== false };
}
async function importPage(
deps: GitHubSyncDeps,
filePath: string,
activePack: { page_types: ReadonlyArray<{ name: string; path_prefixes: ReadonlyArray<string> }> } | undefined,
relPathOverride?: string,
content?: string,
): Promise<{ slug: string; chunks: number; status: 'imported' | 'skipped' }> {
const { importFile, importFromContent } = await import('./import-file.ts');
const rel = relPathOverride ?? relative(deps.cfg.dir, filePath).replace(/\\/g, '/');
const opts = {
noEmbed: true, // embeddings handled by the size gate below, like sync
sourceId: deps.sourceId,
activePack,
};
const result = content === undefined ? await importFile(deps.engine, filePath, rel, opts)
: await importFromContent(deps.engine, slugifyPath(rel), content, { ...opts, sourcePath: rel, preserveGateMarkers: true, holdNoncanonicalPrivate: true });
if (result.status === 'error' || result.error) {
throw new Error(result.error ?? `Import failed for ${rel}`);
}
return { slug: result.slug, chunks: result.chunks, status: result.status === 'imported' ? 'imported' : 'skipped' };
}
async function deleteStalePages(
deps: GitHubSyncDeps,
keepPaths: Set<string>,
summary: GitHubSyncSummary,
succeededRepos: ReadonlySet<string>,
): Promise<void> {
const { planReconcileDeletes } = await import('../commands/sync.ts');
const allRows = await deps.engine.executeRaw<{ slug: string; source_path: string | null }>(
`SELECT slug, source_path FROM pages WHERE source_id = $1 AND deleted_at IS NULL`,
[deps.sourceId],
);
// Data-loss guard: a repo that errored mid-sweep contributes no keepPaths,
// so its pages must be EXCLUDED from the reconcile entirely, otherwise a
// transient API failure would look like a bulk deletion.
const rows = allRows.filter((r) => {
if (r.source_path === null) return false;
const parts = r.source_path.split('/');
const ownerRepo = parts.length >= 3 ? parts.slice(0, 3).join('/') : '';
// Case-folded: legacy rows may carry canonical-case source_paths while
// succeededRepos keys on the lowercase form.
return ownerRepo !== '' && succeededRepos.has(ownerRepo.toLowerCase());
});
const plan = planReconcileDeletes(
rows,
keepPaths,
(p) => p.startsWith('gh/') && p.endsWith('.md'),
);
if (plan.staleSlugs.length === 0) return;
const { massReconcileAllowed } = await import('../commands/sync.ts');
if (plan.massDelete && !massReconcileAllowed()) {
deps.client.log?.(`[github] mass-delete guard refused ${plan.staleSlugs.length} deletes for source ${deps.sourceId}`);
return;
}
const bySlug = new Map(rows.map((r) => [r.slug, r.source_path]));
if (deps.managed) {
for (const slug of plan.staleSlugs) if (await deps.managed.delete(slug, bySlug.get(slug) ?? null)) summary.deleted++;
return;
}
const batchSize = 500;
for (let i = 0; i < plan.staleSlugs.length; i += batchSize) {
const batch = plan.staleSlugs.slice(i, i + batchSize);
await deps.engine.deletePages(batch, { sourceId: deps.sourceId });
// We own these files (unlike git sources): remove them so a re-add of
// the same number starts from a clean page.
for (const slug of batch) {
const rel = bySlug.get(slug);
if (!rel) continue;
try {
const { rmSync } = await import('node:fs');
rmSync(join(deps.cfg.dir, rel), { force: true });
} catch { /* best-effort */ }
}
}
summary.deleted += plan.staleSlugs.length;
}
/**
* Main entry for the github source kind. Called from performSyncInner when
* the resolved source is kind=github. Handles:
* - opts.githubItem -> single-item refresh (webhook path)
* - opts.full -> full reconcile (re-enumerate everything + delete)
* - otherwise -> delta sweep since last run
*/
export async function runGitHubSync(
engine: BrainEngine,
sourceId: string,
cfg: GitHubSourceConfig,
opts: SyncOpts,
fetchImpl?: FetchImpl,
): Promise<import('../commands/sync.ts').SyncResult> {
return withConnectorSync(engine, sourceId, 'github', cfg, opts,
async (managed, options) => options.dryRun ? connectorDryRunResult('github', sourceId) : runGitHubSyncInner(engine, sourceId, cfg, options, managed, fetchImpl), pendingConnectorResult);
}
async function runGitHubSyncInner(engine: BrainEngine, sourceId: string, cfg: GitHubSourceConfig, opts: SyncOpts,
managed: ManagedConnectorSync | null, fetchImpl?: FetchImpl): Promise<import('../commands/sync.ts').SyncResult> {
// Credential source: a GitHub App (auto-minted hourly installation tokens)
// wins when configured; otherwise cfg.tokenEnv is the single source of
// truth (the default is GH_TOKEN; a custom --token-env that is unset fails
// loudly instead of silently using a different token scope, codex LOW).
if (!cfg.app && !process.env[cfg.tokenEnv]) {
throw new Error(
`GitHub source "${sourceId}" has no token. Set ${cfg.tokenEnv} in the environment or configure a GitHub App (gh_app_id + gh_app_pem_path).`,
);
}
const appTokens = cfg.app ? new AppTokenProvider(cfg.app, fetchImpl ?? fetch) : null;
const client = new GitHubClient(appTokens ?? process.env[cfg.tokenEnv] ?? '', fetchImpl, (message) => console.error(message));
if (managed) await managed.assertAccount(await resolveGitHubAccount(cfg, client, appTokens, opts.signal)); // #5686 installation/login pin
const deps: GitHubSyncDeps = { engine, sourceId, cfg, opts, client, managed };
const summary: GitHubSyncSummary = {
status: 'synced',
added: 0,
modified: 0,
deleted: 0,
chunksCreated: 0,
embedded: 0,
pagesAffected: [],
itemsSeen: 0,
itemDetailFetches: 0,
failedFiles: 0,
};
// Active pack for pack-aware typing, mirroring performSyncInner.
let activePack: GitHubSyncDeps['opts'] extends never ? never : { page_types: ReadonlyArray<{ name: string; path_prefixes: ReadonlyArray<string> }> } | undefined;
activePack = undefined;
if (!opts.noSchemaPack) {
try {
const { loadActivePack } = await import('./schema-pack/load-active.ts');
const { loadConfig } = await import('./config.ts');
const resolved = await loadActivePack({ cfg: loadConfig(), remote: false, sourceId });
activePack = { page_types: resolved.manifest.page_types };
} catch { /* fall back to legacy typing */ }
}
// Keep previous scope before this run resolves its refreshed repo list.
// A repo added to an existing source needs a history bootstrap even when
// the source-wide cursor is already ahead of its old items.
const state: GitHubState = managed ? managed.state({ last_sweep_at: null, repos: [] }) : readState(cfg.dir);
const previousRepos = new Set(state.repos);
// Shared bulk-progress reporter (docs/progress-events.md, phase
// sync.github_materialize): stderr-only heartbeats during the network
// phases, one tick per item. finish() is guaranteed by the finally so a
// thrown error never leaves a live phase behind.
const progress = createProgress(cliOptsToProgressOptions(getCliOptions()));
progress.start('sync.github_materialize');
try {
const stopScopeHb = startHeartbeat(progress, 'resolving repo scope');
let repos: string[];
try {
repos = await resolveScopeRepos(cfg, client, opts.signal);
} finally {
stopScopeHb();
}
if (opts.githubItem) {
// Scope guard: only refresh items in the resolved scope. Webhooks for
// out-of-scope repos are acknowledged upstream but never materialized.
// Repo casing is normalized so a webhook payload that differs in case
// from the stored repo still matches (GitHub repo names are
// case-insensitive; the managed dir layout is lowercase).
const item = { ...opts.githubItem, repo: opts.githubItem.repo.toLowerCase() };
if (!repos.includes(item.repo)) {
return syncResult({ ...summary, status: 'up_to_date' }, opts);
}
const stopItemHb = startHeartbeat(progress, `refreshing ${item.repo}#${item.number}`);
try {
await refreshSingleItem(deps, item, activePack, summary);
} finally {
stopItemHb();
}
if (managed) await managed.saveState(state);
else await touchSourceRow(deps, new Date().toISOString());
return syncResult(summary, opts);
}
const since = opts.full ? undefined : state.last_sweep_at ?? undefined;
const keepPaths = new Set<string>();
const succeededRepos = new Set<string>();
// Pages counted in pass 1 must not be counted again when pass 2 replaces
// them in the same run (added/modified describe distinct page outcomes).
const countedSlugs = new Set<string>();
let maxUpdatedAt = state.last_sweep_at ?? '';
const repoMeta = new Map<string, RawRepo>();
// Fix wave 4 (#5740): an item failing 3 consecutive runs is held, so one
// permanently failing item no longer pins the watermark; held items stay
// visible (sources status, doctor, sync summary) until retried.
const holds = await ConnectorHoldSession.open(engine, sourceId, managed, { last_sweep_at: null, repos: [] }, state.item_holds, { full: opts.full });
const itemFailed = async (repo: string, item: { number: number; kind: 'issue' | 'pr'; updated_at: string; list?: { title?: string } }, err: unknown, what = '') => {
const key = `${repo}#${item.number}`;
const blocking = !holds.isHeld(key);
await holds.fail(key, err, { version: item.updated_at || null, ref: item.kind, slug: slugifyPath(`gh/${repo}/${item.number}`),
meta: { title: item.list?.title ?? null, upstream_at: item.updated_at || null } });
deps.client.log?.(`[github] item ${repo}#${item.number}${what} failed: ${err instanceof Error ? err.message : String(err)}`);
summary.failedFiles++;
if (blocking) summary.status = 'partial';
progress.tick(1, `${repo}#${item.number} failed`);
};
for (const repo of repos) {
if (opts.signal?.aborted) break;
try {
const repoSince = opts.full || !previousRepos.has(repo) ? undefined : since;
const stopListHb = startHeartbeat(progress, `listing ${repo}`);
let issues: RawIssueListItem[];
let prs: RawPullListItem[];
try {
({ issues, prs } = await enumerateRepoItems(repo, client, { since: repoSince, signal: opts.signal }));
} finally {
stopListHb();
}
const items: Array<{
repo: string;
number: number;
kind: 'issue' | 'pr';
state: string;
updated_at: string;
list: RawIssueListItem | RawPullListItem;
}> = [
...issues.map((i) => ({ repo, number: i.number, kind: 'issue' as const, state: i.state, updated_at: i.updated_at, list: i as RawIssueListItem })),
...prs.map((p) => ({ repo, number: p.number, kind: 'pr' as const, state: p.state, updated_at: p.updated_at, list: p as RawPullListItem })),
];
// Item count known: restart the phase with this repo's total. Every
// item ticks exactly once (fresh skip, pass-1 failure, or pass 2).
progress.start('sync.github_materialize', items.length);
const pendingDetail: typeof items = [];
for (const item of items) {
const filePath = itemPagePath(cfg.dir, repo, item.number);
keepPaths.add(relative(cfg.dir, filePath).replace(/\\/g, '/'));
summary.itemsSeen++;
if (!holds.shouldAttempt(`${repo}#${item.number}`, item.updated_at)) {
progress.tick(1, `${repo}#${item.number} held`);
continue;
}
// Open PRs are never fresh: their check state can change without
// touching the PR's updated_at (a new check run does not bump it),
// so they are re-fetched every sweep. Cost is bounded by the number
// of open PRs, which is small in practice.
const isOpenPr = item.kind === 'pr' && item.state === 'open';
// #9062: a stored page this read cannot load (one stored in segments over the single-string ceiling) fails this item, not the run.
let stored: Awaited<ReturnType<typeof storedPage>>;
try {
stored = await storedPage(deps, filePath, item.updated_at);
} catch (err) {
await itemFailed(repo, item, err, ' stored page read');
continue;
}
const { fresh, hasDetail } = stored;
if (!opts.full && !isOpenPr && fresh && hasDetail) {
// Cursor accounting: a fresh skip is a success too. Without this,
// a repo whose newest item is always fresh stays re-listed on
// every sweep (the since filter never passes it).
if (item.updated_at > maxUpdatedAt) maxUpdatedAt = item.updated_at;
holds.succeed(`${repo}#${item.number}`);
progress.tick(1, `${repo}#${item.number} fresh`);
continue;
}
// Pass 1 (cheap): materialize from the list payload when the page is
// missing or never detail-fetched. Pages that already carry detail
// are left intact until pass 2 replaces them, so comments never
// vanish mid-sweep.
if (!hasDetail) {
try {
const imported = await materializePage(deps, filePath, renderListItemPage(repo, item.kind, item.list, managed ? null : undefined), activePack);
if (imported.status === 'imported') {
summary.pagesAffected.push(imported.slug);
summary.chunksCreated += imported.chunks;
if (!countedSlugs.has(imported.slug)) {
if (imported.created) summary.added++; else summary.modified++;
countedSlugs.add(imported.slug);
}
}
} catch (err) {
await itemFailed(repo, item, err, ' list render');
continue;
}
}
pendingDetail.push(item);
}
// Held items the listing no longer returns are re-attempted when
// retry-held asked for them or a transient reconsideration is due.
const listed = new Set(items.map((i) => i.number));
for (const key of holds.dueHeldKeys()) {
const number = Number(key.slice(repo.length + 1));
if (!key.startsWith(`${repo}#`) || listed.has(number) || !holds.shouldAttempt(key)) continue;
keepPaths.add(relative(cfg.dir, itemPagePath(cfg.dir, repo, number)).replace(/\\/g, '/'));
pendingDetail.push({ repo, number, kind: holds.holds.record(key)?.ref === 'pr' ? 'pr' : 'issue', state: 'open', updated_at: '', list: {} as RawIssueListItem });
}
// Pass 2 (expensive): comments, reviews, checks, exact merge state.
// Runs in the same sweep so the final page is complete; an item that
// fails here keeps its pass-1 (or previous) page and the cursor does
// not advance past it, so the next sweep retries just that item.
const stopDetailHb = startHeartbeat(progress, `fetching item detail for ${repo}`);
try {
for (const item of pendingDetail) {
if (opts.signal?.aborted) {
summary.status = 'partial';
break;
}
const filePath = itemPagePath(cfg.dir, repo, item.number);
summary.itemDetailFetches++;
try {
const data = await fetchItemData(repo, item.number, item.kind, client, { signal: opts.signal });
const imported = await materializePage(deps, filePath, renderItemPage(data, true, managed ? null : undefined), activePack);
if (imported.status === 'imported') {
summary.pagesAffected.push(imported.slug);
summary.chunksCreated += imported.chunks;
if (!countedSlugs.has(imported.slug)) {
if (imported.created) summary.added++; else summary.modified++;
countedSlugs.add(imported.slug);
}
}
} catch (err) {
await itemFailed(repo, item, err);
continue;
}
holds.succeed(`${repo}#${item.number}`);
progress.tick(1, `${repo}#${item.number}`);
// Cursor advances only for items that fully succeeded.
if (item.updated_at > maxUpdatedAt) maxUpdatedAt = item.updated_at;
}
} finally {
stopDetailHb();
}
// Repo card, refreshed once per repo.
const cardPath = repoCardPath(cfg.dir, repo);
keepPaths.add(relative(cfg.dir, cardPath).replace(/\\/g, '/'));
if (opts.full || !(await storedPage(deps, cardPath, '')).exists) {
try {
const meta = await client.fetchJSON<RawRepo>(`/repos/${repo}`, { signal: opts.signal });
repoMeta.set(repo, meta);
const imported = await materializePage(deps, cardPath, renderRepoCard(repo, meta, managed ? null : undefined), activePack);
if (!summary.pagesAffected.includes(imported.slug)) summary.pagesAffected.push(imported.slug);
if (imported.status === 'imported') {
if (imported.created) summary.added++; else summary.modified++;
}
} catch (error) {
if (managed) { rethrowConnectorWriteError(error); summary.failedFiles++; summary.status = 'partial'; }
}
} else {
keepPaths.add(relative(cfg.dir, cardPath).replace(/\\/g, '/'));
}
succeededRepos.add(`gh/${repo}`);
} catch (err) {
if (managed) rethrowConnectorWriteError(err);
deps.client.log?.(`[github] repo ${repo} failed: ${err instanceof Error ? err.message : String(err)}`);
summary.failedFiles++;
summary.status = 'partial';
}
}
// A page can be imported twice in a run (pass 1 list render, then pass 2
// detail render): dedupe before extract/embed so each page is processed once.
summary.pagesAffected = [...new Set(summary.pagesAffected)];
if (opts.full) {
await deleteStalePages(deps, keepPaths, summary, succeededRepos);
}
// Size-gated extract + embed, mirroring performSyncInner's gates.
await runExtractAndEmbed(deps, summary, activePack);
// Cursor discipline: the sweep cursor only advances on a fully successful
// run. On a partial run (item or repo failures) or an aborted signal we
// keep the previous cursor so the next sweep re-enumerates from the old
// point; fresh pages are still skipped by content hash, so the only real
// cost is the retry of whatever failed.
if (opts.signal?.aborted) summary.status = 'partial';
// A repo enters state.repos only once a sweep fully succeeded for it (or
// it was already listed): state-listing an unswept repo would skip its
// history bootstrap on the next run (previousRepos gates the since filter).
state.repos = repos.filter((r) => succeededRepos.has(`gh/${r}`) || previousRepos.has(r));
// Throws connector_holds_exhausted before any save, so the cursor stays.
state.item_holds = holds.finish();
if (summary.status === 'synced') {
state.last_sweep_at = maxUpdatedAt || (managed ? null : new Date().toISOString());
if (managed) await managed.saveState(state, true, state.last_sweep_at ?? undefined);
else { writeState(cfg.dir, state); await touchSourceRow(deps, state.last_sweep_at!); }
} else if (managed) {
// A partial run leaves the cursor at its last committed position and publishes only changed holds.
if (!opts.signal?.aborted) await managed.publishHolds({ last_sweep_at: null, repos: [] }, state.item_holds);
} else {
// #5740 (#5741): a partial sweep persists its state but never stamps last_sync_at.
writeState(cfg.dir, state);
}
await holds.complete();
return { ...syncResult(summary, opts), ...connectorHoldsResult(sourceId, holds.summary()) };
} finally {
progress.finish();
}
}
async function refreshSingleItem(
deps: GitHubSyncDeps,
item: GitHubItemRef,
activePack: { page_types: ReadonlyArray<{ name: string; path_prefixes: ReadonlyArray<string> }> } | undefined,
summary: GitHubSyncSummary,
): Promise<void> {
const repo = item.repo.toLowerCase();
const filePath = itemPagePath(deps.cfg.dir, repo, item.number);
const slug = slugifyPath(`gh/${repo}/${item.number}`);
if (item.deleted) {
const rows = await deps.engine.executeRaw<{ slug: string }>(
`SELECT slug FROM pages WHERE source_id = $1 AND slug = $2 AND deleted_at IS NULL`,
[deps.sourceId, slug],
);
if (deps.managed) {
for (const row of rows) if (await deps.managed.delete(row.slug, relative(deps.cfg.dir, filePath).replace(/\\/g, '/'))) summary.deleted++;
return;
}
if (rows.length > 0) {
await deps.engine.deletePages([slug], { sourceId: deps.sourceId });
summary.deleted += rows.length;
}
rmSync(filePath, { force: true });
return;
}
const data = await fetchItemData(repo, item.number, item.kind, deps.client, { signal: deps.opts.signal });
const imported = await materializePage(deps, filePath, renderItemPage(data, true, deps.managed ? null : undefined), activePack);
if (imported.status === 'imported') {
summary.pagesAffected.push(imported.slug);
summary.chunksCreated += imported.chunks;
if (deps.managed && imported.created) summary.added++; else summary.modified++;
}
await runExtractAndEmbed(deps, summary, activePack);
}
async function runExtractAndEmbed(
deps: GitHubSyncDeps,
summary: GitHubSyncSummary,
activePack: { page_types: ReadonlyArray<{ name: string; path_prefixes: ReadonlyArray<string> }> } | undefined,
): Promise<void> {
if (deps.managed) return;
const totalChanges = summary.added + summary.modified;
const pagesAffected = summary.pagesAffected;
if (totalChanges === 0 || pagesAffected.length === 0) return;
if (!deps.opts.noExtract && totalChanges <= 100) {
try {
const { extractLinksForSlugs, extractTimelineForSlugs, stampExtracted, slugsSafeToStamp } = await import('../commands/extract.ts');
const extractOpts = { sourceId: deps.sourceId };
const linksResult = await extractLinksForSlugs(deps.engine, deps.cfg.dir, pagesAffected, extractOpts);
const timelineResult = await extractTimelineForSlugs(deps.engine, deps.cfg.dir, pagesAffected, extractOpts);
// Stamp only the slugs both hooks actually read from disk; a page the
// extractor skipped stays stale for 'gbrain extract --stale'.
await stampExtracted(
deps.engine,
slugsSafeToStamp(linksResult, timelineResult)
.map((slug) => ({ slug, source_id: deps.sourceId })),
);
} catch { /* extraction is best-effort */ }
} else if (totalChanges > 100 && !deps.opts.noExtract) {
process.stderr.write(`[github] large sync (${totalChanges} pages); extraction deferred to 'gbrain extract --stale --source-id ${deps.sourceId}'\n`);
}
if (!deps.opts.noEmbed && totalChanges <= 100 && pagesAffected.length > 0) {
try {
const { runEmbedCore } = await import('../commands/embed.ts');
await runEmbedCore(deps.engine, { slugs: pagesAffected, sourceId: deps.sourceId });
summary.embedded = pagesAffected.length;
} catch { /* embed is best-effort */ }
} else if (!deps.opts.noEmbed && totalChanges > 100) {
// Large sync skips the inline embed; queue a capped backfill job so the
// mirrored pages don't sit unembedded until someone notices recall is
// keyword-only. Mirrors performSync's auto-defer chaser.
// #3697: embed's source filter is `--source` (embed.ts parses no --source-id).
const drainHint = `run 'gbrain embed --stale --source ${deps.sourceId}' to drain now`;
try {
const { submitEmbedBackfill } = await import('./embed-backfill-submit.ts');
const sub = await submitEmbedBackfill(deps.engine, deps.sourceId, { reason: 'github_sync_defer' });
if (sub.status === 'submitted') {
process.stderr.write(`[github] large sync (${totalChanges} pages); embeds deferred to embed-backfill job ${sub.jobId} — or ${drainHint}\n`);
} else if (sub.status === 'cooldown' || sub.status === 'spend_capped' || sub.status === 'no_worker_surface' || sub.status === 'stalled') {
process.stderr.write(`[github] large sync (${totalChanges} pages); embed-backfill not queued (${sub.status}) — ${drainHint}\n`);
} else if (sub.status === 'embedding_disabled') {
process.stderr.write(`[github] large sync (${totalChanges} pages); embed-backfill not queued: embedding is disabled on this brain (keyless by choice)\n`);
} else { sub satisfies never; }
} catch (err) {
process.stderr.write(`[github] embed-backfill submission failed: ${err instanceof Error ? err.message : String(err)} — ${drainHint}\n`);
}
}
}
async function touchSourceRow(deps: GitHubSyncDeps, newestContentAt: string): Promise<void> {
try {
await deps.engine.executeRaw(
`UPDATE sources SET last_sync_at = now(), newest_content_at = $1::timestamptz WHERE id = $2`,
[newestContentAt, deps.sourceId],
);
} catch { /* best-effort */ }
}
function syncResult(
summary: GitHubSyncSummary,
opts: SyncOpts,
): import('../commands/sync.ts').SyncResult {
const first = summary.added > 0 && summary.modified === 0 && summary.deleted === 0
&& summary.itemDetailFetches === summary.itemsSeen;
return {
status: summary.status === 'partial' ? 'partial' : (first ? 'first_sync' : summary.added + summary.modified + summary.deleted > 0 ? 'synced' : 'up_to_date'),
fromCommit: null,
toCommit: '',
added: summary.added,
modified: summary.modified,
deleted: summary.deleted,
renamed: 0,
chunksCreated: summary.chunksCreated,
embedded: summary.embedded,
pagesAffected: summary.pagesAffected,
...(summary.failedFiles > 0 ? { failedFiles: summary.failedFiles } : {}),
// #5012: a partial sweep names its real cause and counts what it wrote.
...(summary.status === 'partial' ? {
filesImported: summary.added + summary.modified,
reason: summary.failedFiles > 0 ? 'connector_item_failures' as const : opts.signal?.aborted ? 'timeout' as const : 'connector_partial' as const,
} : {}),
};
}