Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

37.5 KB
Raw Blame
/**
 * Reference Repos service.
 *
 * Each PortOS-managed app can list upstream repos it watches for clean-room
 * reimplementation — the agent reads upstream commits and proposes which
 * features/fixes are worth re-building in the app's OWN code, never
 * copying upstream verbatim.
 * (e.g., PortOS itself watches `phosphene` for video-gen ideas). The
 * `reference-watch` scheduled task asks this service to fetch each ref
 * and find commits since `lastReviewedSha`. The CoS sub-agent then records
 * proposals in the target app's CONFIGURED work tracker — PLAN.md, GitHub
 * Issues, GitLab Issues, or JIRA (resolved via `resolveAppWorkTracker`). PLAN.md
 * keeps its checklist IDs; forge titles are human-readable and use the issue
 * number/key as the ID. `/claim` / `plan-task` / `claim-issue*` picks up later.
 * The destination-specific guidance is injected into the
 * prompt's `{trackerInstructions}` block by BOTH dispatch paths —
 * `triggerReferenceAnalysis` here (the on-commit trigger) and
 * `resolveReferenceWatchBlock` in cosTaskPreStepBlocks.js (the WEEKLY scheduled
 * task) — via the shared `formatTrackerInstructions` below (#3140).
 *
 * Storage: refs live inline on each app in data/apps.json under the
 * `referenceRepos` array — fits the existing per-app config model and
 * keeps the schedule task's per-app dispatch simple.
 *
 * Clones: managed under data/cos/reference-repos/<refId>/. The user
 * never has to clone manually; first `check` initializes the clone.
 */

import { existsSync, statSync } from 'fs';
import { join } from 'path';
import { v4 as uuidv4 } from '../lib/uuid.js';
import { ensureDir, expandHome, PATHS } from '../lib/fileUtils.js';
import { ServerError } from '../lib/errorHandler.js';
import { applyAppPlaceholders } from '../lib/appPromptPlaceholders.js';
import { execGit } from '../lib/execGit.js';
import { isSafeGitRef } from '../lib/gitArgs.js';
import {
  getAppById,
  updateApp,
} from './apps.js';
import { DEFAULT_REVIEWER } from '../lib/validation.js';
import { clampText, fenceBlock, UNTRUSTED_CONTENT_NOTICE } from '../lib/promptFencing.js';
import { resolveAppWorkTracker, isFileTracker, formatTrackerInstructions } from '../lib/workTracker.js';

const REFERENCE_REPOS_ROOT = join(PATHS.data, 'cos', 'reference-repos');

// 40-char hex SHA the same way git outputs it. Used by every callsite that
// reads a SHA back from `git rev-parse` — git is supposed to give us 40 hex
// chars, but if the working tree is corrupt we should fail loud rather than
// quietly write garbage to apps.json.
const SHA_RE = /^[0-9a-f]{40}$/i;

const SHORT_SHA = (sha) => (sha && SHA_RE.test(sha) ? sha.slice(0, 8) : null);

// addReferenceRepo always assigns refIds via uuidv4(), but apps.json is a
// hand-editable JSON file — a corrupted or maliciously-edited entry could
// set `id` to "../.." and have cloneDir escape REFERENCE_REPOS_ROOT,
// letting git operations read/write arbitrary paths under PATHS.data.
// Enforce a strict id format here so the assumption "refId is a safe
// path segment" holds at every callsite.
const REF_ID_RE = /^[A-Za-z0-9_-]+$/;
const cloneDir = (refId) => {
  if (typeof refId !== 'string' || !REF_ID_RE.test(refId)) {
    throw new ServerError(
      `Reference repo id has an invalid format (must match ${REF_ID_RE.source}): ${refId}`,
      { status: 500, code: 'REFERENCE_REPO_BAD_ID' },
    );
  }
  return join(REFERENCE_REPOS_ROOT, refId);
};

// scp-style remotes — git accepts both `user@host:path` and the bare
// `host:path` form. Disambiguating from a Windows drive path (`C:\foo`,
// `C:/foo`) requires the segment before `:` to be longer than one char
// (excluding Windows drive letters). The segment after `:` must NOT start
// with `\` or `/` (excluding `C:\path` / `C:/path` style local paths).
// This matches how git itself parses scp-syntax in `connect.c`. Examples:
//   [email protected]:owner/repo.git   → remote (scp with user, dotted host)
//   github.com:owner/repo.git       → remote (scp without user, dotted host)
//   gitserver:owner/repo.git        → remote (scp without user, intranet host
//                                     with no dot — common on internal forges)
//   user@gitserver:owner/repo.git   → remote (scp with user, host has no dot)
//   C:\Users\me\repo                → local (host-segment single char)
//   C:/Users/me/repo                → local (same; segment after `:` is `/`)
const SCP_REMOTE_RE = /^([^/@:]+@)?[^/@:]{2,}:[^\\/].*$/;

const isLocalPath = (urlOrPath) => {
  if (!urlOrPath) return false;
  // scheme:// and scp-style remotes (`user@host:path` and bare `host:path`)
  // are remote; everything else (including ~ and absolute paths) is local.
  // The user can pass `/Users/.../phosphene` or `~/phosphene` to skip the
  // clone and reuse an existing working tree.
  if (urlOrPath.includes('://')) return false;
  if (SCP_REMOTE_RE.test(urlOrPath)) return false;
  return true;
};

// Strip userinfo (`user:token@`) from URL-shaped args so a bad HTTPS remote
// doesn't surface its embedded PAT in the persisted `lastError` field (which
// the UI renders verbatim). Also catches scp-style `user@host:path` — but
// keeps the host so the user can still see what failed.
const redactUrlCreds = (s) => {
  if (typeof s !== 'string') return s;
  // scheme://user:token@host/...  → scheme://host/...
  let out = s.replace(/(\w+:\/\/)[^\s/@]+@/g, '$1');
  // user@host:path  → host:path  (scp-style only, no scheme)
  if (!out.includes('://')) {
    out = out.replace(/(^|\s)[^\s/@:]+@([^\s/@:]+:[^\s]+)/g, '$1$2');
  }
  return out;
};
const redactArgsForError = (args) => args.map(redactUrlCreds);

// Reject any HTTPS URL with credentials in userinfo (e.g.
// `https://token@host/repo` or `https://user:token@host/repo`). Throws a
// 400 ServerError on rejection so route handlers don't need to repeat the
// check. scp-style `git@host:repo` is NOT credentials (SSH key auth) and
// is allowed through. Used by both addReferenceRepo (ingest) and
// updateReferenceRepo (PATCH) so a tokened URL can't sneak in either way.
const rejectCredentialedUrl = (url) => {
  if (typeof url !== 'string') return;
  if (/^\w+:\/\/[^\s/@]+@/.test(url)) {
    throw new ServerError(
      'repoUrl must not embed credentials in the URL (e.g. https://token@host/repo). Configure git credentials separately (credential helper, SSH keys).',
      { status: 400, code: 'REFERENCE_REPO_URL_HAS_CREDS' },
    );
  }
};

// `repoUrl` and `branch` are caller text that becomes bare git argv. A value
// shaped like an option (`--upload-pack=<cmd>` as the branch of
// `git fetch --prune origin <branch>`) is executed by git's local/ssh
// transport — host code execution from a route that only edits a record.
// Validated when written AND when used, since apps.json already holds records
// saved before this check (and is hand-editable).
const assertGitArgsSafe = ({ repoUrl, branch }) => {
  if (typeof repoUrl === 'string' && repoUrl.trim().startsWith('-')) {
    throw new ServerError('repoUrl must not start with "-"', { status: 400, code: 'REFERENCE_REPO_BAD_URL' });
  }
  if (typeof branch === 'string' && branch.trim() && !isSafeGitRef(branch.trim())) {
    throw new ServerError('branch is not a valid git branch name', { status: 400, code: 'REFERENCE_REPO_BAD_BRANCH' });
  }
};

// Patterns that indicate the user's config is wrong (so the right HTTP
// status is 4xx, not 5xx). Order matters — first match wins. Codes here
// surface to the UI via `lastError`, so the strings are user-facing.
const GIT_USER_ERROR_PATTERNS = [
  { pattern: /authentication failed|could not read username|terminal prompts disabled/i, code: 'REFERENCE_REPO_AUTH_FAILED', status: 400 },
  { pattern: /could not resolve host|repository not found|not found\.\s*$/i, code: 'REFERENCE_REPO_NOT_FOUND_REMOTE', status: 400 },
  { pattern: /unknown revision|bad revision|unknown ref/i, code: 'REFERENCE_REPO_BAD_REF', status: 400 },
  { pattern: /not a git repository/i, code: 'REFERENCE_REPO_NOT_A_REPO', status: 400 },
];

// Wrap the shared execGit helper so the rest of this module gets the same
// `(cwd, args) => stdout` shape it had before — and so any git failure here
// surfaces as a typed ServerError instead of a bare Error.
const runGit = async (cwd, args, { timeoutMs = 60_000 } = {}) => {
  const result = await execGit(args, cwd, { timeout: timeoutMs }).catch((err) => ({ error: err }));
  if (result.error) {
    // Redact `user:token@host` userinfo and scp-style `user@host` from BOTH
    // the persisted/thrown message AND the server log line — credentials
    // embedded in repoUrl must not land anywhere they could later be read
    // (logs, persisted lastError, or API responses).
    const safeArgs = redactArgsForError(args);
    const safeMessage = redactUrlCreds(String(result.error.message || ''));
    console.error(`❌ git ${safeArgs.join(' ')} (cwd=${cwd}) failed: ${safeMessage}`);
    // Map well-known stderr patterns to 4xx so the UI/route layer can
    // distinguish "fix your config" from "server problem". checkReferenceRepo
    // preserves these codes when re-throwing.
    const userError = GIT_USER_ERROR_PATTERNS.find((p) => p.pattern.test(safeMessage));
    const status = userError ? userError.status : 500;
    const code = userError ? userError.code : 'REFERENCE_REPO_GIT_FAILED';
    throw new ServerError(`git ${safeArgs.join(' ')}: ${safeMessage}`, { status, code });
  }
  return String(result.stdout || '').trim();
};

/**
 * Resolve the working directory for a ref. For URL-based refs we use the
 * managed clone path under data/cos/reference-repos/<refId>/; for local
 * refs (user-supplied path), we shell directly into that path so the user
 * can keep using their normal working tree. `~` in a local path is
 * expanded so cwd ends up on the actual filesystem path.
 */
const workingDirectory = (ref) => (
  isLocalPath(ref.repoUrl) ? expandHome(ref.repoUrl) : cloneDir(ref.id)
);

/**
 * Make sure the managed clone exists for a URL-based ref. No-op for local
 * refs (the user maintains those themselves).
 */
const ensureClone = async (ref) => {
  if (isLocalPath(ref.repoUrl)) {
    const localPath = expandHome(ref.repoUrl);
    if (!existsSync(localPath)) {
      throw new ServerError(`Local reference path not found: ${ref.repoUrl}`, { status: 400, code: 'REFERENCE_REPO_LOCAL_MISSING' });
    }
    // A regular file (or symlink to one) at the path will surface later as
    // an opaque "not a git repository" error from `git rev-parse`. Catch it
    // here with a clear message so the user knows to fix the URL.
    let stat;
    try { stat = statSync(localPath); } catch {
      throw new ServerError(`Local reference path not accessible: ${ref.repoUrl}`, { status: 400, code: 'REFERENCE_REPO_LOCAL_INACCESSIBLE' });
    }
    if (!stat.isDirectory()) {
      throw new ServerError(`Local reference path is not a directory: ${ref.repoUrl}`, { status: 400, code: 'REFERENCE_REPO_LOCAL_NOT_DIR' });
    }
    if (!existsSync(join(localPath, '.git'))) {
      throw new ServerError(`Local reference path is not a git working tree (no .git): ${ref.repoUrl}`, { status: 400, code: 'REFERENCE_REPO_LOCAL_NOT_GIT' });
    }
    return localPath;
  }
  await ensureDir(REFERENCE_REPOS_ROOT);
  const dest = cloneDir(ref.id);
  if (existsSync(join(dest, '.git'))) return dest;
  assertGitArgsSafe(ref);
  // No clone yet → bring one down. --depth 1 would lose the diff window we
  // need for `git log lastReviewedSha..HEAD`, so do a full clone.
  console.log(`📦 Cloning reference repo "${ref.name}" → ${dest}`);
  await runGit(REFERENCE_REPOS_ROOT, ['clone', '--', ref.repoUrl, ref.id], { timeoutMs: 600_000 });
  return dest;
};

/**
 * Fetch the latest commits for a ref's branch (or its remote-tracking
 * default if no branch was set). Returns the head SHA after fetch.
 */
const fetchHead = async (ref) => {
  const cwd = await ensureClone(ref);
  assertGitArgsSafe(ref);
  const branch = ref.branch || 'main';
  // For URL-based refs, fetch the latest from origin so rev-parse picks
  // up new commits. Local-path refs are the user's own working tree —
  // skip fetch (the user manages their tree directly) and read the
  // branch tip in place.
  if (!isLocalPath(ref.repoUrl)) {
    await runGit(cwd, ['fetch', '--prune', 'origin', '--', branch]);
  }
  const headRef = isLocalPath(ref.repoUrl) ? branch : `origin/${branch}`;
  const head = await runGit(cwd, ['rev-parse', headRef]);
  if (!SHA_RE.test(head)) {
    throw new ServerError(`git rev-parse returned non-SHA for ${headRef}: "${head}"`, { status: 500, code: 'REFERENCE_REPO_GIT_FAILED' });
  }
  return { cwd, head, headRef };
};

/**
 * Build a structured commit list since `sinceSha` (exclusive) up to the
 * ref tip. Returned shape is JSON-friendly so the UI / agent prompt can
 * render it directly.
 */
// Hard cap on commits returned per check. Without this, a ref left
// unreviewed for a long time (a year of upstream activity, e.g. ~3000
// commits) generates an oversized agent prompt, a huge UI payload, and
// a slow git log. 200 is plenty for the agent to prioritize commits
// against the user's notes; the user can pin lastReviewedSha forward
// to drop older items if needed.
const COMMIT_LIST_CAP = 200;
const LAST_KNOWN_GOOD_SNAPSHOT_VERSION = 1;
const SNAPSHOT_AUTHOR_MAX_LENGTH = 200;
const SNAPSHOT_SUBJECT_MAX_LENGTH = 500;

const boundedText = (value, maxLength) => (
  typeof value === 'string' ? value.slice(0, maxLength) : ''
);

// The live snapshot contains a clone working directory because the internal
// reference-watch prompt needs it. The persisted fallback deliberately does
// not: clone paths are machine-local implementation details, and commit email
// addresses are not needed to explain a reference change to the user.
const persistableSnapshot = (snapshot, fetchedAt) => ({
  schemaVersion: LAST_KNOWN_GOOD_SNAPSHOT_VERSION,
  fetchedAt,
  head: snapshot.head,
  headShort: snapshot.headShort,
  sinceSha: snapshot.sinceSha,
  sinceShort: snapshot.sinceShort,
  branch: snapshot.branch,
  commitCount: snapshot.commitCount,
  truncated: snapshot.truncated === true,
  totalCommitCount: snapshot.totalCommitCount,
  commits: snapshot.commits.slice(0, COMMIT_LIST_CAP).map((commit) => ({
    sha: commit.sha,
    author: boundedText(commit.author, SNAPSHOT_AUTHOR_MAX_LENGTH),
    date: commit.date,
    subject: boundedText(commit.subject, SNAPSHOT_SUBJECT_MAX_LENGTH),
  })),
});

const lastKnownGoodSnapshot = (ref) => {
  const snapshot = ref?.lastKnownGoodSnapshot;
  if (
    !snapshot
    || snapshot.schemaVersion !== LAST_KNOWN_GOOD_SNAPSHOT_VERSION
    || !SHA_RE.test(snapshot.head || '')
    || typeof snapshot.fetchedAt !== 'string'
    || !Number.isFinite(Date.parse(snapshot.fetchedAt))
    || !Array.isArray(snapshot.commits)
  ) return null;
  return snapshot;
};

const staleAgeMs = (fetchedAt) => Math.max(0, Date.now() - Date.parse(fetchedAt));

// Returns { commits, truncated, totalCommitCount } where:
//   - commits.length <= COMMIT_LIST_CAP
//   - truncated is true when totalCommitCount > COMMIT_LIST_CAP
//   - totalCommitCount is null when truncation isn't possible (no sinceSha)
const listCommits = async (cwd, sinceSha, headRef) => {
  if (!sinceSha) {
    // No prior review — surface only the most recent 25 commits to keep
    // the first-scan prompt bounded. The UI tells the user they can pin
    // a SHA manually if they want to start from a specific point.
    const out = await runGit(cwd, ['log', '-n', '25', '--pretty=format:%H%x09%an%x09%ae%x09%aI%x09%s', headRef]);
    return { commits: parseCommitLog(out), truncated: false, totalCommitCount: null };
  }
  // For incremental scans we DO know the total — check it cheaply with
  // `rev-list --count` so the UI can show "showing 200 of N" honestly,
  // then fetch only the most recent COMMIT_LIST_CAP entries.
  const range = `${sinceSha}..${headRef}`;
  const totalRaw = await runGit(cwd, ['rev-list', '--count', range]);
  const totalCommitCount = Number(totalRaw) || 0;
  const out = await runGit(cwd, ['log', '-n', String(COMMIT_LIST_CAP), '--pretty=format:%H%x09%an%x09%ae%x09%aI%x09%s', range]);
  const commits = parseCommitLog(out);
  return { commits, truncated: totalCommitCount > COMMIT_LIST_CAP, totalCommitCount };
};

const parseCommitLog = (raw) => {
  if (!raw) return [];
  return raw.split('\n').filter(Boolean).map((line) => {
    const [sha, author, email, date, ...rest] = line.split('\t');
    return { sha, author, email, date, subject: rest.join('\t') };
  });
};

/**
 * Public API ─────────────────────────────────────────────────────────────
 */

export async function listReferenceRepos(appId) {
  const app = await getAppById(appId);
  if (!app) throw new ServerError(`App not found: ${appId}`, { status: 404, code: 'APP_NOT_FOUND' });
  return Array.isArray(app.referenceRepos) ? app.referenceRepos : [];
}

export async function addReferenceRepo(appId, { name, repoUrl, branch, notes }) {
  const app = await getAppById(appId);
  if (!app) throw new ServerError(`App not found: ${appId}`, { status: 404, code: 'APP_NOT_FOUND' });
  const trimmedUrl = repoUrl.trim();
  rejectCredentialedUrl(trimmedUrl);
  assertGitArgsSafe({ repoUrl: trimmedUrl, branch });
  const existing = Array.isArray(app.referenceRepos) ? app.referenceRepos : [];
  const ref = {
    id: uuidv4(),
    name: name.trim(),
    repoUrl: trimmedUrl,
    branch: (branch || 'main').trim(),
    notes: (notes || '').trim(),
    lastReviewedSha: null,
    lastCheckedAt: null,
    status: 'needs-clone',
    lastError: null,
    lastKnownGoodSnapshot: null,
    createdAt: new Date().toISOString(),
  };
  await updateApp(appId, { referenceRepos: [...existing, ref] });
  return ref;
}

export async function updateReferenceRepo(appId, refId, patch) {
  const app = await getAppById(appId);
  if (!app) throw new ServerError(`App not found: ${appId}`, { status: 404, code: 'APP_NOT_FOUND' });
  const refs = Array.isArray(app.referenceRepos) ? app.referenceRepos : [];
  const idx = refs.findIndex((r) => r.id === refId);
  if (idx < 0) throw new ServerError(`Reference repo not found: ${refId}`, { status: 404, code: 'REFERENCE_REPO_NOT_FOUND' });
  // Allow only known fields through — guards against a bad client payload
  // resetting status/lastError/etc. Trim the same string fields that
  // addReferenceRepo() trims so we don't end up with mismatched shapes
  // (e.g. " main " as a branch name causing confusing git failures).
  //
  // Defensive SHA validation: the route's Zod schema already hex-validates
  // lastReviewedSha, but a non-route caller (CoS task runner, future
  // internal API) could bypass that and persist garbage. Re-check here so
  // the service contract is self-enforcing. null clears the SHA cleanly
  // and is allowed.
  if (patch.lastReviewedSha !== undefined && patch.lastReviewedSha !== null && !SHA_RE.test(patch.lastReviewedSha)) {
    throw new ServerError(`lastReviewedSha must be a 40-char hex SHA: ${patch.lastReviewedSha}`, { status: 400, code: 'REFERENCE_REPO_BAD_SHA' });
  }
  // PATCH must enforce the same credential-in-URL rejection as POST —
  // otherwise a user could sidestep the ingest check by editing the URL
  // afterwards and persisting `https://token@host/repo` into apps.json.
  assertGitArgsSafe(patch);
  if (typeof patch.repoUrl === 'string') {
    rejectCredentialedUrl(patch.repoUrl.trim());
  }
  // When a SHA is being pinned, verify it actually resolves to a commit
  // in the clone. Without this, a PATCH could persist a syntactically valid
  // but nonexistent SHA, and the next `checkReferenceRepo` would die in
  // `git log <bad>..HEAD` with a confusing "unknown revision" error.
  // Skipped when:
  //   - clearing (null) — that always succeeds
  //   - the ref has never been cloned yet (fresh add + manual SHA pin)
  //   - the same PATCH is also changing repoUrl: the clone hasn't been
  //     re-fetched against the new origin yet, so its commits reflect
  //     the OLD repo. Verifying against stale state would produce
  //     misleading results (false positives where the SHA happens to
  //     exist in the old repo, or false negatives where it doesn't).
  //     Let the next checkReferenceRepo surface any real mismatch.
  const repoUrlChanging = typeof patch.repoUrl === 'string' && patch.repoUrl.trim() !== refs[idx].repoUrl;
  if (
    typeof patch.lastReviewedSha === 'string'
    && SHA_RE.test(patch.lastReviewedSha)
    && !repoUrlChanging
  ) {
    const ref = refs[idx];
    const dest = isLocalPath(ref.repoUrl) ? expandHome(ref.repoUrl) : cloneDir(ref.id);
    // Same .git-existence check applies to both URL-based managed clones
    // and user-supplied local paths (ensureClone validates `.git` for local
    // refs too), so no per-branch logic needed.
    const cloneExists = existsSync(join(dest, '.git'));
    if (cloneExists) {
      const verify = await execGit(['cat-file', '-e', `${patch.lastReviewedSha}^{commit}`], dest, { timeout: 10_000 })
        .catch((err) => ({ error: err }));
      if (verify && verify.error) {
        // `cat-file -e` exits non-zero for two distinct reasons: (a) the
        // object isn't there, and (b) the repo is corrupt / unreadable /
        // permissions / timeout. The user-facing fix is different in each
        // case — distinguish via stderr text so the UI can show actionable
        // detail.
        const msg = String(verify.error.message || '').toLowerCase();
        const looksLikeMissing = /not a valid object|missing|does not exist|bad object|unable to read tree/.test(msg);
        if (looksLikeMissing) {
          throw new ServerError(
            `lastReviewedSha ${patch.lastReviewedSha.slice(0, 8)} not found in reference repo "${ref.name}"`,
            { status: 400, code: 'REFERENCE_REPO_SHA_NOT_FOUND' },
          );
        }
        // Anything else (permission denied, corrupt index, timeout) — surface
        // the underlying message so the user can act on it. Redact in case
        // the error includes the URL with credentials.
        throw new ServerError(
          `Failed to verify lastReviewedSha against reference repo "${ref.name}": ${redactUrlCreds(verify.error.message || 'unknown git error')}`,
          { status: 500, code: 'REFERENCE_REPO_SHA_VERIFY_FAILED' },
        );
      }
    }
  }
  // If repoUrl is changing on a URL-based ref with an existing managed
  // clone, point the clone's origin remote at the new URL — otherwise
  // subsequent fetches keep pulling from the old URL and the ref appears
  // stuck. Skipped when: same URL, ref is local-path (no managed clone),
  // or no clone exists yet (next check will clone fresh).
  const oldRef = refs[idx];
  if (typeof patch.repoUrl === 'string' && patch.repoUrl.trim() !== oldRef.repoUrl && !isLocalPath(patch.repoUrl)) {
    const dest = cloneDir(oldRef.id);
    if (existsSync(join(dest, '.git'))) {
      const result = await execGit(['remote', 'set-url', 'origin', '--', patch.repoUrl.trim()], dest, { timeout: 10_000 })
        .catch((err) => ({ error: err }));
      if (result && result.error) {
        // Don't fail the patch outright — the user can manually delete
        // and re-add the ref to recover. But surface a warning so the
        // next check doesn't silently fetch from the old URL.
        console.warn(`⚠️ Failed to update origin remote for ref "${oldRef.name}": ${redactUrlCreds(result.error.message || '')}. Next check may use the previous URL — delete and re-add the ref to recover.`);
      }
    }
  }
  const TRIMMED_KEYS = new Set(['name', 'repoUrl', 'branch', 'notes']);
  const updated = { ...refs[idx] };
  for (const key of ['name', 'repoUrl', 'branch', 'notes', 'lastReviewedSha']) {
    if (patch[key] !== undefined) {
      updated[key] = (TRIMMED_KEYS.has(key) && typeof patch[key] === 'string')
        ? patch[key].trim()
        : patch[key];
    }
  }
  // Manual SHA pin counts as a review — record the time so "last reviewed"
  // doesn't silently lie. Skip the bump when the SHA is being CLEARED
  // (lastReviewedSha=null), otherwise "last checked" would look fresh
  // immediately after a reset.
  if (typeof patch.lastReviewedSha === 'string' && SHA_RE.test(patch.lastReviewedSha)) {
    updated.lastCheckedAt = new Date().toISOString();
  }
  const next = [...refs];
  next[idx] = updated;
  await updateApp(appId, { referenceRepos: next });
  return updated;
}

export async function deleteReferenceRepo(appId, refId) {
  const app = await getAppById(appId);
  if (!app) throw new ServerError(`App not found: ${appId}`, { status: 404, code: 'APP_NOT_FOUND' });
  const refs = Array.isArray(app.referenceRepos) ? app.referenceRepos : [];
  const next = refs.filter((r) => r.id !== refId);
  if (next.length === refs.length) {
    throw new ServerError(`Reference repo not found: ${refId}`, { status: 404, code: 'REFERENCE_REPO_NOT_FOUND' });
  }
  await updateApp(appId, { referenceRepos: next });
  // Clone is best-effort to leave on disk — user might re-add the ref by
  // URL and we'd save them the re-clone. UI offers a "purge clones" action
  // separately if disk pressure becomes a thing.
  return { ok: true };
}

/**
 * Fetch the ref, compute commits since lastReviewedSha, and return a
 * structured snapshot. Does NOT update lastReviewedSha — that happens
 * after the user / scheduled task has reviewed the proposal.
 */
export async function checkReferenceRepo(appId, refId) {
  const app = await getAppById(appId);
  if (!app) throw new ServerError(`App not found: ${appId}`, { status: 404, code: 'APP_NOT_FOUND' });
  const refs = Array.isArray(app.referenceRepos) ? app.referenceRepos : [];
  const ref = refs.find((r) => r.id === refId);
  if (!ref) throw new ServerError(`Reference repo not found: ${refId}`, { status: 404, code: 'REFERENCE_REPO_NOT_FOUND' });

  const checkedAt = new Date().toISOString();
  let snapshot;
  let nextStatus = 'ok';
  let nextError = null;
  let originalError = null;
  try {
    const { cwd, head, headRef } = await fetchHead(ref);
    const { commits, truncated, totalCommitCount } = await listCommits(cwd, ref.lastReviewedSha, headRef);
    snapshot = {
      head,
      headShort: SHORT_SHA(head),
      sinceSha: ref.lastReviewedSha,
      sinceShort: SHORT_SHA(ref.lastReviewedSha),
      // Visible-on-this-page count, capped by COMMIT_LIST_CAP.
      commitCount: commits.length,
      // True when more commits exist upstream than we returned. UI shows
      // "X of Y new" when this is set.
      truncated,
      // The full count (only available for incremental scans). null on
      // first scan since we don't pay for an extra rev-list there.
      totalCommitCount,
      commits,
      cwd,
      branch: ref.branch || 'main',
      fetchedAt: checkedAt,
      stale: false,
    };
  } catch (err) {
    nextStatus = 'error';
    nextError = err instanceof ServerError ? err.message : String(err.message || err);
    originalError = err;
  }
  const storedSnapshot = snapshot ? persistableSnapshot(snapshot, checkedAt) : null;
  const fallbackSnapshot = lastKnownGoodSnapshot(ref);
  const errorCode = originalError instanceof ServerError
    ? originalError.code || 'REFERENCE_REPO_CHECK_FAILED'
    : 'REFERENCE_REPO_CHECK_FAILED';
  // Persist status + lastCheckedAt regardless of success — UI surfaces the
  // error inline so the user can fix bad URL / branch. A failed refresh keeps
  // the last successful snapshot untouched so it remains a trustworthy fallback.
  const next = refs.map((r) => (r.id === refId
    ? {
      ...r,
      status: nextStatus === 'error' && fallbackSnapshot ? 'stale' : nextStatus,
      lastError: nextError,
      lastCheckedAt: checkedAt,
      ...(storedSnapshot ? { lastKnownGoodSnapshot: storedSnapshot } : {}),
    }
    : r));
  await updateApp(appId, { referenceRepos: next });
  if (nextStatus === 'error') {
    if (fallbackSnapshot) {
      return {
        ...fallbackSnapshot,
        stale: true,
        staleAgeMs: staleAgeMs(fallbackSnapshot.fetchedAt),
        error: { code: errorCode, message: nextError },
      };
    }
    // Preserve the original ServerError's status/code (e.g. 400
    // REFERENCE_REPO_LOCAL_MISSING) so user-fixable config problems don't
    // become opaque 500s. Fall back to a generic 500 only for truly
    // unexpected errors.
    if (originalError instanceof ServerError) {
      throw new ServerError(originalError.message, {
        status: originalError.status,
        code: originalError.code || 'REFERENCE_REPO_CHECK_FAILED',
      });
    }
    throw new ServerError(nextError, { status: 500, code: 'REFERENCE_REPO_CHECK_FAILED' });
  }
  return snapshot;
}

/**
 * Mark a ref as reviewed up to the given SHA — called after a CoS
 * sub-agent finishes recording proposals (PLAN.md items / GitHub|GitLab
 * issues / JIRA tickets) for the new commits, or by
 * the UI's "mark as reviewed" button. SHA must match a real commit visible from
 * the ref's working tree (verified via `git cat-file -e <sha>^{commit}`
 * against the managed clone or the user-supplied local path).
 */
export async function markReferenceRepoReviewed(appId, refId, sha) {
  if (!SHA_RE.test(sha || '')) {
    throw new ServerError(`Invalid SHA: ${sha}`, { status: 400, code: 'REFERENCE_REPO_BAD_SHA' });
  }
  const app = await getAppById(appId);
  if (!app) throw new ServerError(`App not found: ${appId}`, { status: 404, code: 'APP_NOT_FOUND' });
  const refs = Array.isArray(app.referenceRepos) ? app.referenceRepos : [];
  const ref = refs.find((r) => r.id === refId);
  if (!ref) throw new ServerError(`Reference repo not found: ${refId}`, { status: 404, code: 'REFERENCE_REPO_NOT_FOUND' });
  // Make sure the clone exists, then verify the SHA resolves to a real
  // commit object. `cat-file -e <sha>^{commit}` exits non-zero (and our
  // execGit wrapper rejects) if the object is missing or not a commit.
  const cwd = await ensureClone(ref);
  const verify = await execGit(['cat-file', '-e', `${sha}^{commit}`], cwd, { timeout: 10_000 })
    .catch((err) => ({ error: err }));
  if (verify && verify.error) {
    throw new ServerError(
      `SHA ${sha.slice(0, 8)} not found in reference repo "${ref.name}"`,
      { status: 400, code: 'REFERENCE_REPO_SHA_NOT_FOUND' },
    );
  }
  return updateReferenceRepo(appId, refId, { lastReviewedSha: sha });
}

/**
 * Render a reference's commit list + notes into a Markdown chunk that
 * the reference-watch task injects into its agent prompt. Kept here
 * (not in the prompt template) so we can iterate the format without
 * touching cos.js.
 */
// Generous cap: COMMIT_LIST_CAP subjects at typical length fit; a crafted flood cannot dominate the prompt.
const COMMIT_LIST_FENCE_CHARS = 20000;
const COMMIT_SUBJECT_CHARS = 200;

export function formatReferenceForPrompt(ref, snapshot) {
  const lines = [];
  lines.push(`## Reference: ${ref.name}`);
  // Redact userinfo before emitting into the prompt — even though we strip
  // credentials at ingest now, older entries persisted before that fix may
  // still carry tokens in apps.json.
  lines.push(`- Repo: ${redactUrlCreds(ref.repoUrl)}`);
  lines.push(`- Branch: ${ref.branch || 'main'}`);
  lines.push(`- Last reviewed: ${SHORT_SHA(ref.lastReviewedSha) || '(none — first scan)'}`);
  // When truncated, surface "showing X of Y" so the agent knows it's
  // looking at a head-end slice (most recent COMMIT_LIST_CAP commits).
  // Older commits are still in the upstream; user can pin a SHA forward
  // to drop them or run another check after reviewing this batch.
  const countLabel = snapshot.truncated
    ? `${snapshot.commitCount} of ${snapshot.totalCommitCount} new commits — older ones omitted; review these first then pin SHA forward`
    : `${snapshot.commitCount} new commits`;
  lines.push(`- Current head: ${snapshot.headShort} (${countLabel})`);
  if (ref.notes) {
    lines.push('');
    // User-provided context: how the app relates to this upstream — what
    // features the app maintains in its own code, what bugs to watch for,
    // what areas of upstream are out of scope. The agent uses this to
    // prioritize commits and decide what's worth reimplementing.
    lines.push('### Context (user-supplied — how this app relates to the repo)');
    lines.push(ref.notes);
  }
  if (snapshot.commits.length === 0) {
    lines.push('');
    lines.push('_No new commits since last review._');
    return lines.join('\n');
  }
  lines.push('');
  lines.push('### Commits to review');
  lines.push(UNTRUSTED_CONTENT_NOTICE);
  // Subjects and authors are upstream-controlled; fence them so they stay data.
  // Per-field clamps keep every SHA inside the overall cap, so one huge subject cannot hide later commits.
  const commitLines = snapshot.commits.map((c) => `${c.sha.slice(0, 8)} ${clampText(c.subject, COMMIT_SUBJECT_CHARS)} (by ${clampText(c.author, 100)}, ${c.date.slice(0, 10)})`);
  lines.push(fenceBlock('Upstream commits', commitLines.join('\n'), COMMIT_LIST_FENCE_CHARS));
  lines.push('');
  lines.push(`Source clone is at: \`${snapshot.cwd}\` — use \`git -C ${snapshot.cwd} show <sha>\` to read each commit's diff.`);
  return lines.join('\n');
}

// The reference-watch prompt's {trackerInstructions} block — where the agent
// records its Adopt/Maybe proposals — now lives in lib/workTracker.js alongside
// the tracker resolution it keys off, shared with the other tracker-filing task
// types (`ux`). Re-exported here so existing `referenceRepos.formatTrackerInstructions`
// callers keep working; the reference-watch wording is the function's DEFAULT
// option set, so a bare call still returns the v3 blocks byte-for-byte.
export { formatTrackerInstructions };

/**
 * Trigger a CoS analysis task for a single reference repo after a successful
 * check found new commits. Builds the reference-watch prompt with the snapshot
 * data and queues it as an internal task for the next available agent slot.
 *
 * Returns `{ queued: true, taskId }` on success, or `{ queued: false, reason }`
 * when the task can't be created (e.g. no commits, app not found).
 */
export async function triggerReferenceAnalysis(app, ref, snapshot) {
  if (snapshot?.stale) {
    return { queued: false, reason: 'stale-snapshot' };
  }
  if (!snapshot || snapshot.commitCount === 0) {
    return { queued: false, reason: 'no-new-commits' };
  }
  if (!app) return { queued: false, reason: 'app-not-found' };

  const { addTask } = await import('./cosTaskStore.js');
  const { getTaskPrompt } = await import('./taskPromptService.js');

  const referenceDataBlock = formatReferenceForPrompt(ref, snapshot);
  // Resolve where THIS app records autonomous work (PLAN.md / GitHub / GitLab /
  // JIRA) so the agent files proposals into the configured tracker instead of
  // always appending to PLAN.md. Never throws — degrades to the PLAN.md block.
  const workTracker = await resolveAppWorkTracker(app).catch(() => ({ resolved: 'plan' }));
  const trackerInstructionsBlock = formatTrackerInstructions(workTracker.resolved);
  const promptTemplate = await getTaskPrompt('reference-watch');
  // Use arrow replacers to avoid $& / $1 interpretation in replacement strings.
  // {trackerInstructions} is substituted FIRST so the {appName}/{repoPath}
  // placeholders inside the injected block are expanded by the later replacers.
  const fullPrompt = applyAppPlaceholders(
    promptTemplate.replace(/\{trackerInstructions\}/g, () => trackerInstructionsBlock),
    app,
  )
    .replace(/\{reviewers\}/g, () => DEFAULT_REVIEWER)
    .replace(/\{referenceData\}/g, () => referenceDataBlock)
    .replace(/\{planConstraint\}/g, () => '');

  const task = await addTask({
    id: `sys-ref-analysis-${app.id}-${ref.id}-${Date.now().toString(36)}`,
    status: 'pending',
    priority: 'MEDIUM',
    priorityValue: 2,
    description: `Reference-watch analysis: ${ref.name} (${redactUrlCreds(ref.repoUrl)}) for ${app.name}`,
    metadata: {
      app: app.id,
      appName: app.name,
      repoPath: app.repoPath,
      analysisType: 'reference-watch',
      autoGenerated: true,
      // Resolved work tracker the prompt told the agent to file proposals into
      // (PLAN.md / GitHub / GitLab / JIRA) — recorded for traceability.
      workTracker: workTracker.resolved,
      // The reference-watch prompt v4 instructs the agent to record proposals in
      // the resolved tracker: PLAN.md commits checklist items; GitHub/GitLab
      // use plain issue titles and the shared `gh`/`glab` filing instructions.
      // readOnly:true would inject the "do not modify or commit
      // files" guard into the system prompt and the agent would refuse to
      // write/commit/shell — defeating the whole flow. Mark writable.
      readOnly: false,
      // Whether the run is expected to dirty the worktree. Derived from the SAME
      // `workTracker.resolved` value that selected the {trackerInstructions}
      // block above, so the flag can never disagree with the instructions the
      // agent actually received: the PLAN.md path commits checklist items (dirty
      // tree), while the github/gitlab/jira paths file issues/tickets and — per
      // the prompt's "no source-code edits" contract — leave the tree CLEAN.
      // Without this downstream task bookkeeping treats a report-shaped run that
      // did its whole job with a clean worktree as missing code work (#3102).
      worktreeChangesExpected: isFileTracker(workTracker.resolved),
      // A multi-line payload here is the agent PROMPT, not the one-line human
      // note — `cosTaskStore.addTask` routes it to `metadata.prompt` on write
      // (#4153, server/lib/cosTaskPrompt.js).
      context: fullPrompt,
    },
    autoApproved: true,
    section: 'pending',
  }, 'internal', { raw: true });

  if (task?.duplicate) {
    return { queued: false, reason: 'duplicate', taskId: task.id };
  }
  return { queued: true, taskId: task.id };
}

// Exported for tests + reference-watch task type lookup.
export const __test = {
  REFERENCE_REPOS_ROOT,
  cloneDir,
  isLocalPath,
  workingDirectory,
  parseCommitLog,
};