Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Repository Cloner Service
*
* Handles cloning repositories (github.com / gitlab.com — see `REPO_HOSTS` in
* lib/repoUrl.js) to a local directory for reference. Supports shallow clones to
* save space and provides progress tracking.
*/
import { spawn } from '../lib/childProcess.js';
import { existsSync } from 'fs';
import { access, mkdtemp, readdir, rename, rm } from 'fs/promises';
import { dirname, join, resolve } from 'path';
import { ensureDir, PATHS } from '../lib/fileUtils.js';
import { MAX_REPO_PATH_DEPTH, REPO_HOSTS, parseRepoUrl, repoCloneUrl } from '../lib/repoUrl.js';
// Default directory for cloned repos (can be configured in settings)
const DEFAULT_CLONE_DIR = PATHS.repos;
const CLONE_STAGING_PREFIX = '.portos-clone-';
const CLONE_STAGING_MAX_AGE_MS = 10 * 60 * 1000;
const CLONE_STAGING_RE = /^\.portos-clone-(\d{13})-[A-Za-z0-9]+$/;
/**
* The clone directory for a parsed repo, relative to the repos root.
*
* A host flagged `flatClonePath` keeps the historical `<owner>/<repo>` layout so
* every clone made before PortOS supported a second host stays exactly where its
* link record says it is. Every other host is namespaced under its hostname,
* which is also what keeps `gitlab.com/acme/widgets` from colliding with
* `github.com/acme/widgets`. A GitLab `owner` may itself be a `group/subgroup`
* path; every segment is validated by parseRepoUrl, so the join stays inside
* the root.
*/
export function repoSubPath({ host, owner, repo }) {
return REPO_HOSTS[host]?.flatClonePath ? join(owner, repo) : join(host, owner, repo);
}
/**
* Get clone directory path
*/
export function getCloneDir(customDir) {
return customDir || DEFAULT_CLONE_DIR;
}
/**
* True when `localPath` is exactly where cloneRepo puts `url`. Every action
* that reads or pulls a recorded clone checks this first: the recorded path is
* stored data, so it is only trusted when it equals the path derived from the
* link's own URL under the clone directory.
*/
export function isDerivedClonePath(url, localPath, customDir) {
const parsed = parseRepoUrl(url);
if (!parsed || typeof localPath !== 'string' || !localPath) return false;
return resolve(localPath) === resolve(join(getCloneDir(customDir), repoSubPath(parsed)));
}
/**
* Ensure clone directory exists
*/
export async function ensureCloneDir(cloneDir) {
const dir = getCloneDir(cloneDir);
if (!existsSync(dir)) {
await ensureDir(dir);
console.log(`📁 Created repos directory: ${dir}`);
}
return dir;
}
/**
* Clone a repository from a supported host.
* Returns the local path where the repo was cloned.
*/
export async function cloneRepo(url, options = {}) {
const parsed = parseRepoUrl(url);
if (!parsed) {
throw new Error('Invalid repository URL');
}
const { owner, repo } = parsed;
const cloneDir = await ensureCloneDir(options.cloneDir);
const localPath = join(cloneDir, repoSubPath(parsed));
// Boot recovery marks only a known interrupted attempt. Old PortOS versions
// cloned straight into localPath, so their partial checkout must be replaced
// before the normal already-cloned compatibility check can trust `.git`.
if (options.replaceIncomplete === true) {
await rm(localPath, { recursive: true, force: true });
}
// Check if already cloned
if (existsSync(join(localPath, '.git'))) {
console.log(`📦 Repo already cloned: ${owner}/${repo}`);
return {
localPath,
owner,
repo,
alreadyCloned: true
};
}
// Ensure the owner directory exists (for a namespaced host, and for a GitLab
// subgroup path, that is several levels below the repos root).
const ownerDir = dirname(localPath);
if (!existsSync(ownerDir)) {
await ensureDir(ownerDir);
}
// Clone into attempt-specific staging, then publish it only after git exits.
// An abruptly orphaned git child can keep writing its private directory, but
// a retry gets a different directory and cannot race the live checkout.
const stagingRoot = await mkdtemp(join(ownerDir, `${CLONE_STAGING_PREFIX}${Date.now()}-`));
const stagingPath = join(stagingRoot, repo);
// Build clone command with shallow clone for space efficiency
const httpsUrl = repoCloneUrl(parsed);
const args = [
'clone',
'--depth', '1',
'--single-branch',
...(options.branch ? ['--branch', options.branch] : []),
httpsUrl,
stagingPath
];
console.log(`📥 Cloning ${owner}/${repo}...`);
const clone = new Promise((resolve, reject) => {
const child = spawn('git', args, {
env: process.env,
shell: false
});
let stderr = '';
child.stderr.on('data', (data) => {
stderr += data.toString();
});
const timeout = setTimeout(() => {
child.kill();
reject(new Error('Clone timed out after 5 minutes'));
}, 300000);
child.on('close', (code) => {
clearTimeout(timeout);
if (code === 0) {
resolve();
} else {
console.error(`❌ Failed to clone ${owner}/${repo}: ${stderr}`);
reject(new Error(`Git clone failed: ${stderr || `exit code ${code}`}`));
}
});
child.on('error', (err) => {
clearTimeout(timeout);
console.error(`❌ Git clone error: ${err.message}`);
reject(err);
});
});
const cleanupStaging = () => rm(stagingRoot, { recursive: true, force: true })
.catch(err => console.error(`❌ Failed to clean clone staging directory: ${err.message}`));
return clone.then(async () => {
// A concurrent attempt may have won while this one was cloning. Keep the
// completed checkout and discard only this attempt's private staging dir.
if (existsSync(join(localPath, '.git'))) {
await cleanupStaging();
return { localPath, owner, repo, alreadyCloned: true };
}
await rename(stagingPath, localPath);
await cleanupStaging();
console.log(`✅ Cloned ${owner}/${repo} to ${localPath}`);
return { localPath, owner, repo, alreadyCloned: false };
}).catch(async (err) => {
await cleanupStaging();
throw err;
});
}
/**
* Remove only PortOS-owned clone staging directories older than twice the git
* timeout. A freshly orphaned child may still be writing; the age gate leaves
* it alone while bounding disk retained across repeated interrupted attempts.
*
* The sweep RECURSES because staging sits beside the checkout it will become,
* and that is no longer always one level down: a namespaced host adds a
* hostname level and each GitLab subgroup adds another. The bound is DERIVED
* from the host table's namespace caps rather than hardcoded — a hand-picked
* number silently stops reaping the moment a host's `maxDepth` is raised, and
* an unreaped interrupted clone is hundreds of MB that nothing ever frees.
*
* Staging sits beside the repo segment, so the sweep must be able to LIST the
* directory one level above the deepest repo — and `sweep` is called with
* `depth = level + 1` (the repos root itself is depth 1). Recursing into that
* directory therefore needs the guard to still pass at depth
* `MAX_REPO_PATH_DEPTH - 1`, which makes the bound `MAX_REPO_PATH_DEPTH` exactly.
*/
const MAX_STAGING_DEPTH = MAX_REPO_PATH_DEPTH;
// A directory holding `.git` is a finished clone, not a namespace level —
// descending into it would walk the studied repo's whole source tree.
const isCheckout = (dir) => access(join(dir, '.git')).then(() => true, () => false);
export async function reapStaleCloneStaging({ cloneDir = DEFAULT_CLONE_DIR, now = Date.now() } = {}) {
// One unreadable directory (removed mid-sweep, or not ours to read) must not
// abort the sweep and leave every later branch's staging behind.
const sweep = async (dir, depth) => {
const entries = await readdir(dir, { withFileTypes: true })
.catch(err => {
if (err.code !== 'ENOENT') console.error(`⚠️ Skipped clone staging sweep for ${dir}: ${err.message}`);
return [];
});
let reaped = 0;
for (const entry of entries) {
if (!entry.isDirectory()) continue;
const match = entry.name.match(CLONE_STAGING_RE);
if (match) {
if (now - Number(match[1]) < CLONE_STAGING_MAX_AGE_MS) continue;
await rm(join(dir, entry.name), { recursive: true, force: true });
reaped++;
} else if (depth < MAX_STAGING_DEPTH && !await isCheckout(join(dir, entry.name))) {
reaped += await sweep(join(dir, entry.name), depth + 1);
}
}
return reaped;
};
const reaped = await sweep(cloneDir, 1);
if (reaped > 0) console.log(`🧹 Reaped ${reaped} stale repository clone staging director${reaped === 1 ? 'y' : 'ies'}`);
return reaped;
}
/**
* Pull latest changes for an existing repo
*/
export async function pullRepo(localPath) {
if (!existsSync(join(localPath, '.git'))) {
throw new Error('Not a git repository');
}
console.log(`🔄 Pulling latest for ${localPath}...`);
return new Promise((resolve, reject) => {
let settled = false;
const child = spawn('git', ['pull', '--ff-only'], {
cwd: localPath,
env: process.env,
shell: false
});
let stdout = '';
let stderr = '';
child.stdout.on('data', (data) => {
stdout += data.toString();
});
child.stderr.on('data', (data) => {
stderr += data.toString();
});
const timeout = setTimeout(() => {
if (!settled) {
settled = true;
child.kill();
reject(new Error('Pull timed out after 2 minutes'));
}
}, 120000);
timeout.unref?.();
child.on('close', (code) => {
clearTimeout(timeout);
if (!settled) {
settled = true;
if (code === 0) {
console.log(`✅ Pulled latest for ${localPath}`);
resolve({ stdout, stderr, success: true });
} else {
console.error(`❌ Failed to pull ${localPath}: ${stderr}`);
reject(new Error(`Git pull failed: ${stderr || `exit code ${code}`}`));
}
}
});
child.on('error', (err) => {
clearTimeout(timeout);
if (!settled) {
settled = true;
reject(err);
}
});
});
}