Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Persistent Mind screenshot attachment storage and lifecycle.
*
* Owns image upload validation, storage in the screenshots directory,
* pending attachment markers, crash recovery, and expiration cleanup
* independently of the supervisor turn execution loop.
*/
import { randomUUID } from 'crypto';
import { mkdir, readFile, readdir, stat } from 'fs/promises';
import { join } from 'path';
import {
PERSISTENT_MIND_LIMITS,
PERSISTENT_MIND_IMAGE_EXTENSIONS,
isPersistentMindAttachmentId,
normalizePersistentMindAttachment,
normalizePersistentMindState,
publicPersistentMindAttachment,
} from '../lib/persistentMind.js';
import {
detectImageFormat,
PATHS,
resolveScreenshot,
sanitizeFilename,
saveImageUpload,
unlinkGuarded,
writeFileGuarded,
} from '../lib/fileUtils.js';
import { backupSnapshotCutPending, withBackupAssetPublication } from '../lib/backupSnapshotBoundary.js';
import { loadState, saveState, withStateLock } from './cosState.js';
import { isUpdateInProgress } from './updateChecker.js';
import {
MIND_ATTACHMENT_FILENAME_PREFIX,
PENDING_ATTACHMENT_MARKER_PREFIX,
PENDING_ATTACHMENT_MARKER_PATTERN,
} from '../lib/persistentMindAttachmentOwnership.js';
export { PENDING_ATTACHMENT_MARKER_PREFIX, PENDING_ATTACHMENT_MARKER_PATTERN } from '../lib/persistentMindAttachmentOwnership.js';
const nowIso = () => new Date().toISOString();
export const attachmentFailure = (error, { code = 'INVALID_ATTACHMENT', status = 400 } = {}) => ({
success: false,
error,
code,
status,
});
export const pendingAttachmentMarkerPath = (attachmentId) => join(
PATHS.screenshots,
`${PENDING_ATTACHMENT_MARKER_PREFIX}${attachmentId}`,
);
export const removePendingAttachmentMarker = async (attachmentId) => unlinkGuarded(pendingAttachmentMarkerPath(attachmentId)).then(
() => true,
(error) => {
if (error?.code === 'ENOENT') return true;
console.error(`❌ Failed to remove Persistent Mind upload marker ${attachmentId}: ${error.message}`);
return false;
},
);
export const removeStoredFilename = async (filename) => {
const filePath = resolveScreenshot(filename);
if (!filePath) return true;
return unlinkGuarded(filePath).then(
() => true,
(error) => {
if (error?.code === 'ENOENT') return true;
console.error(`❌ Failed to remove Persistent Mind attachment file: ${error.message}`);
return false;
},
);
};
export const screenshotEntries = async () => readdir(PATHS.screenshots).then(
(entries) => entries.filter((entry) => typeof entry === 'string'),
(error) => {
if (error?.code === 'ENOENT') return [];
console.error(`❌ Failed to inspect Persistent Mind upload markers: ${error.message}`);
return null;
},
);
export const removePendingAttachmentFiles = async (attachmentId, entries) => {
const prefix = `${MIND_ATTACHMENT_FILENAME_PREFIX}${attachmentId}-`;
const candidates = entries.filter((entry) => (
entry.startsWith(prefix)
&& PERSISTENT_MIND_IMAGE_EXTENSIONS.some((extension) => entry.toLowerCase().endsWith(extension))
));
let removed = true;
for (const filename of candidates) {
if (!await removeStoredFilename(filename)) removed = false;
}
return removed;
};
/** Reap marker-backed files left before their pending state could be indexed. */
export const cleanupUnindexedPendingAttachments = async ({ knownAttachments, now, limit }) => {
const entries = await screenshotEntries();
if (!entries) return { examined: 0, removed: 0 };
const recordsById = new Map(knownAttachments.map((attachment) => [attachment.attachmentId, attachment]));
const markers = entries
.map((entry) => ({ entry, match: PENDING_ATTACHMENT_MARKER_PATTERN.exec(entry) }))
.filter(({ match }) => Boolean(match))
.slice(0, limit);
let removed = 0;
for (const { entry, match } of markers) {
const attachmentId = match[1];
const known = recordsById.get(attachmentId);
if (known) {
// A claimed asset is durable even if its metadata is later pruned. The
// marker is only a pending-upload sentinel, so removing it is safe.
if (known.claimedBy && await removePendingAttachmentMarker(attachmentId)) removed += 1;
continue;
}
const markerAge = await stat(join(PATHS.screenshots, entry)).then(
(value) => value.mtimeMs,
() => null,
);
if (!Number.isFinite(markerAge) || markerAge > now - PERSISTENT_MIND_LIMITS.PENDING_ATTACHMENT_TTL_MS) continue;
if (await removePendingAttachmentFiles(attachmentId, entries)
&& await removePendingAttachmentMarker(attachmentId)) {
removed += 1;
}
}
return { examined: markers.length, removed };
};
export const ATTACHMENT_VERIFICATION = Object.freeze({
VALID: 'valid',
INVALID: 'invalid',
UNAVAILABLE: 'unavailable',
});
/**
* Classify a stored image. Only proof of absence (ENOENT / unresolvable path)
* or a successful read whose bytes mismatch is INVALID. Any other read error
* (EACCES, EIO, EBUSY, ...) says nothing about the bytes, so it is UNAVAILABLE
* and callers must preserve the record and file.
*/
export const verifyStoredAttachment = async (attachment) => {
const filePath = resolveScreenshot(attachment.filename);
if (!filePath) return ATTACHMENT_VERIFICATION.INVALID;
const read = await readFile(filePath).then(
(bytes) => ({ bytes }),
(error) => ({ error }),
);
if (read.error) {
if (read.error.code === 'ENOENT') return ATTACHMENT_VERIFICATION.INVALID;
console.warn(`⚠️ Persistent Mind attachment ${attachment.attachmentId} could not be read (${read.error.code || read.error.message}); preserving it`);
return ATTACHMENT_VERIFICATION.UNAVAILABLE;
}
const detected = detectImageFormat(read.bytes);
return detected
&& detected.mime === attachment.mimeType
&& read.bytes.length === attachment.size
? ATTACHMENT_VERIFICATION.VALID
: ATTACHMENT_VERIFICATION.INVALID;
};
export const removeStoredAttachmentFile = async (attachment) => {
return removeStoredFilename(attachment.filename);
};
export const removeUploadAfterStateFailure = async (filePath, attachmentId) => unlinkGuarded(filePath).then(
async () => removePendingAttachmentMarker(attachmentId),
(error) => {
if (error?.code === 'ENOENT') return removePendingAttachmentMarker(attachmentId).then(() => false);
console.error(`❌ Failed to clean up Persistent Mind upload ${attachmentId}: ${error.message}`);
return false;
},
);
// A validation/write rejection happens before `saveImageUpload` can return its
// stored path. Remove every file with this upload's generated prefix as well as
// the marker, so a partial write cannot survive without the marker-based crash
// recovery path.
export const removeRejectedUpload = async (attachmentId) => {
const entries = await screenshotEntries();
if (!entries) return false;
if (!await removePendingAttachmentFiles(attachmentId, entries)) return false;
return removePendingAttachmentMarker(attachmentId);
};
export const resolveMessageAttachments = async (mind, attachmentIds, messageId) => {
const byId = new Map(mind.pendingAttachments.map((attachment) => [attachment.attachmentId, attachment]));
const attachments = [];
for (const attachmentId of attachmentIds) {
const attachment = byId.get(attachmentId);
if (!attachment) {
return { error: attachmentFailure('Persistent mind attachment was not found', { code: 'ATTACHMENT_NOT_FOUND' }) };
}
if (attachment.claimedBy && attachment.claimedBy !== messageId) {
return { error: attachmentFailure('Persistent mind attachment is already claimed by another message', { code: 'ATTACHMENT_ALREADY_CLAIMED', status: 409 }) };
}
const expiresAt = attachment.expiresAt ? Date.parse(attachment.expiresAt) : null;
if (!attachment.claimedBy && (!Number.isFinite(expiresAt) || expiresAt <= Date.now())) {
return { error: attachmentFailure('Persistent mind attachment has expired', { code: 'ATTACHMENT_EXPIRED' }) };
}
const verification = await verifyStoredAttachment(attachment);
if (verification === ATTACHMENT_VERIFICATION.UNAVAILABLE) {
return {
error: attachmentFailure(
'The stored image could not be read right now. Your upload is kept; retry sending once storage access recovers.',
{ code: 'ATTACHMENT_UNAVAILABLE', status: 503 },
),
};
}
if (verification !== ATTACHMENT_VERIFICATION.VALID) {
return { error: attachmentFailure('Persistent mind attachment is missing or invalid', { code: 'INVALID_ATTACHMENT' }) };
}
attachments.push(attachment);
}
return { attachments };
};
// Every caller either records an image written just before it or removes image
// bytes before dropping the record that names them. The CoS state file and the
// screenshots are copied at different moments of one backup rsync pass, so the
// whole mutation holds the backup lease (#9982), taken before the state lock.
async function mutateMindAttachments(mutator) {
return withBackupAssetPublication(() => withStateLock(async () => {
const root = await loadState();
const mind = normalizePersistentMindState(root.persistentMind);
const result = await mutator(mind, root);
const next = result?.mind || mind;
root.persistentMind = normalizePersistentMindState(next);
await saveState(root);
return { state: root.persistentMind, value: result?.value };
}));
}
/** Remove expired or invalid unclaimed files in one bounded maintenance pass. */
export async function cleanupPersistentMindAttachments({ now = Date.now() } = {}) {
// Housekeeping only (message sends and uploads run it first): defer the pass
// to the next call rather than make them wait out a backup cut.
if (backupSnapshotCutPending()) return { success: true, removed: 0, examined: 0, deferred: true };
const result = await mutateMindAttachments(async (mind) => {
let examined = 0;
let removed = 0;
const pendingAttachments = [];
for (const attachment of mind.pendingAttachments) {
if (attachment.claimedBy) {
await removePendingAttachmentMarker(attachment.attachmentId);
pendingAttachments.push(attachment);
continue;
}
if (examined >= PERSISTENT_MIND_LIMITS.MAX_ATTACHMENT_CLEANUP_PER_PASS) {
pendingAttachments.push(attachment);
continue;
}
examined += 1;
const expiresAt = attachment.expiresAt ? Date.parse(attachment.expiresAt) : null;
const expired = !attachment.claimedBy && Number.isFinite(expiresAt) && expiresAt <= now;
const verification = expired ? ATTACHMENT_VERIFICATION.INVALID : await verifyStoredAttachment(attachment);
// Unreadable is not invalid: keep the record and bytes for a later pass.
if (verification !== ATTACHMENT_VERIFICATION.INVALID) {
pendingAttachments.push(attachment);
continue;
}
if (await removeStoredAttachmentFile(attachment)) removed += 1;
else pendingAttachments.push(attachment);
}
const orphaned = await cleanupUnindexedPendingAttachments({
knownAttachments: pendingAttachments,
now,
limit: Math.max(0, PERSISTENT_MIND_LIMITS.MAX_ATTACHMENT_CLEANUP_PER_PASS - examined),
});
removed += orphaned.removed;
return {
mind: removed > 0 ? { ...mind, pendingAttachments } : mind,
value: { success: true, removed, examined: examined + orphaned.examined },
};
});
return result.value;
}
/** Store one validated image and register its server-owned pending record. */
export async function createPersistentMindAttachment({ filename, data } = {}) {
if (typeof filename !== 'string' || !filename.trim() || typeof data !== 'string' || !data.trim()) {
return attachmentFailure('Image filename and base64 data are required', { code: 'VALIDATION_ERROR' });
}
if (isUpdateInProgress()) {
return attachmentFailure('Persistent Mind image admission is paused during a PortOS update', {
code: 'UPDATE_IN_PROGRESS',
status: 409,
});
}
await cleanupPersistentMindAttachments();
const attachmentId = randomUUID();
const originalName = sanitizeFilename(filename).slice(0, PERSISTENT_MIND_LIMITS.MAX_ATTACHMENT_NAME_CHARS) || `image-${attachmentId}`;
// Create the marker BEFORE the image write. If the process dies after the
// write but before the state record is saved, boot/activity cleanup can find
// and reap the otherwise-unindexed file without ever scanning durable assets.
await mkdir(PATHS.screenshots, { recursive: true });
await writeFileGuarded(pendingAttachmentMarkerPath(attachmentId), '', { flag: 'wx' });
const saved = await saveImageUpload(PATHS.screenshots, {
filename: `${MIND_ATTACHMENT_FILENAME_PREFIX}${attachmentId}-${originalName}`,
data,
}, { maxBytes: PERSISTENT_MIND_LIMITS.MAX_ATTACHMENT_BYTES, stagePrefix: `${MIND_ATTACHMENT_FILENAME_PREFIX}${attachmentId}-` }).then(
(value) => value,
async (error) => {
await removeRejectedUpload(attachmentId);
throw error;
},
);
const uploadedAt = nowIso();
const attachment = normalizePersistentMindAttachment({
attachmentId,
filename: saved.filename,
originalName,
mimeType: saved.mime,
size: saved.size,
uploadedAt,
expiresAt: new Date(Date.now() + PERSISTENT_MIND_LIMITS.PENDING_ATTACHMENT_TTL_MS).toISOString(),
});
if (!attachment) {
await removeUploadAfterStateFailure(saved.filePath, attachmentId);
return attachmentFailure('Stored image metadata was invalid', { code: 'INVALID_ATTACHMENT' });
}
const result = await mutateMindAttachments(async (mind) => {
if (isUpdateInProgress()) {
return {
mind,
value: attachmentFailure('Persistent Mind image admission is paused during a PortOS update', {
code: 'UPDATE_IN_PROGRESS',
status: 409,
}),
};
}
const retainedMessageIds = new Set([
...mind.recentMessageIds,
...mind.queuedMessages.map((message) => message.id),
...(mind.activeTurn?.wake.kind === 'message' ? [mind.activeTurn.wake.message.id] : []),
]);
// Old claimed records are metadata only once their message leaves the
// idempotency window. Keep their files and message references durable, but
// do not let the pending-record index grow without bound. If removing the
// claim marker fails, retain the metadata: without it the marker-based
// orphan sweep could mistake the durable image for an unindexed upload.
const pendingAttachments = [];
for (const item of mind.pendingAttachments) {
if (!item.claimedBy || retainedMessageIds.has(item.claimedBy)) {
pendingAttachments.push(item);
} else if (!await removePendingAttachmentMarker(item.attachmentId)) {
pendingAttachments.push(item);
}
}
if (pendingAttachments.length >= PERSISTENT_MIND_LIMITS.MAX_PENDING_ATTACHMENTS) {
return {
mind,
value: attachmentFailure('Persistent mind has too many pending image uploads', {
code: 'ATTACHMENT_QUEUE_FULL',
status: 409,
}),
};
}
return {
mind: { ...mind, pendingAttachments: [...pendingAttachments, attachment] },
value: { success: true, attachment: publicPersistentMindAttachment(attachment) },
};
}).then(
(value) => value,
async (error) => {
await removeUploadAfterStateFailure(saved.filePath, attachmentId);
throw error;
},
);
if (!result.value.success) {
await removeUploadAfterStateFailure(saved.filePath, attachmentId);
} else {
await removePendingAttachmentMarker(attachmentId);
}
return result.value;
}
/** Delete an unclaimed pending image and its machine-local bytes. */
export async function deletePersistentMindAttachment(attachmentId) {
if (!isPersistentMindAttachmentId(attachmentId)) {
return attachmentFailure('Invalid persistent mind attachment id', { code: 'VALIDATION_ERROR' });
}
const result = await mutateMindAttachments(async (mind) => {
const attachment = mind.pendingAttachments.find((item) => item.attachmentId === attachmentId);
if (!attachment) return { mind, value: attachmentFailure('Persistent mind attachment was not found', { code: 'ATTACHMENT_NOT_FOUND', status: 404 }) };
if (attachment.claimedBy) {
return { mind, value: attachmentFailure('Claimed persistent mind attachments cannot be removed', { code: 'ATTACHMENT_ALREADY_CLAIMED', status: 409 }) };
}
if (!await removeStoredAttachmentFile(attachment)) {
return { mind, value: attachmentFailure('Persistent mind attachment could not be removed', { code: 'ATTACHMENT_DELETE_FAILED', status: 500 }) };
}
await removePendingAttachmentMarker(attachmentId);
return {
mind: { ...mind, pendingAttachments: mind.pendingAttachments.filter((item) => item.attachmentId !== attachmentId) },
value: { success: true, attachmentId },
};
});
return result.value;
}