Open-source, local-first social media research agent for Instagram, TikTok, and LinkedIn. Jev routes read-only steps; socai CLI captures cited browser evidence.
JavaScript agent-skillsai-agentsbrowser-agentbrowser-automationdecision-modeldeep-researchinstagramjevlinkedinlocal-firstopen-sourceosintresearch-agentsocaisocial-listeningsocial-mediasocial-media-researchsocmintsystem-onetiktok
import crypto from "node:crypto";
import { constants } from "node:fs";
import { lstat, mkdir, open, readdir, realpath, rename, unlink, writeFile } from "node:fs/promises";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { getHomeDir } from "./config.js";
import { AppError } from "./errors.js";
import { LABEL_VALUES, MAX_REELS_BYTES, REELS_SCHEMA_VERSION, REVIEW_THRESHOLD, SCRIPT_ROLES, SOURCE_STATES, knownMetric } from "./reels-contract.js";
const object = (value) => value !== null && typeof value === "object" && !Array.isArray(value);
const stableId = (value) => typeof value === "string" && /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$/.test(value);
const libraryId = (value) => typeof value === "string" && /^reels-[a-f0-9]{24}$/.test(value);
const invalid = (message) => new AppError(message, { code: "INVALID_REELS_IMPORT" });
const metric = (value) => knownMetric(value) ? value : null;
const datePattern = /^(\d{4})-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01])[Tt](?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:[Zz]|[+-](?:[01]\d|2[0-3]):[0-5]\d)$/;
const creatorName = (value) => typeof value === "string" && /^[a-zA-Z0-9_.]{1,30}$/.test(value);
function date(value) {
const match = typeof value === "string" ? datePattern.exec(value) : null;
if (!match) return null;
const year = Number(match[1]), month = Number(match[2]), day = Number(match[3]);
const leap = year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0);
const days = [31, leap ? 29 : 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
const timestamp = day <= days[month - 1] ? Date.parse(value) : NaN;
return Number.isFinite(timestamp) ? new Date(timestamp).toISOString() : null;
}
function redactPaths(value) {
// Drop the whole line when a local path is detected, including paths with
// spaces. Scan bounded lines/tokens without nested, overlapping quantifiers.
return value.split(/\r?\n/).map((line) => {
const absolute = /(^|[\s("'`=:])(?:file:\/\/\/|[A-Za-z]:[\\/]|\\\\|~[\\/]|\.{1,2}[\\/]|\/(?!\/))\S/.test(line);
const named = /\b(?:run_dir|local_path|output_dir|artifact_path|report_path)\s*[:=]/i.test(line);
const relative = line.split(/\s+/).some((token) => token.includes("/") && !token.includes("://") && /\.[A-Za-z0-9]{1,10}[),;:!?\]}"']*$/.test(token));
return absolute || named || relative ? "[redacted path]" : line;
}).join("\n");
}
function text(value, max = 12000) {
if (value == null) return "";
if (typeof value !== "string" || value.length > max) throw invalid("Invalid or oversized Reel text field.");
return redactPaths(value)
.replace(/\b(?:authorization|cookie|set-cookie|(?:access[_ -]?|refresh[_ -]?|session[_ -]?)?token|api[_ -]?key|password|secret)\s*[:=]\s*[^\r\n]*/gi, "[redacted credential]")
.replace(/\bBearer\s+[A-Za-z0-9._~+\/-]+=*/gi, "[redacted credential]")
.replace(/\b(?:sk-|gsk_|apify_api_|gh[pousr]_|github_pat_)[A-Za-z0-9_-]{8,}\b/g, "[redacted credential]")
.replace(/\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/g, "[redacted credential]")
.replace(/data:[^\s]+/gi, "[media omitted]").slice(0, max);
}
function source(raw) {
if (typeof raw !== "string" || raw.length > 5000) throw invalid("Every Reel needs a valid Instagram post URL.");
let url;
try { url = new URL(raw); } catch { throw invalid("Every Reel needs a valid Instagram post URL."); }
const code = url.pathname.match(/^\/(?:p|reel)\/([A-Za-z0-9_-]+)\/?$/)?.[1];
if (!code || url.protocol !== "https:" || url.username || url.password || url.port
|| !["instagram.com", "www.instagram.com"].includes(url.hostname)) throw invalid("Every Reel needs a valid Instagram post URL.");
return `https://www.instagram.com/p/${code}/`;
}
function segments(raw = [], labeled = false) {
if (!Array.isArray(raw) || raw.length > 200) throw invalid("Invalid Reel transcript segments.");
let previousStart = null;
return raw.map((segment) => {
if (!object(segment) || typeof segment.text !== "string") throw invalid("Invalid Reel transcript segment.");
const start = segment.start ?? null, end = segment.end ?? null;
if ((start !== null && !knownMetric(start)) || (end !== null && !knownMetric(end))
|| (start !== null && end !== null && end < start)
|| (start !== null && previousStart !== null && start < previousStart)) throw invalid("Invalid Reel segment timestamps.");
if (start !== null) previousStart = start;
const result = { text: text(segment.text, 48000), start, end };
if (labeled) {
if (!SCRIPT_ROLES.includes(segment.value)) throw invalid("Invalid Reel script role.");
if (segment.confidence != null && (!knownMetric(segment.confidence) || segment.confidence > 1)) throw invalid("Invalid Reel segment confidence.");
result.value = segment.value; result.confidence = segment.confidence ?? null;
}
return result;
});
}
function classification(raw, transcript) {
if (raw == null) return null;
if (!object(raw) || !object(raw.labels) || !transcript) throw invalid("Classification requires labels and a transcript.");
const labels = {};
for (const [key, values] of Object.entries(LABEL_VALUES)) {
const label = raw.labels[key];
if (!object(label) || !values.includes(label.value) || !knownMetric(label.confidence) || label.confidence > 1) throw invalid("Invalid Reel classification or confidence.");
labels[key] = { value: label.value, confidence: label.confidence };
}
return { labels, anatomy: segments(raw.anatomy, true) };
}
export function normalizeReels(raw) {
if (!object(raw) || raw.schemaVersion !== REELS_SCHEMA_VERSION) throw invalid("Expected Reel import schemaVersion 1.");
if (!stableId(raw.id) || !creatorName(raw.creator)) throw invalid("Invalid Reel collection ID or creator.");
if (!Array.isArray(raw.posts) || !raw.posts.length || raw.posts.length > 1000) throw invalid("Import 1 to 1,000 Reel items.");
if (raw.status != null && !SOURCE_STATES.includes(raw.status)) throw invalid("Invalid Reel collection status.");
const creator = raw.creator.toLowerCase(), seen = new Map(), posts = [];
for (const item of raw.posts) {
if (!object(item) || !stableId(item.id)) throw invalid("Every Reel needs a stable item ID.");
if (item.creator != null && !creatorName(item.creator)) throw invalid("Invalid Reel creator.");
if (item.status != null && !["classified", "unclassified", "pending", "ready", "no_audio", "no_speech", "excluded", "failed", "error"].includes(item.status)) throw invalid("Invalid Reel item status.");
if (item.transcript != null && !object(item.transcript)) throw invalid("Invalid Reel transcript.");
const transcript = { text: text(item.transcript?.text, 48000), segments: segments(item.transcript?.segments) };
const analysis = classification(item.analysis, transcript.text);
if (item.excludedReason != null && typeof item.excludedReason !== "string") throw invalid("Invalid Reel exclusion reason.");
const excluded = Boolean(item.excludedReason) || ["no_audio", "no_speech", "excluded"].includes(item.status);
const failed = ["failed", "error"].includes(item.status);
if (excluded && failed) throw invalid("A Reel cannot be both excluded and failed.");
if ((excluded || failed) && analysis) throw invalid("Excluded or error rows cannot contain a classification.");
if (item.status === "classified" && !analysis) throw invalid("Classified rows require a classification.");
const status = excluded ? "excluded" : failed ? "error" : analysis ? "classified" : "unclassified";
const post = {
id: item.id, url: source(item.url), creator: (item.creator || creator).toLowerCase(),
publishedAt: date(item.publishedAt), capturedAt: date(item.capturedAt || item.scrapedAt),
caption: text(item.caption), transcript, duration: metric(item.duration),
plays: metric(item.plays), views: metric(item.views), likes: metric(item.likes), comments: metric(item.comments),
status, statusNote: excluded ? (item.status === "no_audio" || item.statusNote === "No audio track" ? "No audio track" : "Excluded by source") : failed ? "Source processing failed" : "",
analysis, needsReview: Boolean(analysis) && (Object.values(analysis.labels).some((label) => label.confidence < REVIEW_THRESHOLD || label.value === "unclear")
|| analysis.anatomy.some((segment) => segment.confidence === null || segment.confidence < REVIEW_THRESHOLD || segment.value === "unclear")),
};
const duplicate = seen.get(post.id);
if (duplicate && JSON.stringify(duplicate) !== JSON.stringify(post)) throw invalid("Conflicting duplicate Reel item IDs.");
if (!duplicate) { seen.set(post.id, post); posts.push(post); }
}
posts.sort((a, b) => a.id.localeCompare(b.id, "en"));
return {
schemaVersion: REELS_SCHEMA_VERSION,
id: `reels-${crypto.createHash("sha256").update(`${creator}\0${raw.id}`).digest("hex").slice(0, 24)}`,
sourceId: raw.id, creator, createdAt: date(raw.createdAt), sourceStatus: raw.status || "unknown",
duplicateCount: raw.posts.length - posts.length, posts,
};
}
async function directory(env, create = false) {
const root = getHomeDir(env);
const project = await realpath(fileURLToPath(new URL("..", import.meta.url)));
const insideProject = (candidate) => {
const relative = path.relative(project, candidate);
return !relative || (relative !== ".." && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative));
};
if (insideProject(root)) throw new Error("Reel storage must be outside the project.");
if (create) await mkdir(root, { recursive: true, mode: 0o700 });
if ((await lstat(root)).isSymbolicLink()) throw new Error("State directory must not be a symlink.");
if (insideProject(await realpath(root))) throw new Error("Reel storage must be outside the project.");
let result = root;
for (const name of ["reels", "v1"]) {
result = path.join(result, name);
if (create) await mkdir(result, { mode: 0o700 }).catch((error) => { if (error.code !== "EEXIST") throw error; });
const info = await lstat(result);
if (info.isSymbolicLink() || !info.isDirectory()) throw new Error("Invalid Reel storage directory.");
}
return result;
}
export async function importReels(raw, env = process.env) {
let bytes;
try { bytes = Buffer.byteLength(JSON.stringify(raw)); } catch { throw invalid("Invalid Reel JSON input."); }
if (bytes > MAX_REELS_BYTES) throw invalid("Reel imports must not exceed 10 MB.");
const collection = normalizeReels(raw), serialized = JSON.stringify(collection);
if (Buffer.byteLength(serialized) > MAX_REELS_BYTES) throw invalid("Normalized Reel imports must not exceed 10 MB.");
let temporary;
try {
const root = await directory(env, true);
temporary = path.join(root, `${collection.id}.${crypto.randomUUID()}.tmp`);
await writeFile(temporary, serialized, { flag: "wx", mode: 0o600 });
await rename(temporary, path.join(root, `${collection.id}.json`));
} catch { throw new AppError("The local library could not save this collection.", { code: "REELS_SAVE_FAILED", status: 500 }); }
finally { if (temporary) await unlink(temporary).catch(() => {}); }
return collection;
}
export async function readReels(id, env = process.env) {
if (!libraryId(id)) return null;
let file;
try {
file = await open(path.join(await directory(env), `${id}.json`), constants.O_RDONLY | (constants.O_NOFOLLOW || 0) | (constants.O_NONBLOCK || 0));
const info = await file.stat();
if (!info.isFile() || info.size > MAX_REELS_BYTES) return null;
const chunks = []; let bytes = 0;
for await (const chunk of file.createReadStream({ autoClose: false })) {
bytes += chunk.length; if (bytes > MAX_REELS_BYTES) return null;
chunks.push(chunk);
}
const saved = JSON.parse(Buffer.concat(chunks).toString("utf8"));
const clean = normalizeReels({ ...saved, id: saved.sourceId, status: saved.sourceStatus });
if (clean.id !== id || saved.id !== id) return null;
if (!Number.isInteger(saved.duplicateCount) || saved.duplicateCount < 0 || saved.duplicateCount > 999) return null;
return { ...clean, duplicateCount: saved.duplicateCount };
} catch { return null; }
finally { await file?.close(); }
}
export async function listReels(env = process.env) {
let files;
try { files = await readdir(await directory(env)); }
catch (error) {
if (error.code === "ENOENT") return [];
throw new AppError("The local library could not read saved collections.", { code: "REELS_READ_FAILED", status: 500 });
}
const collections = [];
for (const name of files.filter((name) => /^reels-[a-f0-9]{24}\.json$/.test(name)).sort()) {
const saved = await readReels(name.slice(0, -5), env);
if (saved) collections.push({ id: saved.id, sourceId: saved.sourceId, schemaVersion: saved.schemaVersion, creator: saved.creator, sourceStatus: saved.sourceStatus, itemCount: saved.posts.length });
}
return collections;
}