Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Spotify listening-history ingestion (#2152) — poll the Web API's
* recently-played endpoint and feed the machine-local human-activity timeline
* (#2150) with `media.listen` events.
*
* Design constraints (docs/plans/2026-07-04-human-activity-tracking.md):
*
* - **50-track window ⇒ cadence must beat it.** `GET /v1/me/player/recently-played`
* only exposes the last 50 plays, so the scheduler polls every ~25 min (see
* spotifyScheduler.js) to avoid gaps. An incremental `after` cursor (the newest
* `played_at` seen) keeps each poll to just the new plays.
* - **Machine-local + idempotent.** The cursor lives in a `data/spotify/` JSON
* file (per-machine, unsynced). Each event carries a stable
* `spotify:<played_at>:<trackId>` dedupe key, so re-polls are no-ops.
* - **LLM-free.** Track/artist/genre metadata is stored verbatim for later twin
* enrichment (Phase 7); no AI-provider calls happen on this path.
* - **Raw-response cache.** The latest API payload is cached under
* `data/spotify/cache/` (local, unsynced) for debugging / re-derivation.
*
* The pure mappers (candidate mapping, dedupe keys, cursor advance) are exported
* and unit-tested with fixture payloads — no network or DB required.
*/
import { dataPath, ensureDir, atomicWrite, tryReadFile, safeJSONParse } from '../lib/fileUtils.js';
import { fetchWithTimeout } from '../lib/fetchWithTimeout.js';
import { getSettings } from './settings.js';
import { getAccessToken, getAuthStatus } from './spotifyAuth.js';
// ---------------------------------------------------------------------------
// Constants
// ---------------------------------------------------------------------------
const RECENTLY_PLAYED_URL = 'https://api.spotify.com/v1/me/player/recently-played';
const RECENTLY_PLAYED_TIMEOUT_MS = 15_000;
// The API caps this at 50; the cursor + fast cadence keep us from missing plays.
const SCAN_LIMIT = 50;
const STATE_FILE = 'sync-state.json';
const CACHE_FILE = 'recently-played.json';
// Sync is OFF by default — the user connects Spotify + opts in from Settings.
const DEFAULT_CONFIG = { enabled: false, intervalMinutes: 25 };
// ---------------------------------------------------------------------------
// Pure helpers (exported for unit tests — no network, no filesystem, no DB).
// ---------------------------------------------------------------------------
/**
* Map one recently-played item to a human-activity `media.listen` candidate, or
* `null` when it's missing a track id / valid `played_at`. Dedupe key is
* `spotify:<played_at ISO>:<trackId>` — the same track replayed later is a
* distinct listen, but the same play re-fetched is deduped.
*
* `metadata` keeps track/artist/album pointers (+ isrc/popularity) verbatim for
* Phase 7 taste enrichment; the short `summary` is the artist line only (privacy
* contract — no full payload in the event row).
*/
export function spotifyListenCandidate(item) {
const track = item?.track;
const playedAt = item?.played_at;
if (!track?.id || !playedAt) return null;
const at = new Date(playedAt);
if (Number.isNaN(at.getTime())) return null;
const artists = Array.isArray(track.artists)
? track.artists.map((a) => ({ id: a?.id || null, name: String(a?.name || '').trim() })).filter((a) => a.name)
: [];
const artistNames = artists.map((a) => a.name).join(', ');
const durationS = Number.isFinite(track.duration_ms) ? Math.round(track.duration_ms / 1000) : null;
return {
source: 'spotify',
accountId: null,
kind: 'media.listen',
happenedAt: at.toISOString(),
durationS,
title: track.name || '(unknown track)',
summary: artistNames,
url: track.external_urls?.spotify || null,
dedupeKey: `spotify:${at.toISOString()}:${track.id}`,
metadata: {
trackId: track.id,
trackName: track.name || null,
artists,
album: track.album?.name || null,
albumId: track.album?.id || null,
isrc: track.external_ids?.isrc || null,
popularity: Number.isFinite(track.popularity) ? track.popularity : null,
explicit: typeof track.explicit === 'boolean' ? track.explicit : null,
context: item.context?.type || null,
contextUri: item.context?.uri || null,
},
};
}
export function spotifyListenCandidates(items = []) {
return (items || []).map(spotifyListenCandidate).filter(Boolean);
}
/**
* Advance the `after` cursor to the newest `played_at` (epoch ms) across a batch,
* never regressing below the current cursor. Spotify's `after` param is a Unix-ms
* timestamp that returns plays strictly AFTER it, so the max seen is the next
* cursor. Ignores items with an unparseable `played_at`.
*/
export function maxPlayedAtMs(items = [], current = 0) {
let max = Number.isFinite(current) ? current : 0;
for (const item of items || []) {
const t = new Date(item?.played_at).getTime();
if (Number.isFinite(t) && t > max) max = t;
}
return max;
}
// ---------------------------------------------------------------------------
// Config + machine-local cursor state
// ---------------------------------------------------------------------------
export async function getSpotifyConfig() {
const settings = await getSettings().catch(() => ({}));
const c = settings?.spotify || {};
return {
enabled: typeof c.enabled === 'boolean' ? c.enabled : DEFAULT_CONFIG.enabled,
intervalMinutes: Number.isFinite(c.intervalMinutes) && c.intervalMinutes >= 1
? Math.floor(c.intervalMinutes)
: DEFAULT_CONFIG.intervalMinutes,
};
}
function stateFilePath() {
return dataPath('spotify', STATE_FILE);
}
// Machine-local incremental cursor. NOT federated — the recently-played window
// and its timestamps are per-account, so this must never sync to a peer.
export async function readSyncState() {
const raw = await tryReadFile(stateFilePath());
const parsed = raw ? safeJSONParse(raw, null, { allowArray: false }) : null;
return {
cursorAfter: Number.isFinite(parsed?.cursorAfter) ? parsed.cursorAfter : 0,
lastRunAt: parsed?.lastRunAt || null,
lastResult: parsed?.lastResult || null,
};
}
async function writeSyncState(state) {
await ensureDir(dataPath('spotify'));
await atomicWrite(stateFilePath(), JSON.stringify(state, null, 2));
}
async function writeRawCache(payload) {
await ensureDir(dataPath('spotify', 'cache'));
await atomicWrite(dataPath('spotify', 'cache', CACHE_FILE), JSON.stringify(payload, null, 2));
}
// ---------------------------------------------------------------------------
// Sync (side-effecting — network + DB). Runs outside the request lifecycle
// (scheduler / explicit endpoint), so network/persist failures return an error
// report instead of throwing.
// ---------------------------------------------------------------------------
// Re-entrancy guard: a manual "Sync now" overlapping a scheduler tick would
// double-read the same cursor (deduped, but doubled work, and the slower writer
// could stamp a stale cursor over the fresher one) — so concurrent callers share
// the in-flight pass.
let syncInFlight = null;
export async function runSync() {
if (syncInFlight) return syncInFlight;
syncInFlight = doRunSync().finally(() => { syncInFlight = null; });
return syncInFlight;
}
async function doRunSync() {
const accessToken = await getAccessToken().catch((err) => {
console.error(`❌ Spotify sync auth failed: ${err?.message || err}`);
return null;
});
if (!accessToken) {
return { ok: false, needsAuth: true, error: 'Spotify not connected — authorize in Brain → Spotify.' };
}
const started = Date.now();
const state = await readSyncState();
const url = new URL(RECENTLY_PLAYED_URL);
url.searchParams.set('limit', String(SCAN_LIMIT));
if (state.cursorAfter) url.searchParams.set('after', String(state.cursorAfter));
let res;
try {
res = await fetchWithTimeout(
url,
{ headers: { Authorization: `Bearer ${accessToken}` } },
RECENTLY_PLAYED_TIMEOUT_MS,
);
} catch (err) {
const reason = err?.message || String(err);
console.error(`❌ Spotify recently-played fetch failed: ${reason}`);
return { ok: false, error: `Spotify API request failed: ${reason}` };
}
if (!res.ok) {
const body = await res.text().catch(() => '');
console.error(`❌ Spotify recently-played fetch failed: ${res.status} ${body.slice(0, 200)}`);
return { ok: false, status: res.status, error: `Spotify API returned ${res.status}` };
}
const payload = await res.json().catch(() => null);
const items = Array.isArray(payload?.items) ? payload.items : [];
await writeRawCache(payload).catch((err) => console.error(`⚠️ Spotify raw-cache write failed: ${err?.message || err}`));
const candidates = spotifyListenCandidates(items);
// Persistence failures must NOT advance the cursor: the dedupe keys make a
// re-poll a no-op, but the recently-played window scrolls, so skipping past
// unpersisted plays loses them permanently. Hold the cursor on failure.
let persistFailed = false;
const { recordEvents } = await import('./humanActivity.js');
const recordResult = await recordEvents(candidates).catch((err) => {
console.error(`❌ Spotify activity record failed: ${err?.message || err}`);
persistFailed = true;
return { recorded: 0, skipped: candidates.length };
});
// Incrementally refresh observed twin evidence when new listens landed (#2156).
// LLM-free + self-guarded — never blocks or fails the sync.
if (!persistFailed && recordResult.recorded > 0) {
const { refreshTwinEvidenceAfterSync } = await import('./twinEnrichment.js');
await refreshTwinEvidenceAfterSync();
}
// Prefer the response's own `cursors.after` (Spotify's canonical next cursor),
// falling back to the computed max — but never regress below the stored cursor.
const responseCursor = Number(payload?.cursors?.after);
const computed = maxPlayedAtMs(items, state.cursorAfter);
const advanced = Number.isFinite(responseCursor) && responseCursor > (state.cursorAfter || 0)
? responseCursor
: computed;
const nextCursor = persistFailed ? state.cursorAfter : advanced;
const result = {
ok: !persistFailed,
...(persistFailed ? { error: 'Persistence failed — cursor held so the batch retries next sync' } : {}),
scanned: items.length,
recorded: recordResult.recorded,
skipped: recordResult.skipped,
cursorAfter: nextCursor,
// A full 50-item batch means older plays may remain beyond this pass — the
// caller (UI toast / next scheduler tick) should run again to keep draining.
hasMore: items.length === SCAN_LIMIT,
durationMs: Date.now() - started,
};
await writeSyncState({ cursorAfter: nextCursor, lastRunAt: new Date().toISOString(), lastResult: result });
console.log(`🎧 Spotify sync: scanned ${result.scanned}, recorded ${result.recorded} listen(s), cursor→${result.cursorAfter}${result.hasMore ? ' (more remaining)' : ''}${persistFailed ? ' — PERSIST FAILED, cursor held' : ''} in ${result.durationMs}ms`);
return result;
}
// Status for the settings UI: config + cursor state + auth status (no API call).
export async function getStatus(options = {}) {
const [config, state, auth] = await Promise.all([
getSpotifyConfig(),
readSyncState(),
getAuthStatus(options),
]);
return { config, state, auth };
}