A modern and elegant dashboard for network traffic visualization and analysis.
/**
* Main Fastify Application
*
* This file registers all controllers and services for the API.
*/
import crypto from 'crypto';
import { createGunzip } from 'node:zlib';
import Fastify from 'fastify';
import cors from '@fastify/cors';
import cookie from '@fastify/cookie';
import type { StatsDatabase } from '../db/db.js';
import type { RealtimeStore } from '../realtime/realtime.store.js';
import { buildGatewayHeaders, getGatewayBaseUrl, isAgentBackendUrl, parseSurgeRule } from '@neko-master/shared';
import type { TrafficUpdate } from '../db/db.js';
import { SurgePolicySyncService } from '../surge/surge-policy-sync.js';
import type { GeoIPService } from '../geo/geo.service.js';
import { BatchBuffer } from '../collector/batch-buffer.js';
import { TrafficWriteError } from '../clickhouse/clickhouse.writer.js';
// Import modules
import { BackendService, backendController } from '../backend/index.js';
import { StatsService, statsController } from '../stats/index.js';
import { AuthService, authController } from '../auth/index.js';
import { configController } from '../config/index.js';
import { gatewayFetch } from '../../shared/utils/gateway-fetch.js';
// Extend Fastify instance to include services
declare module 'fastify' {
interface FastifyInstance {
db: StatsDatabase;
realtimeStore: RealtimeStore;
backendService: BackendService;
statsService: StatsService;
clearAgentRuntimeState?: (backendId?: number) => void;
notifyBackendDataCleared?: (backendId: number) => void;
}
}
export interface AppOptions {
port: number;
db: StatsDatabase;
realtimeStore: RealtimeStore;
logger?: boolean;
policySyncService?: SurgePolicySyncService;
geoService?: GeoIPService;
autoListen?: boolean;
onTrafficIngested?: (backendId: number) => void;
onBackendDataCleared?: (backendId: number) => void;
}
type AgentTrafficUpdatePayload = {
domain?: string;
ip?: string;
chain?: string;
chains?: string[];
rule?: string;
rulePayload?: string;
upload?: number;
download?: number;
connections?: number;
sourceIP?: string;
timestampMs?: number;
};
type AgentConfigPayload = {
backendId?: number | string;
agentId?: string;
config: {
rules: Array<{ type: string; payload: string; proxy: string; raw?: string }>;
proxies: Record<string, { name: string; type: string; now?: string }>;
providers: Record<string, { proxies: Array<{ name: string; type: string; now?: string }> }>;
timestamp: number;
hash: string;
};
};
type AgentHeartbeatPayload = {
backendId?: number;
agentId?: string;
protocolVersion?: number;
agentVersion?: string;
hostname?: string;
version?: string;
gatewayType?: string;
gatewayUrl?: string;
gatewayLatencyMs?: number;
serverLatencyMs?: number;
};
type AgentReportPayload = {
backendId?: number;
agentId?: string;
protocolVersion?: number;
agentVersion?: string;
requestId?: string;
updates?: AgentTrafficUpdatePayload[];
};
export async function createApp(options: AppOptions) {
const {
port,
db,
realtimeStore,
logger = false,
policySyncService,
geoService,
autoListen = true,
onTrafficIngested,
onBackendDataCleared,
} = options;
// Create Fastify instance
// Increase body limit to 5MB for agent config sync (large Clash/Surge configs)
const app = Fastify({ logger, bodyLimit: 5 * 1024 * 1024 });
// Decompress gzip-encoded request bodies (used by the agent to reduce upload bandwidth)
// content-length must be removed because it refers to the compressed size; after decompression
// Fastify would compare the larger decompressed bytes against the original header and reject the request.
app.addHook('preParsing', async (request, _reply, payload) => {
if (request.headers['content-encoding'] === 'gzip') {
delete (request.headers as Record<string, unknown>)['content-length'];
return payload.pipe(createGunzip());
}
return payload;
});
// In-memory dedup for agent report requestIds — prevents double-counting on POST retry.
// Keyed by requestId, value is the time it was first seen (ms). TTL: 5 minutes.
const seenRequestIds = new Map<string, number>();
// Per-backend BatchBuffer for agent mode — mirrors direct mode's 30s batch window and
// applies the same deferred flush behavior for sqlite/clickhouse writes.
const agentBatchBuffers = new Map<number, BatchBuffer>();
// Per-backend legacy fallback dedup set. Only used when an older agent payload
// does not provide explicit `connections`.
const agentCountedByBackend = new Map<number, Set<string>>();
// Per-backend flush lock — mirrors direct mode's isFlushing to prevent a slow CH write
// from letting the next timer tick start a second flush before clearTraffic completes.
const agentIsFlushing = new Map<number, boolean>();
const AGENT_FLUSH_INTERVAL_MS = Math.max(
5_000,
Number.parseInt(process.env.AGENT_FLUSH_INTERVAL_MS || '30000', 10) || 30_000,
);
const flushAgentBuffer = async (backendId: number, buffer: BatchBuffer) => {
if (agentIsFlushing.get(backendId) || !buffer.hasPending()) return;
agentIsFlushing.set(backendId, true);
try {
const stats = buffer.flush(db, geoService, backendId, 'Agent');
let trafficDetailOk = true;
let trafficAggOk = true;
if (stats.pendingTrafficWrite) {
try {
const outcome = await stats.pendingTrafficWrite;
trafficDetailOk = outcome.detailOk;
trafficAggOk = outcome.aggOk;
} catch (err) {
if (err instanceof TrafficWriteError) {
trafficDetailOk = err.detailOk;
trafficAggOk = err.aggOk;
} else {
trafficDetailOk = false;
trafficAggOk = false;
}
console.warn(
`[Agent:${backendId}] ClickHouse traffic write failed detail_ok=${trafficDetailOk} agg_ok=${trafficAggOk}`,
err,
);
}
}
// Durable fallback: SQLite writes were skipped because ClickHouse was
// considered healthy, but the ClickHouse write failed. Persist the
// snapshot to SQLite so the data is not silently lost (buffer is already
// cleared by flush()). Mirrors gateway/surge collectors. See C1.
if (
stats.hasTrafficUpdates &&
stats.trafficOk &&
stats.sqliteSkipped &&
(!trafficDetailOk || !trafficAggOk)
) {
try {
db.batchUpdateTrafficStats(backendId, stats.updates, false);
trafficDetailOk = true;
trafficAggOk = true;
console.warn(
`[Agent:${backendId}] ClickHouse traffic write failed; persisted ${stats.updates.length} updates to SQLite as fallback`,
);
} catch (fallbackErr) {
console.error(
`[Agent:${backendId}] SQLite fallback for traffic also failed; retaining realtime store`,
fallbackErr,
);
}
}
if (stats.hasTrafficUpdates && stats.trafficOk) {
if (trafficDetailOk && trafficAggOk) {
realtimeStore.clearTraffic(backendId);
agentCountedByBackend.delete(backendId);
} else if (trafficDetailOk && !trafficAggOk) {
realtimeStore.clearTrafficDimensions(backendId);
agentCountedByBackend.delete(backendId);
} else if (!trafficDetailOk && trafficAggOk) {
realtimeStore.clearTrafficSummary(backendId);
agentCountedByBackend.delete(backendId);
}
}
let countryWriteOk = true;
if (stats.pendingCountryWrite) {
try {
await stats.pendingCountryWrite;
} catch (err) {
countryWriteOk = false;
console.warn(
`[Agent:${backendId}] ClickHouse country write failed, keeping realtime country store`,
err,
);
}
}
// Durable fallback for country stats: same rationale as traffic above. See C1.
if (
stats.hasCountryUpdates &&
stats.countryOk &&
stats.sqliteSkipped &&
!countryWriteOk
) {
try {
db.batchUpdateCountryStats(backendId, stats.countryUpdates);
countryWriteOk = true;
console.warn(
`[Agent:${backendId}] ClickHouse country write failed; persisted to SQLite as fallback`,
);
} catch (fallbackErr) {
console.error(
`[Agent:${backendId}] SQLite fallback for country also failed; retaining realtime store`,
fallbackErr,
);
}
}
if (stats.hasCountryUpdates && stats.countryOk && countryWriteOk) {
realtimeStore.clearCountries(backendId);
}
} finally {
agentIsFlushing.set(backendId, false);
}
// Periodic memory bounds check on realtime store. Without this the
// per-backend realtime maps grow unbounded while ClickHouse writes keep
// failing (the buffer clears each flush but the realtime store is retained
// for the UI), OOM-ing long-lived router/agent deployments. Mirrors the
// gateway/surge collectors. See C2.
realtimeStore.pruneIfNeeded(backendId);
};
const agentFlushIntervalId = setInterval(() => {
for (const [backendId, buffer] of agentBatchBuffers) {
flushAgentBuffer(backendId, buffer).catch((err) => {
console.error(`[Agent:${backendId}] Periodic flush error:`, err);
});
}
}, AGENT_FLUSH_INTERVAL_MS);
const clearAgentRuntimeState = (backendId?: number) => {
if (backendId !== undefined) {
agentBatchBuffers.get(backendId)?.clear();
agentBatchBuffers.delete(backendId);
agentCountedByBackend.delete(backendId);
agentIsFlushing.delete(backendId);
return;
}
for (const buffer of agentBatchBuffers.values()) {
buffer.clear();
}
agentBatchBuffers.clear();
agentCountedByBackend.clear();
agentIsFlushing.clear();
};
const REQUEST_ID_TTL_MS = 5 * 60 * 1000;
let lastRequestIdPruneAt = 0;
function isDuplicateRequestId(id: string): boolean {
const now = Date.now();
if (seenRequestIds.has(id)) {
return true;
}
seenRequestIds.set(id, now);
// Time-based prune: the previous size-modulo check could skip pruning for
// long stretches under steady growth, letting the map balloon.
if (now - lastRequestIdPruneAt > 60_000) {
lastRequestIdPruneAt = now;
const cutoff = now - REQUEST_ID_TTL_MS;
for (const [k, ts] of seenRequestIds) {
if (ts < cutoff) seenRequestIds.delete(k);
}
}
return false;
}
// Register CORS — CORS_ORIGIN restricts allowed origins (comma-separated);
// defaults to permissive for LAN deployments where the dashboard origin is not known.
const corsOrigins = (process.env.CORS_ORIGIN || '')
.split(',')
.map((s) => s.trim())
.filter(Boolean);
// Security: when CORS_ORIGIN is unset, deny cross-origin requests instead of
// reflecting any Origin with credentials — that combination let any website
// read a logged-in user's data cross-site. Same-origin deployments (the
// dashboard and API served from one origin, the default Docker setup) are
// unaffected because browsers do not apply CORS to same-origin requests.
// To allow a separate dashboard origin, set CORS_ORIGIN explicitly.
const hasExplicitOrigins = corsOrigins.length > 0;
await app.register(cors, {
origin: hasExplicitOrigins ? corsOrigins : false,
credentials: hasExplicitOrigins,
});
app.addHook('onSend', async (_request, reply) => {
reply.header('X-Content-Type-Options', 'nosniff');
reply.header('X-Frame-Options', 'DENY');
reply.header('Referrer-Policy', 'strict-origin-when-cross-origin');
});
// Register Cookie — fall back to a secret persisted in the database so sessions
// survive restarts even when COOKIE_SECRET is not configured.
const cookieSecret = process.env.COOKIE_SECRET || db.getOrCreateCookieSecret();
await app.register(cookie, {
secret: cookieSecret,
parseOptions: {},
});
// Create services
const authService = new AuthService(db);
const backendService = new BackendService(
db,
realtimeStore,
authService,
onBackendDataCleared,
);
const statsService = new StatsService(db, realtimeStore);
// Decorate Fastify instance with services
app.decorate('backendService', backendService);
app.decorate('statsService', statsService);
app.decorate('authService', authService);
app.decorate('db', db);
app.decorate('realtimeStore', realtimeStore);
app.decorate('clearAgentRuntimeState', clearAgentRuntimeState);
app.decorate(
'notifyBackendDataCleared',
(backendId: number) => onBackendDataCleared?.(backendId),
);
const getBackendIdFromQuery = (query: Record<string, unknown>): number | null => {
const backendId = typeof query.backendId === 'string' ? query.backendId : undefined;
return statsService.resolveBackendId(backendId);
};
// ...
// Helper to get headers for backend requests
const getHeaders = (backend: { type: 'clash' | 'surge'; token: string }) => {
return buildGatewayHeaders(backend);
};
const parseNonNegativeInt = (value: unknown): number | null => {
const parsed = Number.parseInt(String(value), 10);
if (!Number.isFinite(parsed) || parsed < 0) return null;
return parsed;
};
const loadRuleSetSizeByProvider = async (
gatewayBaseUrl: string,
headers: Record<string, string>,
): Promise<Map<string, number>> => {
const map = new Map<string, number>();
try {
const res = await gatewayFetch(`${gatewayBaseUrl}/providers/rules`, {
headers: { ...headers, 'Content-Type': 'application/json' },
signal: AbortSignal.timeout(5000),
});
if (!res.ok) {
return map;
}
const payload = (await res.json()) as {
providers?: Record<string, Record<string, unknown>>;
};
const providers = payload.providers || {};
for (const [name, data] of Object.entries(providers)) {
const ruleCount =
parseNonNegativeInt(data.ruleCount) ??
parseNonNegativeInt(data.rule_count) ??
parseNonNegativeInt(data.size) ??
parseNonNegativeInt(data.rules);
if (ruleCount !== null) {
map.set(name.toLowerCase(), ruleCount);
}
}
} catch {
// Best-effort enrichment only; ignore provider endpoint failures.
}
return map;
};
const timingSafeStringEqual = (a: string, b: string): boolean => {
const bufA = Buffer.from(a);
const bufB = Buffer.from(b);
if (bufA.length !== bufB.length) return false;
return crypto.timingSafeEqual(bufA, bufB);
};
const parseAgentToken = (request: { headers: Record<string, unknown> }): string => {
const authHeader = request.headers.authorization;
if (typeof authHeader === 'string' && authHeader.startsWith('Bearer ')) {
return authHeader.slice(7).trim();
}
const customHeader = request.headers['x-agent-token'];
return typeof customHeader === 'string' ? customHeader.trim() : '';
};
const parseBackendId = (raw: unknown): number | null => {
const parsed = Number.parseInt(String(raw), 10);
if (!Number.isFinite(parsed) || parsed <= 0) return null;
return parsed;
};
const parseAgentId = (raw: unknown): string | null => {
const value = String(raw || '').trim().slice(0, 128);
return value || null;
};
const getMinAgentProtocolVersion = (): number => {
const v = Number.parseInt(process.env.MIN_AGENT_PROTOCOL_VERSION || '1', 10);
return Number.isFinite(v) && v > 0 ? v : 1;
};
const getMinAgentVersion = (): string => {
return String(process.env.MIN_AGENT_VERSION || '').trim();
};
const parseProtocolVersion = (raw: unknown): number | null => {
if (raw === undefined || raw === null || raw === '') return null;
const parsed = Number.parseInt(String(raw), 10);
if (!Number.isFinite(parsed) || parsed <= 0) return null;
return parsed;
};
const normalizeVersion = (raw: unknown): string => {
return String(raw || '')
.trim()
.replace(/^agent-v/i, '')
.replace(/^v/i, '');
};
const parseVersionParts = (raw: unknown): [number, number, number] | null => {
const normalized = normalizeVersion(raw);
if (!normalized) return null;
const match = normalized.match(/^(\d+)\.(\d+)(?:\.(\d+))?/);
if (!match) return null;
return [
Number.parseInt(match[1], 10),
Number.parseInt(match[2], 10),
Number.parseInt(match[3] || '0', 10),
];
};
const compareVersionParts = (a: [number, number, number], b: [number, number, number]): number => {
for (let i = 0; i < 3; i++) {
if (a[i] > b[i]) return 1;
if (a[i] < b[i]) return -1;
}
return 0;
};
const isAgentCompatible = (
body: AgentHeartbeatPayload | AgentReportPayload,
reply: { status: (code: number) => { send: (payload: Record<string, unknown>) => unknown } },
): boolean => {
const requiredProtocol = getMinAgentProtocolVersion();
const incomingProtocol = parseProtocolVersion(body.protocolVersion) ?? 1;
if (incomingProtocol < requiredProtocol) {
reply.status(426).send({
error: `Agent protocol version ${incomingProtocol} is too old. Minimum required is ${requiredProtocol}.`,
code: 'AGENT_PROTOCOL_TOO_OLD',
minProtocolVersion: requiredProtocol,
receivedProtocolVersion: incomingProtocol,
});
return false;
}
const requiredVersion = getMinAgentVersion();
if (!requiredVersion) {
return true;
}
const incomingVersionRaw = body.agentVersion || (body as AgentHeartbeatPayload).version;
const requiredParts = parseVersionParts(requiredVersion);
const incomingParts = parseVersionParts(incomingVersionRaw);
if (!requiredParts || !incomingParts) {
reply.status(426).send({
error: `Agent version is missing or invalid. Minimum required is ${requiredVersion}.`,
code: 'AGENT_VERSION_REQUIRED',
minAgentVersion: requiredVersion,
});
return false;
}
if (compareVersionParts(incomingParts, requiredParts) < 0) {
reply.status(426).send({
error: `Agent version ${normalizeVersion(incomingVersionRaw)} is too old. Minimum required is ${requiredVersion}.`,
code: 'AGENT_VERSION_TOO_OLD',
minAgentVersion: requiredVersion,
receivedAgentVersion: normalizeVersion(incomingVersionRaw),
});
return false;
}
return true;
};
const sanitizeAgentTrafficUpdate = (raw: AgentTrafficUpdatePayload): TrafficUpdate | null => {
if (!raw || typeof raw !== 'object') return null;
const upload = Number.isFinite(raw.upload) ? Math.max(0, Math.floor(raw.upload || 0)) : 0;
const download = Number.isFinite(raw.download) ? Math.max(0, Math.floor(raw.download || 0)) : 0;
if (upload === 0 && download === 0) return null;
const rawChains = Array.isArray(raw.chains) ? raw.chains : [];
const chains = rawChains
.map((chain) => String(chain || '').trim())
.filter(Boolean)
.slice(0, 12);
const normalizedChains = chains.length > 0 ? chains : [String(raw.chain || 'DIRECT').trim() || 'DIRECT'];
const timestampMs = Number.isFinite(raw.timestampMs)
? Math.max(0, Math.floor(raw.timestampMs || 0))
: Date.now();
const connections = Number.isFinite(raw.connections)
? Math.max(0, Math.floor(raw.connections || 0))
: undefined;
return {
domain: String(raw.domain || '').trim().slice(0, 253),
ip: String(raw.ip || '').trim().slice(0, 64),
chain: normalizedChains[0] || 'DIRECT',
chains: normalizedChains,
rule: String(raw.rule || 'Match').trim().slice(0, 256) || 'Match',
rulePayload: String(raw.rulePayload || '').trim().slice(0, 512),
upload,
download,
connections,
sourceIP: String(raw.sourceIP || '').trim().slice(0, 64),
timestampMs,
};
};
const isAgentBackendAuthorized = (
backendId: number,
request: { headers: Record<string, unknown> },
reply: { status: (code: number) => { send: (payload: Record<string, unknown>) => unknown } },
): backendId is number => {
const backend = db.getBackend(backendId);
if (!backend) {
reply.status(404).send({ error: 'Backend not found' });
return false;
}
if (!isAgentBackendUrl(backend.url)) {
reply.status(400).send({ error: 'Backend is not in agent mode (url must start with agent://)' });
return false;
}
const expected = (backend.token || '').trim();
if (!expected) {
reply.status(403).send({ error: 'Agent backend token is not configured' });
return false;
}
const provided = parseAgentToken(request);
if (!provided || !timingSafeStringEqual(provided, expected)) {
reply.status(401).send({ error: 'Invalid agent token' });
return false;
}
return true;
};
const isAgentBindingAllowed = (
backendId: number,
agentId: string,
reply: { status: (code: number) => { send: (payload: Record<string, unknown>) => unknown } },
): boolean => {
const heartbeat = db.getAgentHeartbeat(backendId);
if (!heartbeat) {
return true;
}
if (heartbeat.agentId === agentId) {
return true;
}
// Allow rebinding if the existing agent has been offline for a while
// Agent heartbeat interval is 30s, so 10s timeout allows quick restart
// while preventing accidental duplicate bindings
const AGENT_BINDING_TIMEOUT_MS = 10000;
const lastSeenMs = new Date(heartbeat.lastSeen).getTime();
const ageMs = Number.isFinite(lastSeenMs) ? Math.max(0, Date.now() - lastSeenMs) : Number.POSITIVE_INFINITY;
if (Number.isFinite(ageMs) && ageMs > AGENT_BINDING_TIMEOUT_MS) {
console.info(`[Agent] Allowing rebinding for backend ${backendId}: previous agent '${heartbeat.agentId}' offline for ${Math.round(ageMs / 1000)}s`);
return true;
}
reply.status(409).send({
error: `Agent token is already bound to '${heartbeat.agentId}'. ` +
`Wait ${Math.round((AGENT_BINDING_TIMEOUT_MS - ageMs) / 1000)}s for previous agent to timeout. ` +
`If you need to run multiple agents on the same backend, use different --agent-id values.`,
code: 'AGENT_TOKEN_ALREADY_BOUND',
boundAgentId: heartbeat.agentId,
remainingSeconds: Math.round((AGENT_BINDING_TIMEOUT_MS - ageMs) / 1000),
});
return false;
};
app.post('/api/agent/heartbeat', async (request, reply) => {
const body = request.body as AgentHeartbeatPayload;
const backendId = parseBackendId(body?.backendId);
if (backendId === null) {
return reply.status(400).send({ error: 'Invalid backendId' });
}
if (!isAgentBackendAuthorized(backendId, request, reply)) {
return;
}
if (!isAgentCompatible(body, reply)) {
return;
}
const agentId = parseAgentId(body.agentId);
if (!agentId) {
return reply.status(400).send({ error: 'Invalid agentId' });
}
if (!isAgentBindingAllowed(backendId, agentId, reply)) {
return;
}
const hostname = String(body.hostname || '').trim().slice(0, 128) || undefined;
const version = String(body.agentVersion || body.version || '').trim().slice(0, 64) || undefined;
const gatewayType = String(body.gatewayType || '').trim().slice(0, 16) || undefined;
const gatewayUrl = String(body.gatewayUrl || '').trim().slice(0, 512) || undefined;
const remoteIP = request.ip || request.headers['x-forwarded-for']?.toString().split(',')[0]?.trim();
const gatewayLatencyMs = typeof body.gatewayLatencyMs === 'number'
&& Number.isFinite(body.gatewayLatencyMs)
&& body.gatewayLatencyMs >= 0
? Math.round(body.gatewayLatencyMs)
: undefined;
const serverLatencyMs = typeof body.serverLatencyMs === 'number'
&& Number.isFinite(body.serverLatencyMs)
&& body.serverLatencyMs >= 0
? Math.round(body.serverLatencyMs)
: undefined;
db.upsertAgentHeartbeat({
backendId,
agentId,
hostname,
version,
gatewayType,
gatewayUrl,
remoteIP,
gatewayLatencyMs,
serverLatencyMs,
lastSeen: new Date().toISOString(),
});
return { success: true, backendId, agentId, serverTime: new Date().toISOString() };
});
app.post('/api/agent/report', async (request, reply) => {
const body = request.body as AgentReportPayload;
const backendId = parseBackendId(body?.backendId);
if (backendId === null) {
return reply.status(400).send({ error: 'Invalid backendId' });
}
if (!isAgentBackendAuthorized(backendId, request, reply)) {
return;
}
if (!isAgentCompatible(body, reply)) {
return;
}
const agentId = parseAgentId(body.agentId);
if (!agentId) {
return reply.status(400).send({ error: 'Invalid agentId' });
}
if (!isAgentBindingAllowed(backendId, agentId, reply)) {
return;
}
// Idempotency: if this requestId has already been processed, skip to avoid double-counting
const requestId = typeof body?.requestId === 'string' ? body.requestId.slice(0, 64) : '';
if (requestId && isDuplicateRequestId(requestId)) {
return { success: true, backendId, accepted: 0, dropped: 0, duplicate: true };
}
const rawUpdates = Array.isArray(body?.updates) ? body.updates : [];
if (rawUpdates.length === 0) {
return { success: true, backendId, accepted: 0, dropped: 0 };
}
const maxBatchSize = Math.max(
1,
Number.parseInt(process.env.AGENT_INGEST_MAX_BATCH_SIZE || '5000', 10) || 5000,
);
const picked = rawUpdates.slice(0, maxBatchSize);
const updates: TrafficUpdate[] = [];
for (const update of picked) {
const sanitized = sanitizeAgentTrafficUpdate(update);
if (sanitized) updates.push(sanitized);
}
if (updates.length === 0) {
return { success: true, backendId, accepted: 0, dropped: picked.length };
}
// Get or create the per-backend BatchBuffer for agent mode.
// This mirrors the BatchBuffer used by direct mode (gateway.collector.ts): updates are
// accumulated over ~30 seconds then flushed in one batch. The BatchBuffer deduplicates
// by (domain, ip, chain, minute) key, so a long-lived connection is counted once per
// minute instead of once per 2-second agent report — keeping connection counts
// semantically consistent with direct mode.
let agentBuffer = agentBatchBuffers.get(backendId);
if (!agentBuffer) {
agentBuffer = new BatchBuffer();
agentBatchBuffers.set(backendId, agentBuffer);
}
const geoBatchByIp = new Map<
string,
{
upload: number;
download: number;
connections: number;
timestampMs: number;
}
>();
// Compatibility fallback for older agents that do not send explicit `connections`.
// For those payloads only, we deduplicate by a composite key within the current
// flush window to avoid counting every 2s report as a new connection.
let agentCounted = agentCountedByBackend.get(backendId);
if (!agentCounted) {
agentCounted = new Set<string>();
agentCountedByBackend.set(backendId, agentCounted);
}
for (const update of updates) {
let connections = Number.isFinite(update.connections)
? Math.max(0, Math.floor(update.connections || 0))
: undefined;
if (connections === undefined) {
// Connection key matches BatchBuffer's deduplication dimensions (minus minuteKey):
// domain, ip, full chain, rule, rulePayload, sourceIP.
const connectionKey = `${update.domain}:${update.ip}:${update.chains.join(' > ')}:${update.rule}:${update.rulePayload}:${update.sourceIP || ''}`;
connections = agentCounted.has(connectionKey) ? 0 : 1;
agentCounted.add(connectionKey);
}
if (update.ip && update.ip !== '0.0.0.0' && update.ip !== '::') {
const existing = geoBatchByIp.get(update.ip);
if (existing) {
existing.upload += update.upload;
existing.download += update.download;
existing.connections += connections;
existing.timestampMs = Math.max(existing.timestampMs, update.timestampMs || 0);
} else {
geoBatchByIp.set(update.ip, {
upload: update.upload,
download: update.download,
connections,
timestampMs: update.timestampMs || Date.now(),
});
}
}
// Accumulate in BatchBuffer for deferred DB write (clears realtime after flush).
agentBuffer.add(backendId, {
domain: update.domain,
ip: update.ip,
chain: update.chains[0] || 'DIRECT',
chains: update.chains,
rule: update.rule,
rulePayload: update.rulePayload,
upload: update.upload,
download: update.download,
connections,
sourceIP: update.sourceIP,
timestampMs: update.timestampMs,
});
realtimeStore.recordTraffic(
backendId,
{
domain: update.domain,
ip: update.ip,
sourceIP: update.sourceIP,
chains: update.chains,
rule: update.rule,
rulePayload: update.rulePayload,
upload: update.upload,
download: update.download,
},
connections,
update.timestampMs || Date.now(),
);
}
// DB write and realtimeStore.clearTraffic are deferred to the periodic flush interval,
// matching direct mode's batch semantics.
if (geoBatchByIp.size > 0 && geoService) {
const capturedBuffer = agentBuffer;
// Process in background without blocking the agent response
Promise.all(
Array.from(geoBatchByIp.entries()).map(async ([ip, stats]) => {
try {
const geo = await geoService.getGeoLocation(ip);
return { ip, stats, geo };
} catch {
return { ip, stats, geo: null };
}
}),
)
.then((results) => {
for (const r of results) {
if (r.geo === null) continue;
realtimeStore.recordCountryTraffic(
backendId,
r.geo,
r.stats.upload,
r.stats.download,
r.stats.connections,
r.stats.timestampMs,
);
// Queue in buffer; DB write and clearCountries happen on next periodic flush.
capturedBuffer.addGeoResult({
ip: r.ip,
geo: r.geo,
upload: r.stats.upload,
download: r.stats.download,
connections: r.stats.connections,
timestampMs: r.stats.timestampMs,
});
}
})
.catch((err) => {
console.error(`[Agent:${backendId}] Background GeoIP batch processing failed:`, err);
});
}
db.upsertAgentHeartbeat({
backendId,
agentId,
lastSeen: new Date().toISOString(),
remoteIP: request.ip || request.headers['x-forwarded-for']?.toString().split(',')[0]?.trim(),
});
onTrafficIngested?.(backendId);
return {
success: true,
backendId,
accepted: updates.length,
dropped: picked.length - updates.length,
};
});
app.post('/api/agent/config', async (request, reply) => {
const body = request.body as AgentConfigPayload;
const backendId = parseBackendId(body?.backendId);
// Agent config received (log removed to reduce noise)
if (backendId === null) {
return reply.status(400).send({ error: 'Invalid backendId' });
}
if (!isAgentBackendAuthorized(backendId, request, reply)) {
// Backend not authorized (error returned to client)
return;
}
const agentId = parseAgentId(body.agentId);
if (!agentId) {
return reply.status(400).send({ error: 'Invalid agentId' });
}
if (!isAgentBindingAllowed(backendId, agentId, reply)) {
// Agent binding not allowed (error returned to client)
return;
}
if (!body.config) {
return reply.status(400).send({ error: 'Missing config payload' });
}
// Config stored successfully (log removed to reduce noise)
realtimeStore.setAgentConfig(backendId, body.config);
return { success: true, backendId, hash: body.config.hash };
});
// Agent policy state endpoint (synced more frequently than config)
app.post('/api/agent/policy-state', async (request, reply) => {
const body = request.body as { backendId: number; agentId: string; policyState: { proxies: Record<string, unknown>; providers: Record<string, unknown>; timestamp: number } };
const backendId = parseBackendId(body?.backendId);
// Agent policy state received (log removed to reduce noise)
if (backendId === null) {
return reply.status(400).send({ error: 'Invalid backendId' });
}
if (!isAgentBackendAuthorized(backendId, request, reply)) {
// Backend not authorized (error returned to client)
return;
}
if (!body.policyState) {
return reply.status(400).send({ error: 'Missing policyState payload' });
}
// Policy state stored successfully (log removed to reduce noise)
realtimeStore.setAgentPolicyState(backendId, body.policyState as import('../realtime/realtime.store.js').AgentPolicyState);
return { success: true, backendId, timestamp: body.policyState.timestamp };
});
// Compatibility routes: Gateway APIs
app.get('/api/gateway/proxies', async (request, reply) => {
const backendId = getBackendIdFromQuery(request.query as Record<string, unknown>);
// Gateway API /proxies called (log removed to reduce noise)
if (backendId === null) {
return reply.status(404).send({ error: 'No backend specified or active' });
}
const backend = db.getBackend(backendId);
if (!backend) {
return reply.status(404).send({ error: 'Backend not found' });
}
if (isAgentBackendUrl(backend.url)) {
const cached = realtimeStore.getAgentConfigWithPolicyState(backendId);
console.info(`[Gateway API /proxies] Agent mode, cached exists: ${!!cached}`);
if (!cached) {
return reply.status(503).send({ error: 'Agent config not yet synced' });
}
return { proxies: cached.proxies || {}, _source: 'agent-cache' };
}
const gatewayBaseUrl = getGatewayBaseUrl(backend.url);
const isSurge = backend.type === 'surge';
const headers = getHeaders(backend);
try {
if (isSurge) {
// Surge: Get policies list and details
const res = await gatewayFetch(`${gatewayBaseUrl}/v1/policies`, { headers });
if (!res.ok) {
return reply.status(res.status).send({ error: `Surge API error: ${res.status}` });
}
const data = await res.json() as { proxies: string[]; 'policy-groups': string[] };
const proxies: Record<string, { name: string; type: string; now?: string }> = {};
// Get current selection for each policy group
const policyGroups = data['policy-groups'] || [];
const groupDetails = await Promise.allSettled(
policyGroups.map(async (groupName: string) => {
try {
const detailRes = await gatewayFetch(
`${gatewayBaseUrl}/v1/policies/${encodeURIComponent(groupName)}`,
{ headers, signal: AbortSignal.timeout(5000) }
);
if (!detailRes.ok) return { groupName, now: null };
const detail = await detailRes.json() as { policy?: string };
return { groupName, now: detail.policy || null };
} catch {
return { groupName, now: null };
}
})
);
for (const result of groupDetails) {
if (result.status === 'fulfilled') {
const { groupName, now } = result.value;
proxies[groupName] = { name: groupName, type: 'Selector', now: now || '' };
}
}
// Add leaf proxies
if (data.proxies) {
for (const name of data.proxies) {
proxies[name] = { name, type: 'Unknown' };
}
}
return { proxies };
} else {
// Clash/OpenClash: Direct proxy to /proxies endpoint
const res = await gatewayFetch(`${gatewayBaseUrl}/proxies`, {
headers: { ...headers, 'Content-Type': 'application/json' }
});
if (!res.ok) {
return reply.status(res.status).send({ error: `Gateway API error: ${res.status}` });
}
return res.json();
}
} catch (error: unknown) {
const message = error instanceof Error ? error.message : 'Failed to reach Gateway API';
return reply.status(502).send({ error: message });
}
});
// Health check endpoint (not part of any module)
app.get('/health', async () => ({ status: 'ok' }));
// Compatibility routes: Gateway APIs
app.get('/api/gateway/providers/proxies', async (request, reply) => {
const query = request.query as Record<string, unknown>;
const backendId = getBackendIdFromQuery(query);
const forceRefresh = query.refresh === 'true';
if (backendId === null) {
return reply.status(404).send({ error: 'No backend specified or active' });
}
const backend = db.getBackend(backendId);
if (!backend) {
return reply.status(404).send({ error: 'Backend not found' });
}
if (isAgentBackendUrl(backend.url)) {
const cached = realtimeStore.getAgentConfigWithPolicyState(backendId);
if (!cached) {
return reply.status(503).send({ error: 'Agent config not yet synced' });
}
return { providers: cached.providers || {}, proxies: cached.proxies || {}, _source: 'agent-cache' };
}
const gatewayBaseUrl = getGatewayBaseUrl(backend.url);
const isSurge = backend.type === 'surge';
const headers = getHeaders(backend);
try {
if (isSurge) {
// Build response from cache or fetch directly
const providers: Record<string, { proxies: { name: string; type: string; now?: string }[] }> = {};
const cacheStatus = policySyncService?.getCacheStatus(backendId);
// Try to use cache first
if (cacheStatus?.cached && !forceRefresh) {
const cachedPolicies = db.getSurgePolicyCache(backendId);
for (const policy of cachedPolicies) {
if (policy.selectedPolicy) {
providers[policy.policyGroup] = {
proxies: [{ name: policy.policyGroup, type: policy.policyType, now: policy.selectedPolicy }]
};
}
}
}
// If no cache or force refresh, fetch directly from Surge
if (Object.keys(providers).length === 0 || forceRefresh) {
try {
const res = await gatewayFetch(`${gatewayBaseUrl}/v1/policies`, {
headers,
signal: AbortSignal.timeout(10000)
});
if (!res.ok) {
throw new Error(`Surge API error: ${res.status}`);
}
const data = await res.json() as {
proxies: string[];
'policy-groups': string[];
};
const policyGroups = data['policy-groups'] || [];
// Fetch details for each policy group
// Surge uses /v1/policy_groups/select?group_name=xxx endpoint
const groupDetails = await Promise.allSettled(
policyGroups.map(async (groupName: string) => {
try {
const detailRes = await gatewayFetch(
`${gatewayBaseUrl}/v1/policy_groups/select?group_name=${encodeURIComponent(groupName)}`,
{ headers, signal: AbortSignal.timeout(5000) }
);
if (!detailRes.ok) return null;
const detail = await detailRes.json() as { policy?: string; type?: string };
return {
name: groupName,
now: detail.policy || '',
type: detail.type || 'Select'
};
} catch {
return null;
}
})
);
// Build providers from fetched data
let successCount = 0;
for (const result of groupDetails) {
if (result.status === 'fulfilled' && result.value && result.value.now) {
providers[result.value.name] = {
proxies: [{
name: result.value.name,
type: result.value.type,
now: result.value.now
}]
};
successCount++;
}
}
// Add standalone proxies
if (data.proxies?.length > 0) {
providers['default'] = {
proxies: data.proxies.map(name => ({ name, type: 'Unknown' }))
};
}
// Also update cache in background
if (policySyncService) {
policySyncService.syncNow(backendId, gatewayBaseUrl, backend.token || undefined)
.catch(err => console.error(`[Gateway] Background sync failed:`, err.message));
}
} catch (error) {
console.error(`[Gateway] Failed to fetch from Surge:`, error);
if (Object.keys(providers).length === 0) {
return reply.status(502).send({
error: 'Failed to fetch policies',
message: error instanceof Error ? error.message : 'Unknown error'
});
}
}
}
return {
providers,
_cache: cacheStatus ? {
cached: cacheStatus.cached,
lastUpdate: cacheStatus.lastUpdate,
policyCount: cacheStatus.policyCount,
} : undefined
};
} else {
// Clash/OpenClash: direct proxy
const res = await gatewayFetch(`${gatewayBaseUrl}/providers/proxies`, {
headers: { ...headers, 'Content-Type': 'application/json' }
});
if (!res.ok) {
return reply.status(res.status).send({ error: `Gateway API error: ${res.status}` });
}
return res.json();
}
} catch (error: unknown) {
const message = error instanceof Error ? error.message : 'Failed to reach Gateway API';
return reply.status(502).send({ error: message });
}
});
// Manual refresh endpoint for Surge policies
app.post('/api/gateway/providers/proxies/refresh', async (request, reply) => {
const backendId = getBackendIdFromQuery(request.query as Record<string, unknown>);
if (backendId === null) {
return reply.status(404).send({ error: 'No backend specified or active' });
}
const backend = db.getBackend(backendId);
if (!backend || backend.type !== 'surge') {
return reply.status(400).send({ error: 'Only Surge backend supports this operation' });
}
if (isAgentBackendUrl(backend.url)) {
return reply.status(400).send({ error: 'Agent mode backend does not support this operation' });
}
if (!policySyncService) {
return reply.status(503).send({ error: 'Policy sync service not available' });
}
const gatewayBaseUrl = getGatewayBaseUrl(backend.url);
const result = await policySyncService.syncNow(
backendId,
gatewayBaseUrl,
backend.token || undefined
);
return {
success: result.success,
message: result.message,
updated: result.updated,
};
});
app.get('/api/gateway/rules', async (request, reply) => {
const backendId = getBackendIdFromQuery(request.query as Record<string, unknown>);
if (backendId === null) {
return reply.status(404).send({ error: 'No backend specified or active' });
}
const backend = db.getBackend(backendId);
if (!backend) {
return reply.status(404).send({ error: 'Backend not found' });
}
if (isAgentBackendUrl(backend.url)) {
const cached = realtimeStore.getAgentConfig(backendId);
if (!cached) {
return reply.status(503).send({ error: 'Agent config not yet synced' });
}
if (backend.type === 'surge') {
const parsedRules = (cached.rules || [])
.map(r => r.raw ? parseSurgeRule(r.raw) : null)
.filter((r): r is NonNullable<typeof r> => r !== null)
.map(r => ({ type: r.type, payload: r.payload, proxy: r.policy, size: 0 }));
// Agent mode Surge rules logging removed
return { rules: parsedRules, _source: 'agent-cache' };
}
return { rules: cached.rules || [], _source: 'agent-cache' };
}
const gatewayBaseUrl = getGatewayBaseUrl(backend.url);
const isSurge = backend.type === 'surge';
const headers = getHeaders(backend);
try {
if (isSurge) {
// Surge uses /v1/rules endpoint
const res = await gatewayFetch(`${gatewayBaseUrl}/v1/rules`, { headers });
if (!res.ok) {
return reply.status(res.status).send({ error: `Surge API error: ${res.status}` });
}
const data = await res.json() as { rules: string[]; 'available-policies': string[] };
// Parse Surge rules to standard format
const parsedRules = data.rules
.map(raw => {
const parsed = parseSurgeRule(raw);
return parsed ? { type: parsed.type, payload: parsed.payload, policy: parsed.policy, raw } : null;
})
.filter((r): r is NonNullable<typeof r> => r !== null);
return {
rules: parsedRules.map(r => ({
type: r.type,
payload: r.payload,
proxy: r.policy,
size: 0,
})),
_source: 'surge' as const,
_availablePolicies: data['available-policies'],
};
} else {
// Clash/OpenClash uses /rules endpoint
const res = await gatewayFetch(`${gatewayBaseUrl}/rules`, {
headers: { ...headers, 'Content-Type': 'application/json' }
});
if (!res.ok) {
return reply.status(res.status).send({ error: `Gateway API error: ${res.status}` });
}
const data = (await res.json()) as {
rules?: Array<Record<string, unknown>>;
[key: string]: unknown;
};
const rules = Array.isArray(data.rules) ? data.rules : [];
if (rules.length === 0) {
return data;
}
const providerSizeMap = await loadRuleSetSizeByProvider(gatewayBaseUrl, headers);
if (providerSizeMap.size === 0) {
return data;
}
const enrichedRules = rules.map((rule) => {
const type = String(rule.type || '');
const payload = String(rule.payload || '').toLowerCase();
const size = parseNonNegativeInt(rule.size);
if (type !== 'RuleSet' || size !== null || !payload) {
return rule;
}
const enrichedSize = providerSizeMap.get(payload);
if (enrichedSize === undefined) {
return rule;
}
return {
...rule,
size: enrichedSize,
};
});
return {
...data,
rules: enrichedRules,
};
}
} catch (error: unknown) {
const message = error instanceof Error ? error.message : 'Failed to reach Gateway API';
return reply.status(502).send({ error: message });
}
});
// Auth middleware - protects API routes
app.addHook('onRequest', async (request, reply) => {
// Skip auth for public routes
const publicRoutes = [
'/health',
'/api/auth/state',
'/api/auth/verify',
'/api/auth/logout', // Add logout as public so we can clear cookies even if invalid
'/api/agent/heartbeat',
'/api/agent/report',
'/api/agent/config',
'/api/agent/policy-state',
];
// Check if route is public
if (publicRoutes.some(route => request.url.startsWith(route))) {
return;
}
// Check if auth is required
if (!authService.isAuthRequired()) {
return;
}
// Try to get token from Cookie first
const cookieToken = request.cookies['neko-session'];
if (cookieToken) {
const verifyResult = await authService.verifyToken(cookieToken);
if (verifyResult.valid) {
return;
}
}
// Fallback: Get token from header (for backward compatibility / API clients)
const authHeader = request.headers.authorization;
if (!authHeader || !authHeader.startsWith('Bearer ')) {
return reply.status(401).send({ error: 'Authentication required' });
}
const token = authHeader.slice(7);
const verifyResult = await authService.verifyToken(token);
if (!verifyResult.valid) {
return reply.status(401).send({ error: verifyResult.message || 'Invalid token' });
}
});
// On server close: stop health checks, flush all pending agent buffers.
app.addHook('onClose', async () => {
backendService.stopHealthChecks();
clearInterval(agentFlushIntervalId);
for (const [backendId, buffer] of agentBatchBuffers) {
try {
await flushAgentBuffer(backendId, buffer);
} catch (err) {
console.error(`[Agent:${backendId}] Final flush error on close:`, err);
}
}
});
// Register controllers
await app.register(backendController, { prefix: '/api/backends' });
await app.register(statsController, { prefix: '/api/stats' });
await app.register(authController, { prefix: '/api/auth' });
await app.register(configController, { prefix: '/api/db' });
if (autoListen) {
// Start server
await app.listen({ port, host: '0.0.0.0' });
console.log(`[API] Server running at http://localhost:${port}`);
// Start automatic health checks for upstream gateways
backendService.startHealthChecks();
}
return app;
}
export class APIServer {
private app: ReturnType<typeof Fastify> | null = null;
private db: StatsDatabase;
private realtimeStore: RealtimeStore;
private port: number;
private policySyncService?: SurgePolicySyncService;
private geoService?: GeoIPService;
private onTrafficIngested?: (backendId: number) => void;
private onBackendDataCleared?: (backendId: number) => void;
constructor(
port: number,
db: StatsDatabase,
realtimeStore: RealtimeStore,
policySyncService?: SurgePolicySyncService,
geoService?: GeoIPService,
onTrafficIngested?: (backendId: number) => void,
onBackendDataCleared?: (backendId: number) => void,
) {
this.port = port;
this.db = db;
this.realtimeStore = realtimeStore;
this.policySyncService = policySyncService;
this.geoService = geoService;
this.onTrafficIngested = onTrafficIngested;
this.onBackendDataCleared = onBackendDataCleared;
}
async start() {
this.app = await createApp({
port: this.port,
db: this.db,
realtimeStore: this.realtimeStore,
policySyncService: this.policySyncService,
geoService: this.geoService,
onTrafficIngested: this.onTrafficIngested,
onBackendDataCleared: this.onBackendDataCleared,
logger: false,
});
return this.app;
}
async stop() {
if (this.app) {
// Awaiting close runs the onClose hook, which flushes pending agent buffers.
await this.app.close();
console.log('[API] Server stopped');
}
}
}
export default createApp;