Skip to content

A modern and elegant dashboard for network traffic visualization and analysis.

feat: Implement frequent agent policy state synchronization for dynamic proxy selections.

luoleiea235e5c6a94
diff --git a/apps/agent/internal/agent/runner.go b/apps/agent/internal/agent/runner.go
index bd68598..b758160 100644
--- a/apps/agent/internal/agent/runner.go
+++ b/apps/agent/internal/agent/runner.go
@@ -50,6 +50,12 @@ type configPayload struct {
Config *domain.GatewayConfigSnapshot `json:"config"`
}
+type policyStatePayload struct {
+ BackendID int `json:"backendId"`
+ AgentID string `json:"agentId"`
+ PolicyState *domain.PolicyStateSnapshot `json:"policyState"`
+}
+
type Runner struct {
cfg config.Config
httpClient *http.Client
@@ -85,11 +91,12 @@ func (r *Runner) Run(ctx context.Context) {
log.Printf("[agent:%s] starting, backend=%d, gateway_type=%s, server=%s", r.cfg.AgentID, r.cfg.BackendID, r.cfg.GatewayType, r.cfg.ServerAPIBase)
var wg sync.WaitGroup
- wg.Add(4)
+ wg.Add(5)
go r.runCollectorLoop(ctx, &wg)
go r.runReportLoop(ctx, &wg)
go r.runHeartbeatLoop(ctx, &wg)
go r.runConfigSyncLoop(ctx, &wg)
+ go r.runPolicyStateSyncLoop(ctx, &wg)
<-ctx.Done()
log.Printf("[agent:%s] stopping...", r.cfg.AgentID)
@@ -248,6 +255,56 @@ func (r *Runner) syncConfig(ctx context.Context) error {
return nil
}
+// runPolicyStateSyncLoop syncs only the dynamic policy selection state (now field)
+// This runs more frequently (30s) than config sync (2min) to keep chain flow visualization accurate
+func (r *Runner) runPolicyStateSyncLoop(ctx context.Context, wg *sync.WaitGroup) {
+ defer wg.Done()
+
+ // Wait a bit for initial config sync to complete
+ time.Sleep(5 * time.Second)
+
+ // Initial sync
+ if err := r.syncPolicyState(ctx); err != nil {
+ log.Printf("[agent:%s] init policy state sync error: %v", r.cfg.AgentID, err)
+ }
+
+ // Then every 30 seconds
+ ticker := time.NewTicker(30 * time.Second)
+ defer ticker.Stop()
+
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-ticker.C:
+ if err := r.syncPolicyState(ctx); err != nil {
+ log.Printf("[agent:%s] policy state sync error: %v", r.cfg.AgentID, err)
+ }
+ }
+ }
+}
+
+func (r *Runner) syncPolicyState(ctx context.Context) error {
+ snap, err := r.gatewayClient.GetPolicyStateSnapshot(ctx)
+ if err != nil {
+ return err
+ }
+
+ snap.Timestamp = time.Now().UnixMilli()
+
+ payload := policyStatePayload{
+ BackendID: r.cfg.BackendID,
+ AgentID: r.cfg.AgentID,
+ PolicyState: snap,
+ }
+
+ if err := r.postJSON(ctx, "/agent/policy-state", payload); err != nil {
+ return err
+ }
+
+ return nil
+}
+
func (r *Runner) ingestSnapshots(snapshots []domain.FlowSnapshot, nowMs int64) {
active := make(map[string]struct{}, len(snapshots))
updates := make([]domain.TrafficUpdate, 0, len(snapshots))
diff --git a/apps/agent/internal/domain/config.go b/apps/agent/internal/domain/config.go
index 7216ad3..82c1204 100644
--- a/apps/agent/internal/domain/config.go
+++ b/apps/agent/internal/domain/config.go
@@ -26,3 +26,11 @@ type GatewayConfigSnapshot struct {
Timestamp int64 `json:"timestamp"`
Hash string `json:"hash"`
}
+
+// PolicyStateSnapshot contains only the dynamic policy selection state (now field)
+// This is used for frequent syncs without re-fetching static rules
+type PolicyStateSnapshot struct {
+ Proxies map[string]GatewayProxy `json:"proxies"`
+ Providers map[string]GatewayProvider `json:"providers"`
+ Timestamp int64 `json:"timestamp"`
+}
diff --git a/apps/agent/internal/gateway/config.go b/apps/agent/internal/gateway/config.go
index 7322715..19db2ee 100644
--- a/apps/agent/internal/gateway/config.go
+++ b/apps/agent/internal/gateway/config.go
@@ -124,6 +124,117 @@ func (c *Client) getClashConfig(ctx context.Context) (*domain.GatewayConfigSnaps
return snap, nil
}
+// GetPolicyStateSnapshot returns only the dynamic policy selection state (now field)
+// This is much lighter than GetConfigSnapshot as it doesn't fetch rules
+func (c *Client) GetPolicyStateSnapshot(ctx context.Context) (*domain.PolicyStateSnapshot, error) {
+ if c.gatewayType == "clash" {
+ return c.getClashPolicyState(ctx)
+ }
+ return c.getSurgePolicyState(ctx)
+}
+
+func (c *Client) getSurgePolicyState(ctx context.Context) (*domain.PolicyStateSnapshot, error) {
+ var policiesData struct {
+ PolicyGroups []string `json:"policy-groups"`
+ Proxies []string `json:"proxies"`
+ }
+ if err := c.getJSON(ctx, "/v1/policies", &policiesData); err != nil {
+ return nil, fmt.Errorf("surge /v1/policies error: %w", err)
+ }
+
+ snap := &domain.PolicyStateSnapshot{
+ Proxies: make(map[string]domain.GatewayProxy),
+ Providers: make(map[string]domain.GatewayProvider),
+ }
+
+ // Add standalone proxies (no 'now' field for these)
+ for _, p := range policiesData.Proxies {
+ snap.Proxies[p] = domain.GatewayProxy{
+ Name: p,
+ Type: "Proxy",
+ }
+ }
+
+ // Build provider proxies slice for policy groups
+ providerProxies := make([]domain.GatewayProxy, 0, len(policiesData.PolicyGroups))
+
+ // Fetch current selection for each policy group
+ for _, g := range policiesData.PolicyGroups {
+ var groupDetail struct {
+ Type string `json:"type"`
+ Policy string `json:"policy"`
+ }
+ if err := c.getJSON(ctx, "/v1/policies/"+url.PathEscape(g), &groupDetail); err != nil {
+ fmt.Printf("[agent] warning: failed to get policy detail for %s: %v\n", g, err)
+ }
+ snap.Proxies[g] = domain.GatewayProxy{
+ Name: g,
+ Type: groupDetail.Type,
+ Now: groupDetail.Policy,
+ }
+ providerProxies = append(providerProxies, domain.GatewayProxy{
+ Name: g,
+ Type: groupDetail.Type,
+ Now: groupDetail.Policy,
+ })
+ }
+
+ // Create default provider
+ if len(providerProxies) > 0 {
+ snap.Providers["default"] = domain.GatewayProvider{
+ Name: "default",
+ Type: "SurgePolicyGroups",
+ Proxies: providerProxies,
+ }
+ }
+
+ return snap, nil
+}
+
+func (c *Client) getClashPolicyState(ctx context.Context) (*domain.PolicyStateSnapshot, error) {
+ var proxiesData struct {
+ Proxies map[string]struct {
+ Name string `json:"name"`
+ Type string `json:"type"`
+ Now string `json:"now"`
+ } `json:"proxies"`
+ }
+ if err := c.getJSON(ctx, "/proxies", &proxiesData); err != nil {
+ return nil, fmt.Errorf("clash /proxies error: %w", err)
+ }
+
+ snap := &domain.PolicyStateSnapshot{
+ Proxies: make(map[string]domain.GatewayProxy),
+ Providers: make(map[string]domain.GatewayProvider),
+ }
+
+ // Group proxies by type for provider structure
+ providerProxies := make(map[string][]domain.GatewayProxy)
+
+ for name, p := range proxiesData.Proxies {
+ proxy := domain.GatewayProxy{
+ Name: p.Name,
+ Type: p.Type,
+ Now: p.Now,
+ }
+ snap.Proxies[name] = proxy
+
+ // Group by type for providers
+ providerProxies[p.Type] = append(providerProxies[p.Type], proxy)
+ }
+
+ // Create providers by type
+ for typ, proxies := range providerProxies {
+ snap.Providers[typ] = domain.GatewayProvider{
+ Name: typ,
+ Type: typ,
+ Proxies: proxies,
+ }
+ }
+
+ return snap, nil
+}
+
func parseSurgeRuleForAgent(raw string) domain.GatewayRule {
// Basic Surge parsing logic. For agent, returning "raw" is often enough as backend parses it.
// However master expects { type, payload, proxy } if we can parse it.
@@ -178,6 +289,9 @@ func (c *Client) getSurgeConfig(ctx context.Context) (*domain.GatewayConfigSnaps
}
}
+ // Build provider proxies slice for policy groups
+ providerProxies := make([]domain.GatewayProxy, 0, len(policiesData.PolicyGroups))
+
for _, g := range policiesData.PolicyGroups {
var groupDetail struct {
Type string `json:"type"`
@@ -191,8 +305,25 @@ func (c *Client) getSurgeConfig(ctx context.Context) (*domain.GatewayConfigSnaps
Type: groupDetail.Type,
Now: groupDetail.Policy,
}
+ // Also add to provider proxies for frontend compatibility
+ providerProxies = append(providerProxies, domain.GatewayProxy{
+ Name: g,
+ Type: groupDetail.Type,
+ Now: groupDetail.Policy,
+ })
fmt.Printf("[agent] policy group: %s, type: %s, now: %s\n", g, groupDetail.Type, groupDetail.Policy)
}
+
+ // Create a default provider containing all policy groups
+ // This ensures frontend's buildGroupNowMap can find the 'now' values
+ if len(providerProxies) > 0 {
+ snap.Providers["default"] = domain.GatewayProvider{
+ Name: "default",
+ Type: "SurgePolicyGroups",
+ Proxies: providerProxies,
+ }
+ fmt.Printf("[agent] created default provider with %d policy groups\n", len(providerProxies))
+ }
return snap, nil
}
diff --git a/apps/collector/src/modules/app/app.ts b/apps/collector/src/modules/app/app.ts
index 933fa77..51aad57 100644
--- a/apps/collector/src/modules/app/app.ts
+++ b/apps/collector/src/modules/app/app.ts
@@ -636,6 +636,29 @@ export async function createApp(options: AppOptions) {
return { success: true, backendId, hash: body.config.hash };
});
+ // Agent policy state endpoint (synced more frequently than config)
+ app.post('/api/agent/policy-state', async (request, reply) => {
+ const body = request.body as { backendId: number; agentId: string; policyState: { proxies: Record<string, unknown>; providers: Record<string, unknown>; timestamp: number } };
+ const backendId = parseBackendId(body?.backendId);
+ console.info(`[Agent Policy State] Received policy state for backendId: ${backendId}, agentId: ${body?.agentId}`);
+ if (backendId === null) {
+ return reply.status(400).send({ error: 'Invalid backendId' });
+ }
+ if (!isAgentBackendAuthorized(backendId, request, reply)) {
+ console.info(`[Agent Policy State] Backend not authorized: ${backendId}`);
+ return;
+ }
+
+ if (!body.policyState) {
+ return reply.status(400).send({ error: 'Missing policyState payload' });
+ }
+
+ console.info(`[Agent Policy State] Storing policy state for backendId: ${backendId}, proxies count: ${Object.keys(body.policyState?.proxies || {}).length}`);
+ realtimeStore.setAgentPolicyState(backendId, body.policyState as import('../realtime/realtime.store.js').AgentPolicyState);
+
+ return { success: true, backendId, timestamp: body.policyState.timestamp };
+ });
+
// Compatibility routes: Gateway APIs
app.get('/api/gateway/proxies', async (request, reply) => {
const backendId = getBackendIdFromQuery(request.query as Record<string, unknown>);
@@ -649,7 +672,7 @@ export async function createApp(options: AppOptions) {
return reply.status(404).send({ error: 'Backend not found' });
}
if (isAgentBackendUrl(backend.url)) {
- const cached = realtimeStore.getAgentConfig(backendId);
+ const cached = realtimeStore.getAgentConfigWithPolicyState(backendId);
console.info(`[Gateway API /proxies] Agent mode, cached exists: ${!!cached}`);
if (!cached) {
return reply.status(503).send({ error: 'Agent config not yet synced' });
@@ -739,7 +762,7 @@ export async function createApp(options: AppOptions) {
return reply.status(404).send({ error: 'Backend not found' });
}
if (isAgentBackendUrl(backend.url)) {
- const cached = realtimeStore.getAgentConfig(backendId);
+ const cached = realtimeStore.getAgentConfigWithPolicyState(backendId);
if (!cached) {
return reply.status(503).send({ error: 'Agent config not yet synced' });
}
@@ -1029,6 +1052,7 @@ export async function createApp(options: AppOptions) {
'/api/agent/heartbeat',
'/api/agent/report',
'/api/agent/config',
+ '/api/agent/policy-state',
];
// Check if route is public
diff --git a/apps/collector/src/modules/realtime/realtime.store.ts b/apps/collector/src/modules/realtime/realtime.store.ts
index df552d7..abc4def 100644
--- a/apps/collector/src/modules/realtime/realtime.store.ts
+++ b/apps/collector/src/modules/realtime/realtime.store.ts
@@ -214,8 +214,16 @@ export type AgentGatewayConfig = {
hash: string;
};
+// Policy state is synced more frequently (30s) than full config (2min)
+export type AgentPolicyState = {
+ proxies: Record<string, { name: string; type: string; now?: string }>;
+ providers: Record<string, { proxies: Array<{ name: string; type: string; now?: string }> }>;
+ timestamp: number;
+};
+
export class RealtimeStore {
private agentConfigByBackend = new Map<number, AgentGatewayConfig>();
+ private agentPolicyStateByBackend = new Map<number, AgentPolicyState>();
public summaryByBackend = new Map<number, SummaryDelta>();
public minuteByBackend = new Map<number, Map<string, MinuteBucket>>();
public domainByBackend = new Map<number, Map<string, DomainDelta>>();
@@ -247,6 +255,30 @@ export class RealtimeStore {
return this.agentConfigByBackend.get(backendId);
}
+ setAgentPolicyState(backendId: number, state: AgentPolicyState): void {
+ this.agentPolicyStateByBackend.set(backendId, state);
+ }
+
+ getAgentPolicyState(backendId: number): AgentPolicyState | undefined {
+ return this.agentPolicyStateByBackend.get(backendId);
+ }
+
+ // Get merged agent config with latest policy state
+ getAgentConfigWithPolicyState(backendId: number): AgentGatewayConfig | undefined {
+ const config = this.agentConfigByBackend.get(backendId);
+ const policyState = this.agentPolicyStateByBackend.get(backendId);
+
+ if (!config) return undefined;
+ if (!policyState) return config;
+
+ // Merge policy state into config (policy state is more recent)
+ return {
+ ...config,
+ proxies: policyState.proxies,
+ providers: policyState.providers,
+ };
+ }
+
recordTraffic(
backendId: number,
meta: TrafficMeta,