Lightweight server monitoring with historical data, docker stats, and alerts.
feat: Add option to define which DNS server a DNS monitor queries (#2389)
Sven van Ginkelf7528a02082a
diff --git a/agent/network_monitor_probe.go b/agent/network_monitor_probe.goindex cee135e08..7d7b6ae30 100644--- a/agent/network_monitor_probe.go+++ b/agent/network_monitor_probe.go@@ -28,7 +28,7 @@ func networkMonitorProbe(client *http.Client) monitorProbe {case "http":return monitorHTTP(ctx, client, config.Target)case "dns":- return monitorDNS(ctx, config.Target)+ return monitorDNS(ctx, config.Target, config.Server)default:return -1, fmt.Errorf("unknown monitor protocol: %s", config.Protocol)}@@ -73,19 +73,43 @@ func monitorTCP(ctx context.Context, target string, port uint16) (int64, error)return -1, err}-// monitorDNS measures DNS resolution response time in microseconds. Returns -1 and an error on failure.-func monitorDNS(ctx context.Context, target string) (int64, error) {+// monitorDNS measures DNS resolution response time in microseconds. If server is+// non-empty, the lookup is sent to that DNS server (host or host:port, default+// port 53) instead of the system resolver. Returns -1 and an error on failure.+func monitorDNS(ctx context.Context, target, server string) (int64, error) {ctx, cancel := context.WithTimeout(ctx, 3*time.Second)defer cancel()+ resolver := net.DefaultResolver+ if server != "" {+ resolver = dnsResolverForServer(server)+ }+start := time.Now()- ips, err := net.DefaultResolver.LookupHost(ctx, target)+ ips, err := resolver.LookupHost(ctx, target)if err != nil || len(ips) == 0 {return -1, err}return time.Since(start).Microseconds(), nil}+// dnsResolverForServer builds a resolver that sends lookups to the given DNS+// server address instead of the system resolver. server may be a bare host or+// host:port; when no port is given, the standard DNS port 53 is used.+func dnsResolverForServer(server string) *net.Resolver {+ address := server+ if _, _, err := net.SplitHostPort(server); err != nil {+ address = net.JoinHostPort(server, "53")+ }+ return &net.Resolver{+ PreferGo: true,+ Dial: func(ctx context.Context, network, _ string) (net.Conn, error) {+ var dialer net.Dialer+ return dialer.DialContext(ctx, network, address)+ },+ }+}+// monitorHTTP measures HTTP GET request response in microseconds. Returns -1 and an error on failure.func monitorHTTP(ctx context.Context, client *http.Client, url string) (int64, error) {if client == nil {diff --git a/agent/network_monitor_test.go b/agent/network_monitor_test.goindex f725441a1..1116ce3ce 100644--- a/agent/network_monitor_test.go+++ b/agent/network_monitor_test.go@@ -376,15 +376,79 @@ func tcpMonitorTestResolver(ips []string) *net.Resolver {}}}+// udpDNSTestServer starts a UDP server on loopback that answers A queries with the+// given IPs, and returns its listen address (host:port).+func udpDNSTestServer(t *testing.T, ips []string) string {+ t.Helper()+ conn, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.ParseIP("127.0.0.1")})+ require.NoError(t, err)+ t.Cleanup(func() { conn.Close() })++ go func() {+ buf := make([]byte, 512)+ for {+ n, addr, err := conn.ReadFromUDP(buf)+ if err != nil {+ return+ }+ var msg dnsmessage.Message+ if err := msg.Unpack(buf[:n]); err != nil {+ continue+ }+ msg.Header.Response = true+ msg.Header.RecursionAvailable = true+ for _, question := range msg.Questions {+ if question.Type != dnsmessage.TypeA {+ continue+ }+ for _, ip := range ips {+ msg.Answers = append(msg.Answers, dnsmessage.Resource{+ Header: dnsmessage.ResourceHeader{Name: question.Name, Type: dnsmessage.TypeA, Class: dnsmessage.ClassINET},+ Body: &dnsmessage.AResource{A: [4]byte(net.ParseIP(ip).To4())},+ })+ }+ }+ packet, err := msg.Pack()+ if err != nil {+ continue+ }+ _, _ = conn.WriteToUDP(packet, addr)+ }+ }()++ return conn.LocalAddr().String()+}+func TestMonitorDNS(t *testing.T) {t.Run("success", func(t *testing.T) {- responseUs, err := monitorDNS(context.Background(), "localhost")+ responseUs, err := monitorDNS(context.Background(), "localhost", "")require.NoError(t, err)assert.GreaterOrEqual(t, responseUs, int64(0))})t.Run("lookup failure", func(t *testing.T) {- responseUs, err := monitorDNS(context.Background(), "")+ responseUs, err := monitorDNS(context.Background(), "", "")+ assert.Equal(t, int64(-1), responseUs)+ require.Error(t, err)+ })++ t.Run("custom server", func(t *testing.T) {+ serverAddr := udpDNSTestServer(t, []string{"192.0.2.10"})+ responseUs, err := monitorDNS(context.Background(), "example.test.", serverAddr)+ require.NoError(t, err)+ assert.GreaterOrEqual(t, responseUs, int64(0))+ })++ t.Run("custom server without port defaults to 53", func(t *testing.T) {+ resolver := dnsResolverForServer("127.0.0.1")+ conn, err := resolver.Dial(context.Background(), "udp", "")+ require.NoError(t, err)+ defer conn.Close()+ assert.Equal(t, "127.0.0.1:53", conn.RemoteAddr().String())+ })++ t.Run("custom server unreachable", func(t *testing.T) {+ responseUs, err := monitorDNS(context.Background(), "example.test.", "127.0.0.1:1")assert.Equal(t, int64(-1), responseUs)require.Error(t, err)})@@ -479,7 +543,7 @@ func TestMonitorResolutionCancellation(t *testing.T) {case "tcp":_, err = monitorTCP(ctx, "monitor-cancellation.invalid.", 80)case "dns":- _, err = monitorDNS(ctx, "monitor-cancellation.invalid.")+ _, err = monitorDNS(ctx, "monitor-cancellation.invalid.", "")case "icmp":_, err = monitorICMP(ctx, "monitor-cancellation.invalid.")}diff --git a/internal/entities/monitor/monitor.go b/internal/entities/monitor/monitor.goindex c0f0915b0..4d41cd6d7 100644--- a/internal/entities/monitor/monitor.go+++ b/internal/entities/monitor/monitor.go@@ -25,6 +25,9 @@ type Config struct {Protocol string `cbor:"2,keyasint"` // "icmp", "tcp", "http", or "dns"Port uint16 `cbor:"3,keyasint,omitempty"`Interval uint16 `cbor:"4,keyasint"` // seconds+ // Server is the DNS server to query (host or host:port, default port 53).+ // Only used when Protocol is "dns"; empty means use the system resolver.+ Server string `cbor:"5,keyasint,omitempty"`}// CertInfo holds details of the leaf TLS certificate presented by a target.diff --git a/internal/hub/network_monitors.go b/internal/hub/network_monitors.goindex 9569d0106..98e9e0e88 100644--- a/internal/hub/network_monitors.go+++ b/internal/hub/network_monitors.go@@ -17,6 +17,10 @@ func generateMonitorID(systemId string, config monitor.Config) string {if config.Protocol == "tcp" {args = append(args, strconv.FormatUint(uint64(config.Port), 10))}+ // only use server for DNS monitors, so the same target queried via different servers gets distinct monitors+ if config.Protocol == "dns" {+ args = append(args, config.Server)+ }return systems.MakeStableHashId(args...)}@@ -53,10 +57,15 @@ func bindNetworkMonitorsEvents(hub *Hub) {// record with the new ID and delete the old one. Otherwise, just update the existing monitor on the agent.hub.OnRecordUpdateRequest("network_monitors").BindFunc(func(e *core.RecordRequestEvent) error {systemID := e.Record.GetString("system")+ protocol := e.Record.GetString("protocol")// only tcp uses port - set other protocols port to zero- if e.Record.GetString("protocol") != "tcp" {+ if protocol != "tcp" {e.Record.Set("port", 0)}+ // only dns uses server - clear it for other protocols+ if protocol != "dns" {+ e.Record.Set("server", "")+ }ID := generateMonitorID(systemID, *monitorConfigFromRecord(e.Record))if ID != e.Record.Id {newRecord := copyMonitorToNewRecord(e.Record, ID)@@ -103,6 +112,7 @@ func monitorConfigFromRecord(record *core.Record) *monitor.Config {Protocol: record.GetString("protocol"),Port: uint16(record.GetInt("port")),Interval: uint16(record.GetInt("interval")),+ Server: record.GetString("server"),}}@@ -127,7 +137,7 @@ func copyMonitorToNewRecord(oldRecord *core.Record, newID string) *core.Record {collection := oldRecord.Collection()newRecord := core.NewRecord(collection)newRecord.Id = newID- fields := []string{"system", "target", "protocol", "port", "interval", "enabled"}+ fields := []string{"system", "target", "protocol", "port", "server", "interval", "enabled"}for _, field := range fields {newRecord.Set(field, oldRecord.Get(field))}diff --git a/internal/hub/network_monitors_test.go b/internal/hub/network_monitors_test.goindex 1907ebb57..c0d60b3d7 100644--- a/internal/hub/network_monitors_test.go+++ b/internal/hub/network_monitors_test.go@@ -174,6 +174,39 @@ func TestGenerateMonitorID(t *testing.T) {},expected: "84167969",},+ {+ name: "DNS monitor on example.com with server 1.1.1.1",+ systemID: "sys999",+ config: monitor.Config{+ Protocol: "dns",+ Target: "example.com",+ Server: "1.1.1.1",+ Interval: 30,+ },+ expected: "2175898b",+ },+ {+ name: "DNS monitor on example.com with different server",+ systemID: "sys999",+ config: monitor.Config{+ Protocol: "dns",+ Target: "example.com",+ Server: "8.8.8.8",+ Interval: 30,+ },+ expected: "ebcd8b33",+ },+ {+ name: "DNS monitor on example.com with no server (system resolver)",+ systemID: "sys999",+ config: monitor.Config{+ Protocol: "dns",+ Target: "example.com",+ Server: "",+ Interval: 30,+ },+ expected: "19476a7",+ },}for _, tt := range tests {@@ -199,6 +232,7 @@ func TestCopyMonitorToNewRecordDropsResultFields(t *testing.T) {"target": "https://example.com","protocol": "http","port": 443,+ "server": "1.1.1.1","interval": 60,"enabled": true,"res": 1200,@@ -216,6 +250,7 @@ func TestCopyMonitorToNewRecordDropsResultFields(t *testing.T) {assert.Equal(t, "https://example.com", newRecord.GetString("target"))assert.Equal(t, "http", newRecord.GetString("protocol"))assert.Equal(t, 443, newRecord.GetInt("port"))+ assert.Equal(t, "1.1.1.1", newRecord.GetString("server"))assert.True(t, newRecord.GetBool("enabled"))assert.Contains(t, []string{"", "null"}, newRecord.GetString("certInfo"))assert.Zero(t, newRecord.GetFloat("res"))diff --git a/internal/migrations/1790273174_network_monitor_server.go b/internal/migrations/1790273174_network_monitor_server.gonew file mode 100644index 000000000..bdcb26f2a--- /dev/null+++ b/internal/migrations/1790273174_network_monitor_server.go@@ -0,0 +1,24 @@+package migrations++import (+ "github.com/pocketbase/pocketbase/core"+ m "github.com/pocketbase/pocketbase/migrations"+)++func init() {+ m.Register(func(app core.App) error {+ collection, err := app.FindCollectionByNameOrId("network_monitors")+ if err != nil {+ return err+ }+ collection.Fields.Add(&core.TextField{Id: "nm_server", Name: "server", Max: 260})+ return app.Save(collection)+ }, func(app core.App) error {+ collection, err := app.FindCollectionByNameOrId("network_monitors")+ if err != nil {+ return err+ }+ collection.Fields.RemoveByName("server")+ return app.Save(collection)+ })+}diff --git a/internal/site/src/components/network-monitors-table/monitor-dialog.tsx b/internal/site/src/components/network-monitors-table/monitor-dialog.tsxindex 830357ef7..1bdfd7f78 100644--- a/internal/site/src/components/network-monitors-table/monitor-dialog.tsx+++ b/internal/site/src/components/network-monitors-table/monitor-dialog.tsx@@ -37,6 +37,7 @@ type MonitorValues = {target: stringprotocol: MonitorProtocolport: number+ server: stringinterval: string}@@ -44,7 +45,7 @@ type NormalizedMonitorValues = Omit<MonitorValues, "system" | "interval"> & {interval: number}-type BulkMonitorLineSource = Pick<NetworkMonitorRecord, "target" | "protocol" | "port" | "interval">+type BulkMonitorLineSource = Pick<NetworkMonitorRecord, "target" | "protocol" | "port" | "interval" | "server">const defaultInterval = 30@@ -59,6 +60,7 @@ const NormalizedMonitorValuesSchema = v.pipe(target: v.pipe(v.string(), v.trim(), v.nonEmpty("target is required")),protocol: MonitorProtocolSchema,port: v.number(),+ server: v.pipe(v.string(), v.trim()),interval: MonitorIntervalSchema,}),v.transform((input): NormalizedMonitorValues => {@@ -78,6 +80,8 @@ const NormalizedMonitorValuesSchema = v.pipe(target: protocol === "http" ? httpTarget : input.target,protocol,port,+ // Only DNS monitors use a custom server; clear it for other protocols.+ server: protocol === "dns" ? input.server : "",interval: input.interval,}}),@@ -100,6 +104,7 @@ const BulkMonitorSchema = v.object({protocol: v.optional(v.pipe(v.string(), v.trim())),port: v.optional(v.pipe(v.string(), v.trim())),interval: v.optional(v.pipe(v.string(), v.trim())),+ server: v.optional(v.pipe(v.string(), v.trim())),})function normalizeHttpTarget(target: string, port = 0) {@@ -152,18 +157,19 @@ function buildMonitorPayload(values: MonitorValues, enabled = true) {return payload}-type MonitorIdentity = Pick<MonitorValues, "system" | "target" | "protocol" | "port">-function getMonitorIdentityKey({ system, target, protocol, port }: MonitorIdentity) {- return `${system}${target}${protocol}${port}`+type MonitorIdentity = Pick<MonitorValues, "system" | "target" | "protocol" | "port" | "server">+function getMonitorIdentityKey({ system, target, protocol, port, server }: MonitorIdentity) {+ return `${system}${target}${protocol}${port}${protocol === "dns" ? server : ""}`}function parseBulkMonitorLine(line: string, lineNumber: number, system: string) {- const [rawTarget = "", rawProtocol = "", rawPort = "", rawInterval = ""] = line.split(",")+ const [rawTarget = "", rawProtocol = "", rawPort = "", rawInterval = "", rawServer = ""] = line.split(",")const parsed = v.safeParse(BulkMonitorSchema, {target: rawTarget,protocol: rawProtocol,port: rawPort,interval: rawInterval,+ server: rawServer,})if (!parsed.success) {throw new Error(`Line ${lineNumber}: ${parsed.issues[0]?.message || "invalid monitor entry"}`)@@ -176,6 +182,7 @@ function parseBulkMonitorLine(line: string, lineNumber: number, system: string)target: parsed.output.target,protocol,port: parsed.output.port ? Number(parsed.output.port) : 0,+ server: parsed.output.server || "",interval: parsed.output.interval || `${defaultInterval}`,})}@@ -183,7 +190,8 @@ function parseBulkMonitorLine(line: string, lineNumber: number, system: string)export function formatBulkMonitorLine(monitor: BulkMonitorLineSource) {const port = monitor.protocol !== "tcp" || monitor.port === 443 ? "" : `${monitor.port}`const interval = monitor.interval === defaultInterval ? "" : `${monitor.interval}`- return trimTrailingEmptyFields([monitor.target, monitor.protocol, port, interval]).join(",")+ const server = monitor.protocol !== "dns" ? "" : monitor.server+ return trimTrailingEmptyFields([monitor.target, monitor.protocol, port, interval, server]).join(",")}function SystemMultiSelect({@@ -498,7 +506,7 @@ export function AddMonitorDialog({ systemId, monitors }: { systemId?: string; mo<Trans>Bulk Add {{ foo: t`Network Monitors` }}</Trans></SheetTitle><SheetDescription>- <Trans>target[,protocol[,port[,interval]]]</Trans>+ <Trans>target[,protocol[,port[,interval[,server]]]]</Trans></SheetDescription></SheetHeader><form ref={bulkFormRef} onSubmit={handleBulkSubmit} className="flex h-full flex-col overflow-hidden">@@ -532,11 +540,16 @@ export function AddMonitorDialog({ systemId, monitors }: { systemId?: string; mo}}}className="font-mono grow text-sm bg-card"- placeholder={["1.1.1.1", "example.com,tcp", "https://example.com,http,,60"].join("\n")}+ placeholder={[+ "1.1.1.1",+ "example.com,tcp",+ "https://example.com,http,,60",+ "example.com,dns,,,1.1.1.1",+ ].join("\n")}required/><p className="text-xs text-muted-foreground">- <Trans>target[,protocol[,port[,interval]]]</Trans>+ <Trans>target[,protocol[,port[,interval[,server]]]]</Trans></p></div></div>@@ -591,6 +604,7 @@ function MonitorDialogContent({const [protocol, setProtocol] = useState<MonitorProtocol>(monitor?.protocol ?? "icmp")const [target, setTarget] = useState(monitor?.target ?? "")const [port, setPort] = useState(monitor?.protocol === "tcp" && monitor.port ? String(monitor.port) : "")+ const [server, setServer] = useState(monitor?.protocol === "dns" ? (monitor.server ?? "") : "")const [monitorInterval, setMonitorInterval] = useState(String(monitor?.interval ?? defaultInterval))const [loading, setLoading] = useState(false)const [selectedSystemId, setSelectedSystemId] = useState(monitor?.system ?? "")@@ -609,6 +623,7 @@ function MonitorDialogContent({setProtocol(monitor?.protocol ?? "icmp")setTarget(monitor?.target ?? "")setPort(monitor?.protocol === "tcp" && monitor.port ? String(monitor.port) : "")+ setServer(monitor?.protocol === "dns" ? (monitor.server ?? "") : "")setMonitorInterval(String(monitor?.interval ?? defaultInterval))setSelectedSystemId(monitor?.system ?? "")setSelectedSystemIds(new Set())@@ -629,6 +644,7 @@ function MonitorDialogContent({target,protocol,port: protocol === "tcp" ? Number(port) : 0,+ server: protocol === "dns" ? server.trim() : "",interval: monitorInterval,},monitor ? monitor.enabled : true@@ -709,7 +725,7 @@ function MonitorDialogContent({<Inputvalue={target}onChange={(e) => setTarget(e.target.value)}- placeholder={protocol === "http" ? "http://localhost:8090" : "1.1.1.1"}+ placeholder={protocol === "http" ? "http://localhost:8090" : protocol === "dns" ? "example.com" : "1.1.1.1"}required/></div>@@ -745,6 +761,21 @@ function MonitorDialogContent({/></div>)}+ {protocol === "dns" && (+ <div className="grid gap-2">+ <Label>+ <Trans>DNS Server</Trans>+ </Label>+ <Input+ value={server}+ onChange={(e) => setServer(e.target.value)}+ placeholder="1.1.1.1"+ />+ <p className="text-xs text-muted-foreground">+ <Trans>Optional. Defaults to the agent's system resolver.</Trans>+ </p>+ </div>+ )}<div className="grid gap-2"><Label><Trans>Interval (seconds)</Trans>diff --git a/internal/site/src/components/network-monitors-table/network-monitors-table.tsx b/internal/site/src/components/network-monitors-table/network-monitors-table.tsxindex 6618d234f..ccfed80da 100644--- a/internal/site/src/components/network-monitors-table/network-monitors-table.tsx+++ b/internal/site/src/components/network-monitors-table/network-monitors-table.tsx@@ -47,6 +47,7 @@ import {ArrowUpIcon,EthernetPortIcon,EyeIcon,+ GlobeIcon,LandmarkIcon,LoaderCircleIcon,ServerIcon,@@ -720,6 +721,13 @@ function NetworkMonitorSheetContent({<span>{monitor.port}</span></>)}+ {monitor.protocol === "dns" && monitor.server && (+ <>+ <Separator orientation="vertical" className="h-2.5 bg-muted-foreground opacity-70" />+ <GlobeIcon className="size-3.5 text-muted-foreground" />+ <span>{monitor.server}</span>+ </>+ )}{monitor.certInfo?.expires ? <CertExpiry cert={monitor.certInfo} /> : null}</SheetDescription></SheetHeader>diff --git a/internal/site/src/lib/use-network-monitors.ts b/internal/site/src/lib/use-network-monitors.tsindex 7ecaf0c97..be3e1456b 100644--- a/internal/site/src/lib/use-network-monitors.ts+++ b/internal/site/src/lib/use-network-monitors.ts@@ -74,7 +74,7 @@ async function fetchMonitorStats(}const NETWORK_MONITOR_FIELDS =- "id,system,target,protocol,port,interval,res,resMin1h,resMax1h,resAvg1h,loss1h,enabled,certInfo,updated"+ "id,system,target,protocol,port,server,interval,res,resMin1h,resMax1h,resAvg1h,loss1h,enabled,certInfo,updated"interface UseNetworkMonitorsProps {systemId?: stringdiff --git a/internal/site/src/types.d.ts b/internal/site/src/types.d.tsindex 7fcf0d841..bdf147071 100644--- a/internal/site/src/types.d.ts+++ b/internal/site/src/types.d.ts@@ -644,6 +644,7 @@ export interface NetworkMonitorRecord {target: stringprotocol: "icmp" | "tcp" | "http" | "dns"port: number+ server: stringres: numberresMin1h: numberresMax1h: number