refactor network io stats collection (ref #150)

beszel/internal/agent/agent.go

@@ -38,6 +38,7 @@ type Agent struct {
 	containerStatsMutex *sync.Mutex
 	fsNames             []string
 	fsStats             map[string]*system.FsStats
+	netInterfaces       map[string]struct{}
 	netIoStats          *system.NetIoStats
 	dockerClient        *http.Client
 	bufferPool          *sync.Pool
@@ -135,10 +136,14 @@ func (a *Agent) getSystemStats() (system.Info, system.Stats) {
 
 	// network stats
 	if netIO, err := psutilNet.IOCounters(true); err == nil {
+		secondsElapsed := time.Since(a.netIoStats.Time).Seconds()
+		a.netIoStats.Time = time.Now()
 		bytesSent := uint64(0)
 		bytesRecv := uint64(0)
+		// sum all bytes sent and received
 		for _, v := range netIO {
-			if skipNetworkInterface(&v) {
+			// skip if not in valid network interfaces list
+			if _, exists := a.netInterfaces[v.Name]; !exists {
 				continue
 			}
 			// log.Printf("%+v: %+v recv, %+v sent\n", v.Name, v.BytesRecv, v.BytesSent)
@@ -146,15 +151,28 @@ func (a *Agent) getSystemStats() (system.Info, system.Stats) {
 			bytesRecv += v.BytesRecv
 		}
 		// add to systemStats
-		secondsElapsed := time.Since(a.netIoStats.Time).Seconds()
 		sentPerSecond := float64(bytesSent-a.netIoStats.BytesSent) / secondsElapsed
 		recvPerSecond := float64(bytesRecv-a.netIoStats.BytesRecv) / secondsElapsed
-		systemStats.NetworkSent = bytesToMegabytes(sentPerSecond)
+		networkSentPs := bytesToMegabytes(sentPerSecond)
-		systemStats.NetworkRecv = bytesToMegabytes(recvPerSecond)
+		networkRecvPs := bytesToMegabytes(recvPerSecond)
-		// update netIoStats
+		// add check for issue (#150) where sent is a massive number
-		a.netIoStats.BytesSent = bytesSent
+		if networkSentPs > 10_000 || networkRecvPs > 10_000 {
-		a.netIoStats.BytesRecv = bytesRecv
+			log.Printf("Warning: network sent/recv is %.2f/%.2f MB/s. Resetting stats.\n", networkSentPs, networkRecvPs)
-		a.netIoStats.Time = time.Now()
+			for _, v := range netIO {
+				if _, exists := a.netInterfaces[v.Name]; !exists {
+					continue
+				}
+				log.Printf("%+s: %v recv, %v sent\n", v.Name, v.BytesRecv, v.BytesSent)
+			}
+			// reset network I/O stats
+			a.initializeNetIoStats()
+		} else {
+			systemStats.NetworkSent = networkSentPs
+			systemStats.NetworkRecv = networkRecvPs
+			// update netIoStats
+			a.netIoStats.BytesSent = bytesSent
+			a.netIoStats.BytesRecv = bytesRecv
+		}
 	}
 
 	// temperatures
@@ -508,20 +526,24 @@ func (a *Agent) initializeDiskIoStats() {
 }
 
 func (a *Agent) initializeNetIoStats() {
+	// reset valid network interfaces
+	a.netInterfaces = make(map[string]struct{}, 0)
+	// reset network I/O stats
+	a.netIoStats.BytesSent = 0
+	a.netIoStats.BytesRecv = 0
+	// get intial network I/O stats
 	if netIO, err := psutilNet.IOCounters(true); err == nil {
-		bytesSent := uint64(0)
+		a.netIoStats.Time = time.Now()
-		bytesRecv := uint64(0)
 		for _, v := range netIO {
-			if skipNetworkInterface(&v) {
+			if skipNetworkInterface(v) {
 				continue
 			}
 			log.Printf("Detected network interface: %+v (%+v recv, %+v sent)\n", v.Name, v.BytesRecv, v.BytesSent)
-			bytesSent += v.BytesSent
+			a.netIoStats.BytesSent += v.BytesSent
-			bytesRecv += v.BytesRecv
+			a.netIoStats.BytesRecv += v.BytesRecv
+			// store as a valid network interface
+			a.netInterfaces[v.Name] = struct{}{}
 		}
-		a.netIoStats.BytesSent = bytesSent
-		a.netIoStats.BytesRecv = bytesRecv
-		a.netIoStats.Time = time.Now()
 	}
 }
 
@@ -537,7 +559,7 @@ func twoDecimals(value float64) float64 {
 	return math.Round(value*100) / 100
 }
 
-func skipNetworkInterface(v *psutilNet.IOCountersStat) bool {
+func skipNetworkInterface(v psutilNet.IOCountersStat) bool {
 	switch {
 	case strings.HasPrefix(v.Name, "lo"),
 		strings.HasPrefix(v.Name, "docker"),