Add 5m and 10m load avg alerts and table values (#816)

Co-authored-by: henrygd <hank@henrygd.me>

beszel/internal/agent/system.go

@@ -14,6 +14,7 @@ import (
 	"github.com/shirou/gopsutil/v4/cpu"
 	"github.com/shirou/gopsutil/v4/disk"
 	"github.com/shirou/gopsutil/v4/host"
+	"github.com/shirou/gopsutil/v4/load"
 	"github.com/shirou/gopsutil/v4/mem"
 	psutilNet "github.com/shirou/gopsutil/v4/net"
 )
@@ -77,6 +78,16 @@ func (a *Agent) getSystemStats() system.Stats {
 		systemStats.Cpu = twoDecimals(cpuPct[0])
 	}
 
+	// load average
+	if avgstat, err := load.Avg(); err == nil {
+		systemStats.LoadAvg1 = twoDecimals(avgstat.Load1)
+		systemStats.LoadAvg5 = twoDecimals(avgstat.Load5)
+		systemStats.LoadAvg15 = twoDecimals(avgstat.Load15)
+		slog.Debug("Load average", "5m", systemStats.LoadAvg5, "15m", systemStats.LoadAvg15)
+	} else {
+		slog.Error("Error getting load average", "err", err)
+	}
+
 	// memory
 	if v, err := mem.VirtualMemory(); err == nil {
 		// swap
@@ -240,6 +251,8 @@ func (a *Agent) getSystemStats() system.Stats {
 
 	// update base system info
 	a.systemInfo.Cpu = systemStats.Cpu
+	a.systemInfo.LoadAvg5 = systemStats.LoadAvg5
+	a.systemInfo.LoadAvg15 = systemStats.LoadAvg15
 	a.systemInfo.MemPct = systemStats.MemPct
 	a.systemInfo.DiskPct = systemStats.DiskPct
 	a.systemInfo.Uptime, _ = host.Uptime()

beszel/internal/alerts/alerts.go

@@ -47,6 +47,8 @@ type SystemAlertStats struct {
 	NetSent      float64            `json:"ns"`
 	NetRecv      float64            `json:"nr"`
 	Temperatures map[string]float32 `json:"t"`
+	LoadAvg5     float64            `json:"l5"`
+	LoadAvg15    float64            `json:"l15"`
 }
 
 type SystemAlertData struct {

beszel/internal/alerts/alerts_system.go

@@ -54,6 +54,12 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
 			}
 			val = data.Info.DashboardTemp
 			unit = "°C"
+		case "LoadAvg5":
+			val = data.Info.LoadAvg5
+			unit = ""
+		case "LoadAvg15":
+			val = data.Info.LoadAvg15
+			unit = ""
 		}
 
 		triggered := alertRecord.GetBool("triggered")
@@ -190,6 +196,10 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
 					}
 					alert.mapSums[key] += temp
 				}
+			case "LoadAvg5":
+				alert.val += stats.LoadAvg5
+			case "LoadAvg15":
+				alert.val += stats.LoadAvg15
 			default:
 				continue
 			}
@@ -247,6 +257,10 @@ func (am *AlertManager) sendSystemAlert(alert SystemAlertData) {
 	if alert.name == "Disk" {
 		alert.name += " usage"
 	}
+	// format LoadAvg5 and LoadAvg15
+	if after, ok := strings.CutPrefix(alert.name, "LoadAvg"); ok {
+		alert.name = after + "m Load"
+	}
 
 	// make title alert name lowercase if not CPU
 	titleAlertName := alert.name

beszel/internal/entities/system/system.go

@@ -31,6 +31,9 @@ type Stats struct {
 	Temperatures   map[string]float64  `json:"t,omitempty" cbor:"20,keyasint,omitempty"`
 	ExtraFs        map[string]*FsStats `json:"efs,omitempty" cbor:"21,keyasint,omitempty"`
 	GPUData        map[string]GPUData  `json:"g,omitempty" cbor:"22,keyasint,omitempty"`
+	LoadAvg1       float64             `json:"l1,omitempty" cbor:"23,keyasint,omitempty,omitzero"`
+	LoadAvg5       float64             `json:"l5,omitempty" cbor:"24,keyasint,omitempty,omitzero"`
+	LoadAvg15      float64             `json:"l15,omitempty" cbor:"25,keyasint,omitempty,omitzero"`
 }
 
 type GPUData struct {
@@ -89,6 +92,8 @@ type Info struct {
 	GpuPct        float64 `json:"g,omitempty" cbor:"12,keyasint,omitempty"`
 	DashboardTemp float64 `json:"dt,omitempty" cbor:"13,keyasint,omitempty"`
 	Os            Os      `json:"os" cbor:"14,keyasint"`
+	LoadAvg5      float64 `json:"l5,omitempty" cbor:"15,keyasint,omitempty,omitzero"`
+	LoadAvg15     float64 `json:"l15,omitempty" cbor:"16,keyasint,omitempty,omitzero"`
 }
 
 // Final data structure to return to the hub

beszel/migrations/0_collections_snapshot_0_12_0.go

@@ -75,7 +75,9 @@ func init() {
 					"Memory",
 					"Disk",
 					"Temperature",
-					"Bandwidth"
+					"Bandwidth",
+					"LoadAvg5",
+					"LoadAvg15"
 				]
 			},
 			{

beszel/migrations/initial-settings.go

@@ -5,7 +5,7 @@ import (
 	m "github.com/pocketbase/pocketbase/migrations"
 )
 
-var (
+const (
 	TempAdminEmail = "_@b.b"
 )
 

beszel/site/src/components/alerts/alerts-system.tsx

@@ -217,7 +217,7 @@ function AlertContent({ data }: { data: AlertData }) {
 
 	const [checked, setChecked] = useState(data.checked || false)
 	const [min, setMin] = useState(data.min || 10)
-	const [value, setValue] = useState(data.val || (singleDescription ? 0 : 80))
+	const [value, setValue] = useState(data.val || (singleDescription ? 0 : data.alert.start ?? 80))
 
 	const Icon = alertInfo[name].icon
 
@@ -268,7 +268,8 @@ function AlertContent({ data }: { data: AlertData }) {
 										onValueChange={(val) => {
 											setValue(val[0])
 										}}
-										min={1}
+										step={data.alert.step ?? 1}
+										min={data.alert.min ?? 1}
 										max={alertInfo[name].max ?? 99}
 									/>
 								</div>

beszel/site/src/components/systems-table/systems-table.tsx

@@ -68,7 +68,7 @@ import { useStore } from "@nanostores/react"
 import { cn, copyToClipboard, decimalString, isReadOnlyUser, useLocalStorage } from "@/lib/utils"
 import AlertsButton from "../alerts/alert-button"
 import { $router, Link, navigate } from "../router"
-import { EthernetIcon, GpuIcon, ThermometerIcon } from "../ui/icons"
+import { EthernetIcon, GpuIcon, HourglassIcon, ThermometerIcon } from "../ui/icons"
 import { useLingui, Trans } from "@lingui/react/macro"
 import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "../ui/card"
 import { Input } from "../ui/input"
@@ -83,8 +83,8 @@ function CellFormatter(info: CellContext<SystemRecord, unknown>) {
 	const val = (info.getValue() as number) || 0
 	return (
 		<div className="flex gap-2 items-center tabular-nums tracking-tight">
-			<span className="min-w-[3.3em]">{decimalString(val, 1)}%</span>
+			<span className="min-w-8">{decimalString(val, 1)}%</span>
-			<span className="grow min-w-10 block bg-muted h-[1em] relative rounded-sm overflow-hidden">
+			<span className="grow min-w-8 block bg-muted h-[1em] relative rounded-sm overflow-hidden">
 				<span
 					className={cn(
 						"absolute inset-0 w-full h-full origin-left",
@@ -144,7 +144,6 @@ export default function SystemsTable() {
 		}
 		return [
 			{
-				// size: 200,
 				size: 200,
 				minSize: 0,
 				accessorKey: "name",
@@ -163,6 +162,7 @@ export default function SystemsTable() {
 					return false
 				},
 				enableHiding: false,
+				invertSorting: false,
 				Icon: ServerIcon,
 				cell: (info) => (
 					<span className="flex gap-0.5 items-center text-base md:pe-5">
@@ -181,28 +181,26 @@ export default function SystemsTable() {
 				header: sortableHeader,
 			},
 			{
-				accessorKey: "info.cpu",
+				accessorFn: (originalRow) => originalRow.info.cpu,
 				id: "cpu",
 				name: () => t`CPU`,
-				invertSorting: true,
 				cell: CellFormatter,
 				Icon: CpuIcon,
 				header: sortableHeader,
 			},
 			{
-				accessorKey: "info.mp",
+				// accessorKey: "info.mp",
+				accessorFn: (originalRow) => originalRow.info.mp,
 				id: "memory",
 				name: () => t`Memory`,
-				invertSorting: true,
 				cell: CellFormatter,
 				Icon: MemoryStickIcon,
 				header: sortableHeader,
 			},
 			{
-				accessorKey: "info.dp",
+				accessorFn: (originalRow) => originalRow.info.dp,
 				id: "disk",
 				name: () => t`Disk`,
-				invertSorting: true,
 				cell: CellFormatter,
 				Icon: HardDriveIcon,
 				header: sortableHeader,
@@ -211,8 +209,6 @@ export default function SystemsTable() {
 				accessorFn: (originalRow) => originalRow.info.g,
 				id: "gpu",
 				name: () => "GPU",
-				invertSorting: true,
-				sortUndefined: -1,
 				cell: CellFormatter,
 				Icon: GpuIcon,
 				header: sortableHeader,
@@ -221,19 +217,50 @@ export default function SystemsTable() {
 				accessorFn: (originalRow) => originalRow.info.b || 0,
 				id: "net",
 				name: () => t`Net`,
-				invertSorting: true,
 				size: 50,
 				Icon: EthernetIcon,
 				header: sortableHeader,
 				cell(info) {
 					const val = info.getValue() as number
+					return <span className="tabular-nums whitespace-nowrap">{decimalString(val, val >= 100 ? 1 : 2)} MB/s</span>
+				},
+			},
+			{
+				accessorKey: "info.l5",
+				id: "l5",
+				name: () => t({ message: "L5", comment: "Load average 5 minutes" }),
+				size: 0,
+				hideSort: true,
+				Icon: HourglassIcon,
+				header: sortableHeader,
+				cell(info) {
+					const val = info.getValue() as number
+					if (!val) {
+						return null
+					}
 					return (
-						<span
+						<span className={cn("tabular-nums whitespace-nowrap", viewMode === "table" && "ps-1")}>
-							className={cn("tabular-nums whitespace-nowrap", {
+							{decimalString(val)}
-								"ps-1": viewMode === "table",
+						</span>
-							})}
+					)
-						>
+				},
-							{decimalString(val, val >= 100 ? 1 : 2)} MB/s
+			},
+			{
+				accessorKey: "info.l15",
+				id: "l15",
+				name: () => t({ message: "L15", comment: "Load average 15 minutes" }),
+				size: 0,
+				hideSort: true,
+				Icon: HourglassIcon,
+				header: sortableHeader,
+				cell(info) {
+					const val = info.getValue() as number
+					if (!val) {
+						return null
+					}
+					return (
+						<span className={cn("tabular-nums whitespace-nowrap", viewMode === "table" && "ps-1")}>
+							{decimalString(val)}
 						</span>
 					)
 				},
@@ -242,8 +269,6 @@ export default function SystemsTable() {
 				accessorFn: (originalRow) => originalRow.info.dt,
 				id: "temp",
 				name: () => t({ message: "Temp", comment: "Temperature label in systems table" }),
-				invertSorting: true,
-				sortUndefined: -1,
 				size: 50,
 				hideSort: true,
 				Icon: ThermometerIcon,
@@ -254,21 +279,17 @@ export default function SystemsTable() {
 						return null
 					}
 					return (
-						<span
+						<span className={cn("tabular-nums whitespace-nowrap", viewMode === "table" && "ps-0.5")}>
-							className={cn("tabular-nums whitespace-nowrap", {
-								"ps-1.5": viewMode === "table",
-							})}
-						>
 							{decimalString(val)} °C
 						</span>
 					)
 				},
 			},
 			{
-				accessorKey: "info.v",
+				accessorFn: (originalRow) => originalRow.info.v,
 				id: "agent",
 				name: () => t`Agent`,
-				invertSorting: true,
+				// invertSorting: true,
 				size: 50,
 				Icon: WifiIcon,
 				hideSort: true,
@@ -280,11 +301,7 @@ export default function SystemsTable() {
 					}
 					const system = info.row.original
 					return (
-						<span
+						<span className={cn("flex gap-2 items-center md:pe-5 tabular-nums", viewMode === "table" && "ps-0.5")}>
-							className={cn("flex gap-2 items-center md:pe-5 tabular-nums", {
-								"ps-1": viewMode === "table",
-							})}
-						>
 							<IndicatorDot
 								system={system}
 								className={
@@ -304,7 +321,7 @@ export default function SystemsTable() {
 				name: () => t({ message: "Actions", comment: "Table column" }),
 				size: 50,
 				cell: ({ row }) => (
-					<div className="flex justify-end items-center gap-1">
+					<div className="flex justify-end items-center gap-1 -ms-3">
 						<AlertsButton system={row.original} />
 						<ActionsButton system={row.original} />
 					</div>
@@ -328,6 +345,9 @@ export default function SystemsTable() {
 			columnVisibility,
 		},
 		defaultColumn: {
+			// sortDescFirst: true,
+			invertSorting: true,
+			sortUndefined: "last",
 			minSize: 0,
 			size: 900,
 			maxSize: 900,
@@ -511,7 +531,7 @@ function SystemsTableHead({ table, colLength }: { table: TableType<SystemRecord>
 					<TableRow key={headerGroup.id}>
 						{headerGroup.headers.map((header) => {
 							return (
-								<TableHead className="px-2" key={header.id}>
+								<TableHead className="px-1" key={header.id}>
 									{flexRender(header.column.columnDef.header, header.getContext())}
 								</TableHead>
 							)

beszel/site/src/components/ui/icons.tsx

@@ -121,3 +121,12 @@ export function GpuIcon(props: SVGProps<SVGSVGElement>) {
 		</svg>
 	)
 }
+
+// Remix icons (Apache 2.0) https://github.com/Remix-Design/RemixIcon/blob/master/License
+export function HourglassIcon(props: SVGProps<SVGSVGElement>) {
+	return (
+		<svg viewBox="0 0 24 24" {...props} fill="currentColor">
+			<path d="M4 2h16v4.5L13.5 12l6.5 5.5V22H4v-4.5l6.5-5.5L4 6.5zm12.3 5L18 5.5V4H6v1.5L7.7 7zM12 13.3l-6 5.2V20h1l5-3 5 3h1v-1.5z" />
+		</svg>
+	)
+}

beszel/site/src/lib/utils.ts

@@ -9,7 +9,7 @@ import { WritableAtom } from "nanostores"
 import { timeDay, timeHour } from "d3-time"
 import { useEffect, useState } from "react"
 import { CpuIcon, HardDriveIcon, MemoryStickIcon, ServerIcon } from "lucide-react"
-import { EthernetIcon, ThermometerIcon } from "@/components/ui/icons"
+import { EthernetIcon, HourglassIcon, ThermometerIcon } from "@/components/ui/icons"
 import { prependBasePath } from "@/components/router"
 
 export function cn(...inputs: ClassValue[]) {
@@ -342,6 +342,26 @@ export const alertInfo: Record<string, AlertInfo> = {
 		icon: ThermometerIcon,
 		desc: () => t`Triggers when any sensor exceeds a threshold`,
 	},
+	LoadAvg5: {
+		name: () => t`Load Average 5m`,
+		unit: "",
+		icon: HourglassIcon,
+		max: 100,
+		min: 0.1,
+		start: 10,
+		step: 0.1,
+		desc: () => t`Triggers when 5 minute load average exceeds a threshold`,
+	},
+	LoadAvg15: {
+		name: () => t`Load Average 15m`,
+		unit: "",
+		icon: HourglassIcon,
+		min: 0.1,
+		max: 100,
+		start: 10,
+		step: 0.1,
+		desc: () => t`Triggers when 15 minute load average exceeds a threshold`,
+	},
 }
 
 /**

beszel/site/src/types.d.ts

@@ -44,6 +44,10 @@ export interface SystemInfo {
 	c: number
 	/** cpu model */
 	m: string
+	/** load average 5 minutes */
+	l5?: number
+	/** load average 15 minutes */
+	l15?: number
 	/** operating system */
 	o?: string
 	/** uptime */
@@ -71,6 +75,12 @@ export interface SystemStats {
 	cpu: number
 	/** peak cpu */
 	cpum?: number
+	/** load average 1 minute */
+	l1?: number
+	/** load average 5 minutes */
+	l5?: number
+	/** load average 15 minutes */
+	l15?: number
 	/** total memory (gb) */
 	m: number
 	/** memory used (gb) */
@@ -218,6 +228,9 @@ interface AlertInfo {
 	icon: any
 	desc: () => string
 	max?: number
+	min?: number
+	step?: number
+	start?: number
 	/** Single value description (when there's only one value, like status) */
 	singleDesc?: () => string
 }