From 25b73bfb85aa33bc7ebc0053a0f8abcacbf7e49d Mon Sep 17 00:00:00 2001 From: henrygd Date: Wed, 5 Mar 2025 23:39:01 -0500 Subject: [PATCH] fix: make sure system alerts are checked after records are committed --- beszel/internal/alerts/alerts_system.go | 44 ++++++++++++++----------- beszel/internal/hub/systems/systems.go | 19 ++++++----- 2 files changed, 34 insertions(+), 29 deletions(-) diff --git a/beszel/internal/alerts/alerts_system.go b/beszel/internal/alerts/alerts_system.go index 0a29404..5a60f68 100644 --- a/beszel/internal/alerts/alerts_system.go +++ b/beszel/internal/alerts/alerts_system.go @@ -4,7 +4,6 @@ import ( "beszel/internal/entities/system" "fmt" "net/url" - "slices" "strings" "time" @@ -51,14 +50,10 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst } val = maxUsedPct case "Temperature": - if data.Stats.Temperatures == nil { + if data.Info.DashboardTemp < 1 { continue } - for _, temp := range data.Stats.Temperatures { - if temp > val { - val = temp - } - } + val = data.Info.DashboardTemp unit = "°C" } @@ -74,13 +69,8 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst } min := max(1, cast.ToUint8(alertRecord.Get("min"))) - // add time to alert time to make sure it's slighty after record creation - time := now.Add(-time.Duration(min) * time.Minute) - if time.Before(oldestTime) { - oldestTime = time - } - validAlerts = append(validAlerts, SystemAlertData{ + alert := SystemAlertData{ systemRecord: systemRecord, alertRecord: alertRecord, name: name, @@ -88,9 +78,22 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst val: val, threshold: threshold, triggered: triggered, - time: time, min: min, - }) + } + + // send alert immediately if min is 1 - no need to sum up values. + if min == 1 { + alert.triggered = val > threshold + go am.sendSystemAlert(alert) + continue + } + + alert.time = now.Add(-time.Duration(min) * time.Minute) + if alert.time.Before(oldestTime) { + oldestTime = alert.time + } + + validAlerts = append(validAlerts, alert) } systemStats := []struct { @@ -119,13 +122,14 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst oldestRecordTime := systemStats[0].Created.Time() // log.Println("oldestRecordTime", oldestRecordTime.String()) - // delete from validAlerts if time is older than oldestRecord - for i := range validAlerts { - if validAlerts[i].time.Before(oldestRecordTime) { - // log.Println("deleting alert - time is older than oldestRecord", validAlerts[i].name, oldestRecordTime, validAlerts[i].time) - validAlerts = slices.Delete(validAlerts, i, i+1) + // Filter validAlerts to keep only those with time newer than oldestRecord + filteredAlerts := make([]SystemAlertData, 0, len(validAlerts)) + for _, alert := range validAlerts { + if alert.time.After(oldestRecordTime) { + filteredAlerts = append(filteredAlerts, alert) } } + validAlerts = filteredAlerts if len(validAlerts) == 0 { // log.Println("no valid alerts found") diff --git a/beszel/internal/hub/systems/systems.go b/beszel/internal/hub/systems/systems.go index 879dab1..e8af7bf 100644 --- a/beszel/internal/hub/systems/systems.go +++ b/beszel/internal/hub/systems/systems.go @@ -106,7 +106,7 @@ func (sm *SystemManager) onRecordCreate(e *core.RecordEvent) error { // Runs after the record is committed to the database func (sm *SystemManager) onRecordAfterCreateSuccess(e *core.RecordEvent) error { if err := sm.AddRecord(e.Record); err != nil { - sm.hub.Logger().Error("Error adding record", "err", err) + e.App.Logger().Error("Error adding record", "err", err) } return e.Next() } @@ -128,7 +128,7 @@ func (sm *SystemManager) onRecordAfterUpdateSuccess(e *core.RecordEvent) error { return e.Next() case pending: if err := sm.AddRecord(e.Record); err != nil { - sm.hub.Logger().Error("Error adding record", "err", err) + e.App.Logger().Error("Error adding record", "err", err) } return e.Next() } @@ -141,12 +141,12 @@ func (sm *SystemManager) onRecordAfterUpdateSuccess(e *core.RecordEvent) error { // system alerts if system is up if system.Status == up { if err := sm.hub.HandleSystemAlerts(e.Record, system.data); err != nil { - sm.hub.Logger().Error("Error handling system alerts", "err", err) + e.App.Logger().Error("Error handling system alerts", "err", err) } } if (system.Status == down && prevStatus == up) || (system.Status == up && prevStatus == down) { if err := sm.hub.HandleStatusAlerts(system.Status, e.Record); err != nil { - sm.hub.Logger().Error("Error handling status alerts", "err", err) + e.App.Logger().Error("Error handling status alerts", "err", err) } } return e.Next() @@ -248,11 +248,6 @@ func (sys *System) createRecords() (*core.Record, error) { return nil, err } hub := sys.manager.hub - systemRecord.Set("status", up) - systemRecord.Set("info", sys.data.Info) - if err := hub.SaveNoValidate(systemRecord); err != nil { - return nil, err - } // add system_stats and container_stats records systemStats, err := hub.FindCachedCollectionByNameOrId("system_stats") if err != nil { @@ -279,6 +274,12 @@ func (sys *System) createRecords() (*core.Record, error) { return nil, err } } + // update system record (do this last because it triggers alerts and we need above records to be inserted first) + systemRecord.Set("status", up) + systemRecord.Set("info", sys.data.Info) + if err := hub.SaveNoValidate(systemRecord); err != nil { + return nil, err + } return systemRecord, nil }