fix: make sure system alerts are checked after records are committed

This commit is contained in:
henrygd
2025-03-05 23:39:01 -05:00
parent 5fbc0de07f
commit 25b73bfb85
2 changed files with 34 additions and 29 deletions

View File

@@ -4,7 +4,6 @@ import (
"beszel/internal/entities/system" "beszel/internal/entities/system"
"fmt" "fmt"
"net/url" "net/url"
"slices"
"strings" "strings"
"time" "time"
@@ -51,14 +50,10 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
} }
val = maxUsedPct val = maxUsedPct
case "Temperature": case "Temperature":
if data.Stats.Temperatures == nil { if data.Info.DashboardTemp < 1 {
continue continue
} }
for _, temp := range data.Stats.Temperatures { val = data.Info.DashboardTemp
if temp > val {
val = temp
}
}
unit = "°C" unit = "°C"
} }
@@ -74,13 +69,8 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
} }
min := max(1, cast.ToUint8(alertRecord.Get("min"))) min := max(1, cast.ToUint8(alertRecord.Get("min")))
// add time to alert time to make sure it's slighty after record creation
time := now.Add(-time.Duration(min) * time.Minute)
if time.Before(oldestTime) {
oldestTime = time
}
validAlerts = append(validAlerts, SystemAlertData{ alert := SystemAlertData{
systemRecord: systemRecord, systemRecord: systemRecord,
alertRecord: alertRecord, alertRecord: alertRecord,
name: name, name: name,
@@ -88,9 +78,22 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
val: val, val: val,
threshold: threshold, threshold: threshold,
triggered: triggered, triggered: triggered,
time: time,
min: min, min: min,
}) }
// send alert immediately if min is 1 - no need to sum up values.
if min == 1 {
alert.triggered = val > threshold
go am.sendSystemAlert(alert)
continue
}
alert.time = now.Add(-time.Duration(min) * time.Minute)
if alert.time.Before(oldestTime) {
oldestTime = alert.time
}
validAlerts = append(validAlerts, alert)
} }
systemStats := []struct { systemStats := []struct {
@@ -119,13 +122,14 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
oldestRecordTime := systemStats[0].Created.Time() oldestRecordTime := systemStats[0].Created.Time()
// log.Println("oldestRecordTime", oldestRecordTime.String()) // log.Println("oldestRecordTime", oldestRecordTime.String())
// delete from validAlerts if time is older than oldestRecord // Filter validAlerts to keep only those with time newer than oldestRecord
for i := range validAlerts { filteredAlerts := make([]SystemAlertData, 0, len(validAlerts))
if validAlerts[i].time.Before(oldestRecordTime) { for _, alert := range validAlerts {
// log.Println("deleting alert - time is older than oldestRecord", validAlerts[i].name, oldestRecordTime, validAlerts[i].time) if alert.time.After(oldestRecordTime) {
validAlerts = slices.Delete(validAlerts, i, i+1) filteredAlerts = append(filteredAlerts, alert)
} }
} }
validAlerts = filteredAlerts
if len(validAlerts) == 0 { if len(validAlerts) == 0 {
// log.Println("no valid alerts found") // log.Println("no valid alerts found")

View File

@@ -106,7 +106,7 @@ func (sm *SystemManager) onRecordCreate(e *core.RecordEvent) error {
// Runs after the record is committed to the database // Runs after the record is committed to the database
func (sm *SystemManager) onRecordAfterCreateSuccess(e *core.RecordEvent) error { func (sm *SystemManager) onRecordAfterCreateSuccess(e *core.RecordEvent) error {
if err := sm.AddRecord(e.Record); err != nil { if err := sm.AddRecord(e.Record); err != nil {
sm.hub.Logger().Error("Error adding record", "err", err) e.App.Logger().Error("Error adding record", "err", err)
} }
return e.Next() return e.Next()
} }
@@ -128,7 +128,7 @@ func (sm *SystemManager) onRecordAfterUpdateSuccess(e *core.RecordEvent) error {
return e.Next() return e.Next()
case pending: case pending:
if err := sm.AddRecord(e.Record); err != nil { if err := sm.AddRecord(e.Record); err != nil {
sm.hub.Logger().Error("Error adding record", "err", err) e.App.Logger().Error("Error adding record", "err", err)
} }
return e.Next() return e.Next()
} }
@@ -141,12 +141,12 @@ func (sm *SystemManager) onRecordAfterUpdateSuccess(e *core.RecordEvent) error {
// system alerts if system is up // system alerts if system is up
if system.Status == up { if system.Status == up {
if err := sm.hub.HandleSystemAlerts(e.Record, system.data); err != nil { if err := sm.hub.HandleSystemAlerts(e.Record, system.data); err != nil {
sm.hub.Logger().Error("Error handling system alerts", "err", err) e.App.Logger().Error("Error handling system alerts", "err", err)
} }
} }
if (system.Status == down && prevStatus == up) || (system.Status == up && prevStatus == down) { if (system.Status == down && prevStatus == up) || (system.Status == up && prevStatus == down) {
if err := sm.hub.HandleStatusAlerts(system.Status, e.Record); err != nil { if err := sm.hub.HandleStatusAlerts(system.Status, e.Record); err != nil {
sm.hub.Logger().Error("Error handling status alerts", "err", err) e.App.Logger().Error("Error handling status alerts", "err", err)
} }
} }
return e.Next() return e.Next()
@@ -248,11 +248,6 @@ func (sys *System) createRecords() (*core.Record, error) {
return nil, err return nil, err
} }
hub := sys.manager.hub hub := sys.manager.hub
systemRecord.Set("status", up)
systemRecord.Set("info", sys.data.Info)
if err := hub.SaveNoValidate(systemRecord); err != nil {
return nil, err
}
// add system_stats and container_stats records // add system_stats and container_stats records
systemStats, err := hub.FindCachedCollectionByNameOrId("system_stats") systemStats, err := hub.FindCachedCollectionByNameOrId("system_stats")
if err != nil { if err != nil {
@@ -279,6 +274,12 @@ func (sys *System) createRecords() (*core.Record, error) {
return nil, err return nil, err
} }
} }
// update system record (do this last because it triggers alerts and we need above records to be inserted first)
systemRecord.Set("status", up)
systemRecord.Set("info", sys.data.Info)
if err := hub.SaveNoValidate(systemRecord); err != nil {
return nil, err
}
return systemRecord, nil return systemRecord, nil
} }