mirror of
https://github.com/fankes/beszel.git
synced 2025-12-14 02:11:02 +08:00
rename /src to /internal (sorry i'll fix the prs)
This commit is contained in:
henrygd
220
internal/alerts/alerts.go
Normal file
220
internal/alerts/alerts.go
Normal file
@@ -0,0 +1,220 @@
|
||||
// Package alerts handles alert management and delivery.
|
||||
package alerts
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/mail"
|
||||
"net/url"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nicholas-fedor/shoutrrr"
|
||||
"github.com/pocketbase/dbx"
|
||||
"github.com/pocketbase/pocketbase/core"
|
||||
"github.com/pocketbase/pocketbase/tools/mailer"
|
||||
)
|
||||
|
||||
type hubLike interface {
|
||||
core.App
|
||||
MakeLink(parts ...string) string
|
||||
}
|
||||
|
||||
type AlertManager struct {
|
||||
hub hubLike
|
||||
alertQueue chan alertTask
|
||||
stopChan chan struct{}
|
||||
pendingAlerts sync.Map
|
||||
}
|
||||
|
||||
type AlertMessageData struct {
|
||||
UserID string
|
||||
Title string
|
||||
Message string
|
||||
Link string
|
||||
LinkText string
|
||||
}
|
||||
|
||||
type UserNotificationSettings struct {
|
||||
Emails []string `json:"emails"`
|
||||
Webhooks []string `json:"webhooks"`
|
||||
}
|
||||
|
||||
type SystemAlertStats struct {
|
||||
Cpu float64 `json:"cpu"`
|
||||
Mem float64 `json:"mp"`
|
||||
Disk float64 `json:"dp"`
|
||||
NetSent float64 `json:"ns"`
|
||||
NetRecv float64 `json:"nr"`
|
||||
Temperatures map[string]float32 `json:"t"`
|
||||
LoadAvg [3]float64 `json:"la"`
|
||||
}
|
||||
|
||||
type SystemAlertData struct {
|
||||
systemRecord *core.Record
|
||||
alertRecord *core.Record
|
||||
name string
|
||||
unit string
|
||||
val float64
|
||||
threshold float64
|
||||
triggered bool
|
||||
time time.Time
|
||||
count uint8
|
||||
min uint8
|
||||
mapSums map[string]float32
|
||||
descriptor string // override descriptor in notification body (for temp sensor, disk partition, etc)
|
||||
}
|
||||
|
||||
// notification services that support title param
|
||||
var supportsTitle = map[string]struct{}{
|
||||
"bark": {},
|
||||
"discord": {},
|
||||
"gotify": {},
|
||||
"ifttt": {},
|
||||
"join": {},
|
||||
"lark": {},
|
||||
"matrix": {},
|
||||
"ntfy": {},
|
||||
"opsgenie": {},
|
||||
"pushbullet": {},
|
||||
"pushover": {},
|
||||
"slack": {},
|
||||
"teams": {},
|
||||
"telegram": {},
|
||||
"zulip": {},
|
||||
}
|
||||
|
||||
// NewAlertManager creates a new AlertManager instance.
|
||||
func NewAlertManager(app hubLike) *AlertManager {
|
||||
am := &AlertManager{
|
||||
hub: app,
|
||||
alertQueue: make(chan alertTask, 5),
|
||||
stopChan: make(chan struct{}),
|
||||
}
|
||||
am.bindEvents()
|
||||
go am.startWorker()
|
||||
return am
|
||||
}
|
||||
|
||||
// Bind events to the alerts collection lifecycle
|
||||
func (am *AlertManager) bindEvents() {
|
||||
am.hub.OnRecordAfterUpdateSuccess("alerts").BindFunc(updateHistoryOnAlertUpdate)
|
||||
am.hub.OnRecordAfterDeleteSuccess("alerts").BindFunc(resolveHistoryOnAlertDelete)
|
||||
}
|
||||
|
||||
// SendAlert sends an alert to the user
|
||||
func (am *AlertManager) SendAlert(data AlertMessageData) error {
|
||||
// get user settings
|
||||
record, err := am.hub.FindFirstRecordByFilter(
|
||||
"user_settings", "user={:user}",
|
||||
dbx.Params{"user": data.UserID},
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// unmarshal user settings
|
||||
userAlertSettings := UserNotificationSettings{
|
||||
Emails: []string{},
|
||||
Webhooks: []string{},
|
||||
}
|
||||
if err := record.UnmarshalJSONField("settings", &userAlertSettings); err != nil {
|
||||
am.hub.Logger().Error("Failed to unmarshal user settings", "err", err)
|
||||
}
|
||||
// send alerts via webhooks
|
||||
for _, webhook := range userAlertSettings.Webhooks {
|
||||
if err := am.SendShoutrrrAlert(webhook, data.Title, data.Message, data.Link, data.LinkText); err != nil {
|
||||
am.hub.Logger().Error("Failed to send shoutrrr alert", "err", err)
|
||||
}
|
||||
}
|
||||
// send alerts via email
|
||||
if len(userAlertSettings.Emails) == 0 {
|
||||
return nil
|
||||
}
|
||||
addresses := []mail.Address{}
|
||||
for _, email := range userAlertSettings.Emails {
|
||||
addresses = append(addresses, mail.Address{Address: email})
|
||||
}
|
||||
message := mailer.Message{
|
||||
To: addresses,
|
||||
Subject: data.Title,
|
||||
Text: data.Message + fmt.Sprintf("\n\n%s", data.Link),
|
||||
From: mail.Address{
|
||||
Address: am.hub.Settings().Meta.SenderAddress,
|
||||
Name: am.hub.Settings().Meta.SenderName,
|
||||
},
|
||||
}
|
||||
err = am.hub.NewMailClient().Send(&message)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
am.hub.Logger().Info("Sent email alert", "to", message.To, "subj", message.Subject)
|
||||
return nil
|
||||
}
|
||||
|
||||
// SendShoutrrrAlert sends an alert via a Shoutrrr URL
|
||||
func (am *AlertManager) SendShoutrrrAlert(notificationUrl, title, message, link, linkText string) error {
|
||||
// Parse the URL
|
||||
parsedURL, err := url.Parse(notificationUrl)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error parsing URL: %v", err)
|
||||
}
|
||||
scheme := parsedURL.Scheme
|
||||
queryParams := parsedURL.Query()
|
||||
|
||||
// Add title
|
||||
if _, ok := supportsTitle[scheme]; ok {
|
||||
queryParams.Add("title", title)
|
||||
} else if scheme == "mattermost" {
|
||||
// use markdown title for mattermost
|
||||
message = "##### " + title + "\n\n" + message
|
||||
} else if scheme == "generic" && queryParams.Has("template") {
|
||||
// add title as property if using generic with template json
|
||||
titleKey := queryParams.Get("titlekey")
|
||||
if titleKey == "" {
|
||||
titleKey = "title"
|
||||
}
|
||||
queryParams.Add("$"+titleKey, title)
|
||||
} else {
|
||||
// otherwise just add title to message
|
||||
message = title + "\n\n" + message
|
||||
}
|
||||
|
||||
// Add link
|
||||
if scheme == "ntfy" {
|
||||
queryParams.Add("Actions", fmt.Sprintf("view, %s, %s", linkText, link))
|
||||
} else if scheme == "lark" {
|
||||
queryParams.Add("link", link)
|
||||
} else if scheme == "bark" {
|
||||
queryParams.Add("url", link)
|
||||
} else {
|
||||
message += "\n\n" + link
|
||||
}
|
||||
|
||||
// Encode the modified query parameters back into the URL
|
||||
parsedURL.RawQuery = queryParams.Encode()
|
||||
// log.Println("URL after modification:", parsedURL.String())
|
||||
|
||||
err = shoutrrr.Send(parsedURL.String(), message)
|
||||
|
||||
if err == nil {
|
||||
am.hub.Logger().Info("Sent shoutrrr alert", "title", title)
|
||||
} else {
|
||||
am.hub.Logger().Error("Error sending shoutrrr alert", "err", err)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (am *AlertManager) SendTestNotification(e *core.RequestEvent) error {
|
||||
var data struct {
|
||||
URL string `json:"url"`
|
||||
}
|
||||
err := e.BindBody(&data)
|
||||
if err != nil || data.URL == "" {
|
||||
return e.BadRequestError("URL is required", err)
|
||||
}
|
||||
err = am.SendShoutrrrAlert(data.URL, "Test Alert", "This is a notification from Beszel.", am.hub.Settings().Meta.AppURL, "View Beszel")
|
||||
if err != nil {
|
||||
return e.JSON(200, map[string]string{"err": err.Error()})
|
||||
}
|
||||
return e.JSON(200, map[string]bool{"err": false})
|
||||
}
|
||||
119
internal/alerts/alerts_api.go
Normal file
119
internal/alerts/alerts_api.go
Normal file
@@ -0,0 +1,119 @@
|
||||
package alerts
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"net/http"
|
||||
|
||||
"github.com/pocketbase/dbx"
|
||||
"github.com/pocketbase/pocketbase/core"
|
||||
)
|
||||
|
||||
// UpsertUserAlerts handles API request to create or update alerts for a user
|
||||
// across multiple systems (POST /api/beszel/user-alerts)
|
||||
func UpsertUserAlerts(e *core.RequestEvent) error {
|
||||
userID := e.Auth.Id
|
||||
|
||||
reqData := struct {
|
||||
Min uint8 `json:"min"`
|
||||
Value float64 `json:"value"`
|
||||
Name string `json:"name"`
|
||||
Systems []string `json:"systems"`
|
||||
Overwrite bool `json:"overwrite"`
|
||||
}{}
|
||||
err := e.BindBody(&reqData)
|
||||
if err != nil || userID == "" || reqData.Name == "" || len(reqData.Systems) == 0 {
|
||||
return e.BadRequestError("Bad data", err)
|
||||
}
|
||||
|
||||
alertsCollection, err := e.App.FindCachedCollectionByNameOrId("alerts")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = e.App.RunInTransaction(func(txApp core.App) error {
|
||||
for _, systemId := range reqData.Systems {
|
||||
// find existing matching alert
|
||||
alertRecord, err := txApp.FindFirstRecordByFilter(alertsCollection,
|
||||
"system={:system} && name={:name} && user={:user}",
|
||||
dbx.Params{"system": systemId, "name": reqData.Name, "user": userID})
|
||||
|
||||
if err != nil && !errors.Is(err, sql.ErrNoRows) {
|
||||
return err
|
||||
}
|
||||
|
||||
// skip if alert already exists and overwrite is not set
|
||||
if !reqData.Overwrite && alertRecord != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// create new alert if it doesn't exist
|
||||
if alertRecord == nil {
|
||||
alertRecord = core.NewRecord(alertsCollection)
|
||||
alertRecord.Set("user", userID)
|
||||
alertRecord.Set("system", systemId)
|
||||
alertRecord.Set("name", reqData.Name)
|
||||
}
|
||||
|
||||
alertRecord.Set("value", reqData.Value)
|
||||
alertRecord.Set("min", reqData.Min)
|
||||
|
||||
if err := txApp.SaveNoValidate(alertRecord); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return e.JSON(http.StatusOK, map[string]any{"success": true})
|
||||
}
|
||||
|
||||
// DeleteUserAlerts handles API request to delete alerts for a user across multiple systems
|
||||
// (DELETE /api/beszel/user-alerts)
|
||||
func DeleteUserAlerts(e *core.RequestEvent) error {
|
||||
userID := e.Auth.Id
|
||||
|
||||
reqData := struct {
|
||||
AlertName string `json:"name"`
|
||||
Systems []string `json:"systems"`
|
||||
}{}
|
||||
err := e.BindBody(&reqData)
|
||||
if err != nil || userID == "" || reqData.AlertName == "" || len(reqData.Systems) == 0 {
|
||||
return e.BadRequestError("Bad data", err)
|
||||
}
|
||||
|
||||
var numDeleted uint16
|
||||
|
||||
err = e.App.RunInTransaction(func(txApp core.App) error {
|
||||
for _, systemId := range reqData.Systems {
|
||||
// Find existing alert to delete
|
||||
alertRecord, err := txApp.FindFirstRecordByFilter("alerts",
|
||||
"system={:system} && name={:name} && user={:user}",
|
||||
dbx.Params{"system": systemId, "name": reqData.AlertName, "user": userID})
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
// alert doesn't exist, continue to next system
|
||||
continue
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
if err := txApp.Delete(alertRecord); err != nil {
|
||||
return err
|
||||
}
|
||||
numDeleted++
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return e.JSON(http.StatusOK, map[string]any{"success": true, "count": numDeleted})
|
||||
}
|
||||
74
internal/alerts/alerts_history.go
Normal file
74
internal/alerts/alerts_history.go
Normal file
@@ -0,0 +1,74 @@
|
||||
package alerts
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/pocketbase/dbx"
|
||||
"github.com/pocketbase/pocketbase/core"
|
||||
)
|
||||
|
||||
// On triggered alert record delete, set matching alert history record to resolved
|
||||
func resolveHistoryOnAlertDelete(e *core.RecordEvent) error {
|
||||
if !e.Record.GetBool("triggered") {
|
||||
return e.Next()
|
||||
}
|
||||
_ = resolveAlertHistoryRecord(e.App, e.Record.Id)
|
||||
return e.Next()
|
||||
}
|
||||
|
||||
// On alert record update, update alert history record
|
||||
func updateHistoryOnAlertUpdate(e *core.RecordEvent) error {
|
||||
original := e.Record.Original()
|
||||
new := e.Record
|
||||
|
||||
originalTriggered := original.GetBool("triggered")
|
||||
newTriggered := new.GetBool("triggered")
|
||||
|
||||
// no need to update alert history if triggered state has not changed
|
||||
if originalTriggered == newTriggered {
|
||||
return e.Next()
|
||||
}
|
||||
|
||||
// if new state is triggered, create new alert history record
|
||||
if newTriggered {
|
||||
_, _ = createAlertHistoryRecord(e.App, new)
|
||||
return e.Next()
|
||||
}
|
||||
|
||||
// if new state is not triggered, check for matching alert history record and set it to resolved
|
||||
_ = resolveAlertHistoryRecord(e.App, new.Id)
|
||||
return e.Next()
|
||||
}
|
||||
|
||||
// resolveAlertHistoryRecord sets the resolved field to the current time
|
||||
func resolveAlertHistoryRecord(app core.App, alertRecordID string) error {
|
||||
alertHistoryRecord, err := app.FindFirstRecordByFilter("alerts_history", "alert_id={:alert_id} && resolved=null", dbx.Params{"alert_id": alertRecordID})
|
||||
if err != nil || alertHistoryRecord == nil {
|
||||
return err
|
||||
}
|
||||
alertHistoryRecord.Set("resolved", time.Now().UTC())
|
||||
err = app.Save(alertHistoryRecord)
|
||||
if err != nil {
|
||||
app.Logger().Error("Failed to resolve alert history", "err", err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// createAlertHistoryRecord creates a new alert history record
|
||||
func createAlertHistoryRecord(app core.App, alertRecord *core.Record) (alertHistoryRecord *core.Record, err error) {
|
||||
alertHistoryCollection, err := app.FindCachedCollectionByNameOrId("alerts_history")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
alertHistoryRecord = core.NewRecord(alertHistoryCollection)
|
||||
alertHistoryRecord.Set("alert_id", alertRecord.Id)
|
||||
alertHistoryRecord.Set("user", alertRecord.GetString("user"))
|
||||
alertHistoryRecord.Set("system", alertRecord.GetString("system"))
|
||||
alertHistoryRecord.Set("name", alertRecord.GetString("name"))
|
||||
alertHistoryRecord.Set("value", alertRecord.GetFloat("value"))
|
||||
err = app.Save(alertHistoryRecord)
|
||||
if err != nil {
|
||||
app.Logger().Error("Failed to save alert history", "err", err)
|
||||
}
|
||||
return alertHistoryRecord, err
|
||||
}
|
||||
172
internal/alerts/alerts_status.go
Normal file
172
internal/alerts/alerts_status.go
Normal file
@@ -0,0 +1,172 @@
|
||||
package alerts
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/pocketbase/dbx"
|
||||
"github.com/pocketbase/pocketbase/core"
|
||||
)
|
||||
|
||||
type alertTask struct {
|
||||
action string // "schedule" or "cancel"
|
||||
systemName string
|
||||
alertRecord *core.Record
|
||||
delay time.Duration
|
||||
}
|
||||
|
||||
type alertInfo struct {
|
||||
systemName string
|
||||
alertRecord *core.Record
|
||||
expireTime time.Time
|
||||
}
|
||||
|
||||
// startWorker is a long-running goroutine that processes alert tasks
|
||||
// every x seconds. It must be running to process status alerts.
|
||||
func (am *AlertManager) startWorker() {
|
||||
tick := time.Tick(15 * time.Second)
|
||||
for {
|
||||
select {
|
||||
case <-am.stopChan:
|
||||
return
|
||||
case task := <-am.alertQueue:
|
||||
switch task.action {
|
||||
case "schedule":
|
||||
am.pendingAlerts.Store(task.alertRecord.Id, &alertInfo{
|
||||
systemName: task.systemName,
|
||||
alertRecord: task.alertRecord,
|
||||
expireTime: time.Now().Add(task.delay),
|
||||
})
|
||||
case "cancel":
|
||||
am.pendingAlerts.Delete(task.alertRecord.Id)
|
||||
}
|
||||
case <-tick:
|
||||
// Check for expired alerts every tick
|
||||
now := time.Now()
|
||||
for key, value := range am.pendingAlerts.Range {
|
||||
info := value.(*alertInfo)
|
||||
if now.After(info.expireTime) {
|
||||
// Downtime delay has passed, process alert
|
||||
am.sendStatusAlert("down", info.systemName, info.alertRecord)
|
||||
am.pendingAlerts.Delete(key)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// StopWorker shuts down the AlertManager.worker goroutine
|
||||
func (am *AlertManager) StopWorker() {
|
||||
close(am.stopChan)
|
||||
}
|
||||
|
||||
// HandleStatusAlerts manages the logic when system status changes.
|
||||
func (am *AlertManager) HandleStatusAlerts(newStatus string, systemRecord *core.Record) error {
|
||||
if newStatus != "up" && newStatus != "down" {
|
||||
return nil
|
||||
}
|
||||
|
||||
alertRecords, err := am.getSystemStatusAlerts(systemRecord.Id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(alertRecords) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
systemName := systemRecord.GetString("name")
|
||||
if newStatus == "down" {
|
||||
am.handleSystemDown(systemName, alertRecords)
|
||||
} else {
|
||||
am.handleSystemUp(systemName, alertRecords)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// getSystemStatusAlerts retrieves all "Status" alert records for a given system ID.
|
||||
func (am *AlertManager) getSystemStatusAlerts(systemID string) ([]*core.Record, error) {
|
||||
alertRecords, err := am.hub.FindAllRecords("alerts", dbx.HashExp{
|
||||
"system": systemID,
|
||||
"name": "Status",
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return alertRecords, nil
|
||||
}
|
||||
|
||||
// Schedules delayed "down" alerts for each alert record.
|
||||
func (am *AlertManager) handleSystemDown(systemName string, alertRecords []*core.Record) {
|
||||
for _, alertRecord := range alertRecords {
|
||||
// Continue if alert is already scheduled
|
||||
if _, exists := am.pendingAlerts.Load(alertRecord.Id); exists {
|
||||
continue
|
||||
}
|
||||
// Schedule by adding to queue
|
||||
min := max(1, alertRecord.GetInt("min"))
|
||||
am.alertQueue <- alertTask{
|
||||
action: "schedule",
|
||||
systemName: systemName,
|
||||
alertRecord: alertRecord,
|
||||
delay: time.Duration(min) * time.Minute,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// handleSystemUp manages the logic when a system status changes to "up".
|
||||
// It cancels any pending alerts and sends "up" alerts.
|
||||
func (am *AlertManager) handleSystemUp(systemName string, alertRecords []*core.Record) {
|
||||
for _, alertRecord := range alertRecords {
|
||||
alertRecordID := alertRecord.Id
|
||||
// If alert exists for record, delete and continue (down alert not sent)
|
||||
if _, exists := am.pendingAlerts.Load(alertRecordID); exists {
|
||||
am.alertQueue <- alertTask{
|
||||
action: "cancel",
|
||||
alertRecord: alertRecord,
|
||||
}
|
||||
continue
|
||||
}
|
||||
// No alert scheduled for this record, send "up" alert
|
||||
if err := am.sendStatusAlert("up", systemName, alertRecord); err != nil {
|
||||
am.hub.Logger().Error("Failed to send alert", "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sendStatusAlert sends a status alert ("up" or "down") to the users associated with the alert records.
|
||||
func (am *AlertManager) sendStatusAlert(alertStatus string, systemName string, alertRecord *core.Record) error {
|
||||
switch alertStatus {
|
||||
case "up":
|
||||
alertRecord.Set("triggered", false)
|
||||
case "down":
|
||||
alertRecord.Set("triggered", true)
|
||||
}
|
||||
am.hub.Save(alertRecord)
|
||||
|
||||
var emoji string
|
||||
if alertStatus == "up" {
|
||||
emoji = "\u2705" // Green checkmark emoji
|
||||
} else {
|
||||
emoji = "\U0001F534" // Red alert emoji
|
||||
}
|
||||
|
||||
title := fmt.Sprintf("Connection to %s is %s %v", systemName, alertStatus, emoji)
|
||||
message := strings.TrimSuffix(title, emoji)
|
||||
|
||||
// if errs := am.hub.ExpandRecord(alertRecord, []string{"user"}, nil); len(errs) > 0 {
|
||||
// return errs["user"]
|
||||
// }
|
||||
// user := alertRecord.ExpandedOne("user")
|
||||
// if user == nil {
|
||||
// return nil
|
||||
// }
|
||||
|
||||
return am.SendAlert(AlertMessageData{
|
||||
UserID: alertRecord.GetString("user"),
|
||||
Title: title,
|
||||
Message: message,
|
||||
Link: am.hub.MakeLink("system", systemName),
|
||||
LinkText: "View " + systemName,
|
||||
})
|
||||
}
|
||||
304
internal/alerts/alerts_system.go
Normal file
304
internal/alerts/alerts_system.go
Normal file
@@ -0,0 +1,304 @@
|
||||
package alerts
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/henrygd/beszel/src/entities/system"
|
||||
|
||||
"github.com/pocketbase/dbx"
|
||||
"github.com/pocketbase/pocketbase/core"
|
||||
"github.com/pocketbase/pocketbase/tools/types"
|
||||
"github.com/spf13/cast"
|
||||
)
|
||||
|
||||
func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *system.CombinedData) error {
|
||||
alertRecords, err := am.hub.FindAllRecords("alerts",
|
||||
dbx.NewExp("system={:system} AND name!='Status'", dbx.Params{"system": systemRecord.Id}),
|
||||
)
|
||||
if err != nil || len(alertRecords) == 0 {
|
||||
// log.Println("no alerts found for system")
|
||||
return nil
|
||||
}
|
||||
|
||||
var validAlerts []SystemAlertData
|
||||
now := systemRecord.GetDateTime("updated").Time().UTC()
|
||||
oldestTime := now
|
||||
|
||||
for _, alertRecord := range alertRecords {
|
||||
name := alertRecord.GetString("name")
|
||||
var val float64
|
||||
unit := "%"
|
||||
|
||||
switch name {
|
||||
case "CPU":
|
||||
val = data.Info.Cpu
|
||||
case "Memory":
|
||||
val = data.Info.MemPct
|
||||
case "Bandwidth":
|
||||
val = data.Info.Bandwidth
|
||||
unit = " MB/s"
|
||||
case "Disk":
|
||||
maxUsedPct := data.Info.DiskPct
|
||||
for _, fs := range data.Stats.ExtraFs {
|
||||
usedPct := fs.DiskUsed / fs.DiskTotal * 100
|
||||
if usedPct > maxUsedPct {
|
||||
maxUsedPct = usedPct
|
||||
}
|
||||
}
|
||||
val = maxUsedPct
|
||||
case "Temperature":
|
||||
if data.Info.DashboardTemp < 1 {
|
||||
continue
|
||||
}
|
||||
val = data.Info.DashboardTemp
|
||||
unit = "°C"
|
||||
case "LoadAvg1":
|
||||
val = data.Info.LoadAvg[0]
|
||||
unit = ""
|
||||
case "LoadAvg5":
|
||||
val = data.Info.LoadAvg[1]
|
||||
unit = ""
|
||||
case "LoadAvg15":
|
||||
val = data.Info.LoadAvg[2]
|
||||
unit = ""
|
||||
}
|
||||
|
||||
triggered := alertRecord.GetBool("triggered")
|
||||
threshold := alertRecord.GetFloat("value")
|
||||
|
||||
// CONTINUE
|
||||
// IF alert is not triggered and curValue is less than threshold
|
||||
// OR alert is triggered and curValue is greater than threshold
|
||||
if (!triggered && val <= threshold) || (triggered && val > threshold) {
|
||||
// log.Printf("Skipping alert %s: val %f | threshold %f | triggered %v\n", name, val, threshold, triggered)
|
||||
continue
|
||||
}
|
||||
|
||||
min := max(1, cast.ToUint8(alertRecord.Get("min")))
|
||||
|
||||
alert := SystemAlertData{
|
||||
systemRecord: systemRecord,
|
||||
alertRecord: alertRecord,
|
||||
name: name,
|
||||
unit: unit,
|
||||
val: val,
|
||||
threshold: threshold,
|
||||
triggered: triggered,
|
||||
min: min,
|
||||
}
|
||||
|
||||
// send alert immediately if min is 1 - no need to sum up values.
|
||||
if min == 1 {
|
||||
alert.triggered = val > threshold
|
||||
go am.sendSystemAlert(alert)
|
||||
continue
|
||||
}
|
||||
|
||||
alert.time = now.Add(-time.Duration(min) * time.Minute)
|
||||
if alert.time.Before(oldestTime) {
|
||||
oldestTime = alert.time
|
||||
}
|
||||
|
||||
validAlerts = append(validAlerts, alert)
|
||||
}
|
||||
|
||||
systemStats := []struct {
|
||||
Stats []byte `db:"stats"`
|
||||
Created types.DateTime `db:"created"`
|
||||
}{}
|
||||
|
||||
err = am.hub.DB().
|
||||
Select("stats", "created").
|
||||
From("system_stats").
|
||||
Where(dbx.NewExp(
|
||||
"system={:system} AND type='1m' AND created > {:created}",
|
||||
dbx.Params{
|
||||
"system": systemRecord.Id,
|
||||
// subtract some time to give us a bit of buffer
|
||||
"created": oldestTime.Add(-time.Second * 90),
|
||||
},
|
||||
)).
|
||||
OrderBy("created").
|
||||
All(&systemStats)
|
||||
if err != nil || len(systemStats) == 0 {
|
||||
return err
|
||||
}
|
||||
|
||||
// get oldest record creation time from first record in the slice
|
||||
oldestRecordTime := systemStats[0].Created.Time()
|
||||
// log.Println("oldestRecordTime", oldestRecordTime.String())
|
||||
|
||||
// Filter validAlerts to keep only those with time newer than oldestRecord
|
||||
filteredAlerts := make([]SystemAlertData, 0, len(validAlerts))
|
||||
for _, alert := range validAlerts {
|
||||
if alert.time.After(oldestRecordTime) {
|
||||
filteredAlerts = append(filteredAlerts, alert)
|
||||
}
|
||||
}
|
||||
validAlerts = filteredAlerts
|
||||
|
||||
if len(validAlerts) == 0 {
|
||||
// log.Println("no valid alerts found")
|
||||
return nil
|
||||
}
|
||||
|
||||
var stats SystemAlertStats
|
||||
|
||||
// we can skip the latest systemStats record since it's the current value
|
||||
for i := range systemStats {
|
||||
stat := systemStats[i]
|
||||
// subtract 10 seconds to give a small time buffer
|
||||
systemStatsCreation := stat.Created.Time().Add(-time.Second * 10)
|
||||
if err := json.Unmarshal(stat.Stats, &stats); err != nil {
|
||||
return err
|
||||
}
|
||||
// log.Println("stats", stats)
|
||||
for j := range validAlerts {
|
||||
alert := &validAlerts[j]
|
||||
// reset alert val on first iteration
|
||||
if i == 0 {
|
||||
alert.val = 0
|
||||
}
|
||||
// continue if system_stats is older than alert time range
|
||||
if systemStatsCreation.Before(alert.time) {
|
||||
continue
|
||||
}
|
||||
// add to alert value
|
||||
switch alert.name {
|
||||
case "CPU":
|
||||
alert.val += stats.Cpu
|
||||
case "Memory":
|
||||
alert.val += stats.Mem
|
||||
case "Bandwidth":
|
||||
alert.val += stats.NetSent + stats.NetRecv
|
||||
case "Disk":
|
||||
if alert.mapSums == nil {
|
||||
alert.mapSums = make(map[string]float32, len(data.Stats.ExtraFs)+1)
|
||||
}
|
||||
// add root disk
|
||||
if _, ok := alert.mapSums["root"]; !ok {
|
||||
alert.mapSums["root"] = 0.0
|
||||
}
|
||||
alert.mapSums["root"] += float32(stats.Disk)
|
||||
// add extra disks
|
||||
for key, fs := range data.Stats.ExtraFs {
|
||||
if _, ok := alert.mapSums[key]; !ok {
|
||||
alert.mapSums[key] = 0.0
|
||||
}
|
||||
alert.mapSums[key] += float32(fs.DiskUsed / fs.DiskTotal * 100)
|
||||
}
|
||||
case "Temperature":
|
||||
if alert.mapSums == nil {
|
||||
alert.mapSums = make(map[string]float32, len(stats.Temperatures))
|
||||
}
|
||||
for key, temp := range stats.Temperatures {
|
||||
if _, ok := alert.mapSums[key]; !ok {
|
||||
alert.mapSums[key] = float32(0)
|
||||
}
|
||||
alert.mapSums[key] += temp
|
||||
}
|
||||
case "LoadAvg1":
|
||||
alert.val += stats.LoadAvg[0]
|
||||
case "LoadAvg5":
|
||||
alert.val += stats.LoadAvg[1]
|
||||
case "LoadAvg15":
|
||||
alert.val += stats.LoadAvg[2]
|
||||
default:
|
||||
continue
|
||||
}
|
||||
alert.count++
|
||||
}
|
||||
}
|
||||
// sum up vals for each alert
|
||||
for _, alert := range validAlerts {
|
||||
switch alert.name {
|
||||
case "Disk":
|
||||
maxPct := float32(0)
|
||||
for key, value := range alert.mapSums {
|
||||
sumPct := float32(value)
|
||||
if sumPct > maxPct {
|
||||
maxPct = sumPct
|
||||
alert.descriptor = fmt.Sprintf("Usage of %s", key)
|
||||
}
|
||||
}
|
||||
alert.val = float64(maxPct / float32(alert.count))
|
||||
case "Temperature":
|
||||
maxTemp := float32(0)
|
||||
for key, value := range alert.mapSums {
|
||||
sumTemp := float32(value) / float32(alert.count)
|
||||
if sumTemp > maxTemp {
|
||||
maxTemp = sumTemp
|
||||
alert.descriptor = fmt.Sprintf("Highest sensor %s", key)
|
||||
}
|
||||
}
|
||||
alert.val = float64(maxTemp)
|
||||
default:
|
||||
alert.val = alert.val / float64(alert.count)
|
||||
}
|
||||
minCount := float32(alert.min) / 1.2
|
||||
// log.Println("alert", alert.name, "val", alert.val, "threshold", alert.threshold, "triggered", alert.triggered)
|
||||
// log.Printf("%s: val %f | count %d | min-count %f | threshold %f\n", alert.name, alert.val, alert.count, minCount, alert.threshold)
|
||||
// pass through alert if count is greater than or equal to minCount
|
||||
if float32(alert.count) >= minCount {
|
||||
if !alert.triggered && alert.val > alert.threshold {
|
||||
alert.triggered = true
|
||||
go am.sendSystemAlert(alert)
|
||||
} else if alert.triggered && alert.val <= alert.threshold {
|
||||
alert.triggered = false
|
||||
go am.sendSystemAlert(alert)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (am *AlertManager) sendSystemAlert(alert SystemAlertData) {
|
||||
// log.Printf("Sending alert %s: val %f | count %d | threshold %f\n", alert.name, alert.val, alert.count, alert.threshold)
|
||||
systemName := alert.systemRecord.GetString("name")
|
||||
|
||||
// change Disk to Disk usage
|
||||
if alert.name == "Disk" {
|
||||
alert.name += " usage"
|
||||
}
|
||||
// format LoadAvg5 and LoadAvg15
|
||||
if after, ok := strings.CutPrefix(alert.name, "LoadAvg"); ok {
|
||||
alert.name = after + "m Load"
|
||||
}
|
||||
|
||||
// make title alert name lowercase if not CPU
|
||||
titleAlertName := alert.name
|
||||
if titleAlertName != "CPU" {
|
||||
titleAlertName = strings.ToLower(titleAlertName)
|
||||
}
|
||||
|
||||
var subject string
|
||||
if alert.triggered {
|
||||
subject = fmt.Sprintf("%s %s above threshold", systemName, titleAlertName)
|
||||
} else {
|
||||
subject = fmt.Sprintf("%s %s below threshold", systemName, titleAlertName)
|
||||
}
|
||||
minutesLabel := "minute"
|
||||
if alert.min > 1 {
|
||||
minutesLabel += "s"
|
||||
}
|
||||
if alert.descriptor == "" {
|
||||
alert.descriptor = alert.name
|
||||
}
|
||||
body := fmt.Sprintf("%s averaged %.2f%s for the previous %v %s.", alert.descriptor, alert.val, alert.unit, alert.min, minutesLabel)
|
||||
|
||||
alert.alertRecord.Set("triggered", alert.triggered)
|
||||
if err := am.hub.Save(alert.alertRecord); err != nil {
|
||||
// app.Logger().Error("failed to save alert record", "err", err)
|
||||
return
|
||||
}
|
||||
am.SendAlert(AlertMessageData{
|
||||
UserID: alert.alertRecord.GetString("user"),
|
||||
Title: subject,
|
||||
Message: body,
|
||||
Link: am.hub.MakeLink("system", systemName),
|
||||
LinkText: "View " + systemName,
|
||||
})
|
||||
}
|
||||
604
internal/alerts/alerts_test.go
Normal file
604
internal/alerts/alerts_test.go
Normal file
@@ -0,0 +1,604 @@
|
||||
//go:build testing
|
||||
// +build testing
|
||||
|
||||
package alerts_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
beszelTests "github.com/henrygd/beszel/src/tests"
|
||||
|
||||
"github.com/pocketbase/dbx"
|
||||
"github.com/pocketbase/pocketbase/core"
|
||||
pbTests "github.com/pocketbase/pocketbase/tests"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
// marshal to json and return an io.Reader (for use in ApiScenario.Body)
|
||||
func jsonReader(v any) io.Reader {
|
||||
data, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return bytes.NewReader(data)
|
||||
}
|
||||
|
||||
func TestUserAlertsApi(t *testing.T) {
|
||||
hub, _ := beszelTests.NewTestHub(t.TempDir())
|
||||
defer hub.Cleanup()
|
||||
|
||||
hub.StartHub()
|
||||
|
||||
user1, _ := beszelTests.CreateUser(hub, "alertstest@example.com", "password")
|
||||
user1Token, _ := user1.NewAuthToken()
|
||||
|
||||
user2, _ := beszelTests.CreateUser(hub, "alertstest2@example.com", "password")
|
||||
user2Token, _ := user2.NewAuthToken()
|
||||
|
||||
system1, _ := beszelTests.CreateRecord(hub, "systems", map[string]any{
|
||||
"name": "system1",
|
||||
"users": []string{user1.Id},
|
||||
"host": "127.0.0.1",
|
||||
})
|
||||
|
||||
system2, _ := beszelTests.CreateRecord(hub, "systems", map[string]any{
|
||||
"name": "system2",
|
||||
"users": []string{user1.Id, user2.Id},
|
||||
"host": "127.0.0.2",
|
||||
})
|
||||
|
||||
userRecords, _ := hub.CountRecords("users")
|
||||
assert.EqualValues(t, 2, userRecords, "all users should be created")
|
||||
|
||||
systemRecords, _ := hub.CountRecords("systems")
|
||||
assert.EqualValues(t, 2, systemRecords, "all systems should be created")
|
||||
|
||||
testAppFactory := func(t testing.TB) *pbTests.TestApp {
|
||||
return hub.TestApp
|
||||
}
|
||||
|
||||
scenarios := []beszelTests.ApiScenario{
|
||||
// {
|
||||
// Name: "GET not implemented - returns index",
|
||||
// Method: http.MethodGet,
|
||||
// URL: "/api/beszel/user-alerts",
|
||||
// ExpectedStatus: 200,
|
||||
// ExpectedContent: []string{"<html ", "globalThis.BESZEL"},
|
||||
// TestAppFactory: testAppFactory,
|
||||
// },
|
||||
{
|
||||
Name: "POST no auth",
|
||||
Method: http.MethodPost,
|
||||
URL: "/api/beszel/user-alerts",
|
||||
ExpectedStatus: 401,
|
||||
ExpectedContent: []string{"requires valid"},
|
||||
TestAppFactory: testAppFactory,
|
||||
},
|
||||
{
|
||||
Name: "POST no body",
|
||||
Method: http.MethodPost,
|
||||
URL: "/api/beszel/user-alerts",
|
||||
Headers: map[string]string{
|
||||
"Authorization": user1Token,
|
||||
},
|
||||
ExpectedStatus: 400,
|
||||
ExpectedContent: []string{"Bad data"},
|
||||
TestAppFactory: testAppFactory,
|
||||
},
|
||||
{
|
||||
Name: "POST bad data",
|
||||
Method: http.MethodPost,
|
||||
URL: "/api/beszel/user-alerts",
|
||||
Headers: map[string]string{
|
||||
"Authorization": user1Token,
|
||||
},
|
||||
ExpectedStatus: 400,
|
||||
ExpectedContent: []string{"Bad data"},
|
||||
TestAppFactory: testAppFactory,
|
||||
Body: jsonReader(map[string]any{
|
||||
"invalidField": "this should cause validation error",
|
||||
"threshold": "not a number",
|
||||
}),
|
||||
},
|
||||
{
|
||||
Name: "POST malformed JSON",
|
||||
Method: http.MethodPost,
|
||||
URL: "/api/beszel/user-alerts",
|
||||
Headers: map[string]string{
|
||||
"Authorization": user1Token,
|
||||
},
|
||||
ExpectedStatus: 400,
|
||||
ExpectedContent: []string{"Bad data"},
|
||||
TestAppFactory: testAppFactory,
|
||||
Body: strings.NewReader(`{"alertType": "cpu", "threshold": 80, "enabled": true,}`),
|
||||
},
|
||||
{
|
||||
Name: "POST valid alert data multiple systems",
|
||||
Method: http.MethodPost,
|
||||
URL: "/api/beszel/user-alerts",
|
||||
Headers: map[string]string{
|
||||
"Authorization": user1Token,
|
||||
},
|
||||
ExpectedStatus: 200,
|
||||
ExpectedContent: []string{"\"success\":true"},
|
||||
TestAppFactory: testAppFactory,
|
||||
Body: jsonReader(map[string]any{
|
||||
"name": "CPU",
|
||||
"value": 69,
|
||||
"min": 9,
|
||||
"systems": []string{system1.Id, system2.Id},
|
||||
"overwrite": false,
|
||||
}),
|
||||
AfterTestFunc: func(t testing.TB, app *pbTests.TestApp, res *http.Response) {
|
||||
// check total alerts
|
||||
alerts, _ := app.CountRecords("alerts")
|
||||
assert.EqualValues(t, 2, alerts, "should have 2 alerts")
|
||||
// check alert has correct values
|
||||
matchingAlerts, _ := app.CountRecords("alerts", dbx.HashExp{"name": "CPU", "user": user1.Id, "system": system1.Id, "value": 69, "min": 9})
|
||||
assert.EqualValues(t, 1, matchingAlerts, "should have 1 alert")
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "POST valid alert data single system",
|
||||
Method: http.MethodPost,
|
||||
URL: "/api/beszel/user-alerts",
|
||||
Headers: map[string]string{
|
||||
"Authorization": user1Token,
|
||||
},
|
||||
ExpectedStatus: 200,
|
||||
ExpectedContent: []string{"\"success\":true"},
|
||||
TestAppFactory: testAppFactory,
|
||||
Body: jsonReader(map[string]any{
|
||||
"name": "Memory",
|
||||
"systems": []string{system1.Id},
|
||||
"value": 90,
|
||||
"min": 10,
|
||||
}),
|
||||
AfterTestFunc: func(t testing.TB, app *pbTests.TestApp, res *http.Response) {
|
||||
user1Alerts, _ := app.CountRecords("alerts", dbx.HashExp{"user": user1.Id})
|
||||
assert.EqualValues(t, 3, user1Alerts, "should have 3 alerts")
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "Overwrite: false, should not overwrite existing alert",
|
||||
Method: http.MethodPost,
|
||||
URL: "/api/beszel/user-alerts",
|
||||
Headers: map[string]string{
|
||||
"Authorization": user1Token,
|
||||
},
|
||||
ExpectedStatus: 200,
|
||||
ExpectedContent: []string{"\"success\":true"},
|
||||
TestAppFactory: testAppFactory,
|
||||
Body: jsonReader(map[string]any{
|
||||
"name": "CPU",
|
||||
"value": 45,
|
||||
"min": 5,
|
||||
"systems": []string{system1.Id},
|
||||
"overwrite": false,
|
||||
}),
|
||||
BeforeTestFunc: func(t testing.TB, app *pbTests.TestApp, e *core.ServeEvent) {
|
||||
beszelTests.ClearCollection(t, app, "alerts")
|
||||
beszelTests.CreateRecord(app, "alerts", map[string]any{
|
||||
"name": "CPU",
|
||||
"system": system1.Id,
|
||||
"user": user1.Id,
|
||||
"value": 80,
|
||||
"min": 10,
|
||||
})
|
||||
},
|
||||
AfterTestFunc: func(t testing.TB, app *pbTests.TestApp, res *http.Response) {
|
||||
alerts, _ := app.CountRecords("alerts")
|
||||
assert.EqualValues(t, 1, alerts, "should have 1 alert")
|
||||
alert, _ := app.FindFirstRecordByFilter("alerts", "name = 'CPU' && user = {:user}", dbx.Params{"user": user1.Id})
|
||||
assert.EqualValues(t, 80, alert.Get("value"), "should have 80 as value")
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "Overwrite: true, should overwrite existing alert",
|
||||
Method: http.MethodPost,
|
||||
URL: "/api/beszel/user-alerts",
|
||||
Headers: map[string]string{
|
||||
"Authorization": user2Token,
|
||||
},
|
||||
ExpectedStatus: 200,
|
||||
ExpectedContent: []string{"\"success\":true"},
|
||||
TestAppFactory: testAppFactory,
|
||||
Body: jsonReader(map[string]any{
|
||||
"name": "CPU",
|
||||
"value": 45,
|
||||
"min": 5,
|
||||
"systems": []string{system2.Id},
|
||||
"overwrite": true,
|
||||
}),
|
||||
BeforeTestFunc: func(t testing.TB, app *pbTests.TestApp, e *core.ServeEvent) {
|
||||
beszelTests.ClearCollection(t, app, "alerts")
|
||||
beszelTests.CreateRecord(app, "alerts", map[string]any{
|
||||
"name": "CPU",
|
||||
"system": system2.Id,
|
||||
"user": user2.Id,
|
||||
"value": 80,
|
||||
"min": 10,
|
||||
})
|
||||
},
|
||||
AfterTestFunc: func(t testing.TB, app *pbTests.TestApp, res *http.Response) {
|
||||
alerts, _ := app.CountRecords("alerts")
|
||||
assert.EqualValues(t, 1, alerts, "should have 1 alert")
|
||||
alert, _ := app.FindFirstRecordByFilter("alerts", "name = 'CPU' && user = {:user}", dbx.Params{"user": user2.Id})
|
||||
assert.EqualValues(t, 45, alert.Get("value"), "should have 45 as value")
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "DELETE no auth",
|
||||
Method: http.MethodDelete,
|
||||
URL: "/api/beszel/user-alerts",
|
||||
ExpectedStatus: 401,
|
||||
ExpectedContent: []string{"requires valid"},
|
||||
TestAppFactory: testAppFactory,
|
||||
Body: jsonReader(map[string]any{
|
||||
"name": "CPU",
|
||||
"systems": []string{system1.Id},
|
||||
}),
|
||||
BeforeTestFunc: func(t testing.TB, app *pbTests.TestApp, e *core.ServeEvent) {
|
||||
beszelTests.ClearCollection(t, app, "alerts")
|
||||
beszelTests.CreateRecord(app, "alerts", map[string]any{
|
||||
"name": "CPU",
|
||||
"system": system1.Id,
|
||||
"user": user1.Id,
|
||||
"value": 80,
|
||||
"min": 10,
|
||||
})
|
||||
},
|
||||
AfterTestFunc: func(t testing.TB, app *pbTests.TestApp, res *http.Response) {
|
||||
alerts, _ := app.CountRecords("alerts")
|
||||
assert.EqualValues(t, 1, alerts, "should have 1 alert")
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "DELETE alert",
|
||||
Method: http.MethodDelete,
|
||||
URL: "/api/beszel/user-alerts",
|
||||
Headers: map[string]string{
|
||||
"Authorization": user1Token,
|
||||
},
|
||||
ExpectedStatus: 200,
|
||||
ExpectedContent: []string{"\"count\":1", "\"success\":true"},
|
||||
TestAppFactory: testAppFactory,
|
||||
Body: jsonReader(map[string]any{
|
||||
"name": "CPU",
|
||||
"systems": []string{system1.Id},
|
||||
}),
|
||||
BeforeTestFunc: func(t testing.TB, app *pbTests.TestApp, e *core.ServeEvent) {
|
||||
beszelTests.ClearCollection(t, app, "alerts")
|
||||
beszelTests.CreateRecord(app, "alerts", map[string]any{
|
||||
"name": "CPU",
|
||||
"system": system1.Id,
|
||||
"user": user1.Id,
|
||||
"value": 80,
|
||||
"min": 10,
|
||||
})
|
||||
},
|
||||
AfterTestFunc: func(t testing.TB, app *pbTests.TestApp, res *http.Response) {
|
||||
alerts, _ := app.CountRecords("alerts")
|
||||
assert.Zero(t, alerts, "should have 0 alerts")
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "DELETE alert multiple systems",
|
||||
Method: http.MethodDelete,
|
||||
URL: "/api/beszel/user-alerts",
|
||||
Headers: map[string]string{
|
||||
"Authorization": user1Token,
|
||||
},
|
||||
ExpectedStatus: 200,
|
||||
ExpectedContent: []string{"\"count\":2", "\"success\":true"},
|
||||
TestAppFactory: testAppFactory,
|
||||
Body: jsonReader(map[string]any{
|
||||
"name": "Memory",
|
||||
"systems": []string{system1.Id, system2.Id},
|
||||
}),
|
||||
BeforeTestFunc: func(t testing.TB, app *pbTests.TestApp, e *core.ServeEvent) {
|
||||
beszelTests.ClearCollection(t, app, "alerts")
|
||||
for _, systemId := range []string{system1.Id, system2.Id} {
|
||||
_, err := beszelTests.CreateRecord(app, "alerts", map[string]any{
|
||||
"name": "Memory",
|
||||
"system": systemId,
|
||||
"user": user1.Id,
|
||||
"value": 90,
|
||||
"min": 10,
|
||||
})
|
||||
assert.NoError(t, err, "should create alert")
|
||||
}
|
||||
alerts, _ := app.CountRecords("alerts")
|
||||
assert.EqualValues(t, 2, alerts, "should have 2 alerts")
|
||||
},
|
||||
AfterTestFunc: func(t testing.TB, app *pbTests.TestApp, res *http.Response) {
|
||||
alerts, _ := app.CountRecords("alerts")
|
||||
assert.Zero(t, alerts, "should have 0 alerts")
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "User 2 should not be able to delete alert of user 1",
|
||||
Method: http.MethodDelete,
|
||||
URL: "/api/beszel/user-alerts",
|
||||
Headers: map[string]string{
|
||||
"Authorization": user2Token,
|
||||
},
|
||||
ExpectedStatus: 200,
|
||||
ExpectedContent: []string{"\"count\":1", "\"success\":true"},
|
||||
TestAppFactory: testAppFactory,
|
||||
Body: jsonReader(map[string]any{
|
||||
"name": "CPU",
|
||||
"systems": []string{system2.Id},
|
||||
}),
|
||||
BeforeTestFunc: func(t testing.TB, app *pbTests.TestApp, e *core.ServeEvent) {
|
||||
beszelTests.ClearCollection(t, app, "alerts")
|
||||
for _, user := range []string{user1.Id, user2.Id} {
|
||||
beszelTests.CreateRecord(app, "alerts", map[string]any{
|
||||
"name": "CPU",
|
||||
"system": system2.Id,
|
||||
"user": user,
|
||||
"value": 80,
|
||||
"min": 10,
|
||||
})
|
||||
}
|
||||
alerts, _ := app.CountRecords("alerts")
|
||||
assert.EqualValues(t, 2, alerts, "should have 2 alerts")
|
||||
user1AlertCount, _ := app.CountRecords("alerts", dbx.HashExp{"user": user1.Id})
|
||||
assert.EqualValues(t, 1, user1AlertCount, "should have 1 alert")
|
||||
user2AlertCount, _ := app.CountRecords("alerts", dbx.HashExp{"user": user2.Id})
|
||||
assert.EqualValues(t, 1, user2AlertCount, "should have 1 alert")
|
||||
},
|
||||
AfterTestFunc: func(t testing.TB, app *pbTests.TestApp, res *http.Response) {
|
||||
user1AlertCount, _ := app.CountRecords("alerts", dbx.HashExp{"user": user1.Id})
|
||||
assert.EqualValues(t, 1, user1AlertCount, "should have 1 alert")
|
||||
user2AlertCount, _ := app.CountRecords("alerts", dbx.HashExp{"user": user2.Id})
|
||||
assert.Zero(t, user2AlertCount, "should have 0 alerts")
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, scenario := range scenarios {
|
||||
scenario.Test(t)
|
||||
}
|
||||
}
|
||||
|
||||
func getHubWithUser(t *testing.T) (*beszelTests.TestHub, *core.Record) {
|
||||
hub, err := beszelTests.NewTestHub(t.TempDir())
|
||||
assert.NoError(t, err)
|
||||
hub.StartHub()
|
||||
|
||||
// Manually initialize the system manager to bind event hooks
|
||||
err = hub.GetSystemManager().Initialize()
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Create a test user
|
||||
user, err := beszelTests.CreateUser(hub, "test@example.com", "password")
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Create user settings for the test user (required for alert notifications)
|
||||
userSettingsData := map[string]any{
|
||||
"user": user.Id,
|
||||
"settings": `{"emails":[test@example.com],"webhooks":[]}`,
|
||||
}
|
||||
_, err = beszelTests.CreateRecord(hub, "user_settings", userSettingsData)
|
||||
assert.NoError(t, err)
|
||||
|
||||
return hub, user
|
||||
}
|
||||
|
||||
func TestStatusAlerts(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
hub, user := getHubWithUser(t)
|
||||
defer hub.Cleanup()
|
||||
|
||||
systems, err := beszelTests.CreateSystems(hub, 4, user.Id, "paused")
|
||||
assert.NoError(t, err)
|
||||
|
||||
var alerts []*core.Record
|
||||
for i, system := range systems {
|
||||
alert, err := beszelTests.CreateRecord(hub, "alerts", map[string]any{
|
||||
"name": "Status",
|
||||
"system": system.Id,
|
||||
"user": user.Id,
|
||||
"min": i + 1,
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
alerts = append(alerts, alert)
|
||||
}
|
||||
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
|
||||
for _, alert := range alerts {
|
||||
assert.False(t, alert.GetBool("triggered"), "Alert should not be triggered immediately")
|
||||
}
|
||||
if hub.TestMailer.TotalSend() != 0 {
|
||||
assert.Zero(t, hub.TestMailer.TotalSend(), "Expected 0 messages, got %d", hub.TestMailer.TotalSend())
|
||||
}
|
||||
for _, system := range systems {
|
||||
assert.EqualValues(t, "paused", system.GetString("status"), "System should be paused")
|
||||
}
|
||||
for _, system := range systems {
|
||||
system.Set("status", "up")
|
||||
err = hub.SaveNoValidate(system)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
assert.EqualValues(t, 0, hub.GetPendingAlertsCount(), "should have 0 alerts in the pendingAlerts map")
|
||||
for _, system := range systems {
|
||||
system.Set("status", "down")
|
||||
err = hub.SaveNoValidate(system)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
// after 30 seconds, should have 4 alerts in the pendingAlerts map, no triggered alerts
|
||||
time.Sleep(time.Second * 30)
|
||||
assert.EqualValues(t, 4, hub.GetPendingAlertsCount(), "should have 4 alerts in the pendingAlerts map")
|
||||
triggeredCount, err := hub.CountRecords("alerts", dbx.HashExp{"triggered": true})
|
||||
assert.NoError(t, err)
|
||||
assert.EqualValues(t, 0, triggeredCount, "should have 0 alert triggered")
|
||||
assert.EqualValues(t, 0, hub.TestMailer.TotalSend(), "should have 0 messages sent")
|
||||
// after 1:30 seconds, should have 1 triggered alert and 3 pending alerts
|
||||
time.Sleep(time.Second * 60)
|
||||
assert.EqualValues(t, 3, hub.GetPendingAlertsCount(), "should have 3 alerts in the pendingAlerts map")
|
||||
triggeredCount, err = hub.CountRecords("alerts", dbx.HashExp{"triggered": true})
|
||||
assert.NoError(t, err)
|
||||
assert.EqualValues(t, 1, triggeredCount, "should have 1 alert triggered")
|
||||
assert.EqualValues(t, 1, hub.TestMailer.TotalSend(), "should have 1 messages sent")
|
||||
// after 2:30 seconds, should have 2 triggered alerts and 2 pending alerts
|
||||
time.Sleep(time.Second * 60)
|
||||
assert.EqualValues(t, 2, hub.GetPendingAlertsCount(), "should have 2 alerts in the pendingAlerts map")
|
||||
triggeredCount, err = hub.CountRecords("alerts", dbx.HashExp{"triggered": true})
|
||||
assert.NoError(t, err)
|
||||
assert.EqualValues(t, 2, triggeredCount, "should have 2 alert triggered")
|
||||
assert.EqualValues(t, 2, hub.TestMailer.TotalSend(), "should have 2 messages sent")
|
||||
// now we will bring the remaning systems back up
|
||||
for _, system := range systems {
|
||||
system.Set("status", "up")
|
||||
err = hub.SaveNoValidate(system)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
// should have 0 alerts in the pendingAlerts map and 0 alerts triggered
|
||||
assert.EqualValues(t, 0, hub.GetPendingAlertsCount(), "should have 0 alerts in the pendingAlerts map")
|
||||
triggeredCount, err = hub.CountRecords("alerts", dbx.HashExp{"triggered": true})
|
||||
assert.NoError(t, err)
|
||||
assert.Zero(t, triggeredCount, "should have 0 alert triggered")
|
||||
// 4 messages sent, 2 down alerts and 2 up alerts for first 2 systems
|
||||
assert.EqualValues(t, 4, hub.TestMailer.TotalSend(), "should have 4 messages sent")
|
||||
})
|
||||
}
|
||||
|
||||
func TestAlertsHistory(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
hub, user := getHubWithUser(t)
|
||||
defer hub.Cleanup()
|
||||
|
||||
// Create systems and alerts
|
||||
systems, err := beszelTests.CreateSystems(hub, 1, user.Id, "up")
|
||||
assert.NoError(t, err)
|
||||
system := systems[0]
|
||||
|
||||
alert, err := beszelTests.CreateRecord(hub, "alerts", map[string]any{
|
||||
"name": "Status",
|
||||
"system": system.Id,
|
||||
"user": user.Id,
|
||||
"min": 1,
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Initially, no alert history records should exist
|
||||
initialHistoryCount, err := hub.CountRecords("alerts_history", nil)
|
||||
assert.NoError(t, err)
|
||||
assert.Zero(t, initialHistoryCount, "Should have 0 alert history records initially")
|
||||
|
||||
// Set system to up initially
|
||||
system.Set("status", "up")
|
||||
err = hub.SaveNoValidate(system)
|
||||
assert.NoError(t, err)
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
|
||||
// Set system to down to trigger alert
|
||||
system.Set("status", "down")
|
||||
err = hub.SaveNoValidate(system)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Wait for alert to trigger (after the downtime delay)
|
||||
// With 1 minute delay, we need to wait at least 1 minute + some buffer
|
||||
time.Sleep(time.Second * 75)
|
||||
|
||||
// Check that alert is triggered
|
||||
triggeredCount, err := hub.CountRecords("alerts", dbx.HashExp{"triggered": true, "id": alert.Id})
|
||||
assert.NoError(t, err)
|
||||
assert.EqualValues(t, 1, triggeredCount, "Alert should be triggered")
|
||||
|
||||
// Check that alert history record was created
|
||||
historyCount, err := hub.CountRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id})
|
||||
assert.NoError(t, err)
|
||||
assert.EqualValues(t, 1, historyCount, "Should have 1 alert history record for triggered alert")
|
||||
|
||||
// Get the alert history record and verify it's not resolved immediately
|
||||
historyRecord, err := hub.FindFirstRecordByFilter("alerts_history", "alert_id={:alert_id}", dbx.Params{"alert_id": alert.Id})
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, historyRecord, "Alert history record should exist")
|
||||
assert.Equal(t, alert.Id, historyRecord.GetString("alert_id"), "Alert history should reference correct alert")
|
||||
assert.Equal(t, system.Id, historyRecord.GetString("system"), "Alert history should reference correct system")
|
||||
assert.Equal(t, "Status", historyRecord.GetString("name"), "Alert history should have correct name")
|
||||
|
||||
// The alert history might be resolved immediately in some cases, so let's check the alert's triggered status
|
||||
alertRecord, err := hub.FindFirstRecordByFilter("alerts", "id={:id}", dbx.Params{"id": alert.Id})
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, alertRecord.GetBool("triggered"), "Alert should still be triggered when checking history")
|
||||
|
||||
// Now resolve the alert by setting system back to up
|
||||
system.Set("status", "up")
|
||||
err = hub.SaveNoValidate(system)
|
||||
assert.NoError(t, err)
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
|
||||
// Check that alert is no longer triggered
|
||||
triggeredCount, err = hub.CountRecords("alerts", dbx.HashExp{"triggered": true, "id": alert.Id})
|
||||
assert.NoError(t, err)
|
||||
assert.Zero(t, triggeredCount, "Alert should not be triggered after system is back up")
|
||||
|
||||
// Check that alert history record is now resolved
|
||||
historyRecord, err = hub.FindFirstRecordByFilter("alerts_history", "alert_id={:alert_id}", dbx.Params{"alert_id": alert.Id})
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, historyRecord, "Alert history record should still exist")
|
||||
assert.NotNil(t, historyRecord.Get("resolved"), "Alert history should be resolved")
|
||||
|
||||
// Test deleting a triggered alert resolves its history
|
||||
// Create another system and alert
|
||||
systems2, err := beszelTests.CreateSystems(hub, 1, user.Id, "up")
|
||||
assert.NoError(t, err)
|
||||
system2 := systems2[0]
|
||||
system2.Set("name", "test-system-2") // Rename for clarity
|
||||
err = hub.SaveNoValidate(system2)
|
||||
assert.NoError(t, err)
|
||||
|
||||
alert2, err := beszelTests.CreateRecord(hub, "alerts", map[string]any{
|
||||
"name": "Status",
|
||||
"system": system2.Id,
|
||||
"user": user.Id,
|
||||
"min": 1,
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Set system2 to down to trigger alert
|
||||
system2.Set("status", "down")
|
||||
err = hub.SaveNoValidate(system2)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Wait for alert to trigger
|
||||
time.Sleep(time.Second * 75)
|
||||
|
||||
// Verify alert is triggered and history record exists
|
||||
triggeredCount, err = hub.CountRecords("alerts", dbx.HashExp{"triggered": true, "id": alert2.Id})
|
||||
assert.NoError(t, err)
|
||||
assert.EqualValues(t, 1, triggeredCount, "Second alert should be triggered")
|
||||
|
||||
historyCount, err = hub.CountRecords("alerts_history", dbx.HashExp{"alert_id": alert2.Id})
|
||||
assert.NoError(t, err)
|
||||
assert.EqualValues(t, 1, historyCount, "Should have 1 alert history record for second alert")
|
||||
|
||||
// Delete the triggered alert
|
||||
err = hub.Delete(alert2)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Check that alert history record is resolved after deletion
|
||||
historyRecord2, err := hub.FindFirstRecordByFilter("alerts_history", "alert_id={:alert_id}", dbx.Params{"alert_id": alert2.Id})
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, historyRecord2, "Alert history record should still exist after alert deletion")
|
||||
assert.NotNil(t, historyRecord2.Get("resolved"), "Alert history should be resolved after alert deletion")
|
||||
|
||||
// Verify total history count is correct (2 records total)
|
||||
totalHistoryCount, err := hub.CountRecords("alerts_history", nil)
|
||||
assert.NoError(t, err)
|
||||
assert.EqualValues(t, 2, totalHistoryCount, "Should have 2 total alert history records")
|
||||
})
|
||||
}
|
||||
55
internal/alerts/alerts_test_helpers.go
Normal file
55
internal/alerts/alerts_test_helpers.go
Normal file
@@ -0,0 +1,55 @@
|
||||
package alerts
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/pocketbase/pocketbase/core"
|
||||
)
|
||||
|
||||
func (am *AlertManager) GetAlertManager() *AlertManager {
|
||||
return am
|
||||
}
|
||||
|
||||
func (am *AlertManager) GetPendingAlerts() *sync.Map {
|
||||
return &am.pendingAlerts
|
||||
}
|
||||
|
||||
func (am *AlertManager) GetPendingAlertsCount() int {
|
||||
count := 0
|
||||
am.pendingAlerts.Range(func(key, value any) bool {
|
||||
count++
|
||||
return true
|
||||
})
|
||||
return count
|
||||
}
|
||||
|
||||
// ProcessPendingAlerts manually processes all expired alerts (for testing)
|
||||
func (am *AlertManager) ProcessPendingAlerts() ([]*core.Record, error) {
|
||||
now := time.Now()
|
||||
var lastErr error
|
||||
var processedAlerts []*core.Record
|
||||
am.pendingAlerts.Range(func(key, value any) bool {
|
||||
info := value.(*alertInfo)
|
||||
if now.After(info.expireTime) {
|
||||
// Downtime delay has passed, process alert
|
||||
if err := am.sendStatusAlert("down", info.systemName, info.alertRecord); err != nil {
|
||||
lastErr = err
|
||||
}
|
||||
processedAlerts = append(processedAlerts, info.alertRecord)
|
||||
am.pendingAlerts.Delete(key)
|
||||
}
|
||||
return true
|
||||
})
|
||||
return processedAlerts, lastErr
|
||||
}
|
||||
|
||||
// ForceExpirePendingAlerts sets all pending alerts to expire immediately (for testing)
|
||||
func (am *AlertManager) ForceExpirePendingAlerts() {
|
||||
now := time.Now()
|
||||
am.pendingAlerts.Range(func(key, value any) bool {
|
||||
info := value.(*alertInfo)
|
||||
info.expireTime = now.Add(-time.Second) // Set to 1 second ago
|
||||
return true
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user