alerts tests and small refactoring

2025-10-19 01:39:34 +08:00 · 2025-09-04 19:09:16 -04:00
parent a162a54a58
commit d1295b7c50
6 changed files with 337 additions and 22 deletions
--- a/beszel/internal/alerts/alerts.go
+++ b/beszel/internal/alerts/alerts.go
@@ -87,7 +87,7 @@ var supportsTitle = map[string]struct{}{
 func NewAlertManager(app hubLike) *AlertManager {
 	am := &AlertManager{
 		hub:        app,
-		alertQueue: make(chan alertTask),
+		alertQueue: make(chan alertTask, 5),
 		stopChan:   make(chan struct{}),
 	}
 	am.bindEvents()
--- a/beszel/internal/alerts/alerts_history.go
+++ b/beszel/internal/alerts/alerts_history.go
@@ -42,21 +42,10 @@ func updateHistoryOnAlertUpdate(e *core.RecordEvent) error {

 // resolveAlertHistoryRecord sets the resolved field to the current time
 func resolveAlertHistoryRecord(app core.App, alertRecordID string) error {
-	alertHistoryRecords, err := app.FindRecordsByFilter(
-		"alerts_history",
-		"alert_id={:alert_id} && resolved=null",
-		"-created",
-		1,
-		0,
-		dbx.Params{"alert_id": alertRecordID},
-	)
-	if err != nil {
+	alertHistoryRecord, err := app.FindFirstRecordByFilter("alerts_history", "alert_id={:alert_id} && resolved=null", dbx.Params{"alert_id": alertRecordID})
+	if err != nil || alertHistoryRecord == nil {
 		return err
 	}
-	if len(alertHistoryRecords) == 0 {
-		return nil
-	}
-	alertHistoryRecord := alertHistoryRecords[0] // there should be only one record
 	alertHistoryRecord.Set("resolved", time.Now().UTC())
 	err = app.Save(alertHistoryRecord)
 	if err != nil {
--- a/beszel/internal/alerts/alerts_test.go
+++ b/beszel/internal/alerts/alerts_test.go
@@ -10,6 +10,8 @@ import (
 	"net/http"
 	"strings"
 	"testing"
+	"testing/synctest"
+	"time"

 	beszelTests "beszel/internal/tests"

@@ -63,14 +65,14 @@ func TestUserAlertsApi(t *testing.T) {
 	}

 	scenarios := []beszelTests.ApiScenario{
-		{
-			Name:            "GET not implemented - returns index",
-			Method:          http.MethodGet,
-			URL:             "/api/beszel/user-alerts",
-			ExpectedStatus:  200,
-			ExpectedContent: []string{"<html ", "globalThis.BESZEL"},
-			TestAppFactory:  testAppFactory,
-		},
+		// {
+		// 	Name:            "GET not implemented - returns index",
+		// 	Method:          http.MethodGet,
+		// 	URL:             "/api/beszel/user-alerts",
+		// 	ExpectedStatus:  200,
+		// 	ExpectedContent: []string{"<html ", "globalThis.BESZEL"},
+		// 	TestAppFactory:  testAppFactory,
+		// },
 		{
 			Name:            "POST no auth",
 			Method:          http.MethodPost,
@@ -366,3 +368,237 @@ func TestUserAlertsApi(t *testing.T) {
 		scenario.Test(t)
 	}
 }
+
+func getHubWithUser(t *testing.T) (*beszelTests.TestHub, *core.Record) {
+	hub, err := beszelTests.NewTestHub(t.TempDir())
+	assert.NoError(t, err)
+	hub.StartHub()
+
+	// Manually initialize the system manager to bind event hooks
+	err = hub.GetSystemManager().Initialize()
+	assert.NoError(t, err)
+
+	// Create a test user
+	user, err := beszelTests.CreateUser(hub, "test@example.com", "password")
+	assert.NoError(t, err)
+
+	// Create user settings for the test user (required for alert notifications)
+	userSettingsData := map[string]any{
+		"user":     user.Id,
+		"settings": `{"emails":[test@example.com],"webhooks":[]}`,
+	}
+	_, err = beszelTests.CreateRecord(hub, "user_settings", userSettingsData)
+	assert.NoError(t, err)
+
+	return hub, user
+}
+
+func TestStatusAlerts(t *testing.T) {
+	synctest.Test(t, func(t *testing.T) {
+		hub, user := getHubWithUser(t)
+		defer hub.Cleanup()
+
+		systems, err := beszelTests.CreateSystems(hub, 4, user.Id, "paused")
+		assert.NoError(t, err)
+
+		var alerts []*core.Record
+		for i, system := range systems {
+			alert, err := beszelTests.CreateRecord(hub, "alerts", map[string]any{
+				"name":   "Status",
+				"system": system.Id,
+				"user":   user.Id,
+				"min":    i + 1,
+			})
+			assert.NoError(t, err)
+			alerts = append(alerts, alert)
+		}
+
+		time.Sleep(10 * time.Millisecond)
+
+		for _, alert := range alerts {
+			assert.False(t, alert.GetBool("triggered"), "Alert should not be triggered immediately")
+		}
+		if hub.TestMailer.TotalSend() != 0 {
+			assert.Zero(t, hub.TestMailer.TotalSend(), "Expected 0 messages, got %d", hub.TestMailer.TotalSend())
+		}
+		for _, system := range systems {
+			assert.EqualValues(t, "paused", system.GetString("status"), "System should be paused")
+		}
+		for _, system := range systems {
+			system.Set("status", "up")
+			err = hub.SaveNoValidate(system)
+			assert.NoError(t, err)
+		}
+		time.Sleep(time.Second)
+		assert.EqualValues(t, 0, hub.GetPendingAlertsCount(), "should have 0 alerts in the pendingAlerts map")
+		for _, system := range systems {
+			system.Set("status", "down")
+			err = hub.SaveNoValidate(system)
+			assert.NoError(t, err)
+		}
+		// after 30 seconds, should have 4 alerts in the pendingAlerts map, no triggered alerts
+		time.Sleep(time.Second * 30)
+		assert.EqualValues(t, 4, hub.GetPendingAlertsCount(), "should have 4 alerts in the pendingAlerts map")
+		triggeredCount, err := hub.CountRecords("alerts", dbx.HashExp{"triggered": true})
+		assert.NoError(t, err)
+		assert.EqualValues(t, 0, triggeredCount, "should have 0 alert triggered")
+		assert.EqualValues(t, 0, hub.TestMailer.TotalSend(), "should have 0 messages sent")
+		// after 1:30 seconds, should have 1 triggered alert and 3 pending alerts
+		time.Sleep(time.Second * 60)
+		assert.EqualValues(t, 3, hub.GetPendingAlertsCount(), "should have 3 alerts in the pendingAlerts map")
+		triggeredCount, err = hub.CountRecords("alerts", dbx.HashExp{"triggered": true})
+		assert.NoError(t, err)
+		assert.EqualValues(t, 1, triggeredCount, "should have 1 alert triggered")
+		assert.EqualValues(t, 1, hub.TestMailer.TotalSend(), "should have 1 messages sent")
+		// after 2:30 seconds, should have 2 triggered alerts and 2 pending alerts
+		time.Sleep(time.Second * 60)
+		assert.EqualValues(t, 2, hub.GetPendingAlertsCount(), "should have 2 alerts in the pendingAlerts map")
+		triggeredCount, err = hub.CountRecords("alerts", dbx.HashExp{"triggered": true})
+		assert.NoError(t, err)
+		assert.EqualValues(t, 2, triggeredCount, "should have 2 alert triggered")
+		assert.EqualValues(t, 2, hub.TestMailer.TotalSend(), "should have 2 messages sent")
+		// now we will bring the remaning systems back up
+		for _, system := range systems {
+			system.Set("status", "up")
+			err = hub.SaveNoValidate(system)
+			assert.NoError(t, err)
+		}
+		time.Sleep(time.Second)
+		// should have 0 alerts in the pendingAlerts map and 0 alerts triggered
+		assert.EqualValues(t, 0, hub.GetPendingAlertsCount(), "should have 0 alerts in the pendingAlerts map")
+		triggeredCount, err = hub.CountRecords("alerts", dbx.HashExp{"triggered": true})
+		assert.NoError(t, err)
+		assert.Zero(t, triggeredCount, "should have 0 alert triggered")
+		// 4 messages sent, 2 down alerts and 2 up alerts for first 2 systems
+		assert.EqualValues(t, 4, hub.TestMailer.TotalSend(), "should have 4 messages sent")
+	})
+}
+
+func TestAlertsHistory(t *testing.T) {
+	synctest.Test(t, func(t *testing.T) {
+		hub, user := getHubWithUser(t)
+		defer hub.Cleanup()
+
+		// Create systems and alerts
+		systems, err := beszelTests.CreateSystems(hub, 1, user.Id, "up")
+		assert.NoError(t, err)
+		system := systems[0]
+
+		alert, err := beszelTests.CreateRecord(hub, "alerts", map[string]any{
+			"name":   "Status",
+			"system": system.Id,
+			"user":   user.Id,
+			"min":    1,
+		})
+		assert.NoError(t, err)
+
+		// Initially, no alert history records should exist
+		initialHistoryCount, err := hub.CountRecords("alerts_history", nil)
+		assert.NoError(t, err)
+		assert.Zero(t, initialHistoryCount, "Should have 0 alert history records initially")
+
+		// Set system to up initially
+		system.Set("status", "up")
+		err = hub.SaveNoValidate(system)
+		assert.NoError(t, err)
+		time.Sleep(10 * time.Millisecond)
+
+		// Set system to down to trigger alert
+		system.Set("status", "down")
+		err = hub.SaveNoValidate(system)
+		assert.NoError(t, err)
+
+		// Wait for alert to trigger (after the downtime delay)
+		// With 1 minute delay, we need to wait at least 1 minute + some buffer
+		time.Sleep(time.Second * 75)
+
+		// Check that alert is triggered
+		triggeredCount, err := hub.CountRecords("alerts", dbx.HashExp{"triggered": true, "id": alert.Id})
+		assert.NoError(t, err)
+		assert.EqualValues(t, 1, triggeredCount, "Alert should be triggered")
+
+		// Check that alert history record was created
+		historyCount, err := hub.CountRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id})
+		assert.NoError(t, err)
+		assert.EqualValues(t, 1, historyCount, "Should have 1 alert history record for triggered alert")
+
+		// Get the alert history record and verify it's not resolved immediately
+		historyRecord, err := hub.FindFirstRecordByFilter("alerts_history", "alert_id={:alert_id}", dbx.Params{"alert_id": alert.Id})
+		assert.NoError(t, err)
+		assert.NotNil(t, historyRecord, "Alert history record should exist")
+		assert.Equal(t, alert.Id, historyRecord.GetString("alert_id"), "Alert history should reference correct alert")
+		assert.Equal(t, system.Id, historyRecord.GetString("system"), "Alert history should reference correct system")
+		assert.Equal(t, "Status", historyRecord.GetString("name"), "Alert history should have correct name")
+
+		// The alert history might be resolved immediately in some cases, so let's check the alert's triggered status
+		alertRecord, err := hub.FindFirstRecordByFilter("alerts", "id={:id}", dbx.Params{"id": alert.Id})
+		assert.NoError(t, err)
+		assert.True(t, alertRecord.GetBool("triggered"), "Alert should still be triggered when checking history")
+
+		// Now resolve the alert by setting system back to up
+		system.Set("status", "up")
+		err = hub.SaveNoValidate(system)
+		assert.NoError(t, err)
+		time.Sleep(200 * time.Millisecond)
+
+		// Check that alert is no longer triggered
+		triggeredCount, err = hub.CountRecords("alerts", dbx.HashExp{"triggered": true, "id": alert.Id})
+		assert.NoError(t, err)
+		assert.Zero(t, triggeredCount, "Alert should not be triggered after system is back up")
+
+		// Check that alert history record is now resolved
+		historyRecord, err = hub.FindFirstRecordByFilter("alerts_history", "alert_id={:alert_id}", dbx.Params{"alert_id": alert.Id})
+		assert.NoError(t, err)
+		assert.NotNil(t, historyRecord, "Alert history record should still exist")
+		assert.NotNil(t, historyRecord.Get("resolved"), "Alert history should be resolved")
+
+		// Test deleting a triggered alert resolves its history
+		// Create another system and alert
+		systems2, err := beszelTests.CreateSystems(hub, 1, user.Id, "up")
+		assert.NoError(t, err)
+		system2 := systems2[0]
+		system2.Set("name", "test-system-2") // Rename for clarity
+		err = hub.SaveNoValidate(system2)
+		assert.NoError(t, err)
+
+		alert2, err := beszelTests.CreateRecord(hub, "alerts", map[string]any{
+			"name":   "Status",
+			"system": system2.Id,
+			"user":   user.Id,
+			"min":    1,
+		})
+		assert.NoError(t, err)
+
+		// Set system2 to down to trigger alert
+		system2.Set("status", "down")
+		err = hub.SaveNoValidate(system2)
+		assert.NoError(t, err)
+
+		// Wait for alert to trigger
+		time.Sleep(time.Second * 75)
+
+		// Verify alert is triggered and history record exists
+		triggeredCount, err = hub.CountRecords("alerts", dbx.HashExp{"triggered": true, "id": alert2.Id})
+		assert.NoError(t, err)
+		assert.EqualValues(t, 1, triggeredCount, "Second alert should be triggered")
+
+		historyCount, err = hub.CountRecords("alerts_history", dbx.HashExp{"alert_id": alert2.Id})
+		assert.NoError(t, err)
+		assert.EqualValues(t, 1, historyCount, "Should have 1 alert history record for second alert")
+
+		// Delete the triggered alert
+		err = hub.Delete(alert2)
+		assert.NoError(t, err)
+
+		// Check that alert history record is resolved after deletion
+		historyRecord2, err := hub.FindFirstRecordByFilter("alerts_history", "alert_id={:alert_id}", dbx.Params{"alert_id": alert2.Id})
+		assert.NoError(t, err)
+		assert.NotNil(t, historyRecord2, "Alert history record should still exist after alert deletion")
+		assert.NotNil(t, historyRecord2.Get("resolved"), "Alert history should be resolved after alert deletion")
+
+		// Verify total history count is correct (2 records total)
+		totalHistoryCount, err := hub.CountRecords("alerts_history", nil)
+		assert.NoError(t, err)
+		assert.EqualValues(t, 2, totalHistoryCount, "Should have 2 total alert history records")
+	})
+}
--- a/beszel/internal/alerts/alerts_test_helpers.go
+++ b/beszel/internal/alerts/alerts_test_helpers.go
@@ -0,0 +1,55 @@
+package alerts
+
+import (
+	"sync"
+	"time"
+
+	"github.com/pocketbase/pocketbase/core"
+)
+
+func (am *AlertManager) GetAlertManager() *AlertManager {
+	return am
+}
+
+func (am *AlertManager) GetPendingAlerts() *sync.Map {
+	return &am.pendingAlerts
+}
+
+func (am *AlertManager) GetPendingAlertsCount() int {
+	count := 0
+	am.pendingAlerts.Range(func(key, value any) bool {
+		count++
+		return true
+	})
+	return count
+}
+
+// ProcessPendingAlerts manually processes all expired alerts (for testing)
+func (am *AlertManager) ProcessPendingAlerts() ([]*core.Record, error) {
+	now := time.Now()
+	var lastErr error
+	var processedAlerts []*core.Record
+	am.pendingAlerts.Range(func(key, value any) bool {
+		info := value.(*alertInfo)
+		if now.After(info.expireTime) {
+			// Downtime delay has passed, process alert
+			if err := am.sendStatusAlert("down", info.systemName, info.alertRecord); err != nil {
+				lastErr = err
+			}
+			processedAlerts = append(processedAlerts, info.alertRecord)
+			am.pendingAlerts.Delete(key)
+		}
+		return true
+	})
+	return processedAlerts, lastErr
+}
+
+// ForceExpirePendingAlerts sets all pending alerts to expire immediately (for testing)
+func (am *AlertManager) ForceExpirePendingAlerts() {
+	now := time.Now()
+	am.pendingAlerts.Range(func(key, value any) bool {
+		info := value.(*alertInfo)
+		info.expireTime = now.Add(-time.Second) // Set to 1 second ago
+		return true
+	})
+}
--- a/beszel/internal/hub/systems/systems_test_helpers.go
+++ b/beszel/internal/hub/systems/systems_test_helpers.go
@@ -100,3 +100,10 @@ func (sm *SystemManager) SetSystemStatusInDB(systemID string, status string) boo

 	return true
 }
+
+// TESTING ONLY: RemoveAllSystems removes all systems from the store
+func (sm *SystemManager) RemoveAllSystems() {
+	for _, system := range sm.systems.GetAll() {
+		sm.RemoveSystem(system.Id)
+	}
+}
--- a/beszel/internal/tests/hub.go
+++ b/beszel/internal/tests/hub.go
@@ -96,3 +96,31 @@ func ClearCollection(t testing.TB, app core.App, collectionName string) error {
 	assert.EqualValues(t, recordCount, 0, "should have 0 records after clearing")
 	return err
 }
+
+func (h *TestHub) Cleanup() {
+	h.GetAlertManager().StopWorker()
+	h.GetSystemManager().RemoveAllSystems()
+	h.TestApp.Cleanup()
+}
+
+func CreateSystems(app core.App, count int, userId string, status string) ([]*core.Record, error) {
+	systems := make([]*core.Record, 0, count)
+	for i := range count {
+		system, err := CreateRecord(app, "systems", map[string]any{
+			"name":  fmt.Sprintf("test-system-%d", i),
+			"host":  fmt.Sprintf("127.0.0.%d", i),
+			"port":  "33914",
+			"users": []string{userId},
+		})
+		if err != nil {
+			return nil, err
+		}
+		system.Set("status", status)
+		err = app.SaveNoValidate(system)
+		if err != nil {
+			return nil, err
+		}
+		systems = append(systems, system)
+	}
+	return systems, nil
+}