refactor: agent and entities

This commit is contained in:
Henry Dollman
2024-08-14 14:14:41 -04:00
parent 083da9598e
commit c7e67a9b63
8 changed files with 168 additions and 171 deletions

View File

@@ -29,20 +29,14 @@ func main() {
log.Fatal("KEY environment variable is not set") log.Fatal("KEY environment variable is not set")
} }
var port string port := ":45876"
if p, exists := os.LookupEnv("PORT"); exists { if p, exists := os.LookupEnv("PORT"); exists {
// allow passing an address in the form of "127.0.0.1:45876" // allow passing an address in the form of "127.0.0.1:45876"
if !strings.Contains(port, ":") { if !strings.Contains(p, ":") {
port = ":" + port p = ":" + p
} }
port = p port = p
} else {
port = ":45876"
} }
a := agent.NewAgent(pubKey, port) agent.NewAgent(pubKey, port).Run()
a.Run()
} }

View File

@@ -35,6 +35,7 @@ type Agent struct {
containerStatsMutex *sync.Mutex containerStatsMutex *sync.Mutex
diskIoStats system.DiskIoStats diskIoStats system.DiskIoStats
netIoStats system.NetIoStats netIoStats system.NetIoStats
dockerClient *http.Client
} }
func NewAgent(pubKey []byte, port string) *Agent { func NewAgent(pubKey []byte, port string) *Agent {
@@ -45,6 +46,7 @@ func NewAgent(pubKey []byte, port string) *Agent {
containerStatsMutex: &sync.Mutex{}, containerStatsMutex: &sync.Mutex{},
diskIoStats: system.DiskIoStats{}, diskIoStats: system.DiskIoStats{},
netIoStats: system.NetIoStats{}, netIoStats: system.NetIoStats{},
dockerClient: newDockerClient(),
} }
} }
@@ -56,11 +58,8 @@ func (a *Agent) releaseSemaphore() {
<-a.sem <-a.sem
} }
// client for docker engine api func (a *Agent) getSystemStats() (*system.Info, *system.Stats) {
var dockerClient = newDockerClient() systemStats := &system.Stats{}
func (a *Agent) getSystemStats() (*system.SystemInfo, *system.SystemStats) {
systemStats := &system.SystemStats{}
// cpu percent // cpu percent
cpuPct, err := cpu.Percent(0, false) cpuPct, err := cpu.Percent(0, false)
@@ -127,7 +126,7 @@ func (a *Agent) getSystemStats() (*system.SystemInfo, *system.SystemStats) {
a.netIoStats.Time = time.Now() a.netIoStats.Time = time.Now()
} }
systemInfo := &system.SystemInfo{ systemInfo := &system.Info{
Cpu: systemStats.Cpu, Cpu: systemStats.Cpu,
MemPct: systemStats.MemPct, MemPct: systemStats.MemPct,
DiskPct: systemStats.DiskPct, DiskPct: systemStats.DiskPct,
@@ -153,21 +152,21 @@ func (a *Agent) getSystemStats() (*system.SystemInfo, *system.SystemStats) {
} }
func (a *Agent) getDockerStats() ([]*container.ContainerStats, error) { func (a *Agent) getDockerStats() ([]*container.Stats, error) {
resp, err := dockerClient.Get("http://localhost/containers/json") resp, err := a.dockerClient.Get("http://localhost/containers/json")
if err != nil { if err != nil {
closeIdleConnections(err) a.closeIdleConnections(err)
return []*container.ContainerStats{}, err return []*container.Stats{}, err
} }
defer resp.Body.Close() defer resp.Body.Close()
var containers []*container.Container var containers []*container.ApiInfo
if err := json.NewDecoder(resp.Body).Decode(&containers); err != nil { if err := json.NewDecoder(resp.Body).Decode(&containers); err != nil {
log.Printf("Error decoding containers: %+v\n", err) log.Printf("Error decoding containers: %+v\n", err)
return []*container.ContainerStats{}, err return []*container.Stats{}, err
} }
containerStats := make([]*container.ContainerStats, 0, len(containers)) containerStats := make([]*container.Stats, 0, len(containers))
// store valid ids to clean up old container ids from map // store valid ids to clean up old container ids from map
validIds := make(map[string]struct{}, len(containers)) validIds := make(map[string]struct{}, len(containers))
@@ -188,12 +187,10 @@ func (a *Agent) getDockerStats() ([]*container.ContainerStats, error) {
defer wg.Done() defer wg.Done()
cstats, err := a.getContainerStats(ctr) cstats, err := a.getContainerStats(ctr)
if err != nil { if err != nil {
// Check if the error is a network timeout // close idle connections if error is a network timeout
if netErr, ok := err.(net.Error); ok && netErr.Timeout() { isTimeout := a.closeIdleConnections(err)
// Close idle connections to prevent reuse of stale connections // delete container from map if not a timeout
closeIdleConnections(err) if !isTimeout {
} else {
// otherwise delete container from map
a.deleteContainerStatsSync(ctr.IdShort) a.deleteContainerStatsSync(ctr.IdShort)
} }
// retry once // retry once
@@ -219,19 +216,19 @@ func (a *Agent) getDockerStats() ([]*container.ContainerStats, error) {
return containerStats, nil return containerStats, nil
} }
func (a *Agent) getContainerStats(ctr *container.Container) (*container.ContainerStats, error) { func (a *Agent) getContainerStats(ctr *container.ApiInfo) (*container.Stats, error) {
// use semaphore to limit concurrency // use semaphore to limit concurrency
a.acquireSemaphore() a.acquireSemaphore()
defer a.releaseSemaphore() defer a.releaseSemaphore()
resp, err := dockerClient.Get("http://localhost/containers/" + ctr.IdShort + "/stats?stream=0&one-shot=1") resp, err := a.dockerClient.Get("http://localhost/containers/" + ctr.IdShort + "/stats?stream=0&one-shot=1")
if err != nil { if err != nil {
return &container.ContainerStats{}, err return &container.Stats{}, err
} }
defer resp.Body.Close() defer resp.Body.Close()
var statsJson system.CStats var statsJson container.ApiStats
if err := json.NewDecoder(resp.Body).Decode(&statsJson); err != nil { if err := json.NewDecoder(resp.Body).Decode(&statsJson); err != nil {
panic(err) log.Fatal(err)
} }
name := ctr.Names[0][1:] name := ctr.Names[0][1:]
@@ -258,7 +255,7 @@ func (a *Agent) getContainerStats(ctr *container.Container) (*container.Containe
systemDelta := statsJson.CPUStats.SystemUsage - stats.Cpu[1] systemDelta := statsJson.CPUStats.SystemUsage - stats.Cpu[1]
cpuPct := float64(cpuDelta) / float64(systemDelta) * 100 cpuPct := float64(cpuDelta) / float64(systemDelta) * 100
if cpuPct > 100 { if cpuPct > 100 {
return &container.ContainerStats{}, fmt.Errorf("%s cpu pct greater than 100: %+v", name, cpuPct) return &container.Stats{}, fmt.Errorf("%s cpu pct greater than 100: %+v", name, cpuPct)
} }
stats.Cpu = [2]uint64{statsJson.CPUStats.CPUUsage.TotalUsage, statsJson.CPUStats.SystemUsage} stats.Cpu = [2]uint64{statsJson.CPUStats.CPUUsage.TotalUsage, statsJson.CPUStats.SystemUsage}
@@ -280,7 +277,7 @@ func (a *Agent) getContainerStats(ctr *container.Container) (*container.Containe
stats.Net.Recv = total_recv stats.Net.Recv = total_recv
stats.Net.Time = time.Now() stats.Net.Time = time.Now()
cStats := &container.ContainerStats{ cStats := &container.Stats{
Name: name, Name: name,
Cpu: twoDecimals(cpuPct), Cpu: twoDecimals(cpuPct),
Mem: bytesToMegabytes(float64(usedMemory)), Mem: bytesToMegabytes(float64(usedMemory)),
@@ -297,19 +294,18 @@ func (a *Agent) deleteContainerStatsSync(id string) {
delete(containerStatsMap, id) delete(containerStatsMap, id)
} }
func (a *Agent) gatherStats() *system.SystemData { func (a *Agent) gatherStats() *system.CombinedData {
systemInfo, systemStats := a.getSystemStats() systemInfo, systemStats := a.getSystemStats()
stats := &system.SystemData{ systemData := &system.CombinedData{
Stats: systemStats, Stats: systemStats,
Info: systemInfo, Info: systemInfo,
Containers: []*container.ContainerStats{}, // Containers: []*container.Stats{},
} }
containerStats, err := a.getDockerStats() if containerStats, err := a.getDockerStats(); err == nil {
if err == nil { systemData.Containers = containerStats
stats.Containers = containerStats
} }
// fmt.Printf("%+v\n", stats) // fmt.Printf("%+v\n", stats)
return stats return systemData
} }
func (a *Agent) startServer(addr string, pubKey []byte) { func (a *Agent) startServer(addr string, pubKey []byte) {
@@ -324,8 +320,7 @@ func (a *Agent) startServer(addr string, pubKey []byte) {
log.Printf("Starting SSH server on %s", addr) log.Printf("Starting SSH server on %s", addr)
if err := sshServer.ListenAndServe(addr, nil, sshServer.NoPty(), if err := sshServer.ListenAndServe(addr, nil, sshServer.NoPty(),
sshServer.PublicKeyAuth(func(ctx sshServer.Context, key sshServer.PublicKey) bool { sshServer.PublicKeyAuth(func(ctx sshServer.Context, key sshServer.PublicKey) bool {
data := []byte(pubKey) allowed, _, _, _, _ := sshServer.ParseAuthorizedKey(pubKey)
allowed, _, _, _, _ := sshServer.ParseAuthorizedKey(data)
return sshServer.KeysEqual(key, allowed) return sshServer.KeysEqual(key, allowed)
}), }),
); err != nil { ); err != nil {
@@ -334,7 +329,6 @@ func (a *Agent) startServer(addr string, pubKey []byte) {
} }
func (a *Agent) Run() { func (a *Agent) Run() {
if filesystem, exists := os.LookupEnv("FILESYSTEM"); exists { if filesystem, exists := os.LookupEnv("FILESYSTEM"); exists {
a.diskIoStats.Filesystem = filesystem a.diskIoStats.Filesystem = filesystem
} else { } else {
@@ -452,7 +446,12 @@ func newDockerClient() *http.Client {
} }
} }
func closeIdleConnections(err error) { // closes idle connections on timeouts to prevent reuse of stale connections
log.Printf("Closing idle connections. Error: %+v\n", err) func (a *Agent) closeIdleConnections(err error) (isTimeout bool) {
dockerClient.Transport.(*http.Transport).CloseIdleConnections() if netErr, ok := err.(net.Error); ok && netErr.Timeout() {
log.Printf("Closing idle connections. Error: %+v\n", err)
a.dockerClient.Transport.(*http.Transport).CloseIdleConnections()
return true
}
return false
} }

View File

@@ -1,3 +1,4 @@
// Package alerts handles alert management and delivery.
package alerts package alerts
import ( import (
@@ -32,7 +33,7 @@ func (am *AlertManager) HandleSystemAlerts(newStatus string, newRecord *models.R
return return
} }
// log.Println("found alerts", len(alertRecords)) // log.Println("found alerts", len(alertRecords))
var systemInfo *system.SystemInfo var systemInfo *system.Info
for _, alertRecord := range alertRecords { for _, alertRecord := range alertRecords {
name := alertRecord.GetString("name") name := alertRecord.GetString("name")
switch name { switch name {
@@ -56,8 +57,8 @@ func (am *AlertManager) HandleSystemAlerts(newStatus string, newRecord *models.R
} }
} }
func getSystemInfo(record *models.Record) *system.SystemInfo { func getSystemInfo(record *models.Record) *system.Info {
var SystemInfo system.SystemInfo var SystemInfo system.Info
record.UnmarshalJSONField("info", &SystemInfo) record.UnmarshalJSONField("info", &SystemInfo)
return &SystemInfo return &SystemInfo
} }

View File

@@ -1,36 +1,50 @@
package system package container
import "time" import "time"
type SystemStats struct { // Docker container info from /containers/json
Cpu float64 `json:"cpu"` type ApiInfo struct {
Mem float64 `json:"m"` Id string
MemUsed float64 `json:"mu"` IdShort string
MemPct float64 `json:"mp"` Names []string
MemBuffCache float64 `json:"mb"` Status string
Swap float64 `json:"s"` // Image string
SwapUsed float64 `json:"su"` // ImageID string
Disk float64 `json:"d"` // Command string
DiskUsed float64 `json:"du"` // Created int64
DiskPct float64 `json:"dp"` // Ports []Port
DiskRead float64 `json:"dr"` // SizeRw int64 `json:",omitempty"`
DiskWrite float64 `json:"dw"` // SizeRootFs int64 `json:",omitempty"`
NetworkSent float64 `json:"ns"` // Labels map[string]string
NetworkRecv float64 `json:"nr"` // State string
// HostConfig struct {
// NetworkMode string `json:",omitempty"`
// Annotations map[string]string `json:",omitempty"`
// }
// NetworkSettings *SummaryNetworkSettings
// Mounts []MountPoint
} }
type DiskIoStats struct { // Docker container resources from /containers/{id}/stats
Read uint64 type ApiStats struct {
Write uint64 // Common stats
Time time.Time // Read time.Time `json:"read"`
Filesystem string // PreRead time.Time `json:"preread"`
}
type NetIoStats struct { // Linux specific stats, not populated on Windows.
BytesRecv uint64 // PidsStats PidsStats `json:"pids_stats,omitempty"`
BytesSent uint64 // BlkioStats BlkioStats `json:"blkio_stats,omitempty"`
Time time.Time
Name string // Windows specific stats, not populated on Linux.
// NumProcs uint32 `json:"num_procs"`
// StorageStats StorageStats `json:"storage_stats,omitempty"`
// Networks request version >=1.21
Networks map[string]NetworkStats
// Shared stats
CPUStats CPUStats `json:"cpu_stats,omitempty"`
// PreCPUStats CPUStats `json:"precpu_stats,omitempty"` // "Pre"="Previous"
MemoryStats MemoryStats `json:"memory_stats,omitempty"`
} }
type CPUStats struct { type CPUStats struct {
@@ -70,27 +84,6 @@ type CPUUsage struct {
// UsageInUsermode uint64 `json:"usage_in_usermode"` // UsageInUsermode uint64 `json:"usage_in_usermode"`
} }
type CStats struct {
// Common stats
// Read time.Time `json:"read"`
// PreRead time.Time `json:"preread"`
// Linux specific stats, not populated on Windows.
// PidsStats PidsStats `json:"pids_stats,omitempty"`
// BlkioStats BlkioStats `json:"blkio_stats,omitempty"`
// Windows specific stats, not populated on Linux.
// NumProcs uint32 `json:"num_procs"`
// StorageStats StorageStats `json:"storage_stats,omitempty"`
// Networks request version >=1.21
Networks map[string]NetworkStats
// Shared stats
CPUStats CPUStats `json:"cpu_stats,omitempty"`
// PreCPUStats CPUStats `json:"precpu_stats,omitempty"` // "Pre"="Previous"
MemoryStats MemoryStats `json:"memory_stats,omitempty"`
}
type MemoryStats struct { type MemoryStats struct {
// current res_counter usage for memory // current res_counter usage for memory
@@ -119,3 +112,22 @@ type NetworkStats struct {
// Bytes sent. Windows and Linux. // Bytes sent. Windows and Linux.
TxBytes uint64 `json:"tx_bytes"` TxBytes uint64 `json:"tx_bytes"`
} }
// Container stats to return to the hub
type Stats struct {
Name string `json:"n"`
Cpu float64 `json:"c"`
Mem float64 `json:"m"`
NetworkSent float64 `json:"ns"`
NetworkRecv float64 `json:"nr"`
}
// Keeps track of container stats from previous run
type PrevContainerStats struct {
Cpu [2]uint64
Net struct {
Sent uint64
Recv uint64
Time time.Time
}
}

View File

@@ -1,45 +0,0 @@
package container
import "time"
// Docker container resources info from /containers/id/stats
type Container struct {
Id string
IdShort string
Names []string
Status string
// Image string
// ImageID string
// Command string
// Created int64
// Ports []Port
// SizeRw int64 `json:",omitempty"`
// SizeRootFs int64 `json:",omitempty"`
// Labels map[string]string
// State string
// HostConfig struct {
// NetworkMode string `json:",omitempty"`
// Annotations map[string]string `json:",omitempty"`
// }
// NetworkSettings *SummaryNetworkSettings
// Mounts []MountPoint
}
// Stats to return to the hub
type ContainerStats struct {
Name string `json:"n"`
Cpu float64 `json:"c"`
Mem float64 `json:"m"`
NetworkSent float64 `json:"ns"`
NetworkRecv float64 `json:"nr"`
}
// Keeps track of container stats from previous run
type PrevContainerStats struct {
Cpu [2]uint64
Net struct {
Sent uint64
Recv uint64
Time time.Time
}
}

View File

@@ -1,8 +1,42 @@
package system package system
import "beszel/internal/entities/container" import (
"beszel/internal/entities/container"
"time"
)
type SystemInfo struct { type Stats struct {
Cpu float64 `json:"cpu"`
Mem float64 `json:"m"`
MemUsed float64 `json:"mu"`
MemPct float64 `json:"mp"`
MemBuffCache float64 `json:"mb"`
Swap float64 `json:"s"`
SwapUsed float64 `json:"su"`
Disk float64 `json:"d"`
DiskUsed float64 `json:"du"`
DiskPct float64 `json:"dp"`
DiskRead float64 `json:"dr"`
DiskWrite float64 `json:"dw"`
NetworkSent float64 `json:"ns"`
NetworkRecv float64 `json:"nr"`
}
type DiskIoStats struct {
Read uint64
Write uint64
Time time.Time
Filesystem string
}
type NetIoStats struct {
BytesRecv uint64
BytesSent uint64
Time time.Time
Name string
}
type Info struct {
Cores int `json:"c"` Cores int `json:"c"`
Threads int `json:"t"` Threads int `json:"t"`
CpuModel string `json:"m"` CpuModel string `json:"m"`
@@ -13,8 +47,9 @@ type SystemInfo struct {
DiskPct float64 `json:"dp"` DiskPct float64 `json:"dp"`
} }
type SystemData struct { // Final data structure to return to the hub
Stats *SystemStats `json:"stats"` type CombinedData struct {
Info *SystemInfo `json:"info"` Stats *Stats `json:"stats"`
Containers []*container.ContainerStats `json:"container"` Info *Info `json:"info"`
Containers []*container.Stats `json:"container"`
} }

View File

@@ -97,7 +97,7 @@ func (h *Hub) Run() {
Scheme: "http", Scheme: "http",
Host: "localhost:5173", Host: "localhost:5173",
}) })
e.Router.GET("/static/*", apis.StaticDirectoryHandler(os.DirFS("./site/public/static"), false)) e.Router.GET("/static/*", apis.StaticDirectoryHandler(os.DirFS("../../site/public/static"), false))
e.Router.Any("/*", echo.WrapHandler(proxy)) e.Router.Any("/*", echo.WrapHandler(proxy))
// e.Router.Any("/", echo.WrapHandler(proxy)) // e.Router.Any("/", echo.WrapHandler(proxy))
default: default:
@@ -162,7 +162,7 @@ func (h *Hub) Run() {
// system creation defaults // system creation defaults
h.app.OnModelBeforeCreate("systems").Add(func(e *core.ModelEvent) error { h.app.OnModelBeforeCreate("systems").Add(func(e *core.ModelEvent) error {
record := e.Model.(*models.Record) record := e.Model.(*models.Record)
record.Set("info", system.SystemInfo{}) record.Set("info", system.Info{})
record.Set("status", "pending") record.Set("status", "pending")
return nil return nil
}) })
@@ -367,10 +367,10 @@ func (h *Hub) createSSHClientConfig() error {
return nil return nil
} }
func requestJson(client *ssh.Client) (system.SystemData, error) { func requestJson(client *ssh.Client) (system.CombinedData, error) {
session, err := client.NewSession() session, err := client.NewSession()
if err != nil { if err != nil {
return system.SystemData{}, errors.New("retry") return system.CombinedData{}, errors.New("retry")
} }
defer session.Close() defer session.Close()
@@ -379,19 +379,19 @@ func requestJson(client *ssh.Client) (system.SystemData, error) {
session.Stdout = &outputBuffer session.Stdout = &outputBuffer
if err := session.Shell(); err != nil { if err := session.Shell(); err != nil {
return system.SystemData{}, err return system.CombinedData{}, err
} }
err = session.Wait() err = session.Wait()
if err != nil { if err != nil {
return system.SystemData{}, err return system.CombinedData{}, err
} }
// Unmarshal the output into our struct // Unmarshal the output into our struct
var systemData system.SystemData var systemData system.CombinedData
err = json.Unmarshal(outputBuffer.Bytes(), &systemData) err = json.Unmarshal(outputBuffer.Bytes(), &systemData)
if err != nil { if err != nil {
return system.SystemData{}, err return system.CombinedData{}, err
} }
return systemData, nil return systemData, nil

View File

@@ -1,3 +1,4 @@
// Package records handles creating longer records and deleting old records.
package records package records
import ( import (
@@ -95,12 +96,12 @@ func (rm *RecordManager) CreateLongerRecords(collectionName string, shorterRecor
} }
// calculate the average stats of a list of system_stats records // calculate the average stats of a list of system_stats records
func (rm *RecordManager) AverageSystemStats(records []*models.Record) system.SystemStats { func (rm *RecordManager) AverageSystemStats(records []*models.Record) system.Stats {
count := float64(len(records)) count := float64(len(records))
sum := reflect.New(reflect.TypeOf(system.SystemStats{})).Elem() sum := reflect.New(reflect.TypeOf(system.Stats{})).Elem()
for _, record := range records { for _, record := range records {
var stats system.SystemStats var stats system.Stats
record.UnmarshalJSONField("stats", &stats) record.UnmarshalJSONField("stats", &stats)
statValue := reflect.ValueOf(stats) statValue := reflect.ValueOf(stats)
for i := 0; i < statValue.NumField(); i++ { for i := 0; i < statValue.NumField(); i++ {
@@ -109,24 +110,24 @@ func (rm *RecordManager) AverageSystemStats(records []*models.Record) system.Sys
} }
} }
average := reflect.New(reflect.TypeOf(system.SystemStats{})).Elem() average := reflect.New(reflect.TypeOf(system.Stats{})).Elem()
for i := 0; i < sum.NumField(); i++ { for i := 0; i < sum.NumField(); i++ {
average.Field(i).SetFloat(twoDecimals(sum.Field(i).Float() / count)) average.Field(i).SetFloat(twoDecimals(sum.Field(i).Float() / count))
} }
return average.Interface().(system.SystemStats) return average.Interface().(system.Stats)
} }
// calculate the average stats of a list of container_stats records // calculate the average stats of a list of container_stats records
func (rm *RecordManager) AverageContainerStats(records []*models.Record) (stats []container.ContainerStats) { func (rm *RecordManager) AverageContainerStats(records []*models.Record) (stats []container.Stats) {
sums := make(map[string]*container.ContainerStats) sums := make(map[string]*container.Stats)
count := float64(len(records)) count := float64(len(records))
for _, record := range records { for _, record := range records {
var stats []container.ContainerStats var stats []container.Stats
record.UnmarshalJSONField("stats", &stats) record.UnmarshalJSONField("stats", &stats)
for _, stat := range stats { for _, stat := range stats {
if _, ok := sums[stat.Name]; !ok { if _, ok := sums[stat.Name]; !ok {
sums[stat.Name] = &container.ContainerStats{Name: stat.Name, Cpu: 0, Mem: 0} sums[stat.Name] = &container.Stats{Name: stat.Name, Cpu: 0, Mem: 0}
} }
sums[stat.Name].Cpu += stat.Cpu sums[stat.Name].Cpu += stat.Cpu
sums[stat.Name].Mem += stat.Mem sums[stat.Name].Mem += stat.Mem
@@ -135,7 +136,7 @@ func (rm *RecordManager) AverageContainerStats(records []*models.Record) (stats
} }
} }
for _, value := range sums { for _, value := range sums {
stats = append(stats, container.ContainerStats{ stats = append(stats, container.Stats{
Name: value.Name, Name: value.Name,
Cpu: twoDecimals(value.Cpu / count), Cpu: twoDecimals(value.Cpu / count),
Mem: twoDecimals(value.Mem / count), Mem: twoDecimals(value.Mem / count),