fankes/beszel
2
0
Fork 0
mirror of https://github.com/fankes/beszel.git synced 2025-10-19 01:39:34 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

add docker container net stats

Browse Source
This commit is contained in:
Henry Dollman
2024-08-04 13:26:17 -04:00
parent c3e3d483b0
commit 2a3b228668
11 changed files with 276 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
agent/main.go
View File

@@ -26,8 +26,8 @@ import (
var Version = "0.1.1"
var containerCpuMap = make(map[string][2]uint64)
var containerCpuMutex = &sync.Mutex{}
var containerStatsMap = make(map[string]*PrevContainerStats)
var containerStatsMutex = &sync.Mutex{}
var sem = make(chan struct{}, 15)
@@ -53,7 +53,7 @@ var netIoStats = NetIoStats{
Name: "",
}
// dockerClient for docker engine api
// client for docker engine api
var dockerClient = newDockerClient()
func getSystemStats() (*SystemInfo, *SystemStats) {
@@ -175,7 +175,7 @@ func getDockerStats() ([]*ContainerStats, error) {
// note: can't use Created field because it's not updated on restart
if strings.HasSuffix(ctr.Status, "seconds") {
// if so, remove old container data
delete(containerCpuMap, ctr.IdShort)
delete(containerStatsMap, ctr.IdShort)
}
wg.Add(1)
go func() {
@@ -183,7 +183,7 @@ func getDockerStats() ([]*ContainerStats, error) {
cstats, err := getContainerStats(ctr)
if err != nil {
// delete container from map and retry once
delete(containerCpuMap, ctr.IdShort)
delete(containerStatsMap, ctr.IdShort)
cstats, err = getContainerStats(ctr)
if err != nil {
log.Printf("Error getting container stats: %+v\n", err)
@@ -196,10 +196,10 @@ func getDockerStats() ([]*ContainerStats, error) {
wg.Wait()
for id := range containerCpuMap {
for id := range containerStatsMap {
if _, exists := validIds[id]; !exists {
// log.Printf("Removing container cpu map entry: %+v\n", id)
delete(containerCpuMap, id)
delete(containerStatsMap, id)
}
}
@@ -229,27 +229,45 @@ func getContainerStats(ctr *Container) (*ContainerStats, error) {
memCache = statsJson.MemoryStats.Stats["cache"]
}
usedMemory := statsJson.MemoryStats.Usage - memCache
// pctMemory := float64(usedMemory) / float64(statsJson.MemoryStats.Limit) * 100
containerStatsMutex.Lock()
defer containerStatsMutex.Unlock()
// add empty values if they doesn't exist in map
_, initialized := containerStatsMap[ctr.IdShort]
if !initialized {
containerStatsMap[ctr.IdShort] = &PrevContainerStats{}
}
// cpu
// add default values to containerCpu if it doesn't exist
containerCpuMutex.Lock()
defer containerCpuMutex.Unlock()
if _, ok := containerCpuMap[ctr.IdShort]; !ok {
containerCpuMap[ctr.IdShort] = [2]uint64{0, 0}
}
cpuDelta := statsJson.CPUStats.CPUUsage.TotalUsage - containerCpuMap[ctr.IdShort][0]
systemDelta := statsJson.CPUStats.SystemUsage - containerCpuMap[ctr.IdShort][1]
cpuDelta := statsJson.CPUStats.CPUUsage.TotalUsage - containerStatsMap[ctr.IdShort].Cpu[0]
systemDelta := statsJson.CPUStats.SystemUsage - containerStatsMap[ctr.IdShort].Cpu[1]
cpuPct := float64(cpuDelta) / float64(systemDelta) * 100
if cpuPct > 100 {
return &ContainerStats{}, fmt.Errorf("%s cpu pct greater than 100: %+v", name, cpuPct)
}
containerCpuMap[ctr.IdShort] = [2]uint64{statsJson.CPUStats.CPUUsage.TotalUsage, statsJson.CPUStats.SystemUsage}
containerStatsMap[ctr.IdShort].Cpu = [2]uint64{statsJson.CPUStats.CPUUsage.TotalUsage, statsJson.CPUStats.SystemUsage}
// network
var total_sent, total_recv, sent_delta, recv_delta uint64
for _, v := range statsJson.Networks {
total_sent += v.TxBytes
total_recv += v.RxBytes
}
// prevent first run from sending all prev sent/recv bytes
if initialized {
sent_delta = total_sent - containerStatsMap[ctr.IdShort].Net[0]
recv_delta = total_recv - containerStatsMap[ctr.IdShort].Net[1]
// log.Printf("sent delta: %+v, recv delta: %+v\n", sent_delta, recv_delta)
}
containerStatsMap[ctr.IdShort].Net = [2]uint64{total_sent, total_recv}
cStats := &ContainerStats{
Name: name,
Cpu: twoDecimals(cpuPct),
Mem: bytesToMegabytes(float64(usedMemory)),
Name: name,
Cpu: twoDecimals(cpuPct),
Mem: bytesToMegabytes(float64(usedMemory)),
NetworkSent: bytesToMegabytes(float64(sent_delta)),
NetworkRecv: bytesToMegabytes(float64(recv_delta)),
// MemPct: twoDecimals(pctMemory),
}
return cStats, nil
@@ -429,10 +447,8 @@ func newDockerClient() *http.Client {
log.Fatal("Unsupported DOCKER_HOST: " + parsedURL.Scheme)
}
client := &http.Client{
return &http.Client{
Timeout: time.Second,
Transport: transport,
}
return client
}

59
agent/types.go
View File

@@ -35,10 +35,11 @@ type SystemStats struct {
}
type ContainerStats struct {
Name string `json:"n"`
Cpu float64 `json:"c"`
Mem float64 `json:"m"`
// MemPct float64 `json:"mp"`
Name string `json:"n"`
Cpu float64 `json:"c"`
Mem float64 `json:"m"`
NetworkSent float64 `json:"ns"`
NetworkRecv float64 `json:"nr"`
}
type Container struct {
@@ -65,20 +66,22 @@ type Container struct {
type CStats struct {
// Common stats
Read time.Time `json:"read"`
PreRead time.Time `json:"preread"`
// Read time.Time `json:"read"`
// PreRead time.Time `json:"preread"`
// Linux specific stats, not populated on Windows.
// PidsStats PidsStats `json:"pids_stats,omitempty"`
// BlkioStats BlkioStats `json:"blkio_stats,omitempty"`
// Windows specific stats, not populated on Linux.
NumProcs uint32 `json:"num_procs"`
// NumProcs uint32 `json:"num_procs"`
// StorageStats StorageStats `json:"storage_stats,omitempty"`
// Networks request version >=1.21
Networks map[string]NetworkStats
// Shared stats
CPUStats CPUStats `json:"cpu_stats,omitempty"`
PreCPUStats CPUStats `json:"precpu_stats,omitempty"` // "Pre"="Previous"
CPUStats CPUStats `json:"cpu_stats,omitempty"`
// PreCPUStats CPUStats `json:"precpu_stats,omitempty"` // "Pre"="Previous"
MemoryStats MemoryStats `json:"memory_stats,omitempty"`
}
@@ -90,7 +93,7 @@ type CPUStats struct {
SystemUsage uint64 `json:"system_cpu_usage,omitempty"`
// Online CPUs. Linux only.
OnlineCPUs uint32 `json:"online_cpus,omitempty"`
// OnlineCPUs uint32 `json:"online_cpus,omitempty"`
// Throttling Data. Linux only.
// ThrottlingData ThrottlingData `json:"throttling_data,omitempty"`
@@ -104,19 +107,19 @@ type CPUUsage struct {
// Total CPU time consumed per core (Linux). Not used on Windows.
// Units: nanoseconds.
PercpuUsage []uint64 `json:"percpu_usage,omitempty"`
// PercpuUsage []uint64 `json:"percpu_usage,omitempty"`
// Time spent by tasks of the cgroup in kernel mode (Linux).
// Time spent by all container processes in kernel mode (Windows).
// Units: nanoseconds (Linux).
// Units: 100's of nanoseconds (Windows). Not populated for Hyper-V Containers.
UsageInKernelmode uint64 `json:"usage_in_kernelmode"`
// UsageInKernelmode uint64 `json:"usage_in_kernelmode"`
// Time spent by tasks of the cgroup in user mode (Linux).
// Time spent by all container processes in user mode (Windows).
// Units: nanoseconds (Linux).
// Units: 100's of nanoseconds (Windows). Not populated for Hyper-V Containers
UsageInUsermode uint64 `json:"usage_in_usermode"`
// UsageInUsermode uint64 `json:"usage_in_usermode"`
}
type MemoryStats struct {
@@ -125,20 +128,27 @@ type MemoryStats struct {
Usage uint64 `json:"usage,omitempty"`
Cache uint64 `json:"cache,omitempty"`
// maximum usage ever recorded.
MaxUsage uint64 `json:"max_usage,omitempty"`
// MaxUsage uint64 `json:"max_usage,omitempty"`
// TODO(vishh): Export these as stronger types.
// all the stats exported via memory.stat.
Stats map[string]uint64 `json:"stats,omitempty"`
// number of times memory usage hits limits.
Failcnt uint64 `json:"failcnt,omitempty"`
Limit uint64 `json:"limit,omitempty"`
// Failcnt uint64 `json:"failcnt,omitempty"`
// Limit uint64 `json:"limit,omitempty"`
// committed bytes
Commit uint64 `json:"commitbytes,omitempty"`
// peak committed bytes
CommitPeak uint64 `json:"commitpeakbytes,omitempty"`
// private working set
PrivateWorkingSet uint64 `json:"privateworkingset,omitempty"`
// // committed bytes
// Commit uint64 `json:"commitbytes,omitempty"`
// // peak committed bytes
// CommitPeak uint64 `json:"commitpeakbytes,omitempty"`
// // private working set
// PrivateWorkingSet uint64 `json:"privateworkingset,omitempty"`
}
type NetworkStats struct {
// Bytes received. Windows and Linux.
RxBytes uint64 `json:"rx_bytes"`
// Bytes sent. Windows and Linux.
TxBytes uint64 `json:"tx_bytes"`
}
type DiskIoStats struct {
@@ -154,3 +164,8 @@ type NetIoStats struct {
Time time.Time
Name string
}
type PrevContainerStats struct {
Cpu [2]uint64
Net [2]uint64
}

20
hub/records.go
View File

@@ -73,11 +73,11 @@ func createLongerRecords(collectionName string, shorterRecord *models.Record) {
stats = averageContainerStats(allShorterRecords)
}
collection, _ := app.Dao().FindCollectionByNameOrId(collectionName)
tenMinRecord := models.NewRecord(collection)
tenMinRecord.Set("system", systemId)
tenMinRecord.Set("stats", stats)
tenMinRecord.Set("type", longerRecordType)
if err := app.Dao().SaveRecord(tenMinRecord); err != nil {
longerRecord := models.NewRecord(collection)
longerRecord.Set("system", systemId)
longerRecord.Set("stats", stats)
longerRecord.Set("type", longerRecordType)
if err := app.Dao().SaveRecord(longerRecord); err != nil {
fmt.Println("failed to save longer record", "err", err.Error())
}
@@ -119,13 +119,17 @@ func averageContainerStats(records []*models.Record) (stats []ContainerStats) {
}
sums[stat.Name].Cpu += stat.Cpu
sums[stat.Name].Mem += stat.Mem
sums[stat.Name].NetworkSent += stat.NetworkSent
sums[stat.Name].NetworkRecv += stat.NetworkRecv
}
}
for _, value := range sums {
stats = append(stats, ContainerStats{
Name: value.Name,
Cpu: twoDecimals(value.Cpu / count),
Mem: twoDecimals(value.Mem / count),
Name: value.Name,
Cpu: twoDecimals(value.Cpu / count),
Mem: twoDecimals(value.Mem / count),
NetworkSent: twoDecimals(value.NetworkSent / count),
NetworkRecv: twoDecimals(value.NetworkRecv / count),
})
}
return stats

3
hub/site/src/components/add-system.tsx
View File

@@ -17,7 +17,6 @@ import { Copy, Plus } from 'lucide-react'
import { useState, useRef, MutableRefObject, useEffect } from 'react'
import { useStore } from '@nanostores/react'
import { copyToClipboard } from '@/lib/utils'
import { SystemStats } from '@/types'
export function AddSystemButton() {
const [open, setOpen] = useState(false)
@@ -75,7 +74,7 @@ export function AddSystemButton() {
<DialogHeader>
<DialogTitle className="mb-2">Add New System</DialogTitle>
<DialogDescription>
The agent must be running on the server to connect. Copy the{' '}
The agent must be running on the system to connect. Copy the{' '}
<code className="bg-muted px-1 rounded-sm">docker-compose.yml</code> for the agent
below.
</DialogDescription>

2
hub/site/src/components/charts/container-cpu-chart.tsx
View File

@@ -1,5 +1,3 @@
'use client'
import { Area, AreaChart, CartesianGrid, XAxis, YAxis } from 'recharts'
import {
ChartConfig,

4
hub/site/src/components/charts/container-mem-chart.tsx
View File

@@ -1,5 +1,3 @@
'use client'
import { Area, AreaChart, CartesianGrid, XAxis, YAxis } from 'recharts'
import {
ChartConfig,
@@ -103,7 +101,7 @@ export default function ContainerMemChart({
labelFormatter={(_, data) => formatShortDate(data[0].payload.time)}
// @ts-ignore
itemSorter={(a, b) => b.value - a.value}
content={<ChartTooltipContent unit=" MiB" indicator="line" />}
content={<ChartTooltipContent unit=" MB" indicator="line" />}
/>
{Object.keys(chartConfig).map((key) => (
<Area

147
hub/site/src/components/charts/container-net-chart.tsx Normal file
View File

@@ -0,0 +1,147 @@
import { Area, AreaChart, CartesianGrid, XAxis, YAxis } from 'recharts'
import {
ChartConfig,
ChartContainer,
ChartTooltip,
ChartTooltipContent,
} from '@/components/ui/chart'
import { useMemo } from 'react'
import { chartTimeData, formatShortDate } from '@/lib/utils'
import Spinner from '../spinner'
import { useStore } from '@nanostores/react'
import { $chartTime } from '@/lib/stores'
import { Separator } from '@/components/ui/separator'
export default function ContainerCpuChart({
chartData,
ticks,
}: {
chartData: Record<string, number | number[]>[]
ticks: number[]
}) {
const chartTime = useStore($chartTime)
const chartConfig = useMemo(() => {
let config = {} as Record<
string,
{
label: string
color: string
}
>
const totalUsage = {} as Record<string, number>
for (let stats of chartData) {
for (let key in stats) {
if (!Array.isArray(stats[key])) {
continue
}
if (!(key in totalUsage)) {
totalUsage[key] = 0
}
totalUsage[key] += stats[key][2] ?? 0
}
}
let keys = Object.keys(totalUsage)
keys.sort((a, b) => (totalUsage[a] > totalUsage[b] ? -1 : 1))
const length = keys.length
for (let i = 0; i < length; i++) {
const key = keys[i]
const hue = ((i * 360) / length) % 360
config[key] = {
label: key,
color: `hsl(${hue}, 60%, 55%)`,
}
}
return config satisfies ChartConfig
}, [chartData])
if (!chartData.length || !ticks.length) {
return <Spinner />
}
return (
<ChartContainer config={{}} className="h-full w-full absolute aspect-auto">
<AreaChart
accessibilityLayer
data={chartData}
margin={{
top: 10,
}}
reverseStackOrder={true}
>
<CartesianGrid vertical={false} />
<YAxis
// domain={[0, (max: number) => Math.max(Math.ceil(max), 0.4)]}
width={75}
tickLine={false}
axisLine={false}
unit={' MB'}
tickFormatter={(x) => (x % 1 === 0 ? x : x.toFixed(1))}
/>
<XAxis
dataKey="time"
domain={[ticks[0], ticks.at(-1)!]}
ticks={ticks}
type="number"
scale={'time'}
minTickGap={35}
tickMargin={8}
axisLine={false}
tickFormatter={chartTimeData[chartTime].format}
/>
<ChartTooltip
// cursor={false}
animationEasing="ease-out"
animationDuration={150}
labelFormatter={(_, data) => {
return (
<span>
{formatShortDate(data[0].payload.time)}
<br />
<small className="opacity-70">Total MB received / transmitted</small>
</span>
)
}}
// @ts-ignore
itemSorter={(a, b) => b.value - a.value}
content={
<ChartTooltipContent
indicator="line"
contentFormatter={(item, key) => {
try {
const sent = item?.payload?.[key][0] ?? 0
const received = item?.payload?.[key][1] ?? 0
return (
<span className="flex">
{received.toLocaleString()} MB<span className="opacity-70 ml-0.5"> rx </span>
<Separator orientation="vertical" className="h-3 mx-1.5 bg-primary/40" />
{sent.toLocaleString()} MB<span className="opacity-70 ml-0.5"> tx</span>
</span>
)
} catch (e) {
return null
}
}}
/>
}
/>
{Object.keys(chartConfig).map((key) => (
<Area
key={key}
name={key}
// isAnimationActive={chartData.length < 20}
animateNewValues={false}
animationDuration={1200}
dataKey={(data) => data?.[key]?.[2] ?? 0}
type="monotoneX"
fill={chartConfig[key].color}
fillOpacity={0.4}
stroke={chartConfig[key].color}
stackId="a"
/>
))}
</AreaChart>
</ChartContainer>
)
}

26
hub/site/src/components/routes/system.tsx
View File

@@ -18,6 +18,7 @@ const ContainerMemChart = lazy(() => import('../charts/container-mem-chart'))
const DiskChart = lazy(() => import('../charts/disk-chart'))
const DiskIoChart = lazy(() => import('../charts/disk-io-chart'))
const BandwidthChart = lazy(() => import('../charts/bandwidth-chart'))
const ContainerNetChart = lazy(() => import('../charts/container-net-chart'))
export default function ServerDetail({ name }: { name: string }) {
const systems = useStore($systems)
@@ -32,6 +33,9 @@ export default function ServerDetail({ name }: { name: string }) {
const [dockerMemChartData, setDockerMemChartData] = useState(
[] as Record<string, number | string>[]
)
const [dockerNetChartData, setDockerNetChartData] = useState(
[] as Record<string, number | number[]>[]
)
useEffect(() => {
document.title = `${name} / Beszel`
@@ -45,6 +49,7 @@ export default function ServerDetail({ name }: { name: string }) {
setSystemStats([])
setDockerCpuChartData([])
setDockerMemChartData([])
setDockerNetChartData([])
}, [])
useEffect(resetCharts, [chartTime])
@@ -124,22 +129,30 @@ export default function ServerDetail({ name }: { name: string }) {
// container stats for charts
const makeContainerData = useCallback((containers: ContainerStatsRecord[]) => {
// console.log('containers', containers)
const dockerCpuData = [] as Record<string, number | string>[]
const dockerMemData = [] as Record<string, number | string>[]
const dockerCpuData = [] as typeof dockerCpuChartData
const dockerMemData = [] as typeof dockerMemChartData
const dockerNetData = [] as typeof dockerNetChartData
for (let { created, stats } of containers) {
const time = new Date(created).getTime()
let cpuData = { time } as (typeof dockerCpuChartData)[0]
let memData = { time } as (typeof dockerMemChartData)[0]
let netData = { time } as (typeof dockerNetChartData)[0]
for (let container of stats) {
cpuData[container.n] = container.c
memData[container.n] = container.m
netData[container.n] = [container.ns, container.nr, container.ns + container.nr] // sent, received, total
}
dockerCpuData.push(cpuData)
dockerMemData.push(memData)
dockerNetData.push(netData)
}
console.log('dockerCpuData', dockerCpuData)
// console.log('dockerMemData', dockerMemData)
console.log('dockerNetData', dockerNetData)
setDockerCpuChartData(dockerCpuData)
setDockerMemChartData(dockerMemData)
setDockerNetChartData(dockerNetData)
}, [])
const uptime = useMemo(() => {
@@ -243,6 +256,15 @@ export default function ServerDetail({ name }: { name: string }) {
<ChartCard title="Bandwidth" description="Network traffic of public interfaces">
<BandwidthChart ticks={ticks} systemData={systemStats} />
</ChartCard>
{dockerNetChartData.length > 0 && (
<ChartCard
title="Docker Network I/O"
description="Includes traffic between internal services"
>
<ContainerNetChart chartData={dockerNetChartData} ticks={ticks} />
</ChartCard>
)}
</div>
)
}

8
hub/site/src/components/ui/chart.tsx
View File

@@ -100,6 +100,7 @@ const ChartTooltipContent = React.forwardRef<
nameKey?: string
labelKey?: string
unit?: string
contentFormatter?: (item: any, key: string) => React.ReactNode | string
}
>(
(
@@ -119,6 +120,7 @@ const ChartTooltipContent = React.forwardRef<
labelKey,
unit,
itemSorter,
contentFormatter: content = undefined,
},
ref
) => {
@@ -180,7 +182,7 @@ const ChartTooltipContent = React.forwardRef<
return (
<div
key={item.dataKey}
key={item?.name || item.dataKey}
className={cn(
'flex w-full items-stretch gap-2 [&>svg]:h-2.5 [&>svg]:w-2.5 [&>svg]:text-muted-foreground',
indicator === 'dot' && 'items-center'
@@ -228,7 +230,9 @@ const ChartTooltipContent = React.forwardRef<
</div>
{item.value !== undefined && (
<span className="font-mono font-medium tabular-nums text-foreground">
{item.value.toLocaleString() + (unit ? unit : '')}
{content && typeof content === 'function'
? content(item, key)
: item.value.toLocaleString() + (unit ? unit : '')}
</span>
)}
</div>

4
hub/site/src/types.d.ts vendored
View File

@@ -67,6 +67,10 @@ interface ContainerStats {
c: number
/** memory used (gb) */
m: number
// network sent (mb)
ns: number
// network received (mb)
nr: number
}
export interface SystemStatsRecord extends RecordModel {

8
hub/types.go
View File

@@ -44,9 +44,11 @@ type SystemStats struct {
}
type ContainerStats struct {
Name string `json:"n"`
Cpu float64 `json:"c"`
Mem float64 `json:"m"`
Name string `json:"n"`
Cpu float64 `json:"c"`
Mem float64 `json:"m"`
NetworkSent float64 `json:"ns"`
NetworkRecv float64 `json:"nr"`
}
type EmailData struct {