gpu usage and vram charts

This commit is contained in:
Henry Dollman
2024-11-08 18:00:30 -05:00
parent 8262a9a45b
commit cd10727795
7 changed files with 82 additions and 16 deletions

View File

@@ -76,10 +76,12 @@ func (a *Agent) Run(pubKey []byte, addr string) {
a.dockerManager = newDockerManager()
// initialize GPU manager
if gm, err := NewGPUManager(); err != nil {
slog.Error("GPU manager", "err", err)
} else {
a.gpuManager = gm
if os.Getenv("GPU") == "true" {
if gm, err := NewGPUManager(); err != nil {
slog.Error("GPU manager", "err", err)
} else {
a.gpuManager = gm
}
}
// if debugging, print stats

View File

@@ -94,8 +94,8 @@ func (gm *GPUManager) parseNvidiaData(output []byte) {
// update gpu data
gpu := gm.GpuDataMap[id]
gpu.Temperature += temp
gpu.MemoryUsed += memoryUsage
gpu.MemoryTotal += totalMemory
gpu.MemoryUsed += memoryUsage / 1.024
gpu.MemoryTotal += totalMemory / 1.024
gpu.Usage += usage
gpu.Power += power
gpu.Count++

View File

@@ -33,11 +33,15 @@ export default memo(function AreaChartDefault({
unit = " MB/s",
chartName,
chartData,
max,
tickFormatter,
}: {
maxToggled?: boolean
unit?: string
chartName: string
chartData: ChartData
max?: number
tickFormatter?: (value: number) => string
}) {
const { yAxisWidth, updateYAxisWidth } = useYAxisWidth()
const { i18n } = useLingui()
@@ -52,19 +56,21 @@ export default memo(function AreaChartDefault({
return [[t`CPU Usage`, "cpu", 1, 0.4]]
} else if (chartName === "dio") {
return [
[t({ message: "Write", comment: "Context is disk write" }), "dw", 3, 0.3],
[t({ message: "Read", comment: "Context is disk read" }), "dr", 1, 0.3],
[t({ message: "Write", comment: "Disk write" }), "dw", 3, 0.3],
[t({ message: "Read", comment: "Disk read" }), "dr", 1, 0.3],
]
} else if (chartName === "bw") {
return [
[t({ message: "Sent", comment: "Context is network bytes sent (upload)" }), "ns", 5, 0.2],
[t({ message: "Received", comment: "Context is network bytes received (download)" }), "nr", 2, 0.2],
[t({ message: "Sent", comment: "Network bytes sent (upload)" }), "ns", 5, 0.2],
[t({ message: "Received", comment: "Network bytes received (download)" }), "nr", 2, 0.2],
]
} else if (chartName.startsWith("efs")) {
return [
[t`Write`, `${chartName}.w`, 3, 0.3],
[t`Read`, `${chartName}.r`, 1, 0.3],
]
} else if (chartName.startsWith("g.")) {
return [chartName.includes("mu") ? [t`Used`, chartName, 2, 0.25] : [t`Usage`, chartName, 1, 0.4]]
}
return []
}, [chartName, i18n.locale])
@@ -89,8 +95,14 @@ export default memo(function AreaChartDefault({
orientation={chartData.orientation}
className="tracking-tighter"
width={yAxisWidth}
domain={[0, max ?? "auto"]}
tickFormatter={(value) => {
const val = toFixedWithoutTrailingZeros(value, 2) + unit
let val: string
if (tickFormatter) {
val = tickFormatter(value)
} else {
val = toFixedWithoutTrailingZeros(value, 2) + unit
}
return updateYAxisWidth(val)
}}
tickLine={false}

View File

@@ -1,12 +1,12 @@
import { $systems, pb, $chartTime, $containerFilter, $userSettings, $direction } from "@/lib/stores"
import { ChartData, ChartTimes, ContainerStatsRecord, SystemRecord, SystemStatsRecord } from "@/types"
import { ChartData, ChartTimes, ContainerStatsRecord, GPUData, SystemRecord, SystemStatsRecord } from "@/types"
import React, { lazy, useCallback, useEffect, useMemo, useRef, useState } from "react"
import { Card, CardHeader, CardTitle, CardDescription } from "../ui/card"
import { useStore } from "@nanostores/react"
import Spinner from "../spinner"
import { ClockArrowUp, CpuIcon, GlobeIcon, LayoutGridIcon, MonitorIcon, XIcon } from "lucide-react"
import ChartTimeSelect from "../charts/chart-time-select"
import { chartTimeData, cn, getPbTimestamp, useLocalStorage } from "@/lib/utils"
import { chartTimeData, cn, getPbTimestamp, getSizeAndUnit, toFixedFloat, useLocalStorage } from "@/lib/utils"
import { Separator } from "../ui/separator"
import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "../ui/tooltip"
import { Button } from "../ui/button"
@@ -478,6 +478,44 @@ export default function SystemDetail({ name }: { name: string }) {
)}
</div>
{/* GPU charts */}
{Object.keys(systemStats.at(-1)?.stats.g ?? {}).length > 0 && (
<div className="grid xl:grid-cols-2 gap-4">
{Object.keys(systemStats.at(-1)?.stats.g ?? {}).map((id) => {
const gpu = systemStats.at(-1)?.stats.g?.[id] as GPUData
return (
<div key={id} className="contents">
<ChartCard
empty={dataEmpty}
grid={grid}
title={`${gpu.n} ${t`Usage`}`}
description={t`Total utilization of ${gpu.n}`}
>
<AreaChartDefault chartData={chartData} chartName={`g.${id}.u`} unit="%" />
</ChartCard>
<ChartCard
empty={dataEmpty}
grid={grid}
title={`${gpu.n} VRAM`}
description={t`VRAM usage of ${gpu.n}`}
>
<AreaChartDefault
chartData={chartData}
chartName={`g.${id}.mu`}
unit=" MB"
max={gpu.mt}
tickFormatter={(value) => {
const { v, u } = getSizeAndUnit(value, false)
return toFixedFloat(v, 1) + u
}}
/>
</ChartCard>
</div>
)
})}
</div>
)}
{/* extra filesystem charts */}
{Object.keys(systemStats.at(-1)?.stats.efs ?? {}).length > 0 && (
<div className="grid xl:grid-cols-2 gap-4">

View File

@@ -4,7 +4,7 @@ export default function ({ msg }: { msg?: string }) {
return (
<div className="flex flex-col items-center justify-center h-full absolute inset-0">
{msg ? (
<p className={"opacity-60 mb-2 text-center px-4"}>{msg}</p>
<p className={"opacity-60 mb-2 text-center text-sm px-4"}>{msg}</p>
) : (
<LoaderCircleIcon className="animate-spin h-10 w-10 opacity-60" />
)}

View File

@@ -81,6 +81,21 @@ export interface SystemStats {
t?: Record<string, number>
/** extra filesystems */
efs?: Record<string, ExtraFsStats>
/** GPU data */
g?: Record<string, GPUData>
}
export interface GPUData {
/** name */
n: string
/** memory used (mb) */
mu?: number
/** memory total (mb) */
mt?: number
/** usage (%) */
u: number
/** power (w) */
p?: number
}
export interface ExtraFsStats {