Files
dokploy/apps/monitoring/monitoring/monitor.go
Mauricio Siu 74a0f5e992 Feat/monitoring (#1267) Cloud Version
* feat: add start monitoring remote servers

* reafctor: update

* refactor: update

* refactor: update

* refactor: update

* refactor: update

* refactor: update

* refactor: update

* refactor:

* refactor: add metrics

* feat: add disk monitoring

* refactor: translate to english

* refacotor: add stats

* refactor: remove color

* feat: add log server metrics

* refactor: remove unused deps

* refactor: add origin

* refactor: add logs

* refactor: update

* feat: add series monitoring

* refactor: add system monitoring

* feat: add benchmark to optimize data

* refactor: update fn

* refactor: remove comments

* refactor: update

* refactor: exclude items

* feat: add refresh rate

* feat: add monitoring remote servers

* refactor: update

* refactor: remove unsued volumes

* refactor: update monitoring

* refactor: add more presets

* feat: add container metrics

* feat: add docker monitoring

* refactor: update conversion

* refactor: remove unused code

* refactor: update

* refactor: add docker compose logs

* refactor: add docker cli

* refactor: add install curl

* refactor: add get update

* refactor: add monitoring remote servers

* refactor: add containers config

* feat: add container specification

* refactor: update path

* refactor: add server filter

* refactor: simplify logic

* fix: verify if file exist before get stats

* refactor: update

* refactor: remove unused deps

* test: add test for containers

* refactor: update

* refactor add memory collector

* refactor: update

* refactor: update

* refactor: update

* refactor: remove

* refactor: add memory

* refactor: add server memory usage

* refactor: change memory

* refactor: update

* refactor: update

* refactor: add container metrics

* refactor: comment code

* refactor: mount proc bind

* refactor: change interval with node cron

* refactor: remove opening file

* refactor: use streams

* refactor: remove unused ws

* refactor: disable live when is all

* refactor: add sqlite

* refactor: update

* feat: add golang benchmark

* refactor: update go

* refactor: update dockerfile

* refactor: update db

* refactor: add env

* refactor: separate logic

* refactor: split logic

* refactor: update logs

* refactor: update dockerfile

* refactor: hide .env

* refactor: update

* chore: hide ,.ebnv

* refactor: add end angle

* refactor: update

* refactor: update

* refactor: update

* refactor: update

* refactor: update

* refactor: update monitoring

* refactor: add mount db

* refactor: add metrics and url callback

* refactor: add middleware

* refactor: add threshold property

* feat: add memory and cpu threshold notification

* feat: send notifications to the server

* feat: add metrics for dokploy server

* refactor: add dokploy server to monitoring

* refactor: update methods

* refactor: add admin to useeffect

* refactor: stop monitoring containers if elements are 0

* refactor: cancel request if appName is empty

* refactor: reuse methods

* chore; add feat monitoring

* refactor: set base url

* refactor: adjust monitoring

* refactor: delete migrations

* feat: add columns

* fix: add missing flag

* refactor: add free metrics

* refactor: add paid monitoring

* refactor: update methods

* feat: improve ui

* feat: add container stats

* refactor: add all container metrics

* refactor: add color primary

* refactor: change default rate limiting refresher

* refactor: update retention days

* refactor: use json instead of individual properties

* refactor: lint

* refactor: pass json env

* refactor: update

* refactor: delete

* refactor: update

* refactor: fix types

* refactor: add retention days

* chore: add license

* refactor: create db

* refactor: update path

* refactor: update setup

* refactor: update

* refactor: create files

* refactor: update

* refactor: delete

* refactor: update

* refactor: update token metrics

* fix: typechecks

* refactor: setup web server

* refactor: update error handling and add monitoring

* refactor: add local storage save

* refactor: add spacing

* refactor: update

* refactor: upgrade drizzle

* refactor: delete

* refactor: uppgrade drizzle kit

* refactor: update search with jsonB

* chore: upgrade drizzle

* chore: update packages

* refactor: add missing type

* refactor: add serverType

* refactor: update url

* refactor: update

* refactor: update

* refactor: hide monitoring on self hosted

* refactor: update server

* refactor: update

* refactor: update

* refactor: pin node version
2025-02-02 14:08:06 -06:00

262 lines
7.8 KiB
Go

package monitoring
import (
"bytes"
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"os/exec"
"runtime"
"strings"
"time"
"github.com/shirou/gopsutil/v3/cpu"
"github.com/shirou/gopsutil/v3/disk"
"github.com/shirou/gopsutil/v3/host"
"github.com/shirou/gopsutil/v3/mem"
"github.com/shirou/gopsutil/v3/net"
"github.com/mauriciogm/dokploy/apps/monitoring/config"
"github.com/mauriciogm/dokploy/apps/monitoring/database"
)
type SystemMetrics struct {
CPU string `json:"cpu"`
CPUModel string `json:"cpuModel"`
CPUCores int32 `json:"cpuCores"`
CPUPhysicalCores int32 `json:"cpuPhysicalCores"`
CPUSpeed float64 `json:"cpuSpeed"`
OS string `json:"os"`
Distro string `json:"distro"`
Kernel string `json:"kernel"`
Arch string `json:"arch"`
MemUsed string `json:"memUsed"`
MemUsedGB string `json:"memUsedGB"`
MemTotal string `json:"memTotal"`
Uptime uint64 `json:"uptime"`
DiskUsed string `json:"diskUsed"`
TotalDisk string `json:"totalDisk"`
NetworkIn string `json:"networkIn"`
NetworkOut string `json:"networkOut"`
Timestamp string `json:"timestamp"`
}
type AlertPayload struct {
ServerType string `json:"ServerType"`
Type string `json:"Type"`
Value float64 `json:"Value"`
Threshold float64 `json:"Threshold"`
Message string `json:"Message"`
Timestamp string `json:"Timestamp"`
Token string `json:"Token"`
}
func getRealOS() string {
if content, err := os.ReadFile("/etc/os-release"); err == nil {
lines := strings.Split(string(content), "\n")
var id, name, version string
for _, line := range lines {
if strings.HasPrefix(line, "PRETTY_NAME=") {
return strings.Trim(strings.TrimPrefix(line, "PRETTY_NAME="), "\"")
} else if strings.HasPrefix(line, "NAME=") {
name = strings.Trim(strings.TrimPrefix(line, "NAME="), "\"")
} else if strings.HasPrefix(line, "VERSION=") {
version = strings.Trim(strings.TrimPrefix(line, "VERSION="), "\"")
} else if strings.HasPrefix(line, "ID=") {
id = strings.Trim(strings.TrimPrefix(line, "ID="), "\"")
}
}
if name != "" && version != "" {
return fmt.Sprintf("%s %s", name, version)
}
if name != "" {
return name
}
if id != "" {
return id
}
}
if content, err := os.ReadFile("/etc/system-release"); err == nil {
text := strings.ToLower(string(content))
switch {
case strings.Contains(text, "red hat"):
return "rhel"
case strings.Contains(text, "centos"):
return "centos"
case strings.Contains(text, "fedora"):
return "fedora"
}
}
cmd := exec.Command("uname", "-a")
if output, err := cmd.Output(); err == nil {
osInfo := strings.ToLower(string(output))
switch {
case strings.Contains(osInfo, "debian"):
return "debian"
case strings.Contains(osInfo, "ubuntu"):
return "ubuntu"
case strings.Contains(osInfo, "centos"):
return "centos"
case strings.Contains(osInfo, "fedora"):
return "fedora"
case strings.Contains(osInfo, "red hat"):
return "rhel"
case strings.Contains(osInfo, "arch"):
return "arch"
case strings.Contains(osInfo, "darwin"):
return "darwin"
}
}
return runtime.GOOS
}
func GetServerMetrics() database.ServerMetric {
v, _ := mem.VirtualMemory()
c, _ := cpu.Percent(0, false)
cpuInfo, _ := cpu.Info()
diskInfo, _ := disk.Usage("/")
netInfo, _ := net.IOCounters(false)
hostInfo, _ := host.Info()
distro := getRealOS()
cpuModel := ""
if len(cpuInfo) > 0 {
cpuModel = fmt.Sprintf("%s %s", cpuInfo[0].VendorID, cpuInfo[0].ModelName)
}
memTotalGB := float64(v.Total) / 1024 / 1024 / 1024
memUsedGB := float64(v.Used) / 1024 / 1024 / 1024
memUsedPercent := (memUsedGB / memTotalGB) * 100
var networkIn, networkOut float64
if len(netInfo) > 0 {
networkIn = float64(netInfo[0].BytesRecv) / 1024 / 1024
networkOut = float64(netInfo[0].BytesSent) / 1024 / 1024
}
return database.ServerMetric{
Timestamp: time.Now().UTC().Format(time.RFC3339Nano),
CPU: c[0],
CPUModel: cpuModel,
CPUCores: int32(runtime.NumCPU()),
CPUPhysicalCores: int32(len(cpuInfo)),
CPUSpeed: float64(cpuInfo[0].Mhz),
OS: getRealOS(),
Distro: distro,
Kernel: hostInfo.KernelVersion,
Arch: hostInfo.KernelArch,
MemUsed: memUsedPercent,
MemUsedGB: memUsedGB,
MemTotal: memTotalGB,
Uptime: hostInfo.Uptime,
DiskUsed: float64(diskInfo.UsedPercent),
TotalDisk: float64(diskInfo.Total) / 1024 / 1024 / 1024,
NetworkIn: networkIn,
NetworkOut: networkOut,
}
}
func ConvertToSystemMetrics(metric database.ServerMetric) SystemMetrics {
return SystemMetrics{
CPU: fmt.Sprintf("%.2f", metric.CPU),
CPUModel: metric.CPUModel,
CPUCores: metric.CPUCores,
CPUPhysicalCores: metric.CPUPhysicalCores,
CPUSpeed: metric.CPUSpeed,
OS: metric.OS,
Distro: metric.Distro,
Kernel: metric.Kernel,
Arch: metric.Arch,
MemUsed: fmt.Sprintf("%.2f", metric.MemUsed),
MemUsedGB: fmt.Sprintf("%.2f", metric.MemUsedGB),
MemTotal: fmt.Sprintf("%.2f", metric.MemTotal),
Uptime: metric.Uptime,
DiskUsed: fmt.Sprintf("%.2f", metric.DiskUsed),
TotalDisk: fmt.Sprintf("%.2f", metric.TotalDisk),
NetworkIn: fmt.Sprintf("%.2f", metric.NetworkIn),
NetworkOut: fmt.Sprintf("%.2f", metric.NetworkOut),
Timestamp: metric.Timestamp,
}
}
func CheckThresholds(metrics database.ServerMetric) error {
cfg := config.GetMetricsConfig()
cpuThreshold := float64(cfg.Server.Thresholds.CPU)
memThreshold := float64(cfg.Server.Thresholds.Memory)
callbackURL := cfg.Server.UrlCallback
metricsToken := cfg.Server.Token
// log.Printf("CPU threshold: %.2f%%", cpuThreshold)
// log.Printf("Current CPU usage: %.2f%%", metrics.CPU)
// log.Printf("Memory threshold: %.2f%%", memThreshold)
// log.Printf("Callback URL: %s", callbackURL)
// log.Printf("Metrics token: %s", metricsToken)
if cpuThreshold == 0 && memThreshold == 0 {
return nil
}
if cpuThreshold > 0 && metrics.CPU > cpuThreshold {
alert := AlertPayload{
ServerType: cfg.Server.ServerType,
Type: "CPU",
Value: metrics.CPU,
Threshold: cpuThreshold,
Message: fmt.Sprintf("CPU usage (%.2f%%) exceeded threshold (%.2f%%)", metrics.CPU, cpuThreshold),
Timestamp: metrics.Timestamp,
Token: metricsToken,
}
if err := sendAlert(callbackURL, alert); err != nil {
return fmt.Errorf("failed to send CPU alert: %v", err)
}
}
if memThreshold > 0 && metrics.MemUsed > memThreshold {
alert := AlertPayload{
ServerType: cfg.Server.ServerType,
Type: "Memory",
Value: metrics.MemUsed,
Threshold: memThreshold,
Message: fmt.Sprintf("Memory usage (%.2f%%) exceeded threshold (%.2f%%)", metrics.MemUsed, memThreshold),
Timestamp: metrics.Timestamp,
Token: metricsToken,
}
if err := sendAlert(callbackURL, alert); err != nil {
return fmt.Errorf("failed to send memory alert: %v", err)
}
}
return nil
}
func sendAlert(callbackURL string, payload AlertPayload) error {
if callbackURL == "" {
return fmt.Errorf("callback URL is not set")
}
wrappedPayload := map[string]interface{}{
"json": payload,
}
jsonData, err := json.Marshal(wrappedPayload)
if err != nil {
return fmt.Errorf("failed to marshal alert payload: %v", err)
}
resp, err := http.Post(callbackURL, "application/json", bytes.NewBuffer(jsonData))
if err != nil {
return fmt.Errorf("failed to send POST request: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
bodyBytes, _ := io.ReadAll(resp.Body)
return fmt.Errorf("received non-OK response status: %s, body: %s", resp.Status, string(bodyBytes))
}
return nil
}