mirror of
https://github.com/Dokploy/dokploy
synced 2025-06-26 18:27:59 +00:00
Feat/monitoring (#1267) Cloud Version
* feat: add start monitoring remote servers * reafctor: update * refactor: update * refactor: update * refactor: update * refactor: update * refactor: update * refactor: update * refactor: * refactor: add metrics * feat: add disk monitoring * refactor: translate to english * refacotor: add stats * refactor: remove color * feat: add log server metrics * refactor: remove unused deps * refactor: add origin * refactor: add logs * refactor: update * feat: add series monitoring * refactor: add system monitoring * feat: add benchmark to optimize data * refactor: update fn * refactor: remove comments * refactor: update * refactor: exclude items * feat: add refresh rate * feat: add monitoring remote servers * refactor: update * refactor: remove unsued volumes * refactor: update monitoring * refactor: add more presets * feat: add container metrics * feat: add docker monitoring * refactor: update conversion * refactor: remove unused code * refactor: update * refactor: add docker compose logs * refactor: add docker cli * refactor: add install curl * refactor: add get update * refactor: add monitoring remote servers * refactor: add containers config * feat: add container specification * refactor: update path * refactor: add server filter * refactor: simplify logic * fix: verify if file exist before get stats * refactor: update * refactor: remove unused deps * test: add test for containers * refactor: update * refactor add memory collector * refactor: update * refactor: update * refactor: update * refactor: remove * refactor: add memory * refactor: add server memory usage * refactor: change memory * refactor: update * refactor: update * refactor: add container metrics * refactor: comment code * refactor: mount proc bind * refactor: change interval with node cron * refactor: remove opening file * refactor: use streams * refactor: remove unused ws * refactor: disable live when is all * refactor: add sqlite * refactor: update * feat: add golang benchmark * refactor: update go * refactor: update dockerfile * refactor: update db * refactor: add env * refactor: separate logic * refactor: split logic * refactor: update logs * refactor: update dockerfile * refactor: hide .env * refactor: update * chore: hide ,.ebnv * refactor: add end angle * refactor: update * refactor: update * refactor: update * refactor: update * refactor: update * refactor: update monitoring * refactor: add mount db * refactor: add metrics and url callback * refactor: add middleware * refactor: add threshold property * feat: add memory and cpu threshold notification * feat: send notifications to the server * feat: add metrics for dokploy server * refactor: add dokploy server to monitoring * refactor: update methods * refactor: add admin to useeffect * refactor: stop monitoring containers if elements are 0 * refactor: cancel request if appName is empty * refactor: reuse methods * chore; add feat monitoring * refactor: set base url * refactor: adjust monitoring * refactor: delete migrations * feat: add columns * fix: add missing flag * refactor: add free metrics * refactor: add paid monitoring * refactor: update methods * feat: improve ui * feat: add container stats * refactor: add all container metrics * refactor: add color primary * refactor: change default rate limiting refresher * refactor: update retention days * refactor: use json instead of individual properties * refactor: lint * refactor: pass json env * refactor: update * refactor: delete * refactor: update * refactor: fix types * refactor: add retention days * chore: add license * refactor: create db * refactor: update path * refactor: update setup * refactor: update * refactor: create files * refactor: update * refactor: delete * refactor: update * refactor: update token metrics * fix: typechecks * refactor: setup web server * refactor: update error handling and add monitoring * refactor: add local storage save * refactor: add spacing * refactor: update * refactor: upgrade drizzle * refactor: delete * refactor: uppgrade drizzle kit * refactor: update search with jsonB * chore: upgrade drizzle * chore: update packages * refactor: add missing type * refactor: add serverType * refactor: update url * refactor: update * refactor: update * refactor: hide monitoring on self hosted * refactor: update server * refactor: update * refactor: update * refactor: pin node version
This commit is contained in:
Mauricio Siu
GitHub
52
apps/monitoring/database/cleanup.go
Normal file
52
apps/monitoring/database/cleanup.go
Normal file
@@ -0,0 +1,52 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/robfig/cron/v3"
|
||||
)
|
||||
|
||||
// CleanupMetrics deletes metrics older than the retention period
|
||||
func CleanupMetrics(db *sql.DB, retentionDays int) error {
|
||||
cutoffDate := time.Now().AddDate(0, 0, -retentionDays)
|
||||
cutoffDateStr := cutoffDate.UTC().Format(time.RFC3339Nano)
|
||||
|
||||
containerQuery := `DELETE FROM container_metrics WHERE timestamp < ?`
|
||||
_, err := db.Exec(containerQuery, cutoffDateStr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
serverQuery := `DELETE FROM server_metrics WHERE timestamp < ?`
|
||||
_, err = db.Exec(serverQuery, cutoffDateStr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Printf("Metrics deleted (older than %d days)", retentionDays)
|
||||
log.Printf("Cutoff date for both tables: %s", cutoffDateStr)
|
||||
return nil
|
||||
}
|
||||
|
||||
// StartMetricsCleanup starts a cron job to periodically clean up metrics
|
||||
func StartMetricsCleanup(db *sql.DB, retentionDays int, cronExpression string) (*cron.Cron, error) {
|
||||
c := cron.New()
|
||||
|
||||
_, err := c.AddFunc(cronExpression, func() {
|
||||
if err := CleanupMetrics(db, retentionDays); err != nil {
|
||||
log.Printf("Error during metrics cleanup: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
c.Start()
|
||||
log.Printf("Started metrics cleanup job (retention: %d days, cron: %s)",
|
||||
retentionDays, cronExpression)
|
||||
|
||||
return c, nil
|
||||
}
|
||||
160
apps/monitoring/database/containers.go
Normal file
160
apps/monitoring/database/containers.go
Normal file
@@ -0,0 +1,160 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func (db *DB) InitContainerMetricsTable() error {
|
||||
_, err := db.Exec(`
|
||||
CREATE TABLE IF NOT EXISTS container_metrics (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
timestamp TEXT NOT NULL,
|
||||
container_id TEXT NOT NULL,
|
||||
container_name TEXT NOT NULL,
|
||||
metrics_json TEXT NOT NULL
|
||||
)
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error creating container_metrics table: %v", err)
|
||||
}
|
||||
|
||||
// Crear índices para mejorar el rendimiento
|
||||
_, err = db.Exec(`CREATE INDEX IF NOT EXISTS idx_container_metrics_timestamp ON container_metrics(timestamp)`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error creating timestamp index: %v", err)
|
||||
}
|
||||
|
||||
_, err = db.Exec(`CREATE INDEX IF NOT EXISTS idx_container_metrics_name ON container_metrics(container_name)`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error creating name index: %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (db *DB) SaveContainerMetric(metric *ContainerMetric) error {
|
||||
metricsJSON, err := json.Marshal(metric)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error marshaling metrics: %v", err)
|
||||
}
|
||||
|
||||
_, err = db.Exec(`
|
||||
INSERT INTO container_metrics (timestamp, container_id, container_name, metrics_json)
|
||||
VALUES (?, ?, ?, ?)
|
||||
`, metric.Timestamp, metric.ID, metric.Name, string(metricsJSON))
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) GetLastNContainerMetrics(containerName string, limit int) ([]ContainerMetric, error) {
|
||||
name := strings.TrimPrefix(containerName, "/")
|
||||
parts := strings.Split(name, "-")
|
||||
if len(parts) > 1 {
|
||||
containerName = strings.Join(parts[:len(parts)-1], "-")
|
||||
}
|
||||
|
||||
query := `
|
||||
WITH recent_metrics AS (
|
||||
SELECT metrics_json
|
||||
FROM container_metrics
|
||||
WHERE container_name LIKE ? || '%'
|
||||
ORDER BY timestamp DESC
|
||||
LIMIT ?
|
||||
)
|
||||
SELECT metrics_json FROM recent_metrics ORDER BY json_extract(metrics_json, '$.timestamp') ASC
|
||||
`
|
||||
rows, err := db.Query(query, containerName, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var metrics []ContainerMetric
|
||||
for rows.Next() {
|
||||
var metricsJSON string
|
||||
err := rows.Scan(&metricsJSON)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var metric ContainerMetric
|
||||
if err := json.Unmarshal([]byte(metricsJSON), &metric); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
metrics = append(metrics, metric)
|
||||
}
|
||||
return metrics, nil
|
||||
}
|
||||
|
||||
func (db *DB) GetAllMetricsContainer(containerName string) ([]ContainerMetric, error) {
|
||||
name := strings.TrimPrefix(containerName, "/")
|
||||
parts := strings.Split(name, "-")
|
||||
if len(parts) > 1 {
|
||||
containerName = strings.Join(parts[:len(parts)-1], "-")
|
||||
}
|
||||
|
||||
query := `
|
||||
WITH recent_metrics AS (
|
||||
SELECT metrics_json
|
||||
FROM container_metrics
|
||||
WHERE container_name LIKE ? || '%'
|
||||
ORDER BY timestamp DESC
|
||||
)
|
||||
SELECT metrics_json FROM recent_metrics ORDER BY json_extract(metrics_json, '$.timestamp') ASC
|
||||
`
|
||||
rows, err := db.Query(query, containerName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var metrics []ContainerMetric
|
||||
for rows.Next() {
|
||||
var metricsJSON string
|
||||
err := rows.Scan(&metricsJSON)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var metric ContainerMetric
|
||||
if err := json.Unmarshal([]byte(metricsJSON), &metric); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
metrics = append(metrics, metric)
|
||||
}
|
||||
return metrics, nil
|
||||
}
|
||||
|
||||
type ContainerMetric struct {
|
||||
Timestamp string `json:"timestamp"`
|
||||
CPU float64 `json:"CPU"`
|
||||
Memory MemoryMetric `json:"Memory"`
|
||||
Network NetworkMetric `json:"Network"`
|
||||
BlockIO BlockIOMetric `json:"BlockIO"`
|
||||
Container string `json:"Container"`
|
||||
ID string `json:"ID"`
|
||||
Name string `json:"Name"`
|
||||
}
|
||||
|
||||
type MemoryMetric struct {
|
||||
Percentage float64 `json:"percentage"`
|
||||
Used float64 `json:"used"`
|
||||
Total float64 `json:"total"`
|
||||
UsedUnit string `json:"usedUnit"`
|
||||
TotalUnit string `json:"totalUnit"`
|
||||
}
|
||||
|
||||
type NetworkMetric struct {
|
||||
Input float64 `json:"input"`
|
||||
Output float64 `json:"output"`
|
||||
InputUnit string `json:"inputUnit"`
|
||||
OutputUnit string `json:"outputUnit"`
|
||||
}
|
||||
|
||||
type BlockIOMetric struct {
|
||||
Read float64 `json:"read"`
|
||||
Write float64 `json:"write"`
|
||||
ReadUnit string `json:"readUnit"`
|
||||
WriteUnit string `json:"writeUnit"`
|
||||
}
|
||||
47
apps/monitoring/database/db.go
Normal file
47
apps/monitoring/database/db.go
Normal file
@@ -0,0 +1,47 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
)
|
||||
|
||||
type DB struct {
|
||||
*sql.DB
|
||||
}
|
||||
|
||||
func InitDB() (*DB, error) {
|
||||
db, err := sql.Open("sqlite3", "./monitoring.db")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Create metrics table if it doesn't exist
|
||||
_, err = db.Exec(`
|
||||
CREATE TABLE IF NOT EXISTS server_metrics (
|
||||
timestamp TEXT PRIMARY KEY,
|
||||
cpu REAL,
|
||||
cpu_model TEXT,
|
||||
cpu_cores INTEGER,
|
||||
cpu_physical_cores INTEGER,
|
||||
cpu_speed REAL,
|
||||
os TEXT,
|
||||
distro TEXT,
|
||||
kernel TEXT,
|
||||
arch TEXT,
|
||||
mem_used REAL,
|
||||
mem_used_gb REAL,
|
||||
mem_total REAL,
|
||||
uptime INTEGER,
|
||||
disk_used REAL,
|
||||
total_disk REAL,
|
||||
network_in REAL,
|
||||
network_out REAL
|
||||
)
|
||||
`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &DB{db}, nil
|
||||
}
|
||||
115
apps/monitoring/database/server.go
Normal file
115
apps/monitoring/database/server.go
Normal file
@@ -0,0 +1,115 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
)
|
||||
|
||||
type ServerMetric struct {
|
||||
Timestamp string `json:"timestamp"`
|
||||
CPU float64 `json:"cpu"`
|
||||
CPUModel string `json:"cpuModel"`
|
||||
CPUCores int32 `json:"cpuCores"`
|
||||
CPUPhysicalCores int32 `json:"cpuPhysicalCores"`
|
||||
CPUSpeed float64 `json:"cpuSpeed"`
|
||||
OS string `json:"os"`
|
||||
Distro string `json:"distro"`
|
||||
Kernel string `json:"kernel"`
|
||||
Arch string `json:"arch"`
|
||||
MemUsed float64 `json:"memUsed"`
|
||||
MemUsedGB float64 `json:"memUsedGB"`
|
||||
MemTotal float64 `json:"memTotal"`
|
||||
Uptime uint64 `json:"uptime"`
|
||||
DiskUsed float64 `json:"diskUsed"`
|
||||
TotalDisk float64 `json:"totalDisk"`
|
||||
NetworkIn float64 `json:"networkIn"`
|
||||
NetworkOut float64 `json:"networkOut"`
|
||||
}
|
||||
|
||||
func (db *DB) SaveMetric(metric ServerMetric) error {
|
||||
if metric.Timestamp == "" {
|
||||
metric.Timestamp = time.Now().UTC().Format(time.RFC3339Nano)
|
||||
}
|
||||
|
||||
_, err := db.Exec(`
|
||||
INSERT INTO server_metrics (timestamp, cpu, cpu_model, cpu_cores, cpu_physical_cores, cpu_speed, os, distro, kernel, arch, mem_used, mem_used_gb, mem_total, uptime, disk_used, total_disk, network_in, network_out)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`, metric.Timestamp, metric.CPU, metric.CPUModel, metric.CPUCores, metric.CPUPhysicalCores, metric.CPUSpeed, metric.OS, metric.Distro, metric.Kernel, metric.Arch, metric.MemUsed, metric.MemUsedGB, metric.MemTotal, metric.Uptime, metric.DiskUsed, metric.TotalDisk, metric.NetworkIn, metric.NetworkOut)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) GetMetricsInRange(start, end time.Time) ([]ServerMetric, error) {
|
||||
rows, err := db.Query(`
|
||||
SELECT timestamp, cpu, cpu_model, cpu_cores, cpu_physical_cores, cpu_speed, os, distro, kernel, arch, mem_used, mem_used_gb, mem_total, uptime, disk_used, total_disk, network_in, network_out
|
||||
FROM server_metrics
|
||||
WHERE timestamp BETWEEN ? AND ?
|
||||
ORDER BY timestamp ASC
|
||||
`, start.UTC().Format(time.RFC3339Nano), end.UTC().Format(time.RFC3339Nano))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var metrics []ServerMetric
|
||||
for rows.Next() {
|
||||
var m ServerMetric
|
||||
err := rows.Scan(&m.Timestamp, &m.CPU, &m.CPUModel, &m.CPUCores, &m.CPUPhysicalCores, &m.CPUSpeed, &m.OS, &m.Distro, &m.Kernel, &m.Arch, &m.MemUsed, &m.MemUsedGB, &m.MemTotal, &m.Uptime, &m.DiskUsed, &m.TotalDisk, &m.NetworkIn, &m.NetworkOut)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
metrics = append(metrics, m)
|
||||
}
|
||||
return metrics, nil
|
||||
}
|
||||
|
||||
func (db *DB) GetLastNMetrics(n int) ([]ServerMetric, error) {
|
||||
rows, err := db.Query(`
|
||||
WITH recent_metrics AS (
|
||||
SELECT timestamp, cpu, cpu_model, cpu_cores, cpu_physical_cores, cpu_speed, os, distro, kernel, arch, mem_used, mem_used_gb, mem_total, uptime, disk_used, total_disk, network_in, network_out
|
||||
FROM server_metrics
|
||||
ORDER BY timestamp DESC
|
||||
LIMIT ?
|
||||
)
|
||||
SELECT * FROM recent_metrics
|
||||
ORDER BY timestamp ASC
|
||||
`, n)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var metrics []ServerMetric
|
||||
for rows.Next() {
|
||||
var m ServerMetric
|
||||
err := rows.Scan(&m.Timestamp, &m.CPU, &m.CPUModel, &m.CPUCores, &m.CPUPhysicalCores, &m.CPUSpeed, &m.OS, &m.Distro, &m.Kernel, &m.Arch, &m.MemUsed, &m.MemUsedGB, &m.MemTotal, &m.Uptime, &m.DiskUsed, &m.TotalDisk, &m.NetworkIn, &m.NetworkOut)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
metrics = append(metrics, m)
|
||||
}
|
||||
return metrics, nil
|
||||
}
|
||||
|
||||
func (db *DB) GetAllMetrics() ([]ServerMetric, error) {
|
||||
rows, err := db.Query(`
|
||||
SELECT timestamp, cpu, cpu_model, cpu_cores, cpu_physical_cores, cpu_speed, os, distro, kernel, arch, mem_used, mem_used_gb, mem_total, uptime, disk_used, total_disk, network_in, network_out
|
||||
FROM server_metrics
|
||||
ORDER BY timestamp ASC
|
||||
`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var metrics []ServerMetric
|
||||
for rows.Next() {
|
||||
var m ServerMetric
|
||||
err := rows.Scan(&m.Timestamp, &m.CPU, &m.CPUModel, &m.CPUCores, &m.CPUPhysicalCores, &m.CPUSpeed, &m.OS, &m.Distro, &m.Kernel, &m.Arch, &m.MemUsed, &m.MemUsedGB, &m.MemTotal, &m.Uptime, &m.DiskUsed, &m.TotalDisk, &m.NetworkIn, &m.NetworkOut)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
metrics = append(metrics, m)
|
||||
}
|
||||
return metrics, nil
|
||||
}
|
||||
Reference in New Issue
Block a user