feat: 脚本/定时任务真实执行+输出捕获+执行历史 — 替换pending空壳为本地执行,采集stdout/stderr,更新统计;修复CronJobManager/Automation字段名不匹配(status→enabled,cron→cron_expr,command→content,language→type)
This commit is contained in:
attacker committed
1 parent
85bc2a8e05
commit
a4fd5fc178
19 files changed
+2715
-114
No files matched your search
@@ -0,0 +1,288 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"ezops/internal/models"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// AlertEngine 告警引擎 — 定时扫描 Agent 上报数据并触发告警 // Logan.Li
|
||||
type AlertEngine struct {
|
||||
db *gorm.DB
|
||||
}
|
||||
|
||||
func NewAlertEngine(db *gorm.DB) *AlertEngine {
|
||||
return &AlertEngine{db: db}
|
||||
}
|
||||
|
||||
// Check 每 30 秒调用一次
|
||||
func (e *AlertEngine) Check() {
|
||||
e.checkResourceAlerts()
|
||||
e.checkOfflineAlerts()
|
||||
e.checkPortAlerts()
|
||||
e.autoResolveAlerts()
|
||||
}
|
||||
|
||||
// 检查 CPU/内存/磁盘 阈值
|
||||
func (e *AlertEngine) checkResourceAlerts() {
|
||||
var rules []models.AgentRule
|
||||
e.db.Where("enabled = ? AND metric IN ?", true, []string{"cpu", "mem", "disk"}).Find(&rules)
|
||||
|
||||
for _, rule := range rules {
|
||||
// 获取目标主机最新上报
|
||||
var report models.AgentReport
|
||||
query := e.db.Model(&models.AgentReport{})
|
||||
if rule.Hostname != "" {
|
||||
query = query.Where("hostname = ?", rule.Hostname)
|
||||
}
|
||||
query.Order("id DESC").First(&report)
|
||||
if report.ID == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
var value float64
|
||||
switch rule.Metric {
|
||||
case "cpu":
|
||||
value = report.CPUUsage
|
||||
case "mem":
|
||||
value = report.MemUsage
|
||||
case "disk":
|
||||
var disks []models.DiskMetric
|
||||
json.Unmarshal([]byte(report.DiskInfo), &disks)
|
||||
for _, d := range disks {
|
||||
if d.UsagePercent > value {
|
||||
value = d.UsagePercent
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 检查是否超阈值
|
||||
if !e.compareOperator(rule.Operator, value, rule.Threshold) {
|
||||
continue
|
||||
}
|
||||
|
||||
// 避免重复告警：如果最近 10 分钟已有同规则未恢复告警，不重复
|
||||
var existing models.AgentAlert
|
||||
tenMinAgo := time.Now().Add(-10 * time.Minute)
|
||||
e.db.Where("rule_name = ? AND hostname = ? AND resolved = ? AND created_at > ?",
|
||||
rule.Name, report.Hostname, false, tenMinAgo).First(&existing)
|
||||
if existing.ID > 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
// 创建告警
|
||||
alert := models.AgentAlert{
|
||||
RuleName: rule.Name,
|
||||
Hostname: report.Hostname,
|
||||
Metric: rule.Metric,
|
||||
Severity: "warning",
|
||||
Message: fmt.Sprintf("%s: %s %.1f%% %s %.1f%%", report.Hostname, rule.Metric, value, rule.Operator, rule.Threshold),
|
||||
Value: value,
|
||||
Threshold: rule.Threshold,
|
||||
}
|
||||
if value > rule.Threshold*1.5 {
|
||||
alert.Severity = "critical"
|
||||
}
|
||||
e.db.Create(&alert)
|
||||
|
||||
// 更新规则最后触发时间
|
||||
now := time.Now()
|
||||
e.db.Model(&rule).Update("last_triggered", now)
|
||||
}
|
||||
}
|
||||
|
||||
// 检查主机离线（超过 3 分钟无上报）— 内置规则，无需手动创建
|
||||
func (e *AlertEngine) checkOfflineAlerts() {
|
||||
offlineThreshold := 3 * time.Minute
|
||||
|
||||
// 查所有曾上报的主机
|
||||
var hostnames []string
|
||||
e.db.Model(&models.AgentReport{}).Distinct("hostname").Pluck("hostname", &hostnames)
|
||||
|
||||
for _, hostname := range hostnames {
|
||||
var report models.AgentReport
|
||||
e.db.Where("hostname = ?", hostname).Order("id DESC").First(&report)
|
||||
if report.ID == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
isOffline := time.Since(report.ReportTime) > offlineThreshold
|
||||
|
||||
// 检查是否有未恢复的离线告警
|
||||
var existing models.AgentAlert
|
||||
e.db.Where("hostname = ? AND metric = ? AND resolved = ?",
|
||||
hostname, "offline", false).First(&existing)
|
||||
|
||||
if isOffline && existing.ID == 0 {
|
||||
alert := models.AgentAlert{
|
||||
RuleName: "主机离线",
|
||||
Hostname: hostname,
|
||||
Metric: "offline",
|
||||
Severity: "critical",
|
||||
Message: fmt.Sprintf("主机 %s 已超过 3 分钟无数据上报，可能已离线", hostname),
|
||||
Value: time.Since(report.ReportTime).Minutes(),
|
||||
Threshold: 3,
|
||||
}
|
||||
e.db.Create(&alert)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 检查端口消失（之前有的监听端口现在没了）
|
||||
func (e *AlertEngine) checkPortAlerts() {
|
||||
var rules []models.AgentRule
|
||||
e.db.Where("enabled = ? AND metric = ?", true, "port_down").Find(&rules)
|
||||
|
||||
for _, rule := range rules {
|
||||
if rule.Port == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
// 获取目标主机最新上报
|
||||
var report models.AgentReport
|
||||
query := e.db.Model(&models.AgentReport{})
|
||||
if rule.Hostname != "" {
|
||||
query = query.Where("hostname = ?", rule.Hostname)
|
||||
}
|
||||
query.Order("id DESC").First(&report)
|
||||
if report.ID == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
// 检查端口是否在最新上报中
|
||||
var ports []models.PortMetric
|
||||
json.Unmarshal([]byte(report.PortInfo), &ports)
|
||||
|
||||
portUp := false
|
||||
for _, p := range ports {
|
||||
if p.Port == rule.Port {
|
||||
portUp = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if portUp {
|
||||
continue // 端口正常
|
||||
}
|
||||
|
||||
// 端口消失了，检查是否已有未恢复告警
|
||||
var existing models.AgentAlert
|
||||
e.db.Where("rule_name = ? AND hostname = ? AND resolved = ?",
|
||||
rule.Name, report.Hostname, false).First(&existing)
|
||||
if existing.ID > 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
alert := models.AgentAlert{
|
||||
RuleName: rule.Name,
|
||||
Hostname: report.Hostname,
|
||||
Metric: "port_down",
|
||||
Severity: "critical",
|
||||
Message: fmt.Sprintf("主机 %s 端口 %d 服务疑似下线", report.Hostname, rule.Port),
|
||||
Value: float64(rule.Port),
|
||||
Threshold: float64(rule.Port),
|
||||
}
|
||||
e.db.Create(&alert)
|
||||
|
||||
now := time.Now()
|
||||
e.db.Model(&rule).Update("last_triggered", now)
|
||||
}
|
||||
}
|
||||
|
||||
// 自动恢复告警（指标恢复正常）
|
||||
func (e *AlertEngine) autoResolveAlerts() {
|
||||
var alerts []models.AgentAlert
|
||||
e.db.Where("resolved = ?", false).Find(&alerts)
|
||||
|
||||
for _, alert := range alerts {
|
||||
resolved := false
|
||||
|
||||
switch alert.Metric {
|
||||
case "offline":
|
||||
// 主机重新上报了
|
||||
var report models.AgentReport
|
||||
e.db.Where("hostname = ?", alert.Hostname).Order("id DESC").First(&report)
|
||||
if report.ID > 0 && time.Since(report.ReportTime) < 3*time.Minute {
|
||||
resolved = true
|
||||
}
|
||||
case "cpu", "mem", "disk":
|
||||
// 获取对应规则，检查是否恢复正常
|
||||
var rule models.AgentRule
|
||||
e.db.Where("name = ?", alert.RuleName).First(&rule)
|
||||
if rule.ID == 0 {
|
||||
continue
|
||||
}
|
||||
var report models.AgentReport
|
||||
if rule.Hostname != "" {
|
||||
e.db.Where("hostname = ?", alert.Hostname).Order("id DESC").First(&report)
|
||||
} else {
|
||||
e.db.Where("hostname = ?", alert.Hostname).Order("id DESC").First(&report)
|
||||
}
|
||||
if report.ID == 0 {
|
||||
continue
|
||||
}
|
||||
var value float64
|
||||
switch alert.Metric {
|
||||
case "cpu":
|
||||
value = report.CPUUsage
|
||||
case "mem":
|
||||
value = report.MemUsage
|
||||
case "disk":
|
||||
var disks []models.DiskMetric
|
||||
json.Unmarshal([]byte(report.DiskInfo), &disks)
|
||||
for _, d := range disks {
|
||||
if d.UsagePercent > value {
|
||||
value = d.UsagePercent
|
||||
}
|
||||
}
|
||||
}
|
||||
if !e.compareOperator(rule.Operator, value, rule.Threshold) {
|
||||
resolved = true
|
||||
}
|
||||
case "port_down":
|
||||
var rule models.AgentRule
|
||||
e.db.Where("name = ?", alert.RuleName).First(&rule)
|
||||
if rule.ID > 0 {
|
||||
var report models.AgentReport
|
||||
e.db.Where("hostname = ?", alert.Hostname).Order("id DESC").First(&report)
|
||||
var ports []models.PortMetric
|
||||
json.Unmarshal([]byte(report.PortInfo), &ports)
|
||||
for _, p := range ports {
|
||||
if p.Port == rule.Port {
|
||||
resolved = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if resolved {
|
||||
now := time.Now()
|
||||
e.db.Model(&alert).Updates(map[string]interface{}{
|
||||
"resolved": true,
|
||||
"resolved_at": now,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (e *AlertEngine) compareOperator(op string, val, threshold float64) bool {
|
||||
switch op {
|
||||
case ">":
|
||||
return val > threshold
|
||||
case ">=":
|
||||
return val >= threshold
|
||||
case "<":
|
||||
return val < threshold
|
||||
case "<=":
|
||||
return val <= threshold
|
||||
case "==":
|
||||
return val == threshold
|
||||
default:
|
||||
return val > threshold
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user