feat: 监控告警体系统一重构 + 自动化任务调度引擎 + 监控大屏拆分
告警中心 - 新增统一告警(alert_unified)、监控告警规则(monitor_alert_rule)、指标目录(alert_metric_catalog) - 通知策略与消息分发(alert_notify_policy/dispatch/message)、静默规则 - 告警引擎按阈值/SNMP 拆分(alert_engine_monitor/snmp) 监控采集 - SNMP 常量/模板与历史指标(snmp templates/constants、snmp_metrics_history、snmp_collect) - 中间件监控 Redis 采集与种子(middleware_monitor_redis/seed) - 站点探测 pkg(siteprobe) + 站点视图(site_view) 自动化任务 - cron 调度器(cronscheduler)、脚本运行器(scriptrunner)、系统任务(systask)及运行记录(cron_job_run、system_task) 审计与凭证 - 审计设置(audit_setting) + 历史清理(auditcleaner) 前端 - 监控页面(host/middleware/site/snmp)拆分为子模块并大屏化 - 告警中心多 Tab(规则/指标目录/通知策略/静默)、任务管理组件重构、Agent CLI Markdown 渲染 - 移除旧脚本卡片/详情/日志组件，统一为 ScriptTable/ScriptCodeEditor/TerminalOutput Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
LoganandClaude committed
1 parent
092d0e4b9e
commit
471353e3d7
190 files changed
+16640
-3484
No files matched your search
@@ -0,0 +1,277 @@
|
||||
package siteprobe
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"ezops/model"
|
||||
)
|
||||
|
||||
// ProbeOutcome 单次拨测结果 // Logan.Li
|
||||
type ProbeOutcome struct {
|
||||
Status model.SiteResultStatus
|
||||
HTTPStatus int
|
||||
ResponseTime float64
|
||||
TotalTime float64
|
||||
RemoteIP string
|
||||
ErrorMessage string
|
||||
// HTTPS 证书信息 // Logan.Li
|
||||
SSLCertExpiry *time.Time
|
||||
SSLCertIssuer string
|
||||
SSLCertSubject string
|
||||
SSLDaysRemaining int
|
||||
}
|
||||
|
||||
// Execute 按任务类型执行拨测 // Logan.Li
|
||||
func Execute(ctx context.Context, task model.SiteTask) ProbeOutcome {
|
||||
timeout := time.Duration(task.Timeout) * time.Second
|
||||
if timeout <= 0 {
|
||||
timeout = 10 * time.Second
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
|
||||
taskType := strings.ToLower(string(task.TaskType))
|
||||
if taskType == "" {
|
||||
taskType = inferTaskType(task)
|
||||
}
|
||||
|
||||
switch taskType {
|
||||
case "ssl":
|
||||
return probeSSL(ctx, task)
|
||||
case "tcp":
|
||||
return probeTCP(ctx, task.URL)
|
||||
case "dns":
|
||||
return probeDNS(ctx, task.URL, task.DNSResolver)
|
||||
case "ping":
|
||||
return probePing(ctx, task.URL, task.Timeout)
|
||||
case "http", "https":
|
||||
return probeHTTP(ctx, task)
|
||||
default:
|
||||
return probeHTTP(ctx, task)
|
||||
}
|
||||
}
|
||||
|
||||
// InferTaskTypePublic 推断任务类型（供列表修复存量数据）// Logan.Li
|
||||
func InferTaskTypePublic(task model.SiteTask) string {
|
||||
return inferTaskType(task)
|
||||
}
|
||||
|
||||
func inferTaskType(task model.SiteTask) string {
|
||||
name := strings.ToLower(task.Name)
|
||||
switch {
|
||||
case strings.Contains(name, "tcp"):
|
||||
return "tcp"
|
||||
case strings.Contains(name, "dns"):
|
||||
return "dns"
|
||||
case strings.Contains(name, "ping"):
|
||||
return "ping"
|
||||
case strings.Contains(name, "ssl"):
|
||||
return "ssl"
|
||||
default:
|
||||
return "http"
|
||||
}
|
||||
}
|
||||
|
||||
func probeHTTP(ctx context.Context, task model.SiteTask) ProbeOutcome {
|
||||
start := time.Now()
|
||||
url := task.URL
|
||||
if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") {
|
||||
url = "http://" + url
|
||||
}
|
||||
|
||||
transport := &http.Transport{
|
||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: !task.VerifySSL},
|
||||
}
|
||||
client := &http.Client{
|
||||
Timeout: time.Until(deadlineFromCtx(ctx)),
|
||||
Transport: transport,
|
||||
}
|
||||
if !task.FollowRedirect {
|
||||
client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
|
||||
return http.ErrUseLastResponse
|
||||
}
|
||||
}
|
||||
|
||||
method := strings.ToUpper(string(task.Method))
|
||||
if method == "" {
|
||||
method = http.MethodGet
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, method, url, nil)
|
||||
if err != nil {
|
||||
return failOutcome(start, err.Error())
|
||||
}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
elapsed := msSince(start)
|
||||
if err != nil {
|
||||
if ctx.Err() == context.DeadlineExceeded {
|
||||
return ProbeOutcome{Status: model.SiteResultTimeout, TotalTime: elapsed, ResponseTime: elapsed, ErrorMessage: "请求超时"}
|
||||
}
|
||||
return failOutcome(start, err.Error())
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
statusOK := resp.StatusCode == task.ExpectedStatus
|
||||
if task.ExpectedStatus == 0 {
|
||||
statusOK = resp.StatusCode >= 200 && resp.StatusCode < 400
|
||||
}
|
||||
if !statusOK {
|
||||
return ProbeOutcome{
|
||||
Status: model.SiteResultFail,
|
||||
HTTPStatus: resp.StatusCode,
|
||||
ResponseTime: elapsed,
|
||||
TotalTime: elapsed,
|
||||
ErrorMessage: fmt.Sprintf("状态码 %d 不符合期望 %d", resp.StatusCode, task.ExpectedStatus),
|
||||
}
|
||||
}
|
||||
return ProbeOutcome{
|
||||
Status: model.SiteResultSuccess,
|
||||
HTTPStatus: resp.StatusCode,
|
||||
ResponseTime: elapsed,
|
||||
TotalTime: elapsed,
|
||||
}
|
||||
}
|
||||
|
||||
func probeTCP(ctx context.Context, target string) ProbeOutcome {
|
||||
start := time.Now()
|
||||
host, port, err := splitHostPort(target, "80")
|
||||
if err != nil {
|
||||
return failOutcome(start, err.Error())
|
||||
}
|
||||
d := net.Dialer{Timeout: time.Until(deadlineFromCtx(ctx))}
|
||||
conn, err := d.DialContext(ctx, "tcp", net.JoinHostPort(host, port))
|
||||
elapsed := msSince(start)
|
||||
if err != nil {
|
||||
if ctx.Err() == context.DeadlineExceeded {
|
||||
return ProbeOutcome{Status: model.SiteResultTimeout, TotalTime: elapsed, ResponseTime: elapsed, ErrorMessage: "TCP 连接超时"}
|
||||
}
|
||||
return failOutcome(start, err.Error())
|
||||
}
|
||||
defer conn.Close()
|
||||
remote := ""
|
||||
if addr := conn.RemoteAddr(); addr != nil {
|
||||
remote = addr.String()
|
||||
}
|
||||
return ProbeOutcome{
|
||||
Status: model.SiteResultSuccess,
|
||||
ResponseTime: elapsed,
|
||||
TotalTime: elapsed,
|
||||
RemoteIP: remote,
|
||||
}
|
||||
}
|
||||
|
||||
func probeDNS(ctx context.Context, target, resolver string) ProbeOutcome {
|
||||
start := time.Now()
|
||||
domain := normalizeDomain(target)
|
||||
if domain == "" {
|
||||
return failOutcome(start, "域名不能为空")
|
||||
}
|
||||
r := &net.Resolver{}
|
||||
if resolver != "" {
|
||||
r = &net.Resolver{
|
||||
PreferGo: true,
|
||||
Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
|
||||
d := net.Dialer{Timeout: time.Until(deadlineFromCtx(ctx))}
|
||||
return d.DialContext(ctx, network, resolver+":53")
|
||||
},
|
||||
}
|
||||
}
|
||||
addrs, err := r.LookupHost(ctx, domain)
|
||||
elapsed := msSince(start)
|
||||
if err != nil {
|
||||
if ctx.Err() == context.DeadlineExceeded {
|
||||
return ProbeOutcome{Status: model.SiteResultTimeout, TotalTime: elapsed, ResponseTime: elapsed, ErrorMessage: "DNS 解析超时"}
|
||||
}
|
||||
return failOutcome(start, err.Error())
|
||||
}
|
||||
return ProbeOutcome{
|
||||
Status: model.SiteResultSuccess,
|
||||
ResponseTime: elapsed,
|
||||
TotalTime: elapsed,
|
||||
RemoteIP: strings.Join(addrs, ","),
|
||||
}
|
||||
}
|
||||
|
||||
func probePing(ctx context.Context, target string, timeoutSec int) ProbeOutcome {
|
||||
start := time.Now()
|
||||
host := normalizeDomain(target)
|
||||
if host == "" {
|
||||
return failOutcome(start, "主机不能为空")
|
||||
}
|
||||
if timeoutSec <= 0 {
|
||||
timeoutSec = 5
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, "ping", "-c", "1", "-W", strconv.Itoa(timeoutSec), host)
|
||||
out, err := cmd.CombinedOutput()
|
||||
elapsed := msSince(start)
|
||||
if err != nil {
|
||||
msg := strings.TrimSpace(string(out))
|
||||
if msg == "" {
|
||||
msg = err.Error()
|
||||
}
|
||||
if ctx.Err() == context.DeadlineExceeded {
|
||||
return ProbeOutcome{Status: model.SiteResultTimeout, TotalTime: elapsed, ResponseTime: elapsed, ErrorMessage: "PING 超时"}
|
||||
}
|
||||
return ProbeOutcome{Status: model.SiteResultFail, TotalTime: elapsed, ResponseTime: elapsed, ErrorMessage: msg}
|
||||
}
|
||||
if m := regexp.MustCompile(`time[=<]([\d.]+)\s*ms`).FindStringSubmatch(string(out)); len(m) == 2 {
|
||||
if v, parseErr := strconv.ParseFloat(m[1], 64); parseErr == nil {
|
||||
elapsed = v
|
||||
}
|
||||
}
|
||||
return ProbeOutcome{Status: model.SiteResultSuccess, ResponseTime: elapsed, TotalTime: elapsed, RemoteIP: host}
|
||||
}
|
||||
|
||||
func splitHostPort(raw, defaultPort string) (string, string, error) {
|
||||
raw = strings.TrimSpace(raw)
|
||||
for _, prefix := range []string{"ssl://", "tcp://", "http://", "https://"} {
|
||||
raw = strings.TrimPrefix(raw, prefix)
|
||||
}
|
||||
if host, port, err := net.SplitHostPort(raw); err == nil {
|
||||
return host, port, nil
|
||||
}
|
||||
if raw == "" {
|
||||
return "", "", fmt.Errorf("TCP 地址不能为空")
|
||||
}
|
||||
return raw, defaultPort, nil
|
||||
}
|
||||
|
||||
func normalizeDomain(raw string) string {
|
||||
raw = strings.TrimSpace(raw)
|
||||
raw = strings.TrimPrefix(raw, "http://")
|
||||
raw = strings.TrimPrefix(raw, "https://")
|
||||
if host, _, err := net.SplitHostPort(raw); err == nil {
|
||||
return host
|
||||
}
|
||||
return strings.TrimSuffix(raw, "/")
|
||||
}
|
||||
|
||||
func failOutcome(start time.Time, msg string) ProbeOutcome {
|
||||
elapsed := msSince(start)
|
||||
return ProbeOutcome{
|
||||
Status: model.SiteResultFail,
|
||||
ResponseTime: elapsed,
|
||||
TotalTime: elapsed,
|
||||
ErrorMessage: msg,
|
||||
}
|
||||
}
|
||||
|
||||
func msSince(start time.Time) float64 {
|
||||
return float64(time.Since(start).Milliseconds())
|
||||
}
|
||||
|
||||
func deadlineFromCtx(ctx context.Context) time.Time {
|
||||
if d, ok := ctx.Deadline(); ok {
|
||||
return d
|
||||
}
|
||||
return time.Now().Add(10 * time.Second)
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package siteprobe
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"ezops/model"
|
||||
)
|
||||
|
||||
// parseSSLTarget 解析 SSL 拨测地址，支持 example.com:443 / https://example.com // Logan.Li
|
||||
func parseSSLTarget(raw string) (host, port string, err error) {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return "", "", fmt.Errorf("SSL 地址不能为空")
|
||||
}
|
||||
lower := strings.ToLower(raw)
|
||||
// 兼容 https:example.com（单斜杠写法）
|
||||
if strings.HasPrefix(lower, "https:") && !strings.HasPrefix(lower, "https://") {
|
||||
raw = "https://" + strings.TrimSpace(raw[len("https:"):])
|
||||
lower = strings.ToLower(raw)
|
||||
}
|
||||
if strings.HasPrefix(lower, "http://") || strings.HasPrefix(lower, "https://") || strings.HasPrefix(lower, "ssl://") {
|
||||
u, parseErr := url.Parse(raw)
|
||||
if parseErr != nil || u.Host == "" {
|
||||
return "", "", fmt.Errorf("SSL 地址格式无效: %s", raw)
|
||||
}
|
||||
if h, p, splitErr := net.SplitHostPort(u.Host); splitErr == nil {
|
||||
return h, p, nil
|
||||
}
|
||||
defPort := "443"
|
||||
if u.Scheme == "http" {
|
||||
defPort = "80"
|
||||
}
|
||||
return u.Hostname(), defPort, nil
|
||||
}
|
||||
// 去掉路径后缀，如 example.com:443/foo
|
||||
if idx := strings.Index(raw, "/"); idx >= 0 {
|
||||
raw = raw[:idx]
|
||||
}
|
||||
return splitHostPort(raw, "443")
|
||||
}
|
||||
|
||||
// probeSSL 纯 TLS 握手检测证书（不发起 HTTP 请求）// Logan.Li
|
||||
func probeSSL(ctx context.Context, task model.SiteTask) ProbeOutcome {
|
||||
start := time.Now()
|
||||
host, port, err := parseSSLTarget(task.URL)
|
||||
if err != nil {
|
||||
return failOutcome(start, err.Error())
|
||||
}
|
||||
addr := net.JoinHostPort(host, port)
|
||||
d := &net.Dialer{Timeout: time.Until(deadlineFromCtx(ctx))}
|
||||
conn, err := tls.DialWithDialer(d, "tcp", addr, &tls.Config{
|
||||
ServerName: host,
|
||||
InsecureSkipVerify: !task.VerifySSL,
|
||||
})
|
||||
elapsed := msSince(start)
|
||||
if err != nil {
|
||||
if ctx.Err() == context.DeadlineExceeded {
|
||||
return ProbeOutcome{Status: model.SiteResultTimeout, TotalTime: elapsed, ResponseTime: elapsed, ErrorMessage: "TLS 握手超时"}
|
||||
}
|
||||
return failOutcome(start, fmt.Sprintf("TLS 握手失败: %s", err.Error()))
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
remote := ""
|
||||
if addr := conn.RemoteAddr(); addr != nil {
|
||||
remote = addr.String()
|
||||
}
|
||||
out := ProbeOutcome{
|
||||
Status: model.SiteResultSuccess,
|
||||
ResponseTime: elapsed,
|
||||
TotalTime: elapsed,
|
||||
RemoteIP: remote,
|
||||
}
|
||||
applyCertsToOutcome(&out, conn.ConnectionState().PeerCertificates)
|
||||
return out
|
||||
}
|
||||
|
||||
// applyCertsToOutcome 从 TLS 证书链填充到期信息 // Logan.Li
|
||||
func applyCertsToOutcome(out *ProbeOutcome, certs []*x509.Certificate) {
|
||||
if len(certs) == 0 {
|
||||
return
|
||||
}
|
||||
cert := certs[0]
|
||||
expiry := cert.NotAfter
|
||||
days := int(time.Until(expiry).Hours() / 24)
|
||||
out.SSLCertExpiry = &expiry
|
||||
out.SSLDaysRemaining = days
|
||||
out.SSLCertIssuer = cert.Issuer.CommonName
|
||||
if out.SSLCertIssuer == "" {
|
||||
out.SSLCertIssuer = cert.Issuer.String()
|
||||
}
|
||||
out.SSLCertSubject = cert.Subject.CommonName
|
||||
if out.SSLCertSubject == "" {
|
||||
out.SSLCertSubject = cert.Subject.String()
|
||||
}
|
||||
}
|
||||
|
||||
// IsSSLProbeTask 是否为 SSL 证书拨测任务 // Logan.Li
|
||||
func IsSSLProbeTask(task model.SiteTask) bool {
|
||||
return strings.EqualFold(string(task.TaskType), "ssl")
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package siteprobe
|
||||
|
||||
import "testing"
|
||||
|
||||
// SSL 地址解析单测 // Logan.Li
|
||||
func TestParseSSLTarget(t *testing.T) {
|
||||
cases := []struct {
|
||||
raw string
|
||||
host string
|
||||
port string
|
||||
wantFail bool
|
||||
}{
|
||||
{"example.com:443", "example.com", "443", false},
|
||||
{"example.com", "example.com", "443", false},
|
||||
{"https://example.com", "example.com", "443", false},
|
||||
{"https://example.com:8443", "example.com", "8443", false},
|
||||
{"https://example.com/path", "example.com", "443", false},
|
||||
{"https:example.com", "example.com", "443", false},
|
||||
{"ssl://example.com:443", "example.com", "443", false},
|
||||
{"", "", "", true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
host, port, err := parseSSLTarget(c.raw)
|
||||
if c.wantFail {
|
||||
if err == nil {
|
||||
t.Fatalf("expected error for %q", c.raw)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("parseSSLTarget(%q): %v", c.raw, err)
|
||||
}
|
||||
if host != c.host || port != c.port {
|
||||
t.Fatalf("parseSSLTarget(%q) = %s:%s, want %s:%s", c.raw, host, port, c.host, c.port)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
package siteprobe
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"ezops/model"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// Worker 站点拨测后台采集器 // Logan.Li
|
||||
type Worker struct {
|
||||
db *gorm.DB
|
||||
stopCh chan struct{}
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
// NewWorker 创建拨测 worker
|
||||
func NewWorker(db *gorm.DB) *Worker {
|
||||
return &Worker{db: db, stopCh: make(chan struct{})}
|
||||
}
|
||||
|
||||
// Start 启动拨测循环
|
||||
func (w *Worker) Start(intervalSec int) {
|
||||
if intervalSec < 5 {
|
||||
intervalSec = 10
|
||||
}
|
||||
w.wg.Add(1)
|
||||
go func() {
|
||||
defer w.wg.Done()
|
||||
ticker := time.NewTicker(time.Duration(intervalSec) * time.Second)
|
||||
defer ticker.Stop()
|
||||
log.Printf("[SiteProbe] 拨测引擎启动，扫描间隔 %ds", intervalSec)
|
||||
w.runOnce()
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
w.runOnce()
|
||||
case <-w.stopCh:
|
||||
log.Printf("[SiteProbe] 拨测引擎停止")
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Stop 停止拨测
|
||||
func (w *Worker) Stop() {
|
||||
close(w.stopCh)
|
||||
w.wg.Wait()
|
||||
}
|
||||
|
||||
// RunTaskNow 立即执行单个任务拨测 // Logan.Li
|
||||
func (w *Worker) RunTaskNow(taskID uint) error {
|
||||
var task model.SiteTask
|
||||
if err := w.db.First(&task, taskID).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
w.probeTask(task)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Worker) runOnce() {
|
||||
var tasks []model.SiteTask
|
||||
w.db.Where("status = ?", model.SiteTaskStatusRunning).Find(&tasks)
|
||||
now := time.Now()
|
||||
for _, task := range tasks {
|
||||
if task.Frequency <= 0 {
|
||||
task.Frequency = 60
|
||||
}
|
||||
if task.LastCheckAt != nil && now.Sub(*task.LastCheckAt) < time.Duration(task.Frequency)*time.Second {
|
||||
continue
|
||||
}
|
||||
w.probeTask(task)
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) probeTask(task model.SiteTask) {
|
||||
if task.TaskType == "" {
|
||||
task.TaskType = model.SiteTaskType(inferTaskType(task))
|
||||
}
|
||||
outcome := Execute(context.Background(), task)
|
||||
now := time.Now()
|
||||
|
||||
result := model.SiteResult{
|
||||
TaskID: task.ID,
|
||||
Status: outcome.Status,
|
||||
HTTPStatusCode: outcome.HTTPStatus,
|
||||
ResponseTime: outcome.ResponseTime,
|
||||
TotalTime: outcome.TotalTime,
|
||||
RemoteIP: outcome.RemoteIP,
|
||||
ErrorMessage: outcome.ErrorMessage,
|
||||
SSLCertExpiry: outcome.SSLCertExpiry,
|
||||
SSLCertIssuer: outcome.SSLCertIssuer,
|
||||
SSLCertSubject: outcome.SSLCertSubject,
|
||||
SSLDaysRemaining: outcome.SSLDaysRemaining,
|
||||
CheckedAt: now,
|
||||
}
|
||||
w.db.Create(&result)
|
||||
|
||||
lastResult := "fail"
|
||||
if outcome.Status == model.SiteResultSuccess {
|
||||
lastResult = "success"
|
||||
}
|
||||
avg := outcome.ResponseTime
|
||||
if task.AvgResponse > 0 {
|
||||
avg = (task.AvgResponse + outcome.ResponseTime) / 2
|
||||
}
|
||||
w.db.Model(&model.SiteTask{}).Where("id = ?", task.ID).Updates(map[string]interface{}{
|
||||
"task_type": task.TaskType,
|
||||
"last_result": lastResult,
|
||||
"last_check_at": now,
|
||||
"avg_response": avg,
|
||||
})
|
||||
}
|
||||
Reference in new issue
Block a user